From 1dee284157675ceef7a57e6e166f719cae10ff3c Mon Sep 17 00:00:00 2001 From: lalit Date: Fri, 3 May 2024 12:25:53 +0530 Subject: [PATCH 01/33] fix get user app problem in search page --- db-connector.go | 49 ++++++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/db-connector.go b/db-connector.go index 1ea5cce..ebc905c 100755 --- a/db-connector.go +++ b/db-connector.go @@ -6106,8 +6106,6 @@ func fixAppAppend(allApps []WorkflowApp, innerApp WorkflowApp) ([]WorkflowApp, W func GetUserApps(ctx context.Context, userId string) ([]WorkflowApp, error) { wrapper := []WorkflowApp{} - var err error - cacheKey := fmt.Sprintf("userapps-%s", userId) if project.CacheDb { cache, err := GetCache(ctx, cacheKey) @@ -6142,11 +6140,11 @@ func GetUserApps(ctx context.Context, userId string) ([]WorkflowApp, error) { { "match": map[string]interface{}{ "contributors": userId, - }, }, }, }, "minimum_should_match": 1, + }, }, } @@ -6172,9 +6170,8 @@ func GetUserApps(ctx context.Context, userId string) ([]WorkflowApp, error) { } if res.StatusCode != 200 && res.StatusCode != 201 { - return []WorkflowApp{}, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode)) + return []WorkflowApp{}, fmt.Errorf("bad statuscode: %d", res.StatusCode) } - respBody, err := ioutil.ReadAll(res.Body) if err != nil { return []WorkflowApp{}, err @@ -6191,38 +6188,44 @@ func GetUserApps(ctx context.Context, userId string) ([]WorkflowApp, error) { userApps = append(userApps, innerApp) } } else { - cursorStr := "" - query := datastore.NewQuery(indexName).Filter("owner =", userId).Filter("contributors IN", []string{userId}).Order("-edited") - for { - it := project.Dbclient.Run(ctx, query) + uniqueDocs := make(map[string]bool) + + query1 := datastore.NewQuery(indexName). + Filter("owner =", userId) + it := project.Dbclient.Run(ctx, query1) for { innerApp := WorkflowApp{} _, err := it.Next(&innerApp) + if err == iterator.Done { + break + } if err != nil { + log.Printf("[ERROR] Failed fetching results: %v", err) break } userApps = append(userApps, innerApp) + uniqueDocs[innerApp.ID] = true + } + + query2 := datastore.NewQuery(indexName). + Filter("contributors =", userId) + it = project.Dbclient.Run(ctx, query2) + for { + innerApp := WorkflowApp{} + _, err := it.Next(&innerApp) + if err == iterator.Done { + break } - if err != iterator.Done { - log.Printf("[ERROR] Failed fetching results: %v", err) - } - // Get the cursor for the next page of results. - nextCursor, err := it.Cursor() if err != nil { - log.Printf("Cursor error: %s", err) - break - } else { - nextStr := fmt.Sprintf("%s", nextCursor) - if cursorStr == nextStr { - // Break the loop if the cursor is the same as the previous one + log.Printf("[ERROR] Failed fetching results: %v", err) break } - - cursorStr = nextStr - query = query.Start(nextCursor) + if _, ok := uniqueDocs[innerApp.ID]; !ok { + userApps = append(userApps, innerApp) } } } + if project.CacheDb { data, err := json.Marshal(userApps) if err == nil { From f801274bd8235d9882ca3e16bad18aeac05e1567 Mon Sep 17 00:00:00 2001 From: satti-hari-krishna-reddy Date: Mon, 6 May 2024 11:21:56 +0530 Subject: [PATCH 02/33] made the delete webhook work even when the id contains webhook as a prefix --- shared.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/shared.go b/shared.go index c73e168..984744d 100755 --- a/shared.go +++ b/shared.go @@ -10069,6 +10069,11 @@ func HandleDeleteHook(resp http.ResponseWriter, request *http.Request) { fileId = location[4] } + // Check if fileId has the prefix "webhook_" + if strings.HasPrefix(fileId, "webhook_") { + fileId = strings.TrimPrefix(fileId, "webhook_") + } + if len(fileId) != 36 { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Workflow ID when deleting hook is not valid"}`)) From 01a9f3c7485eb0f6836df17bcb84d245d5f39a0a Mon Sep 17 00:00:00 2001 From: satti-hari-krishna-reddy Date: Mon, 6 May 2024 11:31:03 +0530 Subject: [PATCH 03/33] added a handler function to return all the active hooks --- shared.go | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/shared.go b/shared.go index 984744d..ff9b02c 100755 --- a/shared.go +++ b/shared.go @@ -4201,6 +4201,47 @@ func HandleGetSchedules(resp http.ResponseWriter, request *http.Request) { resp.Write(newjson) } +func HandleGetHooks(resp http.ResponseWriter, request *http.Request) { + cors := HandleCors(resp, request) + if cors { + return + } + + user, err := HandleApiAuthentication(resp, request) + if err != nil { + log.Printf("[WARNING] Api authentication failed in get hooks: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + if user.Role != "admin" { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Admin required"}`)) + return + } + + ctx := GetContext(request) + hooks, err := GetHooks(ctx, user.ActiveOrg.Id) + if err != nil { + log.Printf("[WARNING] Failed getting hooks: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Couldn't get hooks"}`)) + return + } + + newjson, err := json.Marshal(hooks) + if err != nil { + log.Printf("Failed unmarshal: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking environments"}`))) + return + } + + resp.WriteHeader(200) + resp.Write(newjson) +} + func HandleUpdateUser(resp http.ResponseWriter, request *http.Request) { cors := HandleCors(resp, request) if cors { From 7f9654f232b318c8cd896cb4113c4d73e82af070 Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Sat, 2 Mar 2024 16:25:36 +0530 Subject: [PATCH 04/33] added AddNewEndpoint method --- codegen.go | 118 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) diff --git a/codegen.go b/codegen.go index f181d76..d4abe0b 100755 --- a/codegen.go +++ b/codegen.go @@ -839,6 +839,121 @@ func MakePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet return functionname, data } +func AddNewEndPoint () WorkflowAppAction { + + parameters := []WorkflowAppActionParameter{} + + parameters = append(parameters, WorkflowAppActionParameter{ + Name: "method", + Description: "The http method to use", + Multiline: false, + Required: true, + Example: "GET , POST", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + + parameters = append(parameters, WorkflowAppActionParameter{ + Name: "headers", + Description: "Headers to use", + Multiline: true, + Required: false, + Example: "Content-Type: application/json", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + + parameters = append(parameters, WorkflowAppActionParameter{ + Name: "base_url", + Description: "the base part of your url ", + Multiline: false , + Required: true , + Example: "https://api.example.com", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + + parameters = append(parameters, WorkflowAppActionParameter{ + Name: "path", + Description: "the path to add to the base url", + Multiline: false, + Required: false, + Example: "users/details/profile/settings", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + + parameters = append(parameters, WorkflowAppActionParameter{ + Name: "username", + Description: "username for basic auth", + Multiline: false, + Required: false, + Example: "example_99", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + + parameters = append(parameters, WorkflowAppActionParameter{ + Name: "password", + Description: "password for the basic auth", + Multiline: false, + Required: false, + Example: "******", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + + parameters = append(parameters, WorkflowAppActionParameter{ + Name: "verify", + Description: "", + Multiline: false, + Required: false, + Example: "True", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + + parameters = append(parameters, WorkflowAppActionParameter{ + Name: "queries", + Description: "queries to be included in the url", + Multiline: true, + Required: false, + Example: "user_id: 123\ncategory: tech", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + + parameters = append(parameters, WorkflowAppActionParameter{ + Name: "req_body", + Description: "the path to add to the base url", + Multiline: true, + Required: false, + Example: '{"username": "example_user","email": "user@example.com"}', + Schema: SchemaDefinition{ + Type: "string", + }, + }) + + action := WorkflowAppAction{ + Description: "add a new endpoint for your app", + Name: "New_Endpoint", + NodeType: "action", + Parameters: parameters, + } + + return action + +} + + func GenerateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, WorkflowApp, []string, error) { api := WorkflowApp{} //log.Printf("%#v", swagger.Info) @@ -1382,6 +1497,9 @@ func GenerateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, pythonFunctions = append(pythonFunctions, curCode) } + action : = AddNewEndPoint() + api.Actions = append(api.Actions, action) + // Has to be here because its used differently above. // FIXING this is done during export instead? //log.Printf("OLDPATH: %s", actualPath) From 28b4acc0ee9deacb5e23ce820c48e618c3a86983 Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Sat, 2 Mar 2024 17:16:17 +0530 Subject: [PATCH 05/33] fixing my go syntax errors --- codegen.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codegen.go b/codegen.go index d4abe0b..41f0aab 100755 --- a/codegen.go +++ b/codegen.go @@ -936,7 +936,7 @@ func AddNewEndPoint () WorkflowAppAction { Description: "the path to add to the base url", Multiline: true, Required: false, - Example: '{"username": "example_user","email": "user@example.com"}', + Example: `{"username": "example_user", "email": "user@example.com"}`, Schema: SchemaDefinition{ Type: "string", }, @@ -1497,7 +1497,7 @@ func GenerateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, pythonFunctions = append(pythonFunctions, curCode) } - action : = AddNewEndPoint() + action := AddNewEndPoint() api.Actions = append(api.Actions, action) // Has to be here because its used differently above. From 0420fcc8aa57465c920d69f383efb023a40c00a6 Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Sun, 3 Mar 2024 20:30:33 +0530 Subject: [PATCH 06/33] added the function to make python code for new endpoint --- codegen.go | 131 +++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 122 insertions(+), 9 deletions(-) diff --git a/codegen.go b/codegen.go index 41f0aab..f9992e3 100755 --- a/codegen.go +++ b/codegen.go @@ -2,7 +2,6 @@ package shuffle import ( "archive/zip" - "sort" "bytes" "context" "crypto/md5" @@ -14,11 +13,13 @@ import ( "log" "os" "regexp" + "sort" "strconv" "strings" "cloud.google.com/go/storage" "github.com/frikky/kin-openapi/openapi3" + //"github.com/satori/go.uuid" "gopkg.in/yaml.v2" ) @@ -839,9 +840,118 @@ func MakePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet return functionname, data } -func AddNewEndPoint () WorkflowAppAction { +func NewEndPointPythonCode () string{ + + pythonCode := ` + def fix_url(self, url): + if "hhttp" in url: + url = url.replace("hhttp", "http") + + if "http:/" in url and not "http://" in url: + url = url.replace("http:/", "http://", -1) + if "https:/" in url and not "https://" in url: + url = url.replace("https:/", "https://", -1) + if "http:///" in url: + url = url.replace("http:///", "http://", -1) + if "https:///" in url: + url = url.replace("https:///", "https://", -1) + if not "http://" in url and not "http" in url: + url = f"http://{url}" + + return url + + def checkverify(self, verify): + if str(verify).lower().strip() == "false": + return False + elif verify == None: + return False + elif verify: + return True + elif not verify: + return False + else: + return True + + def is_valid_method(self, method): + valid_methods = ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD"] + method = method.upper() + + if method in valid_methods: + return method + else: + raise ValueError(f"Invalid HTTP method: {method}") + + def parse_content(self, headers): + parsed_headers = {} + if headers: + split_headers = headers.split("\n") + self.logger.info(split_headers) + for header in split_headers: + if ":" in header: + splititem = ":" + elif "=" in header: + splititem = "=" + else: + self.logger.info("Skipping header %s as its invalid" % header) + continue + + splitheader = header.split(splititem) + if len(splitheader) >= 2: + parsed_headers[splitheader[0].strip()] = splititem.join(splitheader[1:]).strip() + else: + self.logger.info("Skipping header %s with split %s cus only one item" % (header, splititem)) + continue + + return parsed_headers + + def new_endpoint(self, method="", headers="", base_url="", path="", username="", password="", verify=True, queries="", req_body=""): + url = self.fix_url(base_url) + + try: + method = self.is_valid_method(method) + except ValueError as e: + self.logger.error(e) + return {"error": str(e)} + + if path: + url += '/' + path + + parsed_headers = self.parse_content(headers) + parsed_queries = self.parse_content(queries) + + verify = self.checkverify(verify) + + if isinstance(req_body, dict): + try: + req_body = json.dumps(req_body) + except json.JSONDecodeError as e: + self.logger.error(f"error : {e}") + return {"error: Invalid JSON format for request body"} + + auth = None + if username or password: + if "Authorization" in headers: + pass + else: + auth = requests.auth.HTTPBasicAuth(username, password) + + try: + response = requests.request(method, url, headers=parsed_headers, params=parsed_queries, data=req_body, auth=auth, verify=verify) + response.raise_for_status() + return response.json() + + except requests.RequestException as e: + self.logger.error(f"Request failed: {e}") + return {"error": f"Request failed: {e}"} + ` + return pythonCode + +} + +func AddNewEndPoint () (WorkflowAppAction , string) { parameters := []WorkflowAppActionParameter{} + pyCode := NewEndPointPythonCode() parameters = append(parameters, WorkflowAppActionParameter{ Name: "method", @@ -859,7 +969,7 @@ func AddNewEndPoint () WorkflowAppAction { Description: "Headers to use", Multiline: true, Required: false, - Example: "Content-Type: application/json", + Example: "Content-Type:application/json\nAccept:application/json", Schema: SchemaDefinition{ Type: "string", }, @@ -925,7 +1035,7 @@ func AddNewEndPoint () WorkflowAppAction { Description: "queries to be included in the url", Multiline: true, Required: false, - Example: "user_id: 123\ncategory: tech", + Example: "user_id:123\ncategory:tech", Schema: SchemaDefinition{ Type: "string", }, @@ -944,12 +1054,13 @@ func AddNewEndPoint () WorkflowAppAction { action := WorkflowAppAction{ Description: "add a new endpoint for your app", - Name: "New_Endpoint", + Name: "new_endpoint", NodeType: "action", + Environment: "Shuffle", Parameters: parameters, } - return action + return action , pyCode } @@ -1434,7 +1545,11 @@ func GenerateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, Type: "string", }, }) - + + action, curCode := AddNewEndPoint() + api.Actions = append(api.Actions, action) + pythonFunctions = append(pythonFunctions, curCode) + // Fixing parameters with : newExtraParams := []WorkflowAppActionParameter{} newOptionalParams := []WorkflowAppActionParameter{} @@ -1497,8 +1612,6 @@ func GenerateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, pythonFunctions = append(pythonFunctions, curCode) } - action := AddNewEndPoint() - api.Actions = append(api.Actions, action) // Has to be here because its used differently above. // FIXING this is done during export instead? From 55002e5337a5e3d3f9a6c14635f064bab9e98cda Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Sun, 3 Mar 2024 22:45:43 +0530 Subject: [PATCH 07/33] trying fix python indentation issues --- codegen.go | 143 ++++++++++++++++++++++++++--------------------------- 1 file changed, 71 insertions(+), 72 deletions(-) diff --git a/codegen.go b/codegen.go index f9992e3..885800e 100755 --- a/codegen.go +++ b/codegen.go @@ -842,8 +842,7 @@ func MakePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet func NewEndPointPythonCode () string{ - pythonCode := ` - def fix_url(self, url): + pythonCode := ` def fix_url(self, url): if "hhttp" in url: url = url.replace("hhttp", "http") @@ -860,90 +859,90 @@ func NewEndPointPythonCode () string{ return url - def checkverify(self, verify): - if str(verify).lower().strip() == "false": - return False - elif verify == None: - return False - elif verify: - return True - elif not verify: - return False - else: - return True + def checkverify(self, verify): + if str(verify).lower().strip() == "false": + return False + elif verify == None: + return False + elif verify: + return True + elif not verify: + return False + else: + return True - def is_valid_method(self, method): - valid_methods = ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD"] - method = method.upper() + def is_valid_method(self, method): + valid_methods = ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD"] + method = method.upper() - if method in valid_methods: - return method - else: - raise ValueError(f"Invalid HTTP method: {method}") + if method in valid_methods: + return method + else: + raise ValueError(f"Invalid HTTP method: {method}") - def parse_content(self, headers): - parsed_headers = {} - if headers: - split_headers = headers.split("\n") - self.logger.info(split_headers) - for header in split_headers: - if ":" in header: - splititem = ":" - elif "=" in header: - splititem = "=" - else: - self.logger.info("Skipping header %s as its invalid" % header) - continue + def parse_content(self, headers): + parsed_headers = {} + if headers: + split_headers = headers.split("\n") + self.logger.info(split_headers) + for header in split_headers: + if ":" in header: + splititem = ":" + elif "=" in header: + splititem = "=" + else: + self.logger.info("Skipping header %s as its invalid" % header) + continue - splitheader = header.split(splititem) - if len(splitheader) >= 2: - parsed_headers[splitheader[0].strip()] = splititem.join(splitheader[1:]).strip() - else: - self.logger.info("Skipping header %s with split %s cus only one item" % (header, splititem)) - continue + splitheader = header.split(splititem) + if len(splitheader) >= 2: + parsed_headers[splitheader[0].strip()] = splititem.join(splitheader[1:]).strip() + else: + self.logger.info("Skipping header %s with split %s cus only one item" % (header, splititem)) + continue - return parsed_headers + return parsed_headers - def new_endpoint(self, method="", headers="", base_url="", path="", username="", password="", verify=True, queries="", req_body=""): - url = self.fix_url(base_url) + def new_endpoint(self, method="", headers="", base_url="", path="", username="", password="", verify=True, queries="", req_body=""): + url = self.fix_url(base_url) - try: - method = self.is_valid_method(method) - except ValueError as e: - self.logger.error(e) - return {"error": str(e)} + try: + method = self.is_valid_method(method) + except ValueError as e: + self.logger.error(e) + return {"error": str(e)} - if path: - url += '/' + path + if path: + url += '/' + path - parsed_headers = self.parse_content(headers) - parsed_queries = self.parse_content(queries) + parsed_headers = self.parse_content(headers) + parsed_queries = self.parse_content(queries) - verify = self.checkverify(verify) + verify = self.checkverify(verify) - if isinstance(req_body, dict): - try: - req_body = json.dumps(req_body) - except json.JSONDecodeError as e: - self.logger.error(f"error : {e}") - return {"error: Invalid JSON format for request body"} + if isinstance(req_body, dict): + try: + req_body = json.dumps(req_body) + except json.JSONDecodeError as e: + self.logger.error(f"error : {e}") + return {"error: Invalid JSON format for request body"} - auth = None - if username or password: - if "Authorization" in headers: - pass - else: - auth = requests.auth.HTTPBasicAuth(username, password) + auth = None + if username or password: + if "Authorization" in headers: + pass + else: + auth = requests.auth.HTTPBasicAuth(username, password) - try: - response = requests.request(method, url, headers=parsed_headers, params=parsed_queries, data=req_body, auth=auth, verify=verify) - response.raise_for_status() - return response.json() + try: + response = requests.request(method, url, headers=parsed_headers, params=parsed_queries, data=req_body, auth=auth, verify=verify) + response.raise_for_status() + return response.json() - except requests.RequestException as e: - self.logger.error(f"Request failed: {e}") - return {"error": f"Request failed: {e}"} - ` + except requests.RequestException as e: + self.logger.error(f"Request failed: {e}") + return {"error": f"Request failed: {e}"} + ` return pythonCode } From 442b9f1e1e427fa615c431aa817c1a21a87db36c Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Mon, 4 Mar 2024 12:44:22 +0530 Subject: [PATCH 08/33] added configuration --- codegen.go | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/codegen.go b/codegen.go index 885800e..b83d386 100755 --- a/codegen.go +++ b/codegen.go @@ -841,8 +841,7 @@ func MakePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet } func NewEndPointPythonCode () string{ - - pythonCode := ` def fix_url(self, url): + pythonCode := ` def fix_url(self, url): if "hhttp" in url: url = url.replace("hhttp", "http") @@ -904,11 +903,11 @@ func NewEndPointPythonCode () string{ return parsed_headers def new_endpoint(self, method="", headers="", base_url="", path="", username="", password="", verify=True, queries="", req_body=""): - url = self.fix_url(base_url) + url = self.fix_url(base_url) - try: + try: method = self.is_valid_method(method) - except ValueError as e: + except ValueError as e: self.logger.error(e) return {"error": str(e)} @@ -957,7 +956,8 @@ func AddNewEndPoint () (WorkflowAppAction , string) { Description: "The http method to use", Multiline: false, Required: true, - Example: "GET , POST", + Example: "GET , POST PUT ...", + Configuration: true, Schema: SchemaDefinition{ Type: "string", }, @@ -969,6 +969,7 @@ func AddNewEndPoint () (WorkflowAppAction , string) { Multiline: true, Required: false, Example: "Content-Type:application/json\nAccept:application/json", + Configuration: true, Schema: SchemaDefinition{ Type: "string", }, @@ -980,6 +981,7 @@ func AddNewEndPoint () (WorkflowAppAction , string) { Multiline: false , Required: true , Example: "https://api.example.com", + Configuration: true, Schema: SchemaDefinition{ Type: "string", }, @@ -991,6 +993,7 @@ func AddNewEndPoint () (WorkflowAppAction , string) { Multiline: false, Required: false, Example: "users/details/profile/settings", + Configuration: true, Schema: SchemaDefinition{ Type: "string", }, @@ -1002,6 +1005,7 @@ func AddNewEndPoint () (WorkflowAppAction , string) { Multiline: false, Required: false, Example: "example_99", + Configuration: true, Schema: SchemaDefinition{ Type: "string", }, @@ -1013,6 +1017,7 @@ func AddNewEndPoint () (WorkflowAppAction , string) { Multiline: false, Required: false, Example: "******", + Configuration: true, Schema: SchemaDefinition{ Type: "string", }, @@ -1024,6 +1029,7 @@ func AddNewEndPoint () (WorkflowAppAction , string) { Multiline: false, Required: false, Example: "True", + Configuration: true, Schema: SchemaDefinition{ Type: "string", }, @@ -1035,6 +1041,7 @@ func AddNewEndPoint () (WorkflowAppAction , string) { Multiline: true, Required: false, Example: "user_id:123\ncategory:tech", + Configuration: true, Schema: SchemaDefinition{ Type: "string", }, @@ -1046,6 +1053,7 @@ func AddNewEndPoint () (WorkflowAppAction , string) { Multiline: true, Required: false, Example: `{"username": "example_user", "email": "user@example.com"}`, + Configuration: true, Schema: SchemaDefinition{ Type: "string", }, @@ -1059,6 +1067,8 @@ func AddNewEndPoint () (WorkflowAppAction , string) { Parameters: parameters, } + action.Returns.Schema.Type = "string" + return action , pyCode } From 20adc7684956ab755d219e0bb5ad3c6ad8ffcb0d Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Mon, 4 Mar 2024 16:22:51 +0530 Subject: [PATCH 09/33] set config to true --- codegen.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/codegen.go b/codegen.go index b83d386..8820a21 100755 --- a/codegen.go +++ b/codegen.go @@ -969,7 +969,6 @@ func AddNewEndPoint () (WorkflowAppAction , string) { Multiline: true, Required: false, Example: "Content-Type:application/json\nAccept:application/json", - Configuration: true, Schema: SchemaDefinition{ Type: "string", }, @@ -993,7 +992,6 @@ func AddNewEndPoint () (WorkflowAppAction , string) { Multiline: false, Required: false, Example: "users/details/profile/settings", - Configuration: true, Schema: SchemaDefinition{ Type: "string", }, @@ -1005,7 +1003,6 @@ func AddNewEndPoint () (WorkflowAppAction , string) { Multiline: false, Required: false, Example: "example_99", - Configuration: true, Schema: SchemaDefinition{ Type: "string", }, @@ -1017,7 +1014,6 @@ func AddNewEndPoint () (WorkflowAppAction , string) { Multiline: false, Required: false, Example: "******", - Configuration: true, Schema: SchemaDefinition{ Type: "string", }, @@ -1029,7 +1025,6 @@ func AddNewEndPoint () (WorkflowAppAction , string) { Multiline: false, Required: false, Example: "True", - Configuration: true, Schema: SchemaDefinition{ Type: "string", }, @@ -1041,7 +1036,6 @@ func AddNewEndPoint () (WorkflowAppAction , string) { Multiline: true, Required: false, Example: "user_id:123\ncategory:tech", - Configuration: true, Schema: SchemaDefinition{ Type: "string", }, @@ -1053,7 +1047,6 @@ func AddNewEndPoint () (WorkflowAppAction , string) { Multiline: true, Required: false, Example: `{"username": "example_user", "email": "user@example.com"}`, - Configuration: true, Schema: SchemaDefinition{ Type: "string", }, From 2631846de216a3288294d3219bd87308341cff4a Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Sun, 17 Mar 2024 14:52:47 +0530 Subject: [PATCH 10/33] correcting order and fixing indentation in python code --- codegen.go | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/codegen.go b/codegen.go index 8820a21..9a7df61 100755 --- a/codegen.go +++ b/codegen.go @@ -841,7 +841,8 @@ func MakePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet } func NewEndPointPythonCode () string{ - pythonCode := ` def fix_url(self, url): + pythonCode := ` + def fix_url(self, url): if "hhttp" in url: url = url.replace("hhttp", "http") @@ -956,8 +957,20 @@ func AddNewEndPoint () (WorkflowAppAction , string) { Description: "The http method to use", Multiline: false, Required: true, + Configuration: true, Example: "GET , POST PUT ...", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + + parameters = append(parameters, WorkflowAppActionParameter{ + Name: "base_url", + Description: "the base part of your url ", + Multiline: false , + Required: true , Configuration: true, + Example: "https://api.example.com", Schema: SchemaDefinition{ Type: "string", }, @@ -974,18 +987,6 @@ func AddNewEndPoint () (WorkflowAppAction , string) { }, }) - parameters = append(parameters, WorkflowAppActionParameter{ - Name: "base_url", - Description: "the base part of your url ", - Multiline: false , - Required: true , - Example: "https://api.example.com", - Configuration: true, - Schema: SchemaDefinition{ - Type: "string", - }, - }) - parameters = append(parameters, WorkflowAppActionParameter{ Name: "path", Description: "the path to add to the base url", @@ -1548,9 +1549,6 @@ func GenerateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, }, }) - action, curCode := AddNewEndPoint() - api.Actions = append(api.Actions, action) - pythonFunctions = append(pythonFunctions, curCode) // Fixing parameters with : newExtraParams := []WorkflowAppActionParameter{} @@ -1626,6 +1624,10 @@ func GenerateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, //newPaths[actualPath] = path } + action, curCode := AddNewEndPoint() + api.Actions = append(api.Actions, action) + pythonFunctions = append(pythonFunctions, curCode) + return swagger, api, pythonFunctions, nil } From a8062e84613fcbfd3bb4b314856157eb37d73d02 Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Mon, 18 Mar 2024 13:40:22 +0000 Subject: [PATCH 11/33] fixing python indentation errors --- codegen.go | 257 +++++++++++++++++++++++++++++------------------------ 1 file changed, 143 insertions(+), 114 deletions(-) diff --git a/codegen.go b/codegen.go index 9a7df61..c906a24 100755 --- a/codegen.go +++ b/codegen.go @@ -840,108 +840,137 @@ func MakePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet return functionname, data } -func NewEndPointPythonCode () string{ +func NewEndPointPythonCode () string{ pythonCode := ` - def fix_url(self, url): - if "hhttp" in url: - url = url.replace("hhttp", "http") - - if "http:/" in url and not "http://" in url: - url = url.replace("http:/", "http://", -1) - if "https:/" in url and not "https://" in url: - url = url.replace("https:/", "https://", -1) - if "http:///" in url: - url = url.replace("http:///", "http://", -1) - if "https:///" in url: - url = url.replace("https:///", "https://", -1) - if not "http://" in url and not "http" in url: - url = f"http://{url}" - - return url - - def checkverify(self, verify): - if str(verify).lower().strip() == "false": - return False - elif verify == None: - return False - elif verify: - return True - elif not verify: - return False - else: - return True - - def is_valid_method(self, method): - valid_methods = ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD"] - method = method.upper() - - if method in valid_methods: - return method - else: - raise ValueError(f"Invalid HTTP method: {method}") - - def parse_content(self, headers): - parsed_headers = {} - if headers: - split_headers = headers.split("\n") - self.logger.info(split_headers) - for header in split_headers: - if ":" in header: - splititem = ":" - elif "=" in header: - splititem = "=" - else: - self.logger.info("Skipping header %s as its invalid" % header) - continue - - splitheader = header.split(splititem) - if len(splitheader) >= 2: - parsed_headers[splitheader[0].strip()] = splititem.join(splitheader[1:]).strip() - else: - self.logger.info("Skipping header %s with split %s cus only one item" % (header, splititem)) - continue - - return parsed_headers - - def new_endpoint(self, method="", headers="", base_url="", path="", username="", password="", verify=True, queries="", req_body=""): - url = self.fix_url(base_url) - - try: - method = self.is_valid_method(method) - except ValueError as e: - self.logger.error(e) - return {"error": str(e)} - - if path: - url += '/' + path - - parsed_headers = self.parse_content(headers) - parsed_queries = self.parse_content(queries) - - verify = self.checkverify(verify) - - if isinstance(req_body, dict): - try: - req_body = json.dumps(req_body) - except json.JSONDecodeError as e: - self.logger.error(f"error : {e}") - return {"error: Invalid JSON format for request body"} - - auth = None - if username or password: - if "Authorization" in headers: - pass - else: - auth = requests.auth.HTTPBasicAuth(username, password) - - try: - response = requests.request(method, url, headers=parsed_headers, params=parsed_queries, data=req_body, auth=auth, verify=verify) - response.raise_for_status() - return response.json() - - except requests.RequestException as e: - self.logger.error(f"Request failed: {e}") - return {"error": f"Request failed: {e}"} + def fix_url(self, url): + if "hhttp" in url: + url = url.replace("hhttp", "http") + + if "http:/" in url and not "http://" in url: + url = url.replace("http:/", "http://", -1) + if "https:/" in url and not "https://" in url: + url = url.replace("https:/", "https://", -1) + if "http:///" in url: + url = url.replace("http:///", "http://", -1) + if "https:///" in url: + url = url.replace("https:///", "https://", -1) + if not "http://" in url and not "http" in url: + url = f"http://{url}" + + return url + + def checkverify(self, verify): + if str(verify).lower().strip() == "false": + return False + elif verify == None: + return False + elif verify: + return True + elif not verify: + return False + else: + return True + + def is_valid_method(self, method): + valid_methods = ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD"] + method = method.upper() + + if method in valid_methods: + return method + else: + raise ValueError(f"Invalid HTTP method: {method}") + + def parse_headers(self, headers): + parsed_headers = {} + if headers: + split_headers = headers.split("\n") + self.logger.info(split_headers) + for header in split_headers: + if ":" in header: + splititem = ":" + elif "=" in header: + splititem = "=" + else: + self.logger.info("Skipping header %s as its invalid" % header) + continue + + splitheader = header.split(splititem) + if len(splitheader) >= 2: + parsed_headers[splitheader[0].strip()] = splititem.join(splitheader[1:]).strip() + else: + self.logger.info("Skipping header %s with split %s cus only one item" % (header, splititem)) + continue + + return parsed_headers + + def parse_queries(self, queries): + parsed_queries = {} + + if not queries: + return parsed_queries + + cleaned_queries = queries.strip() + + if not cleaned_queries: + return parsed_queries + + cleaned_queries = " ".join(cleaned_queries.split()) + splitted_queries = cleaned_queries.split("&") + self.logger.info(splitted_queries) + for query in splitted_queries: + + if "=" not in query: + self.logger.info("Skipping as there is no = in the query") + continue + key, value = query.split("=") + if not key.strip() or not value.strip(): + self.logger.info("Skipping because either key or value is not present in query") + continue + parsed_queries[key.strip()] = value.strip() + + return parsed_queries + + def new_endpoint(self, method="", base_url="", headers="", queries="", path="", username="", password="", verify=False, req_body=""): + url = self.fix_url(base_url) + + try: + method = self.is_valid_method(method) + except ValueError as e: + self.logger.error(e) + return {"error": str(e)} + + if path and not path.startswith('/'): + path = '/' + path + + url += path + + parsed_headers = self.parse_headers(headers) + parsed_queries = self.parse_queries(queries) + + verify = self.checkverify(verify) + + if isinstance(req_body, dict): + try: + req_body = json.dumps(req_body) + except json.JSONDecodeError as e: + self.logger.error(f"error : {e}") + return {"error: Invalid JSON format for request body"} + + auth = None + if username or password: + if "Authorization" in headers: + pass + else: + auth = requests.auth.HTTPBasicAuth(username, password) + + try: + response = requests.request(method, url, headers=parsed_headers, params=parsed_queries, data=req_body, auth=auth, verify=verify) + response.raise_for_status() + return response.json() + + except requests.RequestException as e: + self.logger.error(f"Request failed: {e}") + return {"error": f"Request failed: {e}"} ` return pythonCode @@ -950,7 +979,7 @@ func NewEndPointPythonCode () string{ func AddNewEndPoint () (WorkflowAppAction , string) { parameters := []WorkflowAppActionParameter{} - pyCode := NewEndPointPythonCode() + pyCode := NewEndPointPythonCode() parameters = append(parameters, WorkflowAppActionParameter{ Name: "method", @@ -987,12 +1016,23 @@ func AddNewEndPoint () (WorkflowAppAction , string) { }, }) + parameters = append(parameters, WorkflowAppActionParameter{ + Name: "queries", + Description: "queries to be included in the url", + Multiline: true, + Required: false, + Example: "user_id=123&category=tech", + Schema: SchemaDefinition{ + Type: "string", + }, + }) + parameters = append(parameters, WorkflowAppActionParameter{ Name: "path", Description: "the path to add to the base url", Multiline: false, Required: false, - Example: "users/details/profile/settings", + Example: "users/profile/settings", Schema: SchemaDefinition{ Type: "string", }, @@ -1031,17 +1071,6 @@ func AddNewEndPoint () (WorkflowAppAction , string) { }, }) - parameters = append(parameters, WorkflowAppActionParameter{ - Name: "queries", - Description: "queries to be included in the url", - Multiline: true, - Required: false, - Example: "user_id:123\ncategory:tech", - Schema: SchemaDefinition{ - Type: "string", - }, - }) - parameters = append(parameters, WorkflowAppActionParameter{ Name: "req_body", Description: "the path to add to the base url", From d8f5ea849122a1fb11c8bc2c557fe60f7daa759f Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Mon, 18 Mar 2024 19:33:33 +0530 Subject: [PATCH 12/33] refix : again fixing python indentation errors --- codegen.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codegen.go b/codegen.go index c906a24..a0c363e 100755 --- a/codegen.go +++ b/codegen.go @@ -939,7 +939,7 @@ func NewEndPointPythonCode () string{ self.logger.error(e) return {"error": str(e)} - if path and not path.startswith('/'): + if path and not path.startswith('/'): path = '/' + path url += path From 1c323115b9714bf4feda2c06c92cd55109ff5378 Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Mon, 18 Mar 2024 20:07:23 +0530 Subject: [PATCH 13/33] fixing annoying indentation --- codegen.go | 97 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 50 insertions(+), 47 deletions(-) diff --git a/codegen.go b/codegen.go index a0c363e..1ae5960 100755 --- a/codegen.go +++ b/codegen.go @@ -905,72 +905,75 @@ func NewEndPointPythonCode () string{ def parse_queries(self, queries): parsed_queries = {} - + if not queries: return parsed_queries - + cleaned_queries = queries.strip() - + if not cleaned_queries: return parsed_queries - + cleaned_queries = " ".join(cleaned_queries.split()) splitted_queries = cleaned_queries.split("&") self.logger.info(splitted_queries) for query in splitted_queries: - + if "=" not in query: self.logger.info("Skipping as there is no = in the query") continue key, value = query.split("=") if not key.strip() or not value.strip(): - self.logger.info("Skipping because either key or value is not present in query") + self.logger.info( + "Skipping because either key or value is not present in query" + ) continue parsed_queries[key.strip()] = value.strip() - - return parsed_queries + + return parsed_queries def new_endpoint(self, method="", base_url="", headers="", queries="", path="", username="", password="", verify=False, req_body=""): - url = self.fix_url(base_url) - - try: - method = self.is_valid_method(method) - except ValueError as e: - self.logger.error(e) - return {"error": str(e)} - - if path and not path.startswith('/'): - path = '/' + path - - url += path - - parsed_headers = self.parse_headers(headers) - parsed_queries = self.parse_queries(queries) - - verify = self.checkverify(verify) - - if isinstance(req_body, dict): - try: - req_body = json.dumps(req_body) - except json.JSONDecodeError as e: - self.logger.error(f"error : {e}") - return {"error: Invalid JSON format for request body"} - - auth = None - if username or password: - if "Authorization" in headers: - pass - else: - auth = requests.auth.HTTPBasicAuth(username, password) - - try: - response = requests.request(method, url, headers=parsed_headers, params=parsed_queries, data=req_body, auth=auth, verify=verify) - response.raise_for_status() - return response.json() + url = self.fix_url(base_url) + + try: + method = self.is_valid_method(method) + except ValueError as e: + self.logger.error(e) + return {"error": str(e)} + + if path and not path.startswith('/'): + path = '/' + path + + url += path + + parsed_headers = self.parse_headers(headers) + parsed_queries = self.parse_queries(queries) + + verify = self.checkverify(verify) + + if isinstance(req_body, dict): + try: + req_body = json.dumps(req_body) + except json.JSONDecodeError as e: + self.logger.error(f"error : {e}") + return {"error: Invalid JSON format for request body"} + + auth = None + if username or password: + if "Authorization" in headers: + pass + else: + auth = requests.auth.HTTPBasicAuth(username, password) + + try: + response = requests.request(method, url, headers=parsed_headers, params=parsed_queries, data=req_body, auth=auth, verify=verify) + response.raise_for_status() + return response.json() + + except requests.RequestException as e: + self.logger.error(f"Request failed: {e}") + return {"error": f"Request failed: {e}"} - except requests.RequestException as e: - self.logger.error(f"Request failed: {e}") - return {"error": f"Request failed: {e}"} ` return pythonCode From 906abebad62b47ad5f7393e96a05359dabc2186b Mon Sep 17 00:00:00 2001 From: Hari Krishna Date: Mon, 18 Mar 2024 20:13:49 +0530 Subject: [PATCH 14/33] fixing annoying indentation --- codegen.go | 139 ++++++++++++++++++++++++++--------------------------- 1 file changed, 69 insertions(+), 70 deletions(-) diff --git a/codegen.go b/codegen.go index 1ae5960..d75960a 100755 --- a/codegen.go +++ b/codegen.go @@ -902,80 +902,79 @@ func NewEndPointPythonCode () string{ continue return parsed_headers - + def parse_queries(self, queries): - parsed_queries = {} - - if not queries: - return parsed_queries - - cleaned_queries = queries.strip() - - if not cleaned_queries: - return parsed_queries - - cleaned_queries = " ".join(cleaned_queries.split()) - splitted_queries = cleaned_queries.split("&") - self.logger.info(splitted_queries) - for query in splitted_queries: - - if "=" not in query: - self.logger.info("Skipping as there is no = in the query") - continue - key, value = query.split("=") - if not key.strip() or not value.strip(): - self.logger.info( - "Skipping because either key or value is not present in query" - ) - continue - parsed_queries[key.strip()] = value.strip() - - return parsed_queries + parsed_queries = {} + + if not queries: + return parsed_queries + + cleaned_queries = queries.strip() + + if not cleaned_queries: + return parsed_queries + + cleaned_queries = " ".join(cleaned_queries.split()) + splitted_queries = cleaned_queries.split("&") + self.logger.info(splitted_queries) + for query in splitted_queries: + + if "=" not in query: + self.logger.info("Skipping as there is no = in the query") + continue + key, value = query.split("=") + if not key.strip() or not value.strip(): + self.logger.info( + "Skipping because either key or value is not present in query" + ) + continue + parsed_queries[key.strip()] = value.strip() + + return parsed_queries def new_endpoint(self, method="", base_url="", headers="", queries="", path="", username="", password="", verify=False, req_body=""): - url = self.fix_url(base_url) - - try: - method = self.is_valid_method(method) - except ValueError as e: - self.logger.error(e) - return {"error": str(e)} - - if path and not path.startswith('/'): - path = '/' + path - - url += path - - parsed_headers = self.parse_headers(headers) - parsed_queries = self.parse_queries(queries) - - verify = self.checkverify(verify) - - if isinstance(req_body, dict): - try: - req_body = json.dumps(req_body) - except json.JSONDecodeError as e: - self.logger.error(f"error : {e}") - return {"error: Invalid JSON format for request body"} - - auth = None - if username or password: - if "Authorization" in headers: - pass - else: - auth = requests.auth.HTTPBasicAuth(username, password) - - try: - response = requests.request(method, url, headers=parsed_headers, params=parsed_queries, data=req_body, auth=auth, verify=verify) - response.raise_for_status() - return response.json() - - except requests.RequestException as e: - self.logger.error(f"Request failed: {e}") - return {"error": f"Request failed: {e}"} - + url = self.fix_url(base_url) + + try: + method = self.is_valid_method(method) + except ValueError as e: + self.logger.error(e) + return {"error": str(e)} + + if path and not path.startswith('/'): + path = '/' + path + + url += path + + parsed_headers = self.parse_headers(headers) + parsed_queries = self.parse_queries(queries) + + verify = self.checkverify(verify) + + if isinstance(req_body, dict): + try: + req_body = json.dumps(req_body) + except json.JSONDecodeError as e: + self.logger.error(f"error : {e}") + return {"error: Invalid JSON format for request body"} + + auth = None + if username or password: + if "Authorization" in headers: + pass + else: + auth = requests.auth.HTTPBasicAuth(username, password) + + try: + response = requests.request(method, url, headers=parsed_headers, params=parsed_queries, data=req_body, auth=auth, verify=verify) + response.raise_for_status() + return response.json() + + except requests.RequestException as e: + self.logger.error(f"Request failed: {e}") + return {"error": f"Request failed: {e}"} ` - return pythonCode + return pythonCode } From edb7c801726c9cac15e502614178e38462bf1088 Mon Sep 17 00:00:00 2001 From: Frikky Date: Wed, 15 May 2024 14:34:44 +0200 Subject: [PATCH 15/33] Fixed file similarity search and changed to gpt-4o --- db-connector.go | 4 +++- files.go | 12 ++++++------ go.mod | 6 ++---- kms.go | 3 ++- shared.go | 4 +++- 5 files changed, 16 insertions(+), 13 deletions(-) diff --git a/db-connector.go b/db-connector.go index 005226e..283c3ec 100755 --- a/db-connector.go +++ b/db-connector.go @@ -2357,7 +2357,7 @@ func FindSimilarFile(ctx context.Context, md5, orgId string) ([]File, error) { } } } else { - query := datastore.NewQuery(nameKey).Filter("md5_sum =", md5).Limit(25) + query := datastore.NewQuery(nameKey).Filter("md5_sum =", md5).Limit(250) _, err := project.Dbclient.GetAll(ctx, query, &files) if err != nil { log.Printf("[WARNING] Failed getting deals for org: %s", orgId) @@ -8521,9 +8521,11 @@ func SetFile(ctx context.Context, file File) error { file.CreatedAt = timeNow } + /* if !strings.HasPrefix(file.Id, "file_") { return errors.New("Invalid file ID. Must start with file_") } + */ cacheKey := fmt.Sprintf("%s_%s", nameKey, file.Id) diff --git a/files.go b/files.go index bbfa036..e244bbd 100755 --- a/files.go +++ b/files.go @@ -391,14 +391,14 @@ func HandleDeleteFile(resp http.ResponseWriter, request *http.Request) { file.Status = "deleted" err = SetFile(ctx, *file) if err != nil { - log.Printf("[ERROR] Failed setting file to deleted") + log.Printf("[ERROR] Failed setting file to deleted: %s", err) resp.WriteHeader(500) resp.Write([]byte(`{"success": false, "reason": "Failed setting file to deleted"}`)) return } outputFiles, err := FindSimilarFile(ctx, file.Md5sum, file.OrgId) - log.Printf("[INFO] Found %d similar files", len(outputFiles)) + log.Printf("[INFO] Found %d similar files for Md5 '%s'", len(outputFiles), file.Md5sum) if len(outputFiles) > 0 { for _, item := range outputFiles { item.Status = "deleted" @@ -596,7 +596,7 @@ func HandleGetFileNamespace(resp http.ResponseWriter, request *http.Request) { // also be environment variables / input arguments filename, filenameOk := request.URL.Query()["filename"] if filenameOk && ArrayContains(reservedCategoryNames, namespace) { - log.Printf("[DEBUG] Filename '%s' in URL with reserved category name: %s. Listlength: %d", filename[0], namespace, len(fileResponse.List)) + //log.Printf("[DEBUG] Filename '%s' in URL with reserved category name: %s. Listlength: %d", filename[0], namespace, len(fileResponse.List)) // Load from Github repo https://github.com/Shuffle/standards filenameFound := false @@ -650,7 +650,7 @@ func HandleGetFileNamespace(resp http.ResponseWriter, request *http.Request) { continue } - log.Printf("\n\n\n[DEBUG] Decoded file '%s' with content:\n%s\n\n\n", *item.Path, string(decoded)) + //log.Printf("[DEBUG] Decoded Github file '%s' with content:\n%s", *item.Path, string(decoded)) timeNow := time.Now().Unix() fileId := "file_"+uuid.NewV4().String() @@ -698,7 +698,7 @@ func HandleGetFileNamespace(resp http.ResponseWriter, request *http.Request) { continue } - log.Printf("\n\n[DEBUG] Uploaded file %s with ID %s in category %#v\n\n", file.Filename, fileId, namespace) + log.Printf("[DEBUG] Uploaded file %#v with ID %s in category %#v", file.Filename, fileId, namespace) fileResponse.List = append(fileResponse.List, BaseFile{ Name: file.Filename, @@ -1641,7 +1641,7 @@ func HandleCreateFile(resp http.ResponseWriter, request *http.Request) { orgId := user.ActiveOrg.Id files, err := FindSimilarFilename(ctx, curfile.Filename, orgId) if err != nil { - log.Printf("[ERROR] Failed finding similar files: %s", err) + //log.Printf("[ERROR] Couldn't find any similar files: %s", err) } else { for _, item := range files { diff --git a/go.mod b/go.mod index c012da9..85a31be 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,9 @@ module github.com/shuffle/shuffle-shared -// Keep on 1.11 until AppEngine supports 1.17 or higher - go 1.11 //replace github.com/frikky/kin-openapi => ../kin-openapi -replace github.com/shuffle/opensearch-go => ../opensearch-go +//replace github.com/shuffle/opensearch-go => ../opensearch-go require ( cloud.google.com/go/datastore v1.4.0 @@ -16,7 +14,7 @@ require ( github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874 github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013 github.com/frikky/kin-openapi v0.41.0 - github.com/frikky/schemaless v0.0.9 + github.com/frikky/schemaless v0.0.11 github.com/go-git/go-billy/v5 v5.5.0 github.com/go-git/go-git/v5 v5.11.0 github.com/google/go-github/v28 v28.1.1 diff --git a/kms.go b/kms.go index 6566767..7562719 100644 --- a/kms.go +++ b/kms.go @@ -23,7 +23,8 @@ import ( openai "github.com/sashabaranov/go-openai" ) -var model = "gpt-4-turbo-preview" +//var model = "gpt-4-turbo-preview" +var model = "gpt-4o" func GetKmsCache(ctx context.Context, auth AppAuthenticationStorage, key string) (string, error) { //log.Printf("\n\n[DEBUG] Getting KMS cache for key %s\n\n", key) diff --git a/shared.go b/shared.go index d7fa45b..847b8ea 100755 --- a/shared.go +++ b/shared.go @@ -7080,7 +7080,9 @@ func HandleGetUsers(resp http.ResponseWriter, request *http.Request) { item.Executions = ExecutionInfo{} item.Limits = UserLimits{} item.PrivateApps = []WorkflowApp{} - item.MFA = MFAInfo{} + item.MFA = MFAInfo{ + Active: item.MFA.Active, + } if !user.SupportAccess { item.LoginInfo = []LoginInfo{} From 5c6c3f0a3f806b382ec1d4afbf179279a2129ed2 Mon Sep 17 00:00:00 2001 From: satti-hari-krishna-reddy Date: Wed, 15 May 2024 18:10:11 +0530 Subject: [PATCH 16/33] added list http methods to show --- codegen.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/codegen.go b/codegen.go index d75960a..f960efc 100755 --- a/codegen.go +++ b/codegen.go @@ -932,7 +932,7 @@ func NewEndPointPythonCode () string{ return parsed_queries - def new_endpoint(self, method="", base_url="", headers="", queries="", path="", username="", password="", verify=False, req_body=""): + def new_endpoint(self, method="", base_url="", headers="", queries="", path="", username="", password="", verify=False, request_body=""): url = self.fix_url(base_url) try: @@ -951,9 +951,9 @@ func NewEndPointPythonCode () string{ verify = self.checkverify(verify) - if isinstance(req_body, dict): + if isinstance(request_body, dict): try: - req_body = json.dumps(req_body) + request_body = json.dumps(request_body) except json.JSONDecodeError as e: self.logger.error(f"error : {e}") return {"error: Invalid JSON format for request body"} @@ -966,7 +966,7 @@ func NewEndPointPythonCode () string{ auth = requests.auth.HTTPBasicAuth(username, password) try: - response = requests.request(method, url, headers=parsed_headers, params=parsed_queries, data=req_body, auth=auth, verify=verify) + response = requests.request(method, url, headers=parsed_headers, params=parsed_queries, data=request_body, auth=auth, verify=verify) response.raise_for_status() return response.json() @@ -988,8 +988,8 @@ func AddNewEndPoint () (WorkflowAppAction , string) { Description: "The http method to use", Multiline: false, Required: true, - Configuration: true, - Example: "GET , POST PUT ...", + Options: []string{"GET","POST","PUT","DELETE","PATCH"}, + Example: "GET", Schema: SchemaDefinition{ Type: "string", }, @@ -998,9 +998,8 @@ func AddNewEndPoint () (WorkflowAppAction , string) { parameters = append(parameters, WorkflowAppActionParameter{ Name: "base_url", Description: "the base part of your url ", - Multiline: false , - Required: true , - Configuration: true, + Multiline: false, + Required: true, Example: "https://api.example.com", Schema: SchemaDefinition{ Type: "string", @@ -1064,17 +1063,18 @@ func AddNewEndPoint () (WorkflowAppAction , string) { parameters = append(parameters, WorkflowAppActionParameter{ Name: "verify", - Description: "", + Description: "Check certificate", Multiline: false, + Options: []string{"false","true"}, Required: false, - Example: "True", + Example: "False", Schema: SchemaDefinition{ Type: "string", }, }) parameters = append(parameters, WorkflowAppActionParameter{ - Name: "req_body", + Name: "request_body", Description: "the path to add to the base url", Multiline: true, Required: false, @@ -1086,7 +1086,7 @@ func AddNewEndPoint () (WorkflowAppAction , string) { action := WorkflowAppAction{ Description: "add a new endpoint for your app", - Name: "new_endpoint", + Name: "add_Endpoint", NodeType: "action", Environment: "Shuffle", Parameters: parameters, @@ -1094,7 +1094,7 @@ func AddNewEndPoint () (WorkflowAppAction , string) { action.Returns.Schema.Type = "string" - return action , pyCode + return action, pyCode } From 96051755f4d7f64a1559285b754ee1dceb26c512 Mon Sep 17 00:00:00 2001 From: satti-hari-krishna-reddy Date: Thu, 16 May 2024 15:48:08 +0530 Subject: [PATCH 17/33] removed basic auth and fixed indentation --- codegen.go | 120 +++++++++++++++++++++++------------------------------ 1 file changed, 51 insertions(+), 69 deletions(-) diff --git a/codegen.go b/codegen.go index f960efc..2dd5abe 100755 --- a/codegen.go +++ b/codegen.go @@ -840,7 +840,7 @@ func MakePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet return functionname, data } -func NewEndPointPythonCode () string{ +func GetCustomAction() string{ pythonCode := ` def fix_url(self, url): if "hhttp" in url: @@ -859,10 +859,11 @@ func NewEndPointPythonCode () string{ return url + def checkverify(self, verify): if str(verify).lower().strip() == "false": return False - elif verify == None: + elif verify is None: return False elif verify: return True @@ -871,6 +872,7 @@ func NewEndPointPythonCode () string{ else: return True + def is_valid_method(self, method): valid_methods = ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD"] method = method.upper() @@ -880,6 +882,7 @@ func NewEndPointPythonCode () string{ else: raise ValueError(f"Invalid HTTP method: {method}") + def parse_headers(self, headers): parsed_headers = {} if headers: @@ -896,29 +899,35 @@ func NewEndPointPythonCode () string{ splitheader = header.split(splititem) if len(splitheader) >= 2: - parsed_headers[splitheader[0].strip()] = splititem.join(splitheader[1:]).strip() + parsed_headers[splitheader[0].strip()] = splititem.join( + splitheader[1:] + ).strip() else: - self.logger.info("Skipping header %s with split %s cus only one item" % (header, splititem)) + self.logger.info( + "Skipping header %s with split %s cus only one item" + % (header, splititem) + ) continue return parsed_headers - + + def parse_queries(self, queries): parsed_queries = {} - + if not queries: return parsed_queries - + cleaned_queries = queries.strip() - + if not cleaned_queries: return parsed_queries - + cleaned_queries = " ".join(cleaned_queries.split()) splitted_queries = cleaned_queries.split("&") self.logger.info(splitted_queries) for query in splitted_queries: - + if "=" not in query: self.logger.info("Skipping as there is no = in the query") continue @@ -929,47 +938,41 @@ func NewEndPointPythonCode () string{ ) continue parsed_queries[key.strip()] = value.strip() - - return parsed_queries - def new_endpoint(self, method="", base_url="", headers="", queries="", path="", username="", password="", verify=False, request_body=""): - url = self.fix_url(base_url) - + return parsed_queries + + + def custom_action(self, method="", url="", headers="", queries="", path="", verify=False, body=""): + url = self.fix_url(url) + try: method = self.is_valid_method(method) except ValueError as e: self.logger.error(e) return {"error": str(e)} - + if path and not path.startswith('/'): path = '/' + path - + url += path - + parsed_headers = self.parse_headers(headers) parsed_queries = self.parse_queries(queries) - + verify = self.checkverify(verify) - - if isinstance(request_body, dict): + + if isinstance(body, dict): try: - request_body = json.dumps(request_body) + body = json.dumps(body) except json.JSONDecodeError as e: self.logger.error(f"error : {e}") return {"error: Invalid JSON format for request body"} - - auth = None - if username or password: - if "Authorization" in headers: - pass - else: - auth = requests.auth.HTTPBasicAuth(username, password) - + try: - response = requests.request(method, url, headers=parsed_headers, params=parsed_queries, data=request_body, auth=auth, verify=verify) + response = requests.request(method, url, headers=parsed_headers, params=parsed_queries, data=body, verify=verify) response.raise_for_status() return response.json() - + except requests.RequestException as e: self.logger.error(f"Request failed: {e}") return {"error": f"Request failed: {e}"} @@ -978,10 +981,10 @@ func NewEndPointPythonCode () string{ } -func AddNewEndPoint () (WorkflowAppAction , string) { +func AddCustomAction() (WorkflowAppAction , string) { parameters := []WorkflowAppActionParameter{} - pyCode := NewEndPointPythonCode() + pyCode := GetCustomAction() parameters = append(parameters, WorkflowAppActionParameter{ Name: "method", @@ -996,7 +999,7 @@ func AddNewEndPoint () (WorkflowAppAction , string) { }) parameters = append(parameters, WorkflowAppActionParameter{ - Name: "base_url", + Name: "url", Description: "the base part of your url ", Multiline: false, Required: true, @@ -1006,61 +1009,40 @@ func AddNewEndPoint () (WorkflowAppAction , string) { }, }) - parameters = append(parameters, WorkflowAppActionParameter{ - Name: "headers", - Description: "Headers to use", - Multiline: true, - Required: false, - Example: "Content-Type:application/json\nAccept:application/json", - Schema: SchemaDefinition{ - Type: "string", - }, - }) - - parameters = append(parameters, WorkflowAppActionParameter{ - Name: "queries", - Description: "queries to be included in the url", - Multiline: true, - Required: false, - Example: "user_id=123&category=tech", - Schema: SchemaDefinition{ - Type: "string", - }, - }) - parameters = append(parameters, WorkflowAppActionParameter{ Name: "path", Description: "the path to add to the base url", Multiline: false, Required: false, - Example: "users/profile/settings", + Example: "/users/profile", Schema: SchemaDefinition{ Type: "string", }, }) parameters = append(parameters, WorkflowAppActionParameter{ - Name: "username", - Description: "username for basic auth", - Multiline: false, + Name: "headers", + Description: "Headers to use", + Multiline: true, Required: false, - Example: "example_99", + Example: "Content-Type:application/json\nAccept:application/json", Schema: SchemaDefinition{ Type: "string", }, }) parameters = append(parameters, WorkflowAppActionParameter{ - Name: "password", - Description: "password for the basic auth", - Multiline: false, + Name: "queries", + Description: "queries to be included in the url", + Multiline: true, Required: false, - Example: "******", + Example: "user_id=123&category=tech", Schema: SchemaDefinition{ Type: "string", }, }) + parameters = append(parameters, WorkflowAppActionParameter{ Name: "verify", Description: "Check certificate", @@ -1074,7 +1056,7 @@ func AddNewEndPoint () (WorkflowAppAction , string) { }) parameters = append(parameters, WorkflowAppActionParameter{ - Name: "request_body", + Name: "body", Description: "the path to add to the base url", Multiline: true, Required: false, @@ -1086,7 +1068,7 @@ func AddNewEndPoint () (WorkflowAppAction , string) { action := WorkflowAppAction{ Description: "add a new endpoint for your app", - Name: "add_Endpoint", + Name: "custom_action", NodeType: "action", Environment: "Shuffle", Parameters: parameters, @@ -1655,7 +1637,7 @@ func GenerateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, //newPaths[actualPath] = path } - action, curCode := AddNewEndPoint() + action, curCode := AddCustomAction() api.Actions = append(api.Actions, action) pythonFunctions = append(pythonFunctions, curCode) From 151daf29d72ffdc6ed676a1dad82b4d60fe207f6 Mon Sep 17 00:00:00 2001 From: Frikky Date: Thu, 16 May 2024 23:09:34 +0200 Subject: [PATCH 18/33] Optimized workflow & app loading with X-Shuffle-Truncated header set. May add gzip more --- kms.go | 78 ++++++++++++++++++++++--------------------------------- shared.go | 36 +++++++++++++++++++++---- 2 files changed, 62 insertions(+), 52 deletions(-) diff --git a/kms.go b/kms.go index 7562719..19f5953 100644 --- a/kms.go +++ b/kms.go @@ -683,30 +683,7 @@ func RunSelfCorrectingRequest(action Action, status int, additionalInfo, outputB log.Printf("\n\nTOKENS (AUTOFIX API~): In: %d, Out: %d\n\n", (len(systemMessage)+len(inputData))/4, len(contentOutput)/4) - if strings.Contains(contentOutput, "```") { - // Handle ```json - start := strings.Index(contentOutput, "```json") - end := strings.Index(contentOutput, "```") - if start != -1 { - end = strings.Index(contentOutput[start+7:], "```") - } - - if start != -1 && end != -1 { - contentOutput = contentOutput[start+7 : end+7] - } - } - - if strings.Contains(contentOutput, "```") { - start := strings.Index(contentOutput, "```") - end := strings.Index(contentOutput[start+3:], "```") - if start != -1 { - end = strings.Index(contentOutput[start+3:], "```") - } - - if start != -1 && end != -1 { - contentOutput = contentOutput[start+3 : end+3] - } - } + contentOutput = FixContentOutput(contentOutput) log.Printf("[INFO] Autocorrect output: %s", contentOutput) @@ -926,29 +903,7 @@ func UpdateActionBody(action WorkflowAppAction) (string, error) { return "", err } - if strings.Contains(contentOutput, "```json") { - start := strings.Index(contentOutput, "```json") - end := strings.Index(contentOutput, "```") - if start != -1 { - end = strings.Index(contentOutput[start+8:], "```") - } - - if start != -1 && end != -1 { - contentOutput = contentOutput[start+7 : end+7] - } - } - - if strings.Contains(contentOutput, "```") { - start := strings.Index(contentOutput, "```") - end := strings.Index(contentOutput[start+3:], "```") - if start != -1 { - end = strings.Index(contentOutput[start+3:], "```") - } - - if start != -1 && end != -1 { - contentOutput = contentOutput[start+3 : end+3] - } - } + contentOutput = FixContentOutput(contentOutput) output := map[string]interface{}{} err = json.Unmarshal([]byte(contentOutput), &output) @@ -1214,3 +1169,32 @@ func uploadParameterBase(ctx context.Context, orgId, appId, actionName, paramNam return nil } + +func FixContentOutput(contentOutput string) string { + if strings.Contains(contentOutput, "```") { + // Handle ```json + start := strings.Index(contentOutput, "```json") + end := strings.Index(contentOutput, "```") + if start != -1 { + end = strings.Index(contentOutput[start+7:], "```") + } + + if start != -1 && end != -1 { + contentOutput = contentOutput[start+7 : end+7] + } + } + + if strings.Contains(contentOutput, "```") { + start := strings.Index(contentOutput, "```") + end := strings.Index(contentOutput[start+3:], "```") + if start != -1 { + end = strings.Index(contentOutput[start+3:], "```") + } + + if start != -1 && end != -1 { + contentOutput = contentOutput[start+3 : end+3] + } + } + + return contentOutput +} diff --git a/shared.go b/shared.go index 847b8ea..1a85cf4 100755 --- a/shared.go +++ b/shared.go @@ -3698,6 +3698,10 @@ func GetWorkflows(resp http.ResponseWriter, request *http.Request) { continue } + if project.Environment == "cloud" && workflow.ExecutionEnvironment == "onprem" { + continue + } + newActions := []Action{} for _, action := range workflow.Actions { // Removed because of exports. These are needed there. @@ -3707,18 +3711,40 @@ func GetWorkflows(resp http.ResponseWriter, request *http.Request) { newActions = append(newActions, action) } - workflow.Actions = newActions + //workflow.Actions = newActions + // Skipping these as they're related to onprem workflows in cloud (orborus) - if project.Environment == "cloud" && workflow.ExecutionEnvironment == "onprem" { - continue - } usecaseIds = append(usecaseIds, workflow.UsecaseIds...) newWorkflows = append(newWorkflows, workflow) } - workflows = newWorkflows + if project.Environment == "cloud" && len(newWorkflows) > 15 { + log.Printf("[WARNING] Removed workflow actions for user %s (%s) in org %s (%s)", user.Username, user.Id, user.ActiveOrg.Name, user.ActiveOrg.Id) + + for workflowIndex, _ := range newWorkflows { + newWorkflows[workflowIndex].Actions = []Action{} + newWorkflows[workflowIndex].Triggers = []Trigger{} + newWorkflows[workflowIndex].Branches = []Branch{} + newWorkflows[workflowIndex].VisualBranches = []Branch{} + newWorkflows[workflowIndex].Image = "" + + newWorkflows[workflowIndex].Description = "" + newWorkflows[workflowIndex].Blogpost = "" + + if len(newWorkflows[workflowIndex].Org) > 0 { + for orgIndex, _ := range newWorkflows[workflowIndex].Org { + newWorkflows[workflowIndex].Org[orgIndex].Image = "" + } + } + + newWorkflows[workflowIndex].ExecutingOrg.Image = "" + } + + // Add header that this is a limited response + resp.Header().Set("X-Shuffle-Truncated", "true") + } // Get the org as well to manage priorities // Only happens on first load, so it's like once per session~ From 9b4374867cf1d9d8956ff68353a69abb81a53fff Mon Sep 17 00:00:00 2001 From: satti-hari-krishna-reddy Date: Fri, 17 May 2024 21:24:49 +0530 Subject: [PATCH 19/33] added auto gen to support auth --- codegen.go | 178 +++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 153 insertions(+), 25 deletions(-) diff --git a/codegen.go b/codegen.go index 2dd5abe..a2e9200 100755 --- a/codegen.go +++ b/codegen.go @@ -840,8 +840,53 @@ func MakePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet return functionname, data } -func GetCustomAction() string{ - pythonCode := ` +func GetCustomActionCode(swagger *openapi3.Swagger, api WorkflowApp) string{ + + authenticationParameter := "" + authenticationSetup := "" + authenticationAddin := "" + + if swagger.Components.SecuritySchemes != nil { + if swagger.Components.SecuritySchemes["BearerAuth"] != nil { + authenticationParameter = ", apikey" + authenticationSetup = "if apikey != \" \" and not apikey.startswith(\"Bearer\"): parsed_headers[\"Authorization\"] = f\"Bearer {apikey}\"" + + } else if swagger.Components.SecuritySchemes["BasicAuth"] != nil { + authenticationParameter = ", username_basic, password_basic" + authenticationSetup = "auth=None\n if username_basic or password_basic:\n if \"Authorization\" not in headers and \"Basic\" not in headers and not \"Bearer\" in headers:\n auth = requests.auth.HTTPBasicAuth(username_basic, password_basic)" + authenticationAddin = ", auth=auth" + + } else if swagger.Components.SecuritySchemes["ApiKeyAuth"] != nil { + authenticationParameter = ", apikey" + + if swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.In == "header" { + + authenticationSetup = fmt.Sprintf(`if apikey != " ": parsed_headers["%s"] = apikey`, swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.Name) + + if len(swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.Description) > 0 { + trimmedDescription := strings.Trim(swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.Description, " ") + + authenticationSetup = fmt.Sprintf("if apikey != \" \":\n if apikey.startswith(\"%s\"):\n parsed_headers[\"%s\"] = apikey\n else:\n apikey = apikey.replace(\"%s\", \"\", -1).strip()\n parsed_headers[\"%s\"] = f\"%s{apikey}\"", trimmedDescription, swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.Name, swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.Description, swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.Name, swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.Description) + } + + } else if swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.In == "query" { + + authenticationSetup = fmt.Sprintf("if apikey != \" \": parsed_queries[\"%s\"] = requests.utils.quote(apikey)", swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.Name) + } + + } else if swagger.Components.SecuritySchemes["Oauth2"] != nil { + + authenticationParameter = ", access_token" + authenticationSetup = fmt.Sprintf("if access_token != \" \": parsed_headers[\"Authorization\"] = f\"Bearer {access_token}\"\n #parsed_headers[\"Content-Type\"] = \"application/json\"") + + } else if swagger.Components.SecuritySchemes["jwt"] != nil { + authenticationParameter = ", username_basic, password_basic" + authenticationSetup = fmt.Sprintf("authret = requests.get(f\"{url}%s\", headers=parsed_headers, auth=(username_basic, password_basic), verify=False)\n if 'access_token' in authret.text:\n parsed_headers[\"Authorization\"] = f\"Bearer {authret.json()['access_token']}\"\n elif 'jwt' in authret.text:\n parsed_headers[\"Authorization\"] = f\"Bearer {authret.json()['jwt']}\"\n elif 'accessToken' in authret.text:\n parsed_headers[\"Authorization\"] = f\"Bearer {authret.json()['accessToken']}\"\n else:\n parsed_headers[\"Authorization\"] = f\"Bearer {authret.text}\"\n print(f\"Found Bearer auth: {authret.text}\")", api.Authentication.TokenUri) + } + + } + + pythonCode := fmt.Sprintf(` def fix_url(self, url): if "hhttp" in url: url = url.replace("hhttp", "http") @@ -894,7 +939,6 @@ func GetCustomAction() string{ elif "=" in header: splititem = "=" else: - self.logger.info("Skipping header %s as its invalid" % header) continue splitheader = header.split(splititem) @@ -903,10 +947,6 @@ func GetCustomAction() string{ splitheader[1:] ).strip() else: - self.logger.info( - "Skipping header %s with split %s cus only one item" - % (header, splititem) - ) continue return parsed_headers @@ -942,7 +982,7 @@ func GetCustomAction() string{ return parsed_queries - def custom_action(self, method="", url="", headers="", queries="", path="", verify=False, body=""): + def custom_action(self%s, method="", url="", headers="", queries="", path="", ssl_verify=False, body=""): url = self.fix_url(url) try: @@ -959,7 +999,9 @@ func GetCustomAction() string{ parsed_headers = self.parse_headers(headers) parsed_queries = self.parse_queries(queries) - verify = self.checkverify(verify) + %s + + ssl_verify = self.checkverify(ssl_verify) if isinstance(body, dict): try: @@ -969,22 +1011,109 @@ func GetCustomAction() string{ return {"error: Invalid JSON format for request body"} try: - response = requests.request(method, url, headers=parsed_headers, params=parsed_queries, data=body, verify=verify) + response = requests.request(method, url, headers=parsed_headers, params=parsed_queries, data=body, verify=ssl_verify%s) response.raise_for_status() return response.json() except requests.RequestException as e: self.logger.error(f"Request failed: {e}") return {"error": f"Request failed: {e}"} - ` + `, authenticationParameter, authenticationSetup, authenticationAddin) + return pythonCode - } -func AddCustomAction() (WorkflowAppAction , string) { +func AddCustomAction(swagger *openapi3.Swagger, api WorkflowApp) (WorkflowAppAction, string) { parameters := []WorkflowAppActionParameter{} - pyCode := GetCustomAction() + pyCode := GetCustomActionCode(swagger, api) + + securitySchemes := swagger.Components.SecuritySchemes + if securitySchemes != nil { + + if securitySchemes["BearerAuth"] != nil { + + parameters = append(parameters, WorkflowAppActionParameter{ + Name: "apikey", + Description: "The apikey to use", + Multiline: false, + Required: true, + Example: "The API key to use. Space = skip", + Configuration: true, + Schema: SchemaDefinition{ + Type: "string", + }, + }) + } else if securitySchemes["ApiKeyAuth"] != nil { + + extraParam := WorkflowAppActionParameter{ + Name: "apikey", + Description: "The apikey to use", + Multiline: false, + Required: true, + Example: "**********", + Configuration: true, + Schema: SchemaDefinition{ + Type: "string", + }, + } + + if len(securitySchemes["ApiKeyAuth"].Value.Description) > 0 { + extraParam.Description = fmt.Sprintf("Start with %s", securitySchemes["ApiKeyAuth"].Value.Description) + } + + parameters = append(parameters, extraParam) + + } else if securitySchemes["jwt"] != nil { + + parameters = append(parameters, WorkflowAppActionParameter{ + Name: "username_basic", + Description: "The username to use", + Multiline: false, + Required: true, + Example: "The username to use", + Configuration: true, + Schema: SchemaDefinition{ + Type: "string", + }, + }) + parameters = append(parameters, WorkflowAppActionParameter{ + Name: "password_basic", + Description: "The password to use", + Multiline: false, + Required: true, + Example: "***********", + Configuration: true, + Schema: SchemaDefinition{ + Type: "string", + }, + }) + } else if securitySchemes["BasicAuth"] != nil { + + parameters = append(parameters, WorkflowAppActionParameter{ + Name: "username_basic", + Description: "The username to use", + Multiline: false, + Required: true, + Example: "The username to use", + Configuration: true, + Schema: SchemaDefinition{ + Type: "string", + }, + }) + parameters = append(parameters, WorkflowAppActionParameter{ + Name: "password_basic", + Description: "The password to use", + Multiline: false, + Required: true, + Example: "***********", + Configuration: true, + Schema: SchemaDefinition{ + Type: "string", + }, + }) + } + } parameters = append(parameters, WorkflowAppActionParameter{ Name: "method", @@ -1000,7 +1129,7 @@ func AddCustomAction() (WorkflowAppAction , string) { parameters = append(parameters, WorkflowAppActionParameter{ Name: "url", - Description: "the base part of your url ", + Description: "The URL of the API", Multiline: false, Required: true, Example: "https://api.example.com", @@ -1022,7 +1151,7 @@ func AddCustomAction() (WorkflowAppAction , string) { parameters = append(parameters, WorkflowAppActionParameter{ Name: "headers", - Description: "Headers to use", + Description: "Add or edit headers", Multiline: true, Required: false, Example: "Content-Type:application/json\nAccept:application/json", @@ -1033,10 +1162,10 @@ func AddCustomAction() (WorkflowAppAction , string) { parameters = append(parameters, WorkflowAppActionParameter{ Name: "queries", - Description: "queries to be included in the url", + Description: "Add or edit queries", Multiline: true, Required: false, - Example: "user_id=123&category=tech", + Example: "view=basic&redirect=test", Schema: SchemaDefinition{ Type: "string", }, @@ -1044,10 +1173,10 @@ func AddCustomAction() (WorkflowAppAction , string) { parameters = append(parameters, WorkflowAppActionParameter{ - Name: "verify", - Description: "Check certificate", + Name: "ssl_verify", + Description: "Check if you want to verify request", Multiline: false, - Options: []string{"false","true"}, + Options: []string{"False","True"}, Required: false, Example: "False", Schema: SchemaDefinition{ @@ -1057,7 +1186,7 @@ func AddCustomAction() (WorkflowAppAction , string) { parameters = append(parameters, WorkflowAppActionParameter{ Name: "body", - Description: "the path to add to the base url", + Description: "The body to use", Multiline: true, Required: false, Example: `{"username": "example_user", "email": "user@example.com"}`, @@ -1067,7 +1196,7 @@ func AddCustomAction() (WorkflowAppAction , string) { }) action := WorkflowAppAction{ - Description: "add a new endpoint for your app", + Description: "add a custom action for your app", Name: "custom_action", NodeType: "action", Environment: "Shuffle", @@ -1080,7 +1209,6 @@ func AddCustomAction() (WorkflowAppAction , string) { } - func GenerateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, WorkflowApp, []string, error) { api := WorkflowApp{} //log.Printf("%#v", swagger.Info) @@ -1637,7 +1765,7 @@ func GenerateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, //newPaths[actualPath] = path } - action, curCode := AddCustomAction() + action, curCode := AddCustomAction(swagger, api) api.Actions = append(api.Actions, action) pythonFunctions = append(pythonFunctions, curCode) From 68fac9ed7e23e5d4d742b9270733f08460e85dac Mon Sep 17 00:00:00 2001 From: Frikky Date: Sun, 19 May 2024 13:05:55 +0200 Subject: [PATCH 20/33] Attempting user input fix --- db-connector.go | 13 +++++++------ shared.go | 12 +++++++++++- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/db-connector.go b/db-connector.go index 283c3ec..2d4533d 100755 --- a/db-connector.go +++ b/db-connector.go @@ -3168,7 +3168,7 @@ func GetAllWorkflowsByQuery(ctx context.Context, user User) ([]Workflow, error) for { innerWorkflow := Workflow{} - _, err := it.Next(&innerWorkflow) + _, err = it.Next(&innerWorkflow) if err != nil { if strings.Contains(fmt.Sprintf("%s", err), "cannot load field") { log.Printf("[ERROR] Fixing workflow %s to have proper org (0.8.74)", innerWorkflow.ID) @@ -3197,8 +3197,8 @@ func GetAllWorkflowsByQuery(ctx context.Context, user User) ([]Workflow, error) } if err != iterator.Done { - //log.Printf("[INFO] Failed fetching results: %v", err) - //break + log.Printf("[INFO] Failed fetching workflow results: %v", err) + break } // Get the cursor for the next page of results. @@ -3217,8 +3217,7 @@ func GetAllWorkflowsByQuery(ctx context.Context, user User) ([]Workflow, error) } } - //log.Printf("[INFO] Appending suborg distribution workflows for organization %s (%s)", user.ActiveOrg.Name, user.ActiveOrg.Id) - + log.Printf("[INFO] Appending suborg distribution workflows for organization %s (%s)", user.ActiveOrg.Name, user.ActiveOrg.Id) cursorStr = "" query = datastore.NewQuery(nameKey).Filter("suborg_distribution =", user.ActiveOrg.Id) for { @@ -3226,7 +3225,9 @@ func GetAllWorkflowsByQuery(ctx context.Context, user User) ([]Workflow, error) for { innerWorkflow := Workflow{} - _, err := it.Next(&innerWorkflow) + _, err = it.Next(&innerWorkflow) + log.Printf("[DEBUG] SUBFLOW: %#v", innerWorkflow.ID) + if err != nil { if strings.Contains(fmt.Sprintf("%s", err), "cannot load field") { log.Printf("[ERROR] Error in workflow loading. Migrating workflow to new workflow handler (1): %s", err) diff --git a/shared.go b/shared.go index 1a85cf4..f681b0b 100755 --- a/shared.go +++ b/shared.go @@ -11881,7 +11881,17 @@ func ParsedExecutionResult(ctx context.Context, workflowExecution WorkflowExecut setWorkflow := false if strings.ToLower(actionResult.Action.Environment) != "cloud" { if project.Environment == "worker" { - log.Printf("\n\n\n[DEBUG] NOT modifying workflow based on User Input as we are in worker\n\n\n") + + if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" || os.Getenv("SHUFFLE_SWARM_CONFIG") == "swarm" { + log.Printf("\n\n\n[DEBUG] MODIFYING workflow based on User Input as we are in swarm\n\n\n") + + workflowExecution.Status = "WAITING" + workflowExecution.Results = append(workflowExecution.Results, actionResult) + setWorkflow = true + } else { + log.Printf("\n\n\n[DEBUG] NOT modifying workflow based on User Input as we are in worker\n\n\n") + } + } else { // Find the waiting node and change it to this result workflowExecution.Status = "WAITING" From dc7b58ffda2ee68c528d797105fffc23d808e981 Mon Sep 17 00:00:00 2001 From: Frikky Date: Sun, 19 May 2024 13:48:28 +0200 Subject: [PATCH 21/33] Removed verbosity for user input in swarm --- shared.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/shared.go b/shared.go index f681b0b..ebd95fc 100755 --- a/shared.go +++ b/shared.go @@ -11883,8 +11883,7 @@ func ParsedExecutionResult(ctx context.Context, workflowExecution WorkflowExecut if project.Environment == "worker" { if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" || os.Getenv("SHUFFLE_SWARM_CONFIG") == "swarm" { - log.Printf("\n\n\n[DEBUG] MODIFYING workflow based on User Input as we are in swarm\n\n\n") - + //log.Printf("\n\n\n[DEBUG] MODIFYING workflow based on User Input as we are in swarm\n\n\n") workflowExecution.Status = "WAITING" workflowExecution.Results = append(workflowExecution.Results, actionResult) setWorkflow = true @@ -17153,7 +17152,7 @@ func PrepareWorkflowExecution(ctx context.Context, workflow Workflow, request *h } result.CompletedAt = int64(time.Now().Unix()) * 1000 - log.Printf("\n\n[INFO][%s] Setting result to %s\n\n", oldExecution.ExecutionId, result.Action.Label) + log.Printf("[INFO][%s] Setting result to %s", oldExecution.ExecutionId, result.Action.Label) sendSelfRequest := false if answer[0] == "false" { @@ -17213,7 +17212,6 @@ func PrepareWorkflowExecution(ctx context.Context, workflow Workflow, request *h } } - log.Printf("\n\n\n[DEBUG][%s] SelfReq: %t, Env: %s\n\n\n", result.ExecutionId, sendSelfRequest, result.Action.Environment) if sendSelfRequest || strings.ToLower(result.Action.Environment) == "cloud" { log.Printf("[DEBUG][%s] Sending User Input result to self because we are on cloud env/action is skipped", result.ExecutionId) From b68bb275db27d6278d12589073abab7ea862b5b4 Mon Sep 17 00:00:00 2001 From: Frikky Date: Mon, 20 May 2024 15:01:56 +0200 Subject: [PATCH 22/33] Added more debug info about whether workflows truncate properly or not --- shared.go | 30 ++++++++++++++++++++++-------- structs.go | 2 ++ 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/shared.go b/shared.go index ebd95fc..44fd61d 100755 --- a/shared.go +++ b/shared.go @@ -3675,13 +3675,13 @@ func GetWorkflows(resp http.ResponseWriter, request *http.Request) { workflows, err = GetAllWorkflowsByQuery(ctx, user) if err != nil { log.Printf("[WARNING] Failed getting workflows for user %s (0): %s", user.Username, err) - resp.WriteHeader(401) + resp.WriteHeader(400) resp.Write([]byte(`{"success": false}`)) return } if len(workflows) == 0 { - log.Printf("[INFO] No workflows found for user %s", user.Username) + log.Printf("[INFO] No workflows found for user %s (%s) in org %s (%s)", user.Username, user.Id, user.ActiveOrg.Name, user.ActiveOrg.Id) resp.WriteHeader(200) resp.Write([]byte("[]")) return @@ -3720,8 +3720,9 @@ func GetWorkflows(resp http.ResponseWriter, request *http.Request) { newWorkflows = append(newWorkflows, workflow) } + //log.Printf("[DEBUG] Env: %s, workflows: %d", project.Environment, len(newWorkflows)) if project.Environment == "cloud" && len(newWorkflows) > 15 { - log.Printf("[WARNING] Removed workflow actions for user %s (%s) in org %s (%s)", user.Username, user.Id, user.ActiveOrg.Name, user.ActiveOrg.Id) + log.Printf("[DEBUG] Removed workflow actions & images for user %s (%s) in org %s (%s)", user.Username, user.Id, user.ActiveOrg.Name, user.ActiveOrg.Id) for workflowIndex, _ := range newWorkflows { newWorkflows[workflowIndex].Actions = []Action{} @@ -3744,6 +3745,8 @@ func GetWorkflows(resp http.ResponseWriter, request *http.Request) { // Add header that this is a limited response resp.Header().Set("X-Shuffle-Truncated", "true") + } else { + log.Printf("[DEBUG] Loading without truncating for user %s (%s) in org %s (%s)", user.Username, user.Id, user.ActiveOrg.Name, user.ActiveOrg.Id) } // Get the org as well to manage priorities @@ -3774,7 +3777,7 @@ func GetWorkflows(resp http.ResponseWriter, request *http.Request) { //log.Printf("[INFO] Returning %d workflows", len(newWorkflows)) newjson, err := json.Marshal(newWorkflows) if err != nil { - resp.WriteHeader(401) + resp.WriteHeader(500) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking workflows"}`))) return } @@ -3782,7 +3785,7 @@ func GetWorkflows(resp http.ResponseWriter, request *http.Request) { if project.CacheDb { err = SetCache(ctx, cacheKey, newjson, 30) if err != nil { - log.Printf("[WARNING] Failed updating workflow cache: %s", err) + log.Printf("[ERROR] Failed updating workflow cache for org %s: %s", user.ActiveOrg.Id, err) } } @@ -20770,7 +20773,7 @@ func RunCategoryAction(resp http.ResponseWriter, request *http.Request) { return } - log.Printf("[INFO] Running category-action '%s' in category '%s' for org %s (%s)", value.Label, value.Category, user.ActiveOrg.Name, user.ActiveOrg.Id) + log.Printf("\n\n\n[INFO] Running category-action '%s' in category '%s' for org %s (%s)\n\n\n", value.Label, value.Category, user.ActiveOrg.Name, user.ActiveOrg.Id) if len(value.Query) > 0 { // Check if app authentication. If so, check if intent is to actually authenticate, or find the actual intent @@ -20779,6 +20782,15 @@ func RunCategoryAction(resp http.ResponseWriter, request *http.Request) { } } + if value.Label == "use_app" { + for _, field := range value.Fields { + if field.Key == "action" { + log.Printf("[INFO] NOT IMPLEMENTED: Changing to action label '%s' from use_app", field.Value) + //value.Label = field.Value + break + } + } + } threadId := value.WorkflowId @@ -20833,7 +20845,8 @@ func RunCategoryAction(resp http.ResponseWriter, request *http.Request) { _ = labelIndex - //log.Printf("[INFO] Found label '%s' in category '%s'. Indexes for category: %d, and label: %d", value.Label, value.Category, foundIndex, labelIndex) + log.Printf("\n\n[INFO] Found label '%s' in category '%s'. Indexes for category: %d, and label: %d\n\n", value.Label, value.Category, foundIndex, labelIndex) + newapps, err := GetPrioritizedApps(ctx, user) if err != nil { log.Printf("[WARNING] Failed getting apps in category action: %s", err) @@ -20939,7 +20952,7 @@ func RunCategoryAction(resp http.ResponseWriter, request *http.Request) { if strings.ReplaceAll(strings.ToLower(action.CategoryLabel[0]), " ", "_") == value.Label { selectedAction = action - //log.Printf("[INFO] Found label %s in app %s. ActionName: %s", value.Label, app.Name, action.Name) + log.Printf("[INFO] Found label %s in app %s. ActionName: %s", value.Label, app.Name, action.Name) break } } @@ -21171,6 +21184,7 @@ func RunCategoryAction(resp http.ResponseWriter, request *http.Request) { // E.g. for Jira: Org_id is required. if foundFields != len(baseFields) { log.Printf("[WARNING] Not all required fields were found in category action. Want: %#v, have: %#v", baseFields, value.Fields) + resp.WriteHeader(400) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Not all required fields are set", "label": "%s", "missing_fields": "%s"}`, value.Label, strings.Join(missingFields, ",")))) return diff --git a/structs.go b/structs.go index cfb43f0..5d54104 100755 --- a/structs.go +++ b/structs.go @@ -3792,6 +3792,7 @@ type StructuredCategoryAction struct { WorkflowId string `json:"workflow_id"` ExecutionId string `json:"execution_id"` Action string `json:"action"` + Label string `json:"label"` Category string `json:"category"` Apps []WorkflowApp `json:"apps"` @@ -3800,6 +3801,7 @@ type StructuredCategoryAction struct { AvailableLabels []string `json:"available_labels"` ThreadId string `json:"thread_id"` RunId string `json:"run_id"` + MissingFields []string `json:"missing_fields"` Translated bool `json:"translated"` } From 1ea3f461ee59faacc3ea98e328ab7b1e4340c62f Mon Sep 17 00:00:00 2001 From: Frikky Date: Mon, 20 May 2024 15:11:51 +0200 Subject: [PATCH 23/33] Preparration cache for workflows not in use anymore to prevent weird caching outliers --- shared.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/shared.go b/shared.go index 44fd61d..4b1e346 100755 --- a/shared.go +++ b/shared.go @@ -3658,6 +3658,7 @@ func GetWorkflows(resp http.ResponseWriter, request *http.Request) { ctx := GetContext(request) var workflows []Workflow + /* cacheKey := fmt.Sprintf("%s_workflows", user.ActiveOrg.Id) cache, err := GetCache(ctx, cacheKey) if err == nil { @@ -3671,6 +3672,7 @@ func GetWorkflows(resp http.ResponseWriter, request *http.Request) { } else { //log.Printf("[INFO] Failed getting cache for workflows for user %s", user.Id) } + */ workflows, err = GetAllWorkflowsByQuery(ctx, user) if err != nil { @@ -3782,12 +3784,14 @@ func GetWorkflows(resp http.ResponseWriter, request *http.Request) { return } + /* if project.CacheDb { err = SetCache(ctx, cacheKey, newjson, 30) if err != nil { log.Printf("[ERROR] Failed updating workflow cache for org %s: %s", user.ActiveOrg.Id, err) } } + */ resp.WriteHeader(200) resp.Write(newjson) From bb0f1e5e8f3b7551166dca711e7bf1724064617d Mon Sep 17 00:00:00 2001 From: lalit Date: Mon, 20 May 2024 19:02:43 +0530 Subject: [PATCH 24/33] sso_backend --- oauth2.go | 23 +++-- shared.go | 260 +++++++++++++++++++++++++++++++++++++++-------------- structs.go | 1 + 3 files changed, 209 insertions(+), 75 deletions(-) diff --git a/oauth2.go b/oauth2.go index 7a64422..af59dd9 100755 --- a/oauth2.go +++ b/oauth2.go @@ -4080,19 +4080,32 @@ func VerifyIdToken(ctx context.Context, idToken string) (IdTokenCheck, error) { foundOrg := "" foundChallenge := "" stateSplit := strings.Split(string(parsedState), "&") + regexPattern := `EXTRA string=([A-Za-z0-9~.]+)` + re := regexp.MustCompile(regexPattern) for _, innerstate := range stateSplit { - itemsplit := strings.Split(innerstate, "=") + itemsplit := strings.SplitN(innerstate, "=", 2) if len(itemsplit) <= 1 { log.Printf("[WARNING] No key:value: %s", innerstate) continue } + key := strings.TrimSpace(itemsplit[0]) + value := strings.TrimSpace(itemsplit[1]) if itemsplit[0] == "org" { - foundOrg = strings.TrimSpace(itemsplit[1]) + foundOrg = value } - if itemsplit[0] == "challenge" { - foundChallenge = strings.TrimSpace(itemsplit[1]) + if key == "challenge" { + // Extract the "extra string" value from the challenge value + matches := re.FindStringSubmatch(value) + if len(matches) > 1 { + extractedString := matches[1] + foundChallenge = extractedString + log.Printf("Extracted 'extra string' value is: %s", extractedString) + } else { + foundChallenge = strings.TrimSpace(itemsplit[1]) + log.Printf("No 'extra string' value found in challenge: %s", value) + } } } @@ -4100,13 +4113,11 @@ func VerifyIdToken(ctx context.Context, idToken string) (IdTokenCheck, error) { log.Printf("[ERROR] No org specified in state (2)") return IdTokenCheck{}, err } - org, err := GetOrg(ctx, foundOrg) if err != nil { log.Printf("[WARNING] Error getting org in OpenID (2): %s", err) return IdTokenCheck{}, err } - // Validating the user itself if token.Aud == org.SSOConfig.OpenIdClientId && foundChallenge == org.SSOConfig.OpenIdClientSecret { log.Printf("[DEBUG] Correct token aud & challenge - successful login!") diff --git a/shared.go b/shared.go index 2767a1e..d8162dc 100755 --- a/shared.go +++ b/shared.go @@ -8704,6 +8704,72 @@ func HandleChangeUserOrg(resp http.ResponseWriter, request *http.Request) { return } + updateUser := false + + if org.SSOConfig.SSORequired == true && user.SupportAccess == false { + + baseSSOUrl := org.SSOConfig.SSOEntrypoint + redirectKey := "SSO_REDIRECT" + if len(org.SSOConfig.OpenIdAuthorization) > 0 { + log.Printf("[INFO] OpenID login for %s", org.Id) + redirectKey = "SSO_REDIRECT" + + baseSSOUrl = GetOpenIdUrl(request, *org) + } + + log.Printf("[DEBUG] Should redirect user %s in org %s(%s) to SSO login at %s", user.Username, user.ActiveOrg.Name, user.ActiveOrg.Id, baseSSOUrl) + + // Check if the user has other orgs that can be swapped to - if so SWAP + userDomain := strings.Split(user.Username, "@") + for _, tmporg := range user.Orgs { + innerorg, err := GetOrg(ctx, tmporg) + if err != nil { + continue + } + + if innerorg.Id == user.ActiveOrg.Id { + continue + } + + if len(innerorg.ManagerOrgs) > 0 { + continue + } + + // Not your own org + if innerorg.Org == user.Username || strings.Contains(innerorg.Name, "@") { + continue + } + + if len(userDomain) >= 2 { + if strings.Contains(strings.ToLower(innerorg.Org), strings.ToLower(userDomain[1])) { + continue + } + } + + // Shouldn't contain the domain of the users' email + log.Printf("[ERROR] Found org for %s (%s) to check into instead of running OpenID/SSO: %s.", user.Username, user.Id, innerorg.Name) + user.ActiveOrg.Id = innerorg.Id + user.ActiveOrg.Name = innerorg.Name + + DeleteCache(ctx, fmt.Sprintf("%s_workflows", user.Id)) + DeleteCache(ctx, fmt.Sprintf("apps_%s", user.Id)) + DeleteCache(ctx, fmt.Sprintf("apps_%s", user.ActiveOrg.Id)) + DeleteCache(ctx, fmt.Sprintf("user_%s", user.Username)) + DeleteCache(ctx, fmt.Sprintf("user_%s", user.Id)) + + updateUser = true + break + + } + + // user controllable field hmm :) + if !updateUser { + resp.WriteHeader(200) + resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%s", "url": "%s"}`, redirectKey, baseSSOUrl))) + return + } + } + if project.Environment == "cloud" && len(org.RegionUrl) > 0 && !strings.Contains(org.RegionUrl, "\"") { regionUrl = org.RegionUrl } @@ -9302,6 +9368,16 @@ func HandleEditOrg(resp http.ResponseWriter, request *http.Request) { if len(tmpData.SSOConfig.SSOEntrypoint) > 0 || len(tmpData.SSOConfig.OpenIdClientId) > 0 || len(tmpData.SSOConfig.OpenIdClientSecret) > 0 || len(tmpData.SSOConfig.OpenIdAuthorization) > 0 || len(tmpData.SSOConfig.OpenIdToken) > 0 { org.SSOConfig = tmpData.SSOConfig } + // Check if there are new values for SSO entry point or certificate or SSORequired + if (tmpData.SSOConfig.SSOEntrypoint != org.SSOConfig.SSOEntrypoint) || + (tmpData.SSOConfig.SSOCertificate != org.SSOConfig.SSOCertificate) || + (tmpData.SSOConfig.SSORequired != org.SSOConfig.SSORequired) { + org.SSOConfig = tmpData.SSOConfig + } + + if (tmpData.SSOConfig.OpenIdClientId != org.SSOConfig.OpenIdClientId) || (tmpData.SSOConfig.OpenIdAuthorization != org.SSOConfig.OpenIdAuthorization) { + org.SSOConfig = tmpData.SSOConfig + } if len(tmpData.SSOConfig.SSOCertificate) > 0 { savedCert := fixCertificate(tmpData.SSOConfig.SSOCertificate) @@ -10636,10 +10712,22 @@ func HandleLogin(resp http.ResponseWriter, request *http.Request) { } } } + goThroughSSO := false + if org.SSOConfig.SSORequired { + goThroughSSO = true + } else if len(data.Password) > 0 { + goThroughSSO = false + } else if len(org.SSOConfig.SSOEntrypoint) > 4 || len(org.SSOConfig.OpenIdAuthorization) > 4 { + goThroughSSO = true + } else if !org.SSOConfig.SSORequired && (len(org.SSOConfig.SSOEntrypoint) == 0 || len(org.SSOConfig.OpenIdAuthorization) == 0) { + goThroughSSO = false + } else { + goThroughSSO = false + } log.Printf("[INFO] Inside SSO / OpenID check: %s", org.Id) // has to contain http(s) - if len(org.SSOConfig.SSOEntrypoint) > 4 || len(org.SSOConfig.OpenIdAuthorization) > 4 { + if goThroughSSO == true { baseSSOUrl := org.SSOConfig.SSOEntrypoint redirectKey := "SSO_REDIRECT" if len(org.SSOConfig.OpenIdAuthorization) > 0 { @@ -10704,7 +10792,7 @@ func HandleLogin(resp http.ResponseWriter, request *http.Request) { } } - if len(users) == 1 { + if len(users) == 1 && len(data.Password) > 0 { err = bcrypt.CompareHashAndPassword([]byte(userdata.Password), []byte(data.Password)) if err != nil { userdata = User{} @@ -16044,7 +16132,7 @@ func HandleOpenId(resp http.ResponseWriter, request *http.Request) { if !strings.Contains(userName, "@") { log.Printf("[ERROR] Bad username, but allowing due to OpenID: %s. Full Subject: %#v", userName, openidUser) } - redirectUrl := "/workflows" + redirectUrl := "https://shuffler.io/workflows" users, err := FindGeneratedUser(ctx, strings.ToLower(strings.TrimSpace(userName))) if err == nil && len(users) > 0 { @@ -16054,6 +16142,11 @@ func HandleOpenId(resp http.ResponseWriter, request *http.Request) { log.Printf("[AUDIT] Found user %s (%s) which matches SSO info for %s. Redirecting to login!", user.Username, user.Id, userName) //log.Printf("SESSION: %s", user.Session) + user.ActiveOrg = OrgMini{ + Name: org.Name, + Id: org.Id, + Role: user.Role, + } expiration := time.Now().Add(3600 * time.Second) //if len(user.Session) == 0 { @@ -16114,6 +16207,11 @@ func HandleOpenId(resp http.ResponseWriter, request *http.Request) { log.Printf("[AUDIT] Found user %s (%s) which matches SSO info for %s. Redirecting to login %s!", user.Username, user.Id, userName, redirectUrl) //log.Printf("SESSION: %s", user.Session) + user.ActiveOrg = OrgMini{ + Name: org.Name, + Id: org.Id, + Role: user.Role, + } expiration := time.Now().Add(3600 * time.Second) //if len(user.Session) == 0 { @@ -16203,6 +16301,11 @@ func HandleOpenId(resp http.ResponseWriter, request *http.Request) { newUser.LoginType = "OpenID" newUser.Role = "user" newUser.Session = uuid.NewV4().String() + newUser.ActiveOrg = OrgMini{ + Name: org.Name, + Id: org.Id, + Role: "user", + } verifyToken := uuid.NewV4() ID := uuid.NewV4() @@ -16463,7 +16566,7 @@ func HandleSSO(resp http.ResponseWriter, request *http.Request) { log.Printf("[AUDIT] Found user %s (%s) which matches SSO info for %s. Redirecting to login!", user.Username, user.Id, userName) if project.Environment == "cloud" { - user.ActiveOrg.Id = matchingOrgs[0].Id + // user.ActiveOrg.Id = matchingOrgs[0].Id DeleteCache(ctx, fmt.Sprintf("%s_workflows", user.Id)) DeleteCache(ctx, fmt.Sprintf("apps_%s", user.Id)) @@ -16472,48 +16575,54 @@ func HandleSSO(resp http.ResponseWriter, request *http.Request) { DeleteCache(ctx, fmt.Sprintf("user_%s", user.Id)) } + user.ActiveOrg = OrgMini{ + Name: matchingOrgs[0].Name, + Id: matchingOrgs[0].Id, + Role: user.Role, + } //log.Printf("SESSION: %s", user.Session) expiration := time.Now().Add(3600 * time.Second) - //if len(user.Session) == 0 { - log.Printf("[INFO] User does NOT have session - creating") - sessionToken := uuid.NewV4().String() - newCookie := &http.Cookie{ - Name: "session_token", - Value: sessionToken, - Expires: expiration, - Path: "/", - } + if len(user.Session) == 0 { + log.Printf("[INFO] User does NOT have session - creating") + sessionToken := uuid.NewV4().String() + newCookie := &http.Cookie{ + Name: "session_token", + Value: sessionToken, + Expires: expiration, + Path: "/", + } - if project.Environment == "cloud" { - newCookie.Domain = ".shuffler.io" - newCookie.Secure = true - newCookie.HttpOnly = true - } + if project.Environment == "cloud" { + newCookie.Domain = ".shuffler.io" + newCookie.Secure = true + newCookie.HttpOnly = true + } - http.SetCookie(resp, newCookie) + http.SetCookie(resp, newCookie) - newCookie.Name = "__session" - http.SetCookie(resp, newCookie) + newCookie.Name = "__session" + http.SetCookie(resp, newCookie) - err = SetSession(ctx, user, sessionToken) - if err != nil { - log.Printf("[WARNING] Error creating session for user: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed setting session"}`))) - return - } + err = SetSession(ctx, user, sessionToken) + if err != nil { + log.Printf("[WARNING] Error creating session for user: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed setting session"}`))) + return + } - user.LoginInfo = append(user.LoginInfo, LoginInfo{ - IP: GetRequestIp(request), - Timestamp: time.Now().Unix(), - }) + user.LoginInfo = append(user.LoginInfo, LoginInfo{ + IP: GetRequestIp(request), + Timestamp: time.Now().Unix(), + }) - user.Session = sessionToken - user.LoginInfo = append(user.LoginInfo, LoginInfo{ - IP: GetRequestIp(request), - Timestamp: time.Now().Unix(), - }) + user.Session = sessionToken + user.LoginInfo = append(user.LoginInfo, LoginInfo{ + IP: GetRequestIp(request), + Timestamp: time.Now().Unix(), + }) + } err = SetUser(ctx, &user, false) if err != nil { log.Printf("[WARNING] Failed updating user when setting session: %s", err) @@ -16537,45 +16646,52 @@ func HandleSSO(resp http.ResponseWriter, request *http.Request) { log.Printf("[AUDIT] Found user %s (%s) which matches SSO info for %s. Redirecting to login %s!", user.Username, user.Id, userName, redirectUrl) //log.Printf("SESSION: %s", user.Session) - if project.Environment == "cloud" { - user.ActiveOrg.Id = matchingOrgs[0].Id + // if project.Environment == "cloud" { + // user.ActiveOrg.Id = matchingOrgs[0].Id + // } + + user.ActiveOrg = OrgMini{ + Name: matchingOrgs[0].Name, + Id: matchingOrgs[0].Id, + Role: user.Role, } expiration := time.Now().Add(3600 * time.Second) - //if len(user.Session) == 0 { - log.Printf("[INFO] User does NOT have session - creating") - sessionToken := uuid.NewV4().String() - newCookie := &http.Cookie{ - Name: "session_token", - Value: sessionToken, - Expires: expiration, - Path: "/", - } + if len(user.Session) == 0 { + log.Printf("[INFO] User does NOT have session - creating") + sessionToken := uuid.NewV4().String() + newCookie := &http.Cookie{ + Name: "session_token", + Value: sessionToken, + Expires: expiration, + Path: "/", + } - if project.Environment == "cloud" { - newCookie.Domain = ".shuffler.io" - newCookie.Secure = true - newCookie.HttpOnly = true - } + if project.Environment == "cloud" { + newCookie.Domain = ".shuffler.io" + newCookie.Secure = true + newCookie.HttpOnly = true + } - http.SetCookie(resp, newCookie) + http.SetCookie(resp, newCookie) - newCookie.Name = "__session" - http.SetCookie(resp, newCookie) + newCookie.Name = "__session" + http.SetCookie(resp, newCookie) - err = SetSession(ctx, user, sessionToken) - if err != nil { - log.Printf("[WARNING] Error creating session for user: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed setting session"}`))) - return - } + err = SetSession(ctx, user, sessionToken) + if err != nil { + log.Printf("[WARNING] Error creating session for user: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed setting session"}`))) + return + } - user.Session = sessionToken - user.LoginInfo = append(user.LoginInfo, LoginInfo{ - IP: GetRequestIp(request), - Timestamp: time.Now().Unix(), - }) + user.Session = sessionToken + user.LoginInfo = append(user.LoginInfo, LoginInfo{ + IP: GetRequestIp(request), + Timestamp: time.Now().Unix(), + }) + } err = SetUser(ctx, &user, false) if err != nil { log.Printf("[WARNING] Failed updating user when setting session: %s", err) @@ -16630,7 +16746,13 @@ func HandleSSO(resp http.ResponseWriter, request *http.Request) { newUser.Role = "user" newUser.Session = uuid.NewV4().String() - newUser.ActiveOrg.Id = matchingOrgs[0].Id + // newUser.ActiveOrg.Id = matchingOrgs[0].Id + + newUser.ActiveOrg = OrgMini{ + Name: matchingOrgs[0].Name, + Id: matchingOrgs[0].Id, + Role: "user", + } verifyToken := uuid.NewV4() ID := uuid.NewV4() diff --git a/structs.go b/structs.go index e9a693f..407a97b 100755 --- a/structs.go +++ b/structs.go @@ -2515,6 +2515,7 @@ type SSOConfig struct { OpenIdClientSecret string `json:"client_secret" datastore:"client_secret"` OpenIdAuthorization string `json:"openid_authorization" datastore:"openid_authorization"` OpenIdToken string `json:"openid_token" datastore:"openid_token"` + SSORequired bool `json:"SSORequired" datastore:"SSORequired"` } type SamlRequest struct { From 7c39dacaa3c7d0a7208a78cce6dde7c0e038644c Mon Sep 17 00:00:00 2001 From: yashsinghcodes Date: Mon, 20 May 2024 14:09:24 +0000 Subject: [PATCH 25/33] handle no org, create default at login --- shared.go | 48 ++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/shared.go b/shared.go index 4b1e346..cadffc0 100755 --- a/shared.go +++ b/shared.go @@ -10887,10 +10887,50 @@ func HandleLogin(resp http.ResponseWriter, request *http.Request) { if userdata.ActiveOrg.Id == "" { if len(userdata.Orgs) == 0 { - log.Printf("[ERROR] User %s (%s) has no chosen org. ID: %s, Name: %s", userdata.Username, userdata.Id, userdata.ActiveOrg.Id, userdata.ActiveOrg.Name) - resp.WriteHeader(400) - resp.Write([]byte(`{"success": false, "reason": "No organization available. Please contact your team, or the shuffle if you think there has been a mistake: support@shuffler.io"}`)) - return + log.Printf("[WARNING] User %s (%s) has no chosen org. ID: %s, Name: %s. Creating a default org", userdata.Username, userdata.Id, userdata.ActiveOrg.Id, userdata.ActiveOrg.Name) + orgSetupName := "default" + orgId := uuid.NewV4().String() + newOrg := Org{ + Name: orgSetupName, + Id: orgId, + Org: orgSetupName, + Users: []User{userdata}, + Roles: userdata.Roles, + CloudSync: false, + } + + err := SetOrg(ctx, newOrg, newOrg.Id) + + if err != nil { + log.Printf("[ERROR] Failed setting default org for the user: %s", userdata.Username) + } else { + log.Printf("[DEBUG] Successfully created the default org!") + + defaultEnv := os.Getenv("ORG_ID") + if len(defaultEnv) == 0 { + defaultEnv = "Shuffle" + log.Printf("[DEBUG] Setting default environment for org to %s", defaultEnv) + } + + item := Environment{ + Name: defaultEnv, + Type: "onperm", + OrgId: orgId, + Default: true, + Id: uuid.NewV4().String(), + } + + err := SetEnvironment(ctx, &item) + if err != nil { + log.Printf("[ERROR] Failed setting up new environment for new org: %s", err) + } + + userdata.Orgs = append(userdata.Orgs, newOrg.Id) + } + + // resp.WriteHeader(400) + // resp.Write([]byte(`{"success": false, "reason": "No organization available. Please contact your team, or the shuffle if you think there has been a mistake: support@shuffler.io"}`)) + // return } userdata.ActiveOrg.Id = userdata.Orgs[0] From 67a07d5205ee59873369ffc7ddec56f05cf34849 Mon Sep 17 00:00:00 2001 From: Frikky Date: Mon, 20 May 2024 22:22:35 +0200 Subject: [PATCH 26/33] Fixed subflow position in merge wf --- db-connector.go | 2 +- shared.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/db-connector.go b/db-connector.go index 2d4533d..5121d32 100755 --- a/db-connector.go +++ b/db-connector.go @@ -3226,7 +3226,7 @@ func GetAllWorkflowsByQuery(ctx context.Context, user User) ([]Workflow, error) for { innerWorkflow := Workflow{} _, err = it.Next(&innerWorkflow) - log.Printf("[DEBUG] SUBFLOW: %#v", innerWorkflow.ID) + //log.Printf("[DEBUG] SUBFLOW: %#v", innerWorkflow.ID) if err != nil { if strings.Contains(fmt.Sprintf("%s", err), "cannot load field") { diff --git a/shared.go b/shared.go index 4b1e346..cfaea54 100755 --- a/shared.go +++ b/shared.go @@ -24459,7 +24459,7 @@ func GetStandardDestWorkflow(app *WorkflowApp, action string, enrich bool) *Work TriggerType: "SUBFLOW", Position: Position{ - X: 0, + X: 150, Y: 150, }, From 7f233f3b0cb5a8f6e6b8869b4f8f84af9dff15a3 Mon Sep 17 00:00:00 2001 From: satti-hari-krishna-reddy Date: Tue, 21 May 2024 13:32:00 +0530 Subject: [PATCH 27/33] adding new fields for Pipeline --- structs.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/structs.go b/structs.go index dd01d6c..3c30d5e 100755 --- a/structs.go +++ b/structs.go @@ -20,6 +20,7 @@ type PipelineRequest struct { Command string `json:"command"` Environment string `json:"environment"` WorkflowId string `json:"workflow_id"` + StartNode string `json:"start_node"` PipelineId string `json:"pipeline_id"` TriggerId string `json:"trigger_id"` @@ -35,6 +36,8 @@ type Pipeline struct { OrgId string `json:"org_id"` Status string `json:"status"` Errors []string `json:"errors"` + Url string `json:"url"` + Owner string `json:"owner"` PipelineId string `json:"pipeline_id"` TriggerId string `json:"trigger_id"` From 3d37c605f186a7597aed7b88caec8f7e33b1e392 Mon Sep 17 00:00:00 2001 From: satti-hari-krishna-reddy Date: Tue, 21 May 2024 13:34:01 +0530 Subject: [PATCH 28/33] commented out HandleSavePipelineInfo --- shared.go | 108 +++++++++++++++++++++++++++--------------------------- 1 file changed, 53 insertions(+), 55 deletions(-) diff --git a/shared.go b/shared.go index 616428c..084d3e5 100755 --- a/shared.go +++ b/shared.go @@ -24197,61 +24197,59 @@ func HandleWorkflowRunSearch(resp http.ResponseWriter, request *http.Request) { resp.Write(respBody) } -func HandleSavePipelineInfo(resp http.ResponseWriter, request *http.Request) { - cors := HandleCors(resp, request) - if cors { - return - } - - // How do I make sure that Orborus is the one that made this request? - - var requestBody Pipeline - err := json.NewDecoder(request.Body).Decode(&requestBody) - if err != nil { - log.Printf("[WARNING] Failed to decode request body: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - if len(requestBody.TriggerId) == 0 || len(requestBody.PipelineId) == 0 || len(requestBody.Status) == 0 { - log.Printf("[WARNING] Missing fields in the request body") - resp.WriteHeader(400) - resp.Write([]byte(`{"success": false, "reason": "Missing fields in the request body"}`)) - return - } - - ctx := GetContext(request) - pipeline, err := GetPipeline(ctx, requestBody.TriggerId) - log.Printf("[HARI TESTING] trigger id is %s", requestBody.TriggerId) - if err != nil { - if strings.Contains(fmt.Sprintf("%s", err),"pipeline doesn't exist"){ - log.Printf("[DEBUG] no matching document found for Pipeline: %s", requestBody.PipelineId) - resp.WriteHeader(404) - resp.Write([]byte(`{"success": false, "reason": "pipeline not found"}`)) - return - } else { - log.Printf("[WARNING] Failed getting pipeline: %s due to %s", requestBody.PipelineId, err) - resp.WriteHeader(500) - resp.Write([]byte(`{"success": false}`)) - return - } - } - - pipeline.PipelineId = requestBody.PipelineId - pipeline.Status = requestBody.Status - - - err = savePipelineData(ctx, *pipeline) - if err != nil { - log.Printf("[WARNING] Failed updating pipeline with ID: %s due to %s", pipeline.PipelineId, err) - resp.WriteHeader(500) - resp.Write([]byte(`{"success": false}`)) - return - } - log.Printf("[INFO] Sucessfully saved pipeline: %s", pipeline.PipelineId) - resp.WriteHeader(200) - resp.Write([]byte(`{"success": true}`)) -} +// func HandleSavePipelineInfo(resp http.ResponseWriter, request *http.Request) { +// cors := HandleCors(resp, request) +// if cors { +// return +// } + +// // How do I make sure that Orborus is the one that made this request? + +// var requestBody Pipeline +// err := json.NewDecoder(request.Body).Decode(&requestBody) +// if err != nil { +// log.Printf("[WARNING] Failed to decode request body: %s", err) +// resp.WriteHeader(401) +// resp.Write([]byte(`{"success": false}`)) +// return +// } +// if len(requestBody.TriggerId) == 0 || len(requestBody.PipelineId) == 0 || len(requestBody.Status) == 0 { +// log.Printf("[WARNING] Missing fields in the request body") +// resp.WriteHeader(400) +// resp.Write([]byte(`{"success": false, "reason": "Missing fields in the request body"}`)) +// return +// } + +// ctx := GetContext(request) +// pipeline, err := GetPipeline(ctx, requestBody.TriggerId) +// if err != nil { +// if strings.Contains(fmt.Sprintf("%s", err),"pipeline doesn't exist"){ +// log.Printf("[DEBUG] no matching document found for Pipeline: %s", requestBody.PipelineId) +// resp.WriteHeader(404) +// resp.Write([]byte(`{"success": false, "reason": "pipeline not found"}`)) +// return +// } else { +// log.Printf("[WARNING] Failed getting pipeline: %s due to %s", requestBody.PipelineId, err) +// resp.WriteHeader(500) +// resp.Write([]byte(`{"success": false}`)) +// return +// } +// } + +// pipeline.PipelineId = requestBody.PipelineId +// pipeline.Status = requestBody.Status + +// err = savePipelineData(ctx, *pipeline) +// if err != nil { +// log.Printf("[WARNING] Failed updating pipeline with ID: %s due to %s", pipeline.PipelineId, err) +// resp.WriteHeader(500) +// resp.Write([]byte(`{"success": false}`)) +// return +// } +// log.Printf("[INFO] Sucessfully saved pipeline: %s", pipeline.PipelineId) +// resp.WriteHeader(200) +// resp.Write([]byte(`{"success": true}`)) +// } func LoadUsecases(resp http.ResponseWriter, request *http.Request) { cors := HandleCors(resp, request) From 25040a9007b9675bd692131c72692096ddc051d0 Mon Sep 17 00:00:00 2001 From: satti-hari-krishna-reddy Date: Tue, 21 May 2024 15:12:03 +0530 Subject: [PATCH 29/33] added pipeline delete and update functionality --- db-connector.go | 9 +++-- pipelines.go | 98 ++++++++++++++++++++++++++++++++++++++++++++----- structs.go | 28 +++++++------- 3 files changed, 107 insertions(+), 28 deletions(-) diff --git a/db-connector.go b/db-connector.go index 835e34d..97b052b 100755 --- a/db-connector.go +++ b/db-connector.go @@ -8204,10 +8204,11 @@ func savePipelineData(ctx context.Context, pipeline Pipeline) error { return err } } else { - // key := datastore.NameKey(nameKey, pipelineId, nil) - // if _, err := project.Dbclient.Put(ctx, key, &pipeline); err != nil { - // log.Printf("[ERROR] failed to add pipeline: %s", err) - // return err + key := datastore.NameKey(nameKey, triggerId, nil) + if _, err := project.Dbclient.Put(ctx, key, &pipeline); err != nil { + log.Printf("[ERROR] failed to add pipeline: %s", err) + return err + } } return nil diff --git a/pipelines.go b/pipelines.go index 53150d1..7cadc6f 100644 --- a/pipelines.go +++ b/pipelines.go @@ -2,6 +2,8 @@ package shuffle import ( "encoding/json" + "errors" + "context" "fmt" "io/ioutil" "log" @@ -92,7 +94,7 @@ func HandleNewPipelineRegister(resp http.ResponseWriter, request *http.Request) } } - if !envFound { + if !envFound && pipeline.Type != "delete"{ log.Printf("[WARNING] Environment '%s' is not available", pipeline.Environment) resp.WriteHeader(400) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Environment '%s' is not available. Please make it, or change the environment you want to deploy to."}`, pipeline.Environment))) @@ -118,11 +120,24 @@ func HandleNewPipelineRegister(resp http.ResponseWriter, request *http.Request) return } - // 1. Add to trigger list - /* TBD */ - // Look for PIPELINE_ command that exists in the queue already startCommand := strings.ToUpper(strings.Split(pipeline.Type, " ")[0]) + + //check if this is the first time creating the pipeline + pipelineInfo, err := GetPipeline(ctx, pipeline.TriggerId) + if err != nil { + if (startCommand == "DELETE" || startCommand == "STOP") && err.Error() == "pipeline doesn't exist" { + log.Printf("[WARNING] Failed getting pipeline %s, reason: %s", pipeline.TriggerId, err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } else if startCommand == "START" && err.Error() == "pipeline doesn't exist" { + startCommand = "CREATE" + } + } else if startCommand == "CREATE" { + startCommand = "START" + } + //parsedId := fmt.Sprintf("%s_%s", strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(pipeline.Environment, " ", "-"), "_", "-")), user.ActiveOrg.Id) parsedId := strings.ToLower(pipeline.Environment) formattedType := fmt.Sprintf("PIPELINE_%s", startCommand) @@ -147,26 +162,48 @@ func HandleNewPipelineRegister(resp http.ResponseWriter, request *http.Request) Priority: 11, } - if startCommand == "CREATE" { + pipelineData := Pipeline{} + + if startCommand == "DELETE" { + + err := deletePipeline(ctx, *pipelineInfo) + if err != nil { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Failed deleting the pipeline."}`)) + return + } + + } else if startCommand == "STOP" { + + pipelineInfo.Status = "stopped" + err = setPipelineTrigger(ctx, *pipelineInfo) + if err != nil { + log.Printf("[ERROR] Failed to stop the pipeline with trigger id: %s, reason: %s", pipelineInfo.TriggerId, err) + resp.WriteHeader(500) + resp.Write([]byte(`{"success": false}`)) + return + } + log.Printf("[INFO] Stopped the pipeline %s sucessfully", pipelineInfo.TriggerId) + } else { - pipelineData := Pipeline{} pipelineData.Name = pipeline.Name pipelineData.Type = startCommand pipelineData.Command = pipeline.Command pipelineData.Environment = pipeline.Environment pipelineData.WorkflowId = pipeline.WorkflowId pipelineData.OrgId = user.ActiveOrg.Id - pipelineData.Status = "uninitialized" + pipelineData.Owner = user.Id + pipelineData.Status = "running" pipelineData.TriggerId = pipeline.TriggerId - err = savePipelineData(ctx, pipelineData) + err = setPipelineTrigger(ctx, pipelineData) if err != nil { - log.Printf("[ERROR] Failed to save the pipeline with trigger id: %s into the db: %s", pipeline.TriggerId, err) + log.Printf("[ERROR] Failed to create the pipeline with trigger id: %s, reason: %s", pipeline.TriggerId, err) resp.WriteHeader(500) resp.Write([]byte(`{"success": false}`)) return } - log.Printf("[INFO] Successfully saved the pipeline info") + log.Printf("[INFO] Set up ipeline with trigger ID %s and environment %s", pipeline.TriggerId, pipeline.Environment) } err = SetWorkflowQueue(ctx, execRequest, parsedId) @@ -180,3 +217,44 @@ func HandleNewPipelineRegister(resp http.ResponseWriter, request *http.Request) resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Pipeline will be created"}`))) } + + +func setPipelineTrigger(ctx context.Context, pipeline Pipeline) error{ + + input := pipeline.Command + index := strings.Index(input, "to ") + + if index == -1 { + return errors.New("url not found") + } + extractedURL := input[index+len("to "):] + extractedURL = strings.TrimSpace(extractedURL) + + pipeline.Url = extractedURL + err := savePipelineData(ctx, pipeline) + + if err != nil { + return err + } + + return nil +} + +func deletePipeline(ctx context.Context, pipeline Pipeline) error { + + pipeline.Status = "stopped" + err := savePipelineData(ctx, pipeline) + if err != nil { + log.Printf("[WARNING] Failed saving pipeline: %s", err) + return err + } + + err = DeleteKey(ctx, "pipelines", pipeline.TriggerId) + if err != nil { + log.Printf("[WARNING] Error deleting pipeline %s, reason: %s", pipeline.TriggerId) + return err + } + + log.Printf("[INFO] Successfully deleted pipeline %s", pipeline.TriggerId) + return nil +} \ No newline at end of file diff --git a/structs.go b/structs.go index 3c30d5e..4d7e296 100755 --- a/structs.go +++ b/structs.go @@ -27,20 +27,20 @@ type PipelineRequest struct { } type Pipeline struct { - Name string `json:"name"` - Type string `json:"type"` - Command string `json:"command"` - Environment string `json:"environment"` - WorkflowId string `json:"workflow_id"` - StartNode string `json:"start_node"` - OrgId string `json:"org_id"` - Status string `json:"status"` - Errors []string `json:"errors"` - Url string `json:"url"` - Owner string `json:"owner"` - - PipelineId string `json:"pipeline_id"` - TriggerId string `json:"trigger_id"` + Name string `json:"name" datastore:"name"` + Type string `json:"type" datastore:"type"` + Command string `json:"command" datastore:"command"` + Environment string `json:"environment" datastore:"environment"` + WorkflowId string `json:"workflow_id" datastore:"workflow_id"` + StartNode string `json:"start_node" datastore:"start_node"` + OrgId string `json:"org_id" datastore:"org_id"` + Status string `json:"status" datastore:"status"` + Errors []string `json:"errors" datastore:"errors"` + Url string `json:"url" datastore:"url"` + Owner string `json:"owner" datastore:"owner"` + + PipelineId string `json:"pipeline_id" datastore:"pipeline_id"` + TriggerId string `json:"trigger_id" datastore:"trigger_id"` } type PipelineWrapper struct { From 4c621daa9715c3c9e32782ecc9766b5a1c5ac4d4 Mon Sep 17 00:00:00 2001 From: satti-hari-krishna-reddy Date: Tue, 21 May 2024 15:24:29 +0530 Subject: [PATCH 30/33] fixed a small typo --- pipelines.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines.go b/pipelines.go index 7cadc6f..ed6b696 100644 --- a/pipelines.go +++ b/pipelines.go @@ -203,7 +203,7 @@ func HandleNewPipelineRegister(resp http.ResponseWriter, request *http.Request) resp.Write([]byte(`{"success": false}`)) return } - log.Printf("[INFO] Set up ipeline with trigger ID %s and environment %s", pipeline.TriggerId, pipeline.Environment) + log.Printf("[INFO] Set up pipeline with trigger ID %s and environment %s", pipeline.TriggerId, pipeline.Environment) } err = SetWorkflowQueue(ctx, execRequest, parsedId) From a533894de009ee14436e87d421f23f44f79320d8 Mon Sep 17 00:00:00 2001 From: Frikky Date: Tue, 21 May 2024 13:16:52 +0200 Subject: [PATCH 31/33] Revert "backend for to make sso required or optional" --- db-connector.go | 35 ++++--- oauth2.go | 23 ++--- shared.go | 260 +++++++++++++----------------------------------- structs.go | 1 - 4 files changed, 96 insertions(+), 223 deletions(-) diff --git a/db-connector.go b/db-connector.go index 835e34d..5121d32 100755 --- a/db-connector.go +++ b/db-connector.go @@ -6068,6 +6068,8 @@ func GetPrioritizedApps(ctx context.Context, user User) ([]WorkflowApp, error) { } } + + return allApps, nil } @@ -6166,6 +6168,8 @@ func fixAppAppend(allApps []WorkflowApp, innerApp WorkflowApp) ([]WorkflowApp, W func GetUserApps(ctx context.Context, userId string) ([]WorkflowApp, error) { wrapper := []WorkflowApp{} + var err error + cacheKey := fmt.Sprintf("userapps-%s", userId) if project.CacheDb { cache, err := GetCache(ctx, cacheKey) @@ -6203,8 +6207,8 @@ func GetUserApps(ctx context.Context, userId string) ([]WorkflowApp, error) { }, }, }, - "minimum_should_match": 1, }, + "minimum_should_match": 1, }, } @@ -6230,8 +6234,9 @@ func GetUserApps(ctx context.Context, userId string) ([]WorkflowApp, error) { } if res.StatusCode != 200 && res.StatusCode != 201 { - return []WorkflowApp{}, fmt.Errorf("bad statuscode: %d", res.StatusCode) + return []WorkflowApp{}, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode)) } + respBody, err := ioutil.ReadAll(res.Body) if err != nil { return []WorkflowApp{}, err @@ -7855,17 +7860,17 @@ func GetHooks(ctx context.Context, OrgId string) ([]Hook, error) { } wrapper := AllHooksWrapper{} err = json.Unmarshal(respBody, &wrapper) - + if err != nil { return []Hook{}, err } for _, hit := range wrapper.Hits.Hits { - hook := hit.Source - hooks = append(hooks, hook) + hook := hit.Source + hooks = append(hooks, hook) } return hooks, err - + } else { q := datastore.NewQuery(nameKey).Filter("org_id = ", OrgId).Limit(1000) @@ -7936,7 +7941,7 @@ func GetPipelines(ctx context.Context, OrgId string) ([]Pipeline, error) { if res.StatusCode != 200 && res.StatusCode != 201 { return []Pipeline{}, fmt.Errorf("bad statuscode: %d", res.StatusCode) - + } respBody, err := ioutil.ReadAll(res.Body) @@ -7945,17 +7950,17 @@ func GetPipelines(ctx context.Context, OrgId string) ([]Pipeline, error) { } wrapper := AllPipelinesWrapper{} err = json.Unmarshal(respBody, &wrapper) - + if err != nil { return []Pipeline{}, err } for _, hit := range wrapper.Hits.Hits { - pipeline := hit.Source - pipelines = append(pipelines, pipeline) + pipeline := hit.Source + pipelines = append(pipelines, pipeline) } return pipelines, err - + } else { // q := datastore.NewQuery(nameKey).Filter("org_id = ", OrgId).Limit(1000) @@ -8326,7 +8331,7 @@ func SetHook(ctx context.Context, hook Hook) error { func GetPipeline(ctx context.Context, triggerId string) (*Pipeline, error) { pipeline := &Pipeline{} nameKey := "pipelines" - + triggerId = strings.ToLower(triggerId) if project.DbType == "opensearch" { @@ -8353,7 +8358,7 @@ func GetPipeline(ctx context.Context, triggerId string) (*Pipeline, error) { } pipeline = &wrapped.Source - } else { + } else { // key := datastore.NameKey(nameKey, triggerId, nil) // err := project.Dbclient.Get(ctx, key, pipeline) // if err != nil { @@ -8457,7 +8462,7 @@ func GetFile(ctx context.Context, id string) (*File, error) { fileData, err := json.Marshal(curFile) if err != nil { log.Printf("[WARNING] Failed marshalling in getfile: %s", err) - return curFile, nil + return curFile, nil } err = SetCache(ctx, cacheKey, fileData, 30) @@ -8560,6 +8565,7 @@ func SetFile(ctx context.Context, file File) error { DeleteCache(ctx, fmt.Sprintf("files_%s_%s", file.OrgId, file.Namespace)) DeleteCache(ctx, fmt.Sprintf("files_%s_", file.OrgId)) + return nil } @@ -8990,6 +8996,7 @@ func GetAllFiles(ctx context.Context, orgId, namespace string) ([]File, error) { log.Printf("[WARNING] Failed updating file cache: %s", err) } } + return files, nil } diff --git a/oauth2.go b/oauth2.go index af59dd9..7a64422 100755 --- a/oauth2.go +++ b/oauth2.go @@ -4080,32 +4080,19 @@ func VerifyIdToken(ctx context.Context, idToken string) (IdTokenCheck, error) { foundOrg := "" foundChallenge := "" stateSplit := strings.Split(string(parsedState), "&") - regexPattern := `EXTRA string=([A-Za-z0-9~.]+)` - re := regexp.MustCompile(regexPattern) for _, innerstate := range stateSplit { - itemsplit := strings.SplitN(innerstate, "=", 2) + itemsplit := strings.Split(innerstate, "=") if len(itemsplit) <= 1 { log.Printf("[WARNING] No key:value: %s", innerstate) continue } - key := strings.TrimSpace(itemsplit[0]) - value := strings.TrimSpace(itemsplit[1]) if itemsplit[0] == "org" { - foundOrg = value + foundOrg = strings.TrimSpace(itemsplit[1]) } - if key == "challenge" { - // Extract the "extra string" value from the challenge value - matches := re.FindStringSubmatch(value) - if len(matches) > 1 { - extractedString := matches[1] - foundChallenge = extractedString - log.Printf("Extracted 'extra string' value is: %s", extractedString) - } else { - foundChallenge = strings.TrimSpace(itemsplit[1]) - log.Printf("No 'extra string' value found in challenge: %s", value) - } + if itemsplit[0] == "challenge" { + foundChallenge = strings.TrimSpace(itemsplit[1]) } } @@ -4113,11 +4100,13 @@ func VerifyIdToken(ctx context.Context, idToken string) (IdTokenCheck, error) { log.Printf("[ERROR] No org specified in state (2)") return IdTokenCheck{}, err } + org, err := GetOrg(ctx, foundOrg) if err != nil { log.Printf("[WARNING] Error getting org in OpenID (2): %s", err) return IdTokenCheck{}, err } + // Validating the user itself if token.Aud == org.SSOConfig.OpenIdClientId && foundChallenge == org.SSOConfig.OpenIdClientSecret { log.Printf("[DEBUG] Correct token aud & challenge - successful login!") diff --git a/shared.go b/shared.go index 616428c..133dd89 100755 --- a/shared.go +++ b/shared.go @@ -8807,72 +8807,6 @@ func HandleChangeUserOrg(resp http.ResponseWriter, request *http.Request) { return } - updateUser := false - - if org.SSOConfig.SSORequired == true && user.SupportAccess == false { - - baseSSOUrl := org.SSOConfig.SSOEntrypoint - redirectKey := "SSO_REDIRECT" - if len(org.SSOConfig.OpenIdAuthorization) > 0 { - log.Printf("[INFO] OpenID login for %s", org.Id) - redirectKey = "SSO_REDIRECT" - - baseSSOUrl = GetOpenIdUrl(request, *org) - } - - log.Printf("[DEBUG] Should redirect user %s in org %s(%s) to SSO login at %s", user.Username, user.ActiveOrg.Name, user.ActiveOrg.Id, baseSSOUrl) - - // Check if the user has other orgs that can be swapped to - if so SWAP - userDomain := strings.Split(user.Username, "@") - for _, tmporg := range user.Orgs { - innerorg, err := GetOrg(ctx, tmporg) - if err != nil { - continue - } - - if innerorg.Id == user.ActiveOrg.Id { - continue - } - - if len(innerorg.ManagerOrgs) > 0 { - continue - } - - // Not your own org - if innerorg.Org == user.Username || strings.Contains(innerorg.Name, "@") { - continue - } - - if len(userDomain) >= 2 { - if strings.Contains(strings.ToLower(innerorg.Org), strings.ToLower(userDomain[1])) { - continue - } - } - - // Shouldn't contain the domain of the users' email - log.Printf("[ERROR] Found org for %s (%s) to check into instead of running OpenID/SSO: %s.", user.Username, user.Id, innerorg.Name) - user.ActiveOrg.Id = innerorg.Id - user.ActiveOrg.Name = innerorg.Name - - DeleteCache(ctx, fmt.Sprintf("%s_workflows", user.Id)) - DeleteCache(ctx, fmt.Sprintf("apps_%s", user.Id)) - DeleteCache(ctx, fmt.Sprintf("apps_%s", user.ActiveOrg.Id)) - DeleteCache(ctx, fmt.Sprintf("user_%s", user.Username)) - DeleteCache(ctx, fmt.Sprintf("user_%s", user.Id)) - - updateUser = true - break - - } - - // user controllable field hmm :) - if !updateUser { - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%s", "url": "%s"}`, redirectKey, baseSSOUrl))) - return - } - } - if project.Environment == "cloud" && len(org.RegionUrl) > 0 && !strings.Contains(org.RegionUrl, "\"") { regionUrl = org.RegionUrl } @@ -9471,16 +9405,6 @@ func HandleEditOrg(resp http.ResponseWriter, request *http.Request) { if len(tmpData.SSOConfig.SSOEntrypoint) > 0 || len(tmpData.SSOConfig.OpenIdClientId) > 0 || len(tmpData.SSOConfig.OpenIdClientSecret) > 0 || len(tmpData.SSOConfig.OpenIdAuthorization) > 0 || len(tmpData.SSOConfig.OpenIdToken) > 0 { org.SSOConfig = tmpData.SSOConfig } - // Check if there are new values for SSO entry point or certificate or SSORequired - if (tmpData.SSOConfig.SSOEntrypoint != org.SSOConfig.SSOEntrypoint) || - (tmpData.SSOConfig.SSOCertificate != org.SSOConfig.SSOCertificate) || - (tmpData.SSOConfig.SSORequired != org.SSOConfig.SSORequired) { - org.SSOConfig = tmpData.SSOConfig - } - - if (tmpData.SSOConfig.OpenIdClientId != org.SSOConfig.OpenIdClientId) || (tmpData.SSOConfig.OpenIdAuthorization != org.SSOConfig.OpenIdAuthorization) { - org.SSOConfig = tmpData.SSOConfig - } if len(tmpData.SSOConfig.SSOCertificate) > 0 { savedCert := fixCertificate(tmpData.SSOConfig.SSOCertificate) @@ -10820,22 +10744,10 @@ func HandleLogin(resp http.ResponseWriter, request *http.Request) { } } } - goThroughSSO := false - if org.SSOConfig.SSORequired { - goThroughSSO = true - } else if len(data.Password) > 0 { - goThroughSSO = false - } else if len(org.SSOConfig.SSOEntrypoint) > 4 || len(org.SSOConfig.OpenIdAuthorization) > 4 { - goThroughSSO = true - } else if !org.SSOConfig.SSORequired && (len(org.SSOConfig.SSOEntrypoint) == 0 || len(org.SSOConfig.OpenIdAuthorization) == 0) { - goThroughSSO = false - } else { - goThroughSSO = false - } log.Printf("[INFO] Inside SSO / OpenID check: %s", org.Id) // has to contain http(s) - if goThroughSSO == true { + if len(org.SSOConfig.SSOEntrypoint) > 4 || len(org.SSOConfig.OpenIdAuthorization) > 4 { baseSSOUrl := org.SSOConfig.SSOEntrypoint redirectKey := "SSO_REDIRECT" if len(org.SSOConfig.OpenIdAuthorization) > 0 { @@ -10900,7 +10812,7 @@ func HandleLogin(resp http.ResponseWriter, request *http.Request) { } } - if len(users) == 1 && len(data.Password) > 0 { + if len(users) == 1 { err = bcrypt.CompareHashAndPassword([]byte(userdata.Password), []byte(data.Password)) if err != nil { userdata = User{} @@ -16289,7 +16201,7 @@ func HandleOpenId(resp http.ResponseWriter, request *http.Request) { if !strings.Contains(userName, "@") { log.Printf("[ERROR] Bad username, but allowing due to OpenID: %s. Full Subject: %#v", userName, openidUser) } - redirectUrl := "https://shuffler.io/workflows" + redirectUrl := "/workflows" users, err := FindGeneratedUser(ctx, strings.ToLower(strings.TrimSpace(userName))) if err == nil && len(users) > 0 { @@ -16299,11 +16211,6 @@ func HandleOpenId(resp http.ResponseWriter, request *http.Request) { log.Printf("[AUDIT] Found user %s (%s) which matches SSO info for %s. Redirecting to login!", user.Username, user.Id, userName) //log.Printf("SESSION: %s", user.Session) - user.ActiveOrg = OrgMini{ - Name: org.Name, - Id: org.Id, - Role: user.Role, - } expiration := time.Now().Add(3600 * time.Second) //if len(user.Session) == 0 { @@ -16364,11 +16271,6 @@ func HandleOpenId(resp http.ResponseWriter, request *http.Request) { log.Printf("[AUDIT] Found user %s (%s) which matches SSO info for %s. Redirecting to login %s!", user.Username, user.Id, userName, redirectUrl) //log.Printf("SESSION: %s", user.Session) - user.ActiveOrg = OrgMini{ - Name: org.Name, - Id: org.Id, - Role: user.Role, - } expiration := time.Now().Add(3600 * time.Second) //if len(user.Session) == 0 { @@ -16458,11 +16360,6 @@ func HandleOpenId(resp http.ResponseWriter, request *http.Request) { newUser.LoginType = "OpenID" newUser.Role = "user" newUser.Session = uuid.NewV4().String() - newUser.ActiveOrg = OrgMini{ - Name: org.Name, - Id: org.Id, - Role: "user", - } verifyToken := uuid.NewV4() ID := uuid.NewV4() @@ -16723,7 +16620,7 @@ func HandleSSO(resp http.ResponseWriter, request *http.Request) { log.Printf("[AUDIT] Found user %s (%s) which matches SSO info for %s. Redirecting to login!", user.Username, user.Id, userName) if project.Environment == "cloud" { - // user.ActiveOrg.Id = matchingOrgs[0].Id + user.ActiveOrg.Id = matchingOrgs[0].Id DeleteCache(ctx, fmt.Sprintf("%s_workflows", user.Id)) DeleteCache(ctx, fmt.Sprintf("apps_%s", user.Id)) @@ -16732,54 +16629,48 @@ func HandleSSO(resp http.ResponseWriter, request *http.Request) { DeleteCache(ctx, fmt.Sprintf("user_%s", user.Id)) } - user.ActiveOrg = OrgMini{ - Name: matchingOrgs[0].Name, - Id: matchingOrgs[0].Id, - Role: user.Role, - } //log.Printf("SESSION: %s", user.Session) expiration := time.Now().Add(3600 * time.Second) - if len(user.Session) == 0 { - log.Printf("[INFO] User does NOT have session - creating") - sessionToken := uuid.NewV4().String() - newCookie := &http.Cookie{ - Name: "session_token", - Value: sessionToken, - Expires: expiration, - Path: "/", - } + //if len(user.Session) == 0 { + log.Printf("[INFO] User does NOT have session - creating") + sessionToken := uuid.NewV4().String() + newCookie := &http.Cookie{ + Name: "session_token", + Value: sessionToken, + Expires: expiration, + Path: "/", + } - if project.Environment == "cloud" { - newCookie.Domain = ".shuffler.io" - newCookie.Secure = true - newCookie.HttpOnly = true - } + if project.Environment == "cloud" { + newCookie.Domain = ".shuffler.io" + newCookie.Secure = true + newCookie.HttpOnly = true + } - http.SetCookie(resp, newCookie) + http.SetCookie(resp, newCookie) - newCookie.Name = "__session" - http.SetCookie(resp, newCookie) + newCookie.Name = "__session" + http.SetCookie(resp, newCookie) - err = SetSession(ctx, user, sessionToken) - if err != nil { - log.Printf("[WARNING] Error creating session for user: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed setting session"}`))) - return - } + err = SetSession(ctx, user, sessionToken) + if err != nil { + log.Printf("[WARNING] Error creating session for user: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed setting session"}`))) + return + } - user.LoginInfo = append(user.LoginInfo, LoginInfo{ - IP: GetRequestIp(request), - Timestamp: time.Now().Unix(), - }) + user.LoginInfo = append(user.LoginInfo, LoginInfo{ + IP: GetRequestIp(request), + Timestamp: time.Now().Unix(), + }) - user.Session = sessionToken - user.LoginInfo = append(user.LoginInfo, LoginInfo{ - IP: GetRequestIp(request), - Timestamp: time.Now().Unix(), - }) - } + user.Session = sessionToken + user.LoginInfo = append(user.LoginInfo, LoginInfo{ + IP: GetRequestIp(request), + Timestamp: time.Now().Unix(), + }) err = SetUser(ctx, &user, false) if err != nil { log.Printf("[WARNING] Failed updating user when setting session: %s", err) @@ -16803,52 +16694,45 @@ func HandleSSO(resp http.ResponseWriter, request *http.Request) { log.Printf("[AUDIT] Found user %s (%s) which matches SSO info for %s. Redirecting to login %s!", user.Username, user.Id, userName, redirectUrl) //log.Printf("SESSION: %s", user.Session) - // if project.Environment == "cloud" { - // user.ActiveOrg.Id = matchingOrgs[0].Id - // } - - user.ActiveOrg = OrgMini{ - Name: matchingOrgs[0].Name, - Id: matchingOrgs[0].Id, - Role: user.Role, + if project.Environment == "cloud" { + user.ActiveOrg.Id = matchingOrgs[0].Id } expiration := time.Now().Add(3600 * time.Second) - if len(user.Session) == 0 { - log.Printf("[INFO] User does NOT have session - creating") - sessionToken := uuid.NewV4().String() - newCookie := &http.Cookie{ - Name: "session_token", - Value: sessionToken, - Expires: expiration, - Path: "/", - } - - if project.Environment == "cloud" { - newCookie.Domain = ".shuffler.io" - newCookie.Secure = true - newCookie.HttpOnly = true - } + //if len(user.Session) == 0 { + log.Printf("[INFO] User does NOT have session - creating") + sessionToken := uuid.NewV4().String() + newCookie := &http.Cookie{ + Name: "session_token", + Value: sessionToken, + Expires: expiration, + Path: "/", + } - http.SetCookie(resp, newCookie) + if project.Environment == "cloud" { + newCookie.Domain = ".shuffler.io" + newCookie.Secure = true + newCookie.HttpOnly = true + } - newCookie.Name = "__session" - http.SetCookie(resp, newCookie) + http.SetCookie(resp, newCookie) - err = SetSession(ctx, user, sessionToken) - if err != nil { - log.Printf("[WARNING] Error creating session for user: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed setting session"}`))) - return - } + newCookie.Name = "__session" + http.SetCookie(resp, newCookie) - user.Session = sessionToken - user.LoginInfo = append(user.LoginInfo, LoginInfo{ - IP: GetRequestIp(request), - Timestamp: time.Now().Unix(), - }) + err = SetSession(ctx, user, sessionToken) + if err != nil { + log.Printf("[WARNING] Error creating session for user: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed setting session"}`))) + return } + + user.Session = sessionToken + user.LoginInfo = append(user.LoginInfo, LoginInfo{ + IP: GetRequestIp(request), + Timestamp: time.Now().Unix(), + }) err = SetUser(ctx, &user, false) if err != nil { log.Printf("[WARNING] Failed updating user when setting session: %s", err) @@ -16903,13 +16787,7 @@ func HandleSSO(resp http.ResponseWriter, request *http.Request) { newUser.Role = "user" newUser.Session = uuid.NewV4().String() - // newUser.ActiveOrg.Id = matchingOrgs[0].Id - - newUser.ActiveOrg = OrgMini{ - Name: matchingOrgs[0].Name, - Id: matchingOrgs[0].Id, - Role: "user", - } + newUser.ActiveOrg.Id = matchingOrgs[0].Id verifyToken := uuid.NewV4() ID := uuid.NewV4() diff --git a/structs.go b/structs.go index dd01d6c..5d54104 100755 --- a/structs.go +++ b/structs.go @@ -2516,7 +2516,6 @@ type SSOConfig struct { OpenIdClientSecret string `json:"client_secret" datastore:"client_secret"` OpenIdAuthorization string `json:"openid_authorization" datastore:"openid_authorization"` OpenIdToken string `json:"openid_token" datastore:"openid_token"` - SSORequired bool `json:"SSORequired" datastore:"SSORequired"` } type SamlRequest struct { From be7e7131fc5ddcc043efbe038d7b6c4303e57e20 Mon Sep 17 00:00:00 2001 From: Frikky Date: Tue, 21 May 2024 13:17:54 +0200 Subject: [PATCH 32/33] Minor app load fixes --- app_upload/stitcher.go | 4 ++-- db-connector.go | 16 +++++++++++----- oauth2.go | 3 +++ shared.go | 8 ++++++-- 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/app_upload/stitcher.go b/app_upload/stitcher.go index 2e7b11d..d1fa87b 100755 --- a/app_upload/stitcher.go +++ b/app_upload/stitcher.go @@ -844,8 +844,8 @@ func main() { bucketName = os.Args[5] } - appname := "email" - appversion := "1.3.0" + appname := "shuffle-ai" + appversion := "1.0.0" err := deployConfigToBackend(appfolder, appname, appversion) if err != nil { log.Printf("[WARNING] Failed uploading config: %s", err) diff --git a/db-connector.go b/db-connector.go index 835e34d..c11bb70 100755 --- a/db-connector.go +++ b/db-connector.go @@ -1656,7 +1656,7 @@ func GetWorkflowExecution(ctx context.Context, id string) (*WorkflowExecution, e if err != nil { log.Printf("[DEBUG][%s] Failed to parse in execution file value for exec argument: %s (3)", workflowExecution.ExecutionId, err) } else { - log.Printf("[DEBUG][%s] Found a new value to parse with exec argument", workflowExecution.ExecutionId) + //log.Printf("[DEBUG][%s] Found a new value to parse with exec argument", workflowExecution.ExecutionId) workflowExecution.ExecutionArgument = newValue } } @@ -1721,7 +1721,7 @@ func GetWorkflowExecution(ctx context.Context, id string) (*WorkflowExecution, e // A workaround for large bits of information for execution argument if strings.Contains(workflowExecution.ExecutionArgument, "Result too large to handle") { - log.Printf("[DEBUG] Found prefix %s to be replaced for exec argument (3)", workflowExecution.ExecutionArgument) + //log.Printf("[DEBUG] Found prefix %s to be replaced for exec argument (3)", workflowExecution.ExecutionArgument) baseArgument := &ActionResult{ Result: workflowExecution.ExecutionArgument, Action: Action{ID: "execution_argument"}, @@ -1730,7 +1730,7 @@ func GetWorkflowExecution(ctx context.Context, id string) (*WorkflowExecution, e if err != nil { log.Printf("[DEBUG] Failed to parse in execution file value for exec argument: %s (4)", err) } else { - log.Printf("[DEBUG] Found a new value to parse with exec argument") + //log.Printf("[DEBUG] Found a new value to parse with exec argument") workflowExecution.ExecutionArgument = newValue } } @@ -1865,6 +1865,10 @@ func GetApp(ctx context.Context, id string, user User, skipCache bool) (*Workflo return workflowApp, errors.New("No ID provided to get an app") } + if id == "integration" { + return workflowApp, errors.New("Integration is for the integration framework. Uses the Shuffle-ai app") + } + nameKey := "workflowapp" cacheKey := fmt.Sprintf("%s_%s", nameKey, id) @@ -6252,6 +6256,7 @@ func GetUserApps(ctx context.Context, userId string) ([]WorkflowApp, error) { cursorStr := "" log.Printf("[DEBUG] Getting user apps for %s", userId) + var err error queries := []datastore.Query{} q := datastore.NewQuery(indexName).Filter("contributors =", userId) @@ -6286,9 +6291,10 @@ func GetUserApps(ctx context.Context, userId string) ([]WorkflowApp, error) { } if err != nil { - log.Printf("[ERROR] Failed fetching user apps (1): %v", err) if !strings.Contains(fmt.Sprintf("%s", err), "cannot load field") { + log.Printf("[ERROR] Failed fetching user apps (1): %v", err) + if strings.Contains("no matching index found", fmt.Sprintf("%s", err)) { log.Printf("[ERROR] No more apps for %s in user app load? Breaking: %s.", userId, err) } else { @@ -6310,7 +6316,7 @@ func GetUserApps(ctx context.Context, userId string) ([]WorkflowApp, error) { if err != nil { if !strings.Contains(fmt.Sprintf("%s", err), "no more items") { - log.Printf("[ERROR] Failed fetching user apps (1): %v", err) + log.Printf("[ERROR] Failed fetching user apps (3): %v", err) } break diff --git a/oauth2.go b/oauth2.go index af59dd9..fe56cea 100755 --- a/oauth2.go +++ b/oauth2.go @@ -5,6 +5,7 @@ package shuffle import ( "bytes" "context" + "regexp" "crypto/sha256" "encoding/base64" "encoding/json" @@ -4113,11 +4114,13 @@ func VerifyIdToken(ctx context.Context, idToken string) (IdTokenCheck, error) { log.Printf("[ERROR] No org specified in state (2)") return IdTokenCheck{}, err } + org, err := GetOrg(ctx, foundOrg) if err != nil { log.Printf("[WARNING] Error getting org in OpenID (2): %s", err) return IdTokenCheck{}, err } + // Validating the user itself if token.Aud == org.SSOConfig.OpenIdClientId && foundChallenge == org.SSOConfig.OpenIdClientSecret { log.Printf("[DEBUG] Correct token aud & challenge - successful login!") diff --git a/shared.go b/shared.go index 616428c..8685f85 100755 --- a/shared.go +++ b/shared.go @@ -10428,7 +10428,7 @@ func GetWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { //log.Printf("SHARING: %s. PUBLIC: %s", app.Sharing, app.Public) if app.Sharing || app.Public { if openapiok && len(openapi) > 0 && strings.ToLower(openapi[0]) == "false" { - log.Printf("Should return WITHOUT openapi") + log.Printf("[DEBUG] Returning app '%s' without OpenAPI", fileId) } else { //log.Printf("CAN SHARE APP!") parsedApi, err := GetOpenApiDatastore(ctx, fileId) @@ -21394,14 +21394,18 @@ func RunCategoryAction(resp http.ResponseWriter, request *http.Request) { // FIXME: Check if ALL fields for the target app can be fullfiled // E.g. for Jira: Org_id is required. + + /* if foundFields != len(baseFields) { log.Printf("[WARNING] Not all required fields were found in category action. Want: %#v, have: %#v", baseFields, value.Fields) resp.WriteHeader(400) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Not all required fields are set", "label": "%s", "missing_fields": "%s"}`, value.Label, strings.Join(missingFields, ",")))) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Not all required fields are set. This can be autocompleted from fields you fille in", "label": "%s", "missing_fields": "%s"}`, value.Label, strings.Join(missingFields, ",")))) return } + */ + _ = baseFields /* From 628e28dd59d67baaf5fa1fe96efdb91a32f3b827 Mon Sep 17 00:00:00 2001 From: Frikky Date: Wed, 22 May 2024 02:05:48 +0200 Subject: [PATCH 33/33] Fixed bugs with error handling --- db-connector.go | 2 +- oauth2.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/db-connector.go b/db-connector.go index 39a0f24..4d06a60 100755 --- a/db-connector.go +++ b/db-connector.go @@ -6172,7 +6172,7 @@ func fixAppAppend(allApps []WorkflowApp, innerApp WorkflowApp) ([]WorkflowApp, W func GetUserApps(ctx context.Context, userId string) ([]WorkflowApp, error) { wrapper := []WorkflowApp{} - var err error + //var err error cacheKey := fmt.Sprintf("userapps-%s", userId) if project.CacheDb { diff --git a/oauth2.go b/oauth2.go index 161e118..32ba9e3 100755 --- a/oauth2.go +++ b/oauth2.go @@ -5,7 +5,7 @@ package shuffle import ( "bytes" "context" - "regexp" + //"regexp" "crypto/sha256" "encoding/base64" "encoding/json"