From 971ba03d2059c44d131c6ed40ec89d3c3ab0be78 Mon Sep 17 00:00:00 2001 From: Frikky Date: Tue, 11 Jun 2024 16:31:09 +0200 Subject: [PATCH 1/5] Run update apps with sdk --- shuffle-tools/1.2.0/api.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shuffle-tools/1.2.0/api.yaml b/shuffle-tools/1.2.0/api.yaml index 81e4a7e7..2c202eaa 100644 --- a/shuffle-tools/1.2.0/api.yaml +++ b/shuffle-tools/1.2.0/api.yaml @@ -1,7 +1,7 @@ --- app_version: 1.2.0 name: Shuffle Tools -description: A tool app for Shuffle. Gives access to most missing features along with Liquid. +description: A tool app for Shuffle. Gives access to most missing features along with Liquid. tags: - Testing - Shuffle From 8f5526bceae96d7170e3e9d29530dc3c59e9dcf1 Mon Sep 17 00:00:00 2001 From: Frikky Date: Mon, 17 Jun 2024 00:56:28 +0200 Subject: [PATCH 2/5] Fixed shuffle-ai upload to not time out so fast --- shuffle-ai/1.0.0/src/app.py | 19 +++++++++++++++++++ shuffle-ai/1.0.0/upload.sh | 4 ++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/shuffle-ai/1.0.0/src/app.py b/shuffle-ai/1.0.0/src/app.py index 9bcff4a2..89925445 100644 --- a/shuffle-ai/1.0.0/src/app.py +++ b/shuffle-ai/1.0.0/src/app.py @@ -290,6 +290,25 @@ def run_schemaless(self, category, action, app_name="", fields=""): headers=headers, ) + try: + self.logger.info("Starting url checker") + if "parameters" in self.action: + response_headers = request.headers + for key, value in response_headers.items(): + if not str(key).lower().endswith("-url"): + continue + + self.action["parameters"].append({ + "name": key, + "value": value, + }) + + self.logger.info("[DEBUG] Response header: %s: %s" % (key, value)) + else: + self.logger.info("[DEBUG] No parameters in action. Can't append url headers.") + except Exception as e: + self.logger.info("[ERROR] Failed to get response headers: %s" % e) + try: return request.json() except: diff --git a/shuffle-ai/1.0.0/upload.sh b/shuffle-ai/1.0.0/upload.sh index 6dbdff4d..e5a55fb5 100755 --- a/shuffle-ai/1.0.0/upload.sh +++ b/shuffle-ai/1.0.0/upload.sh @@ -2,5 +2,5 @@ gcloud run deploy shuffle-ai-1-0-0 \ --region=europe-west2 \ --max-instances=5 \ - --set-env-vars=SHUFFLE_APP_EXPOSED_PORT=8080,SHUFFLE_SWARM_CONFIG=run,SHUFFLE_LOGS_DISABLED=true --source=./ \ - --timeout=300s + --set-env-vars=SHUFFLE_APP_EXPOSED_PORT=8080,SHUFFLE_SWARM_CONFIG=run,SHUFFLE_LOGS_DISABLED=true,SHUFFLE_APP_SDK_TIMEOUT=120 --source=./ \ + --timeout=120s From 9247bab81e10e974d6edbe7eb815c3d9541f3a27 Mon Sep 17 00:00:00 2001 From: Frikky Date: Mon, 17 Jun 2024 14:08:57 +0200 Subject: [PATCH 3/5] Added a dedup+merge action that is easy to use --- shuffle-tools/1.2.0/api.yaml | 49 +++++++++++++++++++++++ shuffle-tools/1.2.0/src/app.py | 73 ++++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+) diff --git a/shuffle-tools/1.2.0/api.yaml b/shuffle-tools/1.2.0/api.yaml index 2c202eaa..4e8bf4f7 100644 --- a/shuffle-tools/1.2.0/api.yaml +++ b/shuffle-tools/1.2.0/api.yaml @@ -35,6 +35,55 @@ actions: example: print("hello world") schema: type: string + + - name: dedup_and_merge + description: Merges data from multiple workflows within a set timeframe. Returns action as SKIPPED if the data is a duplicate. Returns with a list of all data if the data at the end + parameters: + - name: key + description: The key to use for deduplication + required: true + multiline: false + example: "ticketname+username" + schema: + type: string + - name: value + description: The full value of the item + required: true + multiline: true + example: "1208301599081" + schema: + type: string + - name: timeout + description: The timeout before returning + required: true + options: + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 15 + - 20 + - 25 + multiline: false + example: "1" + schema: + type: string + - name: set_skipped + description: Whether to set the action SKIPPED or not IF it matches another workflow in the same timeframe + required: true + options: + - true + - false + multiline: false + example: "true" + schema: + type: string + - name: check_cache_contains description: Checks Shuffle cache whether a user-provided key contains a value. Returns ALL the values previously appended. parameters: diff --git a/shuffle-tools/1.2.0/src/app.py b/shuffle-tools/1.2.0/src/app.py index 821d3830..ae1e0f5c 100644 --- a/shuffle-tools/1.2.0/src/app.py +++ b/shuffle-tools/1.2.0/src/app.py @@ -221,6 +221,79 @@ def send_email_shuffle(self, apikey, recipients, subject, body, attachments=""): def repeat_back_to_me(self, call): return call + def dedup_and_merge(self, key, value, timeout, set_skipped=True): + timeout = int(timeout) + key = str(key) + + set_skipped = True + if str(set_skipped).lower() == "false": + set_skipped = False + else: + set_skipped = True + + cachekey = "dedup-%s" % (key) + response = { + "success": False, + "datastore_key": cachekey, + "info": "All keys from the last %d seconds with the key '%s' have been merged. The result was set to SKIPPED in all other actions." % (timeout, key), + "timeout": timeout, + "original_value": value, + "all_values": [], + } + + found_cache = self.get_cache(cachekey) + + if found_cache["success"] == True and len(found_cache["value"]) > 0: + if "value" in found_cache: + if not str(found_cache["value"]).startswith("["): + found_cache["value"] = [found_cache["value"]] + else: + try: + found_cache["value"] = json.loads(found_cache["value"]) + except Exception as e: + self.logger.info("[ERROR] Failed parsing JSON: %s" % e) + else: + found_cache["value"] = [] + + found_cache["value"].append(value) + if "created" in found_cache: + if found_cache["created"] + timeout + 3 < time.time(): + set_skipped = False + response["success"] = True + response["all_values"] = found_cache["value"] + + self.delete_cache(cachekey) + + return json.dumps(response) + else: + self.logger.info("Dedup-key is already handled in another workflow with timeout %d" % timeout) + + self.set_cache(cachekey, json.dumps(found_cache["value"])) + if set_skipped == True: + self.action_result["status"] = "SKIPPED" + self.action_result["result"] = json.dumps({ + "status": False, + "reason": "Dedup-key is already handled in another workflow with timeout %d" % timeout, + }) + + self.send_result(self.action_result, {"Authorization": "Bearer %s" % self.authorization}, "/api/v1/streams") + + return found_cache + + parsedvalue = [value] + resp = self.set_cache(cachekey, json.dumps(parsedvalue)) + + self.logger.info("Sleeping for %d seconds while waiting for cache to fill up elsewhere" % timeout) + time.sleep(timeout) + found_cache = self.get_cache(cachekey) + + response["success"] = True + response["all_values"] = found_cache["value"] + + self.delete_cache(cachekey) + return json.dumps(response) + + # https://github.com/fhightower/ioc-finder def parse_file_ioc(self, file_ids, input_type="all"): def parse(data): From d919784208d2882d24eaf725bba45783a458a09e Mon Sep 17 00:00:00 2001 From: Frikky Date: Wed, 19 Jun 2024 10:18:51 +0200 Subject: [PATCH 4/5] Small api fixes for shuffle tools/ai --- shuffle-ai/1.0.0/src/app.py | 9 ++-- shuffle-tools/1.2.0/api.yaml | 82 +++++++++++++++++----------------- shuffle-tools/1.2.0/src/app.py | 5 ++- 3 files changed, 48 insertions(+), 48 deletions(-) diff --git a/shuffle-ai/1.0.0/src/app.py b/shuffle-ai/1.0.0/src/app.py index 89925445..979a0158 100644 --- a/shuffle-ai/1.0.0/src/app.py +++ b/shuffle-ai/1.0.0/src/app.py @@ -243,6 +243,8 @@ def run_schemaless(self, category, action, app_name="", fields=""): if app_name: data["app_name"] = app_name + self.logger.info(f"\n\nFIELDS MAPPED\n\n: {fields}") + if fields: if isinstance(fields, list): data["fields"] = fields @@ -291,7 +293,6 @@ def run_schemaless(self, category, action, app_name="", fields=""): ) try: - self.logger.info("Starting url checker") if "parameters" in self.action: response_headers = request.headers for key, value in response_headers.items(): @@ -303,11 +304,9 @@ def run_schemaless(self, category, action, app_name="", fields=""): "value": value, }) - self.logger.info("[DEBUG] Response header: %s: %s" % (key, value)) - else: - self.logger.info("[DEBUG] No parameters in action. Can't append url headers.") + #self.logger.info("[DEBUG] Response header: %s: %s" % (key, value)) except Exception as e: - self.logger.info("[ERROR] Failed to get response headers: %s" % e) + self.logger.info("[ERROR] Failed to get response headers (category action url debug mapping): %s" % e) try: return request.json() diff --git a/shuffle-tools/1.2.0/api.yaml b/shuffle-tools/1.2.0/api.yaml index 4e8bf4f7..14f4f258 100644 --- a/shuffle-tools/1.2.0/api.yaml +++ b/shuffle-tools/1.2.0/api.yaml @@ -184,47 +184,47 @@ actions: returns: schema: type: string - - name: send_email_shuffle - description: Send an email from Shuffle - parameters: - - name: apikey - description: Your https://shuffler.io organization apikey - multiline: false - example: "https://shuffler.io apikey" - required: true - schema: - type: string - - name: recipients - description: The recipients of the email - multiline: false - example: "test@example.com,frikky@shuffler.io" - required: true - schema: - type: string - - name: subject - description: The subject to use - multiline: false - example: "SOS this is an alert :o" - required: true - schema: - type: string - - name: body - description: The body to add to the email - multiline: true - example: "This is an email alert from Shuffler.io :)" - required: true - schema: - type: string - - name: attachments - description: The ID of files in Shuffle to add as attachments - multiline: false - example: "file_id1,file_id2,file_id3" - required: false - schema: - type: string - returns: - schema: - type: string + #- name: send_email_shuffle + # description: Send an email from Shuffle + # parameters: + # - name: apikey + # description: Your https://shuffler.io organization apikey + # multiline: false + # example: "https://shuffler.io apikey" + # required: true + # schema: + # type: string + # - name: recipients + # description: The recipients of the email + # multiline: false + # example: "test@example.com,frikky@shuffler.io" + # required: true + # schema: + # type: string + # - name: subject + # description: The subject to use + # multiline: false + # example: "SOS this is an alert :o" + # required: true + # schema: + # type: string + # - name: body + # description: The body to add to the email + # multiline: true + # example: "This is an email alert from Shuffler.io :)" + # required: true + # schema: + # type: string + # - name: attachments + # description: The ID of files in Shuffle to add as attachments + # multiline: false + # example: "file_id1,file_id2,file_id3" + # required: false + # schema: + # type: string + # returns: + # schema: + # type: string - name: filter_list description: Takes a list and filters based on your data skip_multicheck: true diff --git a/shuffle-tools/1.2.0/src/app.py b/shuffle-tools/1.2.0/src/app.py index ae1e0f5c..77b556c8 100644 --- a/shuffle-tools/1.2.0/src/app.py +++ b/shuffle-tools/1.2.0/src/app.py @@ -198,6 +198,7 @@ def send_email_shuffle(self, apikey, recipients, subject, body, attachments=""): "subject": subject, "body": body, "type": "alert", + "email_app": True, } # Read the attachments @@ -214,9 +215,9 @@ def send_email_shuffle(self, apikey, recipients, subject, body, attachments=""): pass - url = "https://shuffler.io/api/v1/functions/sendmail" + url = "https://shuffler.io/functions/sendmail" headers = {"Authorization": "Bearer %s" % apikey} - return requests.post(url, headers=headers, json=data, verify=False).text + return requests.post(url, headers=headers, json=data).text def repeat_back_to_me(self, call): return call From 2cae4b49eb9f91feb92584a327ce159ee5004dcb Mon Sep 17 00:00:00 2001 From: Frikky Date: Wed, 26 Jun 2024 13:15:35 +0200 Subject: [PATCH 5/5] Fixed item problem with parse-ioc --- shuffle-subflow/1.1.0/src/app.py | 52 ++++++++++++++++++++------------ shuffle-tools/1.2.0/src/app.py | 2 ++ 2 files changed, 34 insertions(+), 20 deletions(-) diff --git a/shuffle-subflow/1.1.0/src/app.py b/shuffle-subflow/1.1.0/src/app.py index 4eead3fb..ad741e68 100644 --- a/shuffle-subflow/1.1.0/src/app.py +++ b/shuffle-subflow/1.1.0/src/app.py @@ -29,6 +29,7 @@ def run_userinput(self, user_apikey, sms="", email="", subflow="", information=" "User-Agent": "Shuffle Userinput 1.1.0", } + result = { "success": True, "source": "userinput", @@ -40,6 +41,11 @@ def run_userinput(self, user_apikey, sms="", email="", subflow="", information=" "ip": "", "user": "", "note": "", + }, + "links": { + "frontend_no_answer": "", + "api_continue": "", + "api_abort": "", } } @@ -50,26 +56,32 @@ def run_userinput(self, user_apikey, sms="", email="", subflow="", information=" if len(str(backend_url)) > 0: url = backend_url - print("Found backend url: %s" % url) - #print("AUTH: %s" % self.full_execution["authorization"]) - #if len(information): - # print("Should run arg: %s", information) + frontend_url = url + if ":5001" in frontend_url: + print("Should change port to 3001.") + if "appspot.com" in frontend_url: + frontend_url = "https://shuffler.io" + if "run.app" in frontend_url: + frontend_url = "https://shuffler.io" + if "ngrok" in frontend_url: + frontend_url = "" + + explore_path = "%s/workflows/%s/run?authorization=%s&reference_execution=%s&source_node=%s" % (frontend_url, self.full_execution["workflow"]["id"], self.full_execution["authorization"], self.full_execution["execution_id"], source_node) + frontend_continue_url = "%s/workflows/%s/run?authorization=%s&reference_execution=%s&answer=true&source_node=%s" % (frontend_url, self.full_execution["workflow"]["id"], self.full_execution["authorization"], self.full_execution["execution_id"], source_node) + frontend_abort_url = "%s/workflows/%s/run?authorization=%s&reference_execution=%s&answer=false&source_node=%s" % (frontend_url, self.full_execution["workflow"]["id"], self.full_execution["authorization"], self.full_execution["execution_id"], source_node) + api_continue_url = "%s/api/v1/workflows/%s/execute?authorization=%s&reference_execution=%s&answer=true&source_node=%s" % (frontend_url, self.full_execution["workflow"]["id"], self.full_execution["authorization"], self.full_execution["execution_id"], source_node) + api_abort_url = "%s/api/v1/workflows/%s/execute?authorization=%s&reference_execution=%s&answer=false&source_node=%s" % (frontend_url, self.full_execution["workflow"]["id"], self.full_execution["authorization"], self.full_execution["execution_id"], source_node) + + result["links"]["frontend_no_answer"] = explore_path + result["links"]["frontend_continue"] = frontend_continue_url + result["links"]["frontend_abort"] = frontend_abort_url + result["links"]["api_continue"] = api_continue_url + result["links"]["api_abort"] = api_abort_url + print("Found backend url: %s" % url) if len(subflow) > 0: - #print("Should run subflow: %s", subflow) - - # Missing startnode (user input trigger) - #print("Subflows to run from userinput: ", subflows) subflows = subflow.split(",") - frontend_url = url - if ":5001" in frontend_url: - print("Should change port to 3001.") - if "appspot.com" in frontend_url: - frontend_url = "https://shuffler.io" - if "run.app" in frontend_url: - frontend_url = "https://shuffler.io" - for item in subflows: # In case of URL being passed, and not just ID if "/" in item: @@ -80,10 +92,10 @@ def run_userinput(self, user_apikey, sms="", email="", subflow="", information=" argument = json.dumps({ "information": information, "parent_workflow": self.full_execution["workflow"]["id"], - "frontend_continue": "%s/workflows/%s/run?authorization=%s&reference_execution=%s&answer=true&source_node=%s" % (frontend_url, self.full_execution["workflow"]["id"], self.full_execution["authorization"], self.full_execution["execution_id"], source_node), - "frontend_abort": "%s/workflows/%s/run?authorization=%s&reference_execution=%s&answer=false&source_node=%s" % (frontend_url, self.full_execution["workflow"]["id"], self.full_execution["authorization"], self.full_execution["execution_id"], source_node), - "api_continue": "%s/api/v1/workflows/%s/execute?authorization=%s&reference_execution=%s&answer=true&source_node=%s" % (frontend_url, self.full_execution["workflow"]["id"], self.full_execution["authorization"], self.full_execution["execution_id"], source_node), - "api_abort": "%s/api/v1/workflows/%s/execute?authorization=%s&reference_execution=%s&answer=false&source_node=%s" % (frontend_url, self.full_execution["workflow"]["id"], self.full_execution["authorization"], self.full_execution["execution_id"], source_node), + "frontend_continue": frontend_continue_url, + "frontend_abort": frontend_abort_url, + "api_continue": api_continue_url, + "api_abort": api_abort_url, }) ret = self.run_subflow(user_apikey, item, argument, source_workflow=self.full_execution["workflow"]["id"], source_execution=self.full_execution["execution_id"], source_auth=self.full_execution["authorization"], startnode=startnode, backend_url=backend_url, source_node=source_node) diff --git a/shuffle-tools/1.2.0/src/app.py b/shuffle-tools/1.2.0/src/app.py index 77b556c8..cc713f91 100644 --- a/shuffle-tools/1.2.0/src/app.py +++ b/shuffle-tools/1.2.0/src/app.py @@ -2480,6 +2480,8 @@ def parse_ioc(self, input_string, input_type="all"): else: input_type = input_type.split(",") for i in range(len(input_type)): + item = input_type[i] + item = item.strip() if not item.endswith("s"): item = "%ss" % item