Merge branch 'Shuffle:master' into master

shuffle-ai/1.0.0/src/app.py

@@ -243,6 +243,8 @@ def run_schemaless(self, category, action, app_name="", fields=""):
         if app_name:
             data["app_name"] = app_name
 
+        self.logger.info(f"\n\nFIELDS MAPPED\n\n: {fields}")
+
         if fields:
             if isinstance(fields, list):
                 data["fields"] = fields
@@ -290,6 +292,22 @@ def run_schemaless(self, category, action, app_name="", fields=""):
             headers=headers,
         )
 
+        try: 
+            if "parameters" in self.action:
+                response_headers = request.headers
+                for key, value in response_headers.items():
+                    if not str(key).lower().endswith("-url"):
+                        continue
+
+                    self.action["parameters"].append({
+                        "name": key,
+                        "value": value,
+                    })
+
+                    #self.logger.info("[DEBUG] Response header: %s: %s" % (key, value))
+        except Exception as e:
+            self.logger.info("[ERROR] Failed to get response headers (category action url debug mapping): %s" % e)
+
         try: 
             return request.json()
         except:

shuffle-ai/1.0.0/upload.sh

@@ -2,5 +2,5 @@
 gcloud run deploy shuffle-ai-1-0-0 \
 	--region=europe-west2 \
 	--max-instances=5 \
-	--set-env-vars=SHUFFLE_APP_EXPOSED_PORT=8080,SHUFFLE_SWARM_CONFIG=run,SHUFFLE_LOGS_DISABLED=true --source=./  \
-	--timeout=300s
+	--set-env-vars=SHUFFLE_APP_EXPOSED_PORT=8080,SHUFFLE_SWARM_CONFIG=run,SHUFFLE_LOGS_DISABLED=true,SHUFFLE_APP_SDK_TIMEOUT=120 --source=./  \
+	--timeout=120s

shuffle-subflow/1.1.0/src/app.py

@@ -29,6 +29,7 @@ def run_userinput(self, user_apikey, sms="", email="", subflow="", information="
             "User-Agent": "Shuffle Userinput 1.1.0",
         }
 
+
         result = {
             "success": True,
             "source": "userinput",
@@ -40,6 +41,11 @@ def run_userinput(self, user_apikey, sms="", email="", subflow="", information="
                 "ip": "",
                 "user": "", 
                 "note": "",
+            },
+            "links": {
+                "frontend_no_answer": "",
+                "api_continue": "",
+                "api_abort": "",
             }
         }
 
@@ -50,26 +56,32 @@ def run_userinput(self, user_apikey, sms="", email="", subflow="", information="
         if len(str(backend_url)) > 0:
             url = backend_url
 
-        print("Found backend url: %s" % url)
-        #print("AUTH: %s" % self.full_execution["authorization"])
-        #if len(information):
-        #    print("Should run arg: %s", information)
+        frontend_url = url
+        if ":5001" in frontend_url:
+            print("Should change port to 3001.")
+        if "appspot.com" in frontend_url:
+            frontend_url = "https://shuffler.io"
+        if "run.app" in frontend_url:
+            frontend_url = "https://shuffler.io"
+        if "ngrok" in frontend_url:
+            frontend_url = ""
+
+        explore_path = "%s/workflows/%s/run?authorization=%s&reference_execution=%s&source_node=%s" % (frontend_url, self.full_execution["workflow"]["id"], self.full_execution["authorization"], self.full_execution["execution_id"], source_node)
+        frontend_continue_url = "%s/workflows/%s/run?authorization=%s&reference_execution=%s&answer=true&source_node=%s" % (frontend_url, self.full_execution["workflow"]["id"], self.full_execution["authorization"], self.full_execution["execution_id"], source_node)
+        frontend_abort_url = "%s/workflows/%s/run?authorization=%s&reference_execution=%s&answer=false&source_node=%s" % (frontend_url, self.full_execution["workflow"]["id"], self.full_execution["authorization"], self.full_execution["execution_id"], source_node)
+        api_continue_url = "%s/api/v1/workflows/%s/execute?authorization=%s&reference_execution=%s&answer=true&source_node=%s" % (frontend_url, self.full_execution["workflow"]["id"], self.full_execution["authorization"], self.full_execution["execution_id"], source_node)
+        api_abort_url = "%s/api/v1/workflows/%s/execute?authorization=%s&reference_execution=%s&answer=false&source_node=%s" % (frontend_url, self.full_execution["workflow"]["id"], self.full_execution["authorization"], self.full_execution["execution_id"], source_node)
+
+        result["links"]["frontend_no_answer"] = explore_path
+        result["links"]["frontend_continue"] = frontend_continue_url
+        result["links"]["frontend_abort"] = frontend_abort_url
+        result["links"]["api_continue"] = api_continue_url
+        result["links"]["api_abort"] = api_abort_url
 
+        print("Found backend url: %s" % url)
         if len(subflow) > 0:
-            #print("Should run subflow: %s", subflow) 
-
-            # Missing startnode (user input trigger)
-            #print("Subflows to run from userinput: ", subflows)
 
             subflows = subflow.split(",")
-            frontend_url = url
-            if ":5001" in frontend_url:
-                print("Should change port to 3001.")
-            if "appspot.com" in frontend_url:
-                frontend_url = "https://shuffler.io"
-            if "run.app" in frontend_url:
-                frontend_url = "https://shuffler.io"
-
             for item in subflows: 
                 # In case of URL being passed, and not just ID
                 if "/" in item:
@@ -80,10 +92,10 @@ def run_userinput(self, user_apikey, sms="", email="", subflow="", information="
                 argument = json.dumps({
                     "information": information,
                     "parent_workflow": self.full_execution["workflow"]["id"],
-                    "frontend_continue": "%s/workflows/%s/run?authorization=%s&reference_execution=%s&answer=true&source_node=%s" % (frontend_url, self.full_execution["workflow"]["id"], self.full_execution["authorization"], self.full_execution["execution_id"], source_node),
-                    "frontend_abort": "%s/workflows/%s/run?authorization=%s&reference_execution=%s&answer=false&source_node=%s" % (frontend_url, self.full_execution["workflow"]["id"], self.full_execution["authorization"], self.full_execution["execution_id"], source_node),
-                    "api_continue": "%s/api/v1/workflows/%s/execute?authorization=%s&reference_execution=%s&answer=true&source_node=%s" % (frontend_url, self.full_execution["workflow"]["id"], self.full_execution["authorization"], self.full_execution["execution_id"], source_node),
-                    "api_abort": "%s/api/v1/workflows/%s/execute?authorization=%s&reference_execution=%s&answer=false&source_node=%s" % (frontend_url, self.full_execution["workflow"]["id"], self.full_execution["authorization"], self.full_execution["execution_id"], source_node),
+                    "frontend_continue": frontend_continue_url,
+                    "frontend_abort": frontend_abort_url,
+                    "api_continue": api_continue_url,
+                    "api_abort": api_abort_url,
                 })
 
                 ret = self.run_subflow(user_apikey, item, argument, source_workflow=self.full_execution["workflow"]["id"], source_execution=self.full_execution["execution_id"], source_auth=self.full_execution["authorization"], startnode=startnode, backend_url=backend_url, source_node=source_node)

shuffle-tools/1.2.0/api.yaml

@@ -1,7 +1,7 @@
 ---
 app_version: 1.2.0
 name: Shuffle Tools
-description: A tool app for Shuffle.  Gives access to most missing features along with Liquid.
+description: A tool app for Shuffle. Gives access to most missing features along with Liquid.
 tags:
   - Testing
   - Shuffle
@@ -35,6 +35,55 @@ actions:
         example: print("hello world")
         schema:
           type: string
+
+  - name: dedup_and_merge 
+    description: Merges data from multiple workflows within a set timeframe. Returns action as SKIPPED if the data is a duplicate. Returns with a list of all data if the data at the end
+    parameters:
+      - name: key
+        description: The key to use for deduplication
+        required: true
+        multiline: false
+        example: "ticketname+username"
+        schema:
+          type: string
+      - name: value 
+        description: The full value of the item
+        required: true
+        multiline: true 
+        example: "1208301599081"
+        schema:
+          type: string
+      - name: timeout 
+        description: The timeout before returning
+        required: true
+        options:
+          - 2
+          - 3
+          - 4
+          - 5
+          - 6
+          - 7
+          - 8
+          - 9
+          - 10
+          - 15
+          - 20
+          - 25 
+        multiline: false
+        example: "1"
+        schema:
+          type: string
+      - name: set_skipped 
+        description: Whether to set the action SKIPPED or not IF it matches another workflow in the same timeframe
+        required: true
+        options:
+          - true
+          - false
+        multiline: false
+        example: "true"
+        schema:
+          type: string
+
   - name: check_cache_contains 
     description: Checks Shuffle cache whether a user-provided key contains a value. Returns ALL the values previously appended.
     parameters:
@@ -135,47 +184,47 @@ actions:
     returns:
       schema:
         type: string
-  - name: send_email_shuffle
-    description: Send an email from Shuffle
-    parameters:
-      - name: apikey
-        description: Your https://shuffler.io organization apikey
-        multiline: false
-        example: "https://shuffler.io apikey"
-        required: true
-        schema:
-          type: string
-      - name: recipients
-        description: The recipients of the email
-        multiline: false
-        example: "[email protected],[email protected]"
-        required: true
-        schema:
-          type: string
-      - name: subject
-        description: The subject to use
-        multiline: false
-        example: "SOS this is an alert :o"
-        required: true
-        schema:
-          type: string
-      - name: body
-        description: The body to add to the email
-        multiline: true
-        example: "This is an email alert from Shuffler.io :)"
-        required: true
-        schema:
-          type: string
-      - name: attachments 
-        description: The ID of files in Shuffle to add as attachments 
-        multiline: false
-        example: "file_id1,file_id2,file_id3"
-        required: false 
-        schema:
-          type: string
-    returns:
-      schema:
-        type: string
+  #- name: send_email_shuffle
+  #  description: Send an email from Shuffle
+  #  parameters:
+  #    - name: apikey
+  #      description: Your https://shuffler.io organization apikey
+  #      multiline: false
+  #      example: "https://shuffler.io apikey"
+  #      required: true
+  #      schema:
+  #        type: string
+  #    - name: recipients
+  #      description: The recipients of the email
+  #      multiline: false
+  #      example: "[email protected],[email protected]"
+  #      required: true
+  #      schema:
+  #        type: string
+  #    - name: subject
+  #      description: The subject to use
+  #      multiline: false
+  #      example: "SOS this is an alert :o"
+  #      required: true
+  #      schema:
+  #        type: string
+  #    - name: body
+  #      description: The body to add to the email
+  #      multiline: true
+  #      example: "This is an email alert from Shuffler.io :)"
+  #      required: true
+  #      schema:
+  #        type: string
+  #    - name: attachments 
+  #      description: The ID of files in Shuffle to add as attachments 
+  #      multiline: false
+  #      example: "file_id1,file_id2,file_id3"
+  #      required: false 
+  #      schema:
+  #        type: string
+  #  returns:
+  #    schema:
+  #      type: string
   - name: filter_list
     description: Takes a list and filters based on your data
     skip_multicheck: true

shuffle-tools/1.2.0/src/app.py

@@ -198,6 +198,7 @@ def send_email_shuffle(self, apikey, recipients, subject, body, attachments=""):
             "subject": subject, 
             "body": body, 
             "type": "alert",
+            "email_app": True,
         }
 
         # Read the attachments
@@ -214,13 +215,86 @@ def send_email_shuffle(self, apikey, recipients, subject, body, attachments=""):
                 pass
 
 
-        url = "https://shuffler.io/api/v1/functions/sendmail"
+        url = "https://shuffler.io/functions/sendmail"
         headers = {"Authorization": "Bearer %s" % apikey}
-        return requests.post(url, headers=headers, json=data, verify=False).text
+        return requests.post(url, headers=headers, json=data).text
 
     def repeat_back_to_me(self, call):
         return call
 
+    def dedup_and_merge(self, key, value, timeout, set_skipped=True):
+        timeout = int(timeout)
+        key = str(key)
+
+        set_skipped = True
+        if str(set_skipped).lower() == "false":
+            set_skipped = False
+        else:
+            set_skipped = True
+
+        cachekey = "dedup-%s" % (key)
+        response = {
+            "success": False,
+            "datastore_key": cachekey,
+            "info": "All keys from the last %d seconds with the key '%s' have been merged. The result was set to SKIPPED in all other actions." % (timeout, key),
+            "timeout": timeout,
+            "original_value": value,
+            "all_values": [],
+        }
+
+        found_cache = self.get_cache(cachekey)
+
+        if found_cache["success"] == True and len(found_cache["value"]) > 0:
+            if "value" in found_cache:
+                if not str(found_cache["value"]).startswith("["):
+                    found_cache["value"] = [found_cache["value"]]
+                else:
+                    try:
+                        found_cache["value"] = json.loads(found_cache["value"])
+                    except Exception as e:
+                        self.logger.info("[ERROR] Failed parsing JSON: %s" % e)
+            else:
+                found_cache["value"] = []
+
+            found_cache["value"].append(value)
+            if "created" in found_cache:
+                if found_cache["created"] + timeout + 3 < time.time():
+                    set_skipped = False 
+                    response["success"] = True
+                    response["all_values"] = found_cache["value"]
+
+                    self.delete_cache(cachekey)
+
+                    return json.dumps(response)
+                else:
+                    self.logger.info("Dedup-key is already handled in another workflow with timeout %d" % timeout)
+
+            self.set_cache(cachekey, json.dumps(found_cache["value"]))
+            if set_skipped == True:
+                self.action_result["status"] = "SKIPPED"
+                self.action_result["result"] = json.dumps({
+                    "status": False,
+                    "reason": "Dedup-key is already handled in another workflow with timeout %d" % timeout,
+                })
+
+                self.send_result(self.action_result, {"Authorization": "Bearer %s" % self.authorization}, "/api/v1/streams")
+
+            return found_cache
+
+        parsedvalue = [value]
+        resp = self.set_cache(cachekey, json.dumps(parsedvalue))
+
+        self.logger.info("Sleeping for %d seconds while waiting for cache to fill up elsewhere" % timeout)
+        time.sleep(timeout)
+        found_cache = self.get_cache(cachekey)
+
+        response["success"] = True
+        response["all_values"] = found_cache["value"]
+
+        self.delete_cache(cachekey)
+        return json.dumps(response)
+
+
     # https://github.com/fhightower/ioc-finder
     def parse_file_ioc(self, file_ids, input_type="all"):
         def parse(data):
@@ -2406,6 +2480,8 @@ def parse_ioc(self, input_string, input_type="all"):
         else:
             input_type = input_type.split(",")
             for i in range(len(input_type)):
+                item = input_type[i]
+
                 item = item.strip()
                 if not item.endswith("s"):
                     item = "%ss" % item