diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 00000000..c53d2d66 --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,2 @@ +App: + - '/.*/' diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 53e0cbd7..aa536619 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -64,7 +64,7 @@ jobs: with: context: ${{ matrix.app }}/${{ matrix.version }} file: ${{ matrix.app }}/${{ matrix.version }}/Dockerfile - platforms: linux/amd64,linux/arm64,linux/386 + platforms: linux/amd64,linux/arm64 push: true tags: | ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.app }}:${{ matrix.version }} diff --git a/.github/workflows/project_automation.yml b/.github/workflows/project_automation.yml index 0e3bf945..837957f6 100644 --- a/.github/workflows/project_automation.yml +++ b/.github/workflows/project_automation.yml @@ -6,6 +6,16 @@ on: - opened jobs: + add-label: + name: Add label to issue + runs-on: ubuntu-latest + steps: + - uses: github/issue-labeler@v3.3 #May not be the latest version + with: + configuration-path: .github/labeler.yml + repo-token: ${{ secrets.ADD_TO_PROJECT_PAT }} + enable-versioned-regex: 0 + add-to-project: name: Add issue to project runs-on: ubuntu-latest diff --git a/README.md b/README.md index 3ed3879d..8d19233d 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Shuffle Apps -This is a repository for apps to be used in [Shuffle](https://github.com/frikky/shuffle) +All public apps are available in the search, engine either in your local instance or on [https://shuffler.io/search?tab=apps](https://shuffler.io/search?tab=apps). This is a repository for apps to be used in [Shuffle](https://github.com/frikky/shuffle) -**PS:** These apps should be valid with WALKOFF, but the SDK is different, meaning you have to change the FIRST line in each Dockerfile (FROM frikky/shuffle:app_sdk). +**PS:** These apps should be valid with WALKOFF (from NSA), but the SDK is different, meaning you have to change the FIRST line in each Dockerfile (FROM frikky/shuffle:app_sdk) to make it compatible with Shuffle. ## App Creation App creation can be done with the Shuffle App Creator (exports as OpenAPI) or Python, which makes it possible to connect _literally_ any tool. Always prioritize using the App Creator when applicable. diff --git a/email/1.2.0/src/app.py b/email/1.2.0/src/app.py index 182d72d9..4a27c9b2 100644 --- a/email/1.2.0/src/app.py +++ b/email/1.2.0/src/app.py @@ -394,7 +394,7 @@ def parse_email_file(self, file_id, file_extension): print("File: %s" % file_path) if file_extension.lower() == 'eml': print('working with .eml file') - ep = eml_parser.EmlParser(include_attachment_data=True, include_raw_body=True) + ep = eml_parser.EmlParser(include_attachment_data=True, include_raw_body=True, parse_attachment=True) try: parsed_eml = ep.decode_email_bytes(file_path['data']) if str(parsed_eml["header"]["date"]) == "1970-01-01 00:00:00+00:00": diff --git a/servicenow/1.0.0/Dockerfile b/email/1.3.0/Dockerfile similarity index 99% rename from servicenow/1.0.0/Dockerfile rename to email/1.3.0/Dockerfile index 364e1531..bcc1273d 100644 --- a/servicenow/1.0.0/Dockerfile +++ b/email/1.3.0/Dockerfile @@ -24,3 +24,4 @@ COPY src /app # Finally, lets run our app! WORKDIR /app CMD python app.py --log-level DEBUG + diff --git a/email/1.3.0/api.yaml b/email/1.3.0/api.yaml new file mode 100644 index 00000000..40998adf --- /dev/null +++ b/email/1.3.0/api.yaml @@ -0,0 +1,280 @@ +walkoff_version: 1.3.0 +app_version: 1.3.0 +name: email +description: Email app +tags: + - email +categories: + - communication +contact_info: + name: "@frikkylikeme" + url: https://github.com/frikky + email: "frikky@shuffler.io" +actions: + - name: send_email_shuffle + description: Send an email from Shuffle + parameters: + - name: apikey + description: Your https://shuffler.io apikey + multiline: false + example: "https://shuffler.io apikey" + required: true + schema: + type: string + - name: recipients + description: The recipients of the email + multiline: false + example: "test@example.com,frikky@shuffler.io" + required: true + schema: + type: string + - name: subject + description: The subject to use + multiline: false + example: "SOS this is an alert :o" + required: true + schema: + type: string + - name: body + description: The body to add to the email + multiline: true + example: "This is an email alert from Shuffler.io :)" + required: true + schema: + type: string + returns: + schema: + type: string + - name: send_email_smtp + description: Send an email with SMTP + parameters: + - name: username + description: The SMTP login username + multiline: false + example: "frikky@shuffler.io" + required: false + schema: + type: string + - name: password + description: The password to log in with SMTP + multiline: false + example: "******************" + required: false + schema: + type: string + - name: smtp_host + description: The host of the SMTP + multiline: false + example: "smtp-mail.outlook.com" + required: true + schema: + type: string + - name: smtp_port + description: The port to use for SMTP + multiline: false + example: "587" + required: true + schema: + type: string + - name: recipient + description: The receiver(s) of the email + multiline: false + example: "frikky@shuffler.io,frikky@shuffler.io" + required: true + schema: + type: string + - name: subject + description: The subject of the email + multiline: false + example: "This is a subject, hello there :)" + required: true + schema: + type: string + - name: body + description: The body to add to the email + multiline: true + example: "This is an email alert from Shuffler.io :)" + required: true + schema: + type: string + - name: attachments + description: Send files from shuffle as part of the email + multiline: false + example: "file_id1,file_id2,file_id3" + required: false + schema: + type: string + - name: ssl_verify + description: Whether to use TLS or not + example: "true" + required: false + options: + - true + - false + schema: + type: string + - name: body_type + description: The type of body to send. HTML by default + example: "true" + required: false + options: + - "html" + - "plain" + schema: + type: string + returns: + schema: + type: string + - name: get_emails_imap + description: Get emails using IMAP (e.g. imap.gmail.com / Outlook.office365.com) + parameters: + - name: username + description: The SMTP login username + multiline: false + example: "frikky@shuffler.io" + required: true + schema: + type: string + - name: password + description: The password to log in with SMTP + multiline: false + example: "******************" + required: true + schema: + type: string + - name: imap_server + description: The imap server host + multiline: false + example: "Outlook.office365.com" + required: true + schema: + type: string + - name: foldername + description: The folder to use, e.g. "inbox" + multiline: false + example: "inbox" + required: true + schema: + type: string + - name: amount + description: Amount of emails to retrieve + multiline: false + example: "10" + required: true + schema: + type: string + - name: unread + description: Retrieve just unread emails + multiline: false + options: + - "false" + - "true" + required: true + schema: + type: bool + - name: fields + description: Comma separated list of fields to be exported + multiline: false + example: "body, header.subject, header.header.message-id" + required: false + schema: + type: string + - name: include_raw_body + description: Include raw body in email export + multiline: false + options: + - "true" + - "false" + required: true + schema: + type: bool + - name: include_attachment_data + description: Include raw attachments in email export + multiline: false + options: + - "false" + - "true" + required: true + schema: + type: bool + - name: upload_email_shuffle + description: Upload email in shuffle, return uid + multiline: false + options: + - "false" + - "true" + required: true + schema: + type: bool + - name: upload_attachments_shuffle + description: Upload attachments in shuffle, return uids + multiline: false + options: + - "false" + - "true" + required: true + schema: + type: bool + - name: ssl_verify + description: Whether to use TLS or not + example: "true" + required: false + options: + - true + - false + schema: + type: string + - name: mark_as_read + description: Mark email as read or not + multiline: false + options: + - "false" + - "true" + required: false + schema: + type: bool + - name: parse_email_file + description: Takes a file from shuffle and analyzes it if it's a valid .eml or .msg + parameters: + - name: file_id + description: file id + required: true + multiline: true + example: 'adf5e3d0fd85633be17004735a0a119e' + schema: + type: string + - name: extract_attachments + description: Whether to extract the attachments straight into files + required: true + options: + - true + - false + example: 'true' + schema: + type: string + - name: parse_email_headers + description: + parameters: + - name: email_headers + description: Email headers + required: true + multiline: true + example: 'Email Headers' + schema: + type: string + returns: + schema: + type: string + - name: analyze_headers + description: + parameters: + - name: headers + description: Email headers in any format + required: true + multiline: true + schema: + type: string + returns: + schema: + type: string +large_image: data:image/png;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4QAYRXhpZgAASUkqAAgAAAAAAAAAAAAAAP/hAytodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuMy1jMDExIDY2LjE0NTY2MSwgMjAxMi8wMi8wNi0xNDo1NjoyNyAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIiB4bWxuczp4bXBNTT0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wL21tLyIgeG1sbnM6c3RSZWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9zVHlwZS9SZXNvdXJjZVJlZiMiIHhtcDpDcmVhdG9yVG9vbD0iQWRvYmUgUGhvdG9zaG9wIENTNiAoV2luZG93cykiIHhtcE1NOkluc3RhbmNlSUQ9InhtcC5paWQ6Nzg4QTJBMjVEMDI1MTFFN0EwQUVDODc5QjYyQkFCMUQiIHhtcE1NOkRvY3VtZW50SUQ9InhtcC5kaWQ6Nzg4QTJBMjZEMDI1MTFFN0EwQUVDODc5QjYyQkFCMUQiPiA8eG1wTU06RGVyaXZlZEZyb20gc3RSZWY6aW5zdGFuY2VJRD0ieG1wLmlpZDo3ODhBMkEyM0QwMjUxMUU3QTBBRUM4NzlCNjJCQUIxRCIgc3RSZWY6ZG9jdW1lbnRJRD0ieG1wLmRpZDo3ODhBMkEyNEQwMjUxMUU3QTBBRUM4NzlCNjJCQUIxRCIvPiA8L3JkZjpEZXNjcmlwdGlvbj4gPC9yZGY6UkRGPiA8L3g6eG1wbWV0YT4gPD94cGFja2V0IGVuZD0iciI/Pv/bAEMAAwICAgICAwICAgMDAwMEBgQEBAQECAYGBQYJCAoKCQgJCQoMDwwKCw4LCQkNEQ0ODxAQERAKDBITEhATDxAQEP/AAAsIAGQAZAEBEQD/xAAeAAABAwUBAQAAAAAAAAAAAAAAAQgJAgQFBwoGA//EAEoQAAECBAMEBwMDEQgDAAAAAAECAwAEBREGBxIIITFRCRMiMkFSYRRicSNCQxUWGBk2U1dYc3WBlJWzwdLTJDNjZXKDkeEmgqH/2gAIAQEAAD8Ak8JKipSlBwuDSpSeDw8qeRguQQrUAQNAV4JH3s+sA7OnT8n1fcv9Bfzc7wWAAToIAOsJ8Uq++H0gI1XSUlYWdSkji6fMnkBAdS9SidesWUpI3OjyjkYXtXCgbEDSFW3JT5D6+sIOzp0/J9X3NX0F/NzvBYABOggA6wnxSr74fSAjVdJSVhZ1KSOLp8yeQEBJUSVKCysaVKHB0eVPIwXIIVqAIGgK8Ej72fWFS640kNtzjcukcGli6k/GENwVagAQO0EcEjmj1g33AATe1wD3SnmffgG/Tp337mv535T+EaB2k9tzInZilVyuMq+up4iUjWxh+mFLs8s23dbv0stnmsgnwBiNPOHpddozHExMSmWsrSsA0tZIaMs0JudCfV50FKf/AFQPjDYsT7S+0LjJ8zGJc68aTqiSrSqtPoQD6ISoJH6BGFkM5c3qW8Jim5qYvlXArUFNVuZSb89y43Tlv0ju15ltMtKZzUmsRSbZGuSxC0mebcHIrV8r/wALEP02c+l2ywzAmZXDGeND+saqvEIRVWXFP0x1Z3AOE9thPx1JHioQ/un1Kn1eQZqtLn2ZuSmkJdamZVwOIWlQuktKTuUg8xFybgnUACB2gjgkc0e9BvuAAm9rgHulPM+/CpDik3bal1p8FPd8/GEtp7Ojq+r36OPUe96wWv2dF79vR5/8T/qI/ukM6RMZMGcyWyUqDMzjZ5vRWKwmy26UlQ3IQOBmLHx3IFibncIeqvWKtiCqTVbrtSmahUJ11T0zNTLqnHXnFG5UpSiSSeZi0ggggh2GxRt8Y92X69K4cr83N1zLuadCZumLXrcp4Ue0/KXPZPiW+6r0O+JxsF4zwvmFhSl42wXV5eo0Sqy6ZySmWFakJbUO/wDHiCk7wQQd4jN2v2dF79vR5v8AE/6g6rrflPYfar/S69Or9EIAAE6QoAHshfFJ5r9Ibpt3bTrOzBkbPYhpb7f11V5aqZh9le8iZUm65i3i20ntcirQPGIA6nU6jWqlNVirzr05PTzy5iZmHllTjrqyVKWoneSSSSYtoIIIIIIkI6J/axmsA4+Rs84yqZ+tvFb5VQ1vL7EjVCNyN/Bt4C1vOEn5xiYndYghVr3IHeKuY9yKVJbJu63MrV4qY7h+EVA6rEKKwvclSuLp8quQiDnpWM5ZnMrafn8HS04pykZfy6KMw2D2BNEByZUPXWQj/aEM0h3uwvk5PYmoeLM4HcnqHmth/CU1KytdwrNy5M+uUdQtZmZBYIu83oN2z3wbcbWk7ym2Zej6zuwXJ49y5yXwbUqXNgpUPZlpelnh32XmyrU24k7ik7/0WMey+wI2OPxesJfqyv5oPsCNjj8XrCX6sr+aD7AjY4/F6wl+rq/mhs2b2SGzXjPGc3s9bKuzngiq4zZGjEWJ3pRTlKwkyrcVOKCrOzVr6GRex73AiIe65TvqRWqhSet632Kadl9enTq0LKb28L24R86ZUp6jVKUrFMmVy85IvtzMu8g2U24hQUlQPMEAx0h7O+abGdWR+DM0mnAF16lMuzRTxamgNDzQHIOJWI2IpxDZ0Lm3ZdQ4tti6U/CEdeDaHJhxYWAklahwdAF9KeRjmXzRxJMYxzLxXiyacUt2sVqdnlFRuflHlq/jHmIlv6D37is1T/mlM/cvQ5zNnZ7xtl/jWc2htlFUtIYrfs7iXCLy+rpeLGk7zcDczN2vpdFrnvcSTsrIPaHwNtBYcmKlhsv02t0h4ydfw7UE9XUKPOJJC2Xmzv3KBAWNyrbvEDaDjjbTa3XVpQhAKlKUbBIHEk+ENLxdnDj/AGr8T1LJ7Zgra6NgymPmSxhmU0LhB+kkaV4OPkGynu6gG4PAlwGUuT2AMjsDy2BMuqGin06XBcdWTrfm3j3333D2nHFHeVH/AOCwjmnxr92Ve/Oc1+9VGGibDogMVP1vZWmKI+4b4dxJOybS1G4S06ht7QPipxf/ADD4kurbGhE21LpHBtwXUn4xbVNpb9OnGAAFrl3EkI4JukgFHrHMFWpdyUrE/KvAhxmZdbUDxBCyDFnEuHQej/wjNU/5rTP3L0PQ2hs1cR4f+pOUWU/VTGZeOtbFK1jW3SZNO6YqkwPBtlJ7IPfcKUi++NdYh2H5LB1CoWLtnXFD2Fs1sLS6rV+ZUXG8SqWouPtVVP0yXnCo6+8gqFtwAGAbkdpzbFcTgXNLB1Qyay9pBEri1iXm9VQxPNo/vJeVdT/dyJ3XcG9YNgTvt6TFODKZsY4nlc2MsaEmRypn25em45oMi2eqpiUANsVllA8gsiYtvUiyzcpJhz8rOylSkGqjITTUzKzTKXmHmlhSHG1C6VJI3EEEEGOXnGv3ZV785zX71UYaJjehfk3mdn/GM4oHRM4sWEBfcsiUZ1Eeu+JBUhxSbttS60+Cnu+fjCAaDbR1fV9rRx6n3vW8c5+2Bl1MZV7TGYmDXmlIaZrkxNyhIsFy0wrr2lD00OJjT0SfdE5mphvJnIrOXHmJutdalatSmZSSlxqmKhNuNOpYlWU8VOOLISAOdzuBiQLZ5yrxJQTVs382Q0/mVjrQ9VAk6m6RJp3y9Llz4NtA9ojvuFSjfdG54N8fGekZOpyUxTajKtTMrNNLYfYdSFIdbULKSpJ3EEEgiG4ZXz07s0Zis7OuJpp1zAmJFvP5cVSYWSJVYut2hurPzkC62Ce83dHFFogGxr92Ve/Oc1+9VGGiezo0MupnLzY/wezPy5bm8RqmMROsqFiUvr+SWf8AaQ2besOl6rrflPYfar/S69Or9EIAAE6QoAHshfFJ5r9Ii76Y7Z4mJgUHaSw7IqWhlCKHiLQN6RcmWmP9Nypsn8mIizj3GWGdOY2T9Xka1gSuJlH6bO/VKWbflm5hlubDam0v9U4lSC4lClBKiLp1G1iY3v8AbSdtr8LTP7Ekf6UL9tJ22vwss/sOR/pQfbSdtr8LLP7Dkf6UH20nba/C0z+w5H+lHmMxOkB2qc1MOKwrjjMNmfkPaGZxrTSZRl1iYZWFtPNOobC21pULhSSDx8DDe5uamJ6aenZt1Tr8w4p11auKlqNyT8SY2ZszZH1raIzqw1ldSG1hqozSXKlMAHTKyLZCn3VHwsi4HvKSPGOjSj0im0CjyVBpMqJen06XalZZhG7Q22kJQE+4AAIulJbJu63MrV4qY7h+EVA6rEKKwvclSuLp8quQjB45wVhrMbB9YwNjGnIn6JW5VyQnWVjihYtoTysbEKHAgGOf7a72U8abKeZszhStMOzVAnVreoNXCfk5uXvuSojcHUAgLTz3jcRGi4IIIIIuqVSqnXKnK0ajSD89PzzyJeWlpdsrcecUbJQlI3kkkAAROd0eGxqjZiy7XiLF8sy5j/FjaFVEiyhJMDeiSB9D2lkbiqw4JEO58CrUQAdJV4pPkHu+sIpxDZ0Lm3ZdQ4tti6U/CFJKiVKUFle5Sk8HR5U8jBcghWoAgaArwSPIfe9Y8PnJkvl1nzgScy7zMoDVQpMyLtlXZekHfmvNucULHgR8DcEiIZtqro1s58gZucxFg6QmsbYJQVOonpFkqnJNrw9pYTcgAfSJuk8Tp4Qz9SSklKgQQbEHwgggjYeTOz9m7n/iFGHMq8Fz1YdCgJiZSjRKSiT8955XYQB6m58AYmN2LejtwJsyoYxri52XxVmC43unA3/ZpAEb0ygVvv4F09ojgEgm7wSSq5Kgsr3KUODo8qeRguQQrUAQNAV4JHkPvesKl1bY0Im2pdI4NuC6k/GENwVagAQO0EcEjmj1g33AATe1wD3SnmffgG/Tp337mv535T+EG4i4KiCbAnvE8j7kaHzf2Hdl/O1+YqONcraexU3jd6p0i8jNFfPU1ZLnxWlUNjxL0LOTs6+tzC+bWLKSkHUWpmXl5xKR4BJAbJjD0/oTcEoeH1Uz5rbzfe0sUZlolH+pTigFelo3Nlr0UuyZgSYZn6vQ6zjKaQQpr6uz3yBI462WQhNvRVxDscMYVwvgujMYfwfh+n0Wly/ZZlJCVQw2k8tCABp9YypsL6iQAe0U8Unkj3YDcE6gAQO0EcEjmj3oN9wAE3tcA90p5n34VIcUm7bUutPgp7vn4wOpS05MNtiyZdAW0PKo+MASkuIbIulbPXKHNfOEa+V9m6zf7Vq633rcIpSoqbQ6T2lvdQo80coVxRbRMLRuVLuBts+VJ4iKnEhtb6ECwl0BxseVR8YAlJcQ2RdK2euUOa+cI18r7N1m/wBq1db71uEUpUVNodJ7S3uoUeaOUK4otomFo3Kl3A22fKk8RFTiQ2t9CBYS6A42PKo+MASkuIbIulbPXKHNfOPtKSkvNy6JiYaC3F71KJO+P//Z diff --git a/email/1.3.0/requirements.txt b/email/1.3.0/requirements.txt new file mode 100644 index 00000000..926027e8 --- /dev/null +++ b/email/1.3.0/requirements.txt @@ -0,0 +1,8 @@ +requests==2.25.1 +glom==20.11.0 +eml-parser==1.17.0 +msg-parser==1.2.0 +mail-parser==3.15.0 +extract-msg==0.30.9 +jsonpickle==2.0.0 + diff --git a/email/1.3.0/src/app.py b/email/1.3.0/src/app.py new file mode 100644 index 00000000..62b3a26d --- /dev/null +++ b/email/1.3.0/src/app.py @@ -0,0 +1,617 @@ +import json +import uuid +import socket +import asyncio +import requests +import tempfile +import datetime +import base64 +import imaplib +import smtplib +import time +import random +import eml_parser +import mailparser +import extract_msg +import jsonpickle + +from glom import glom +from msg_parser import MsOxMessage +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from email.mime.application import MIMEApplication + +from walkoff_app_sdk.app_base import AppBase + +def json_serial(obj): + if isinstance(obj, datetime.datetime): + serial = obj.isoformat() + return serial + +def default(o): + """helpers to store item in json + arguments: + - o: field of the object to serialize + returns: + - valid serialized value for unserializable fields + """ + if isinstance(o, (datetime.date, datetime.datetime)): + return o.isoformat() + if isinstance(o, set): + return list(o) + if isinstance(o, bytes): + try: + return o.decode("utf-8") + except: + print("Failed parsing utf-8 string") + return o + + +class Email(AppBase): + __version__ = "1.3.0" + app_name = "email" + + def __init__(self, redis, logger, console_logger=None): + """ + Each app should have this __init__ to set up Redis and logging. + :param redis: + :param logger: + :param console_logger: + """ + super().__init__(redis, logger, console_logger) + + # This is an email function of Shuffle + def send_email_shuffle(self, apikey, recipients, subject, body): + targets = [recipients] + if ", " in recipients: + targets = recipients.split(", ") + elif "," in recipients: + targets = recipients.split(",") + + data = {"targets": targets, "body": body, "subject": subject, "type": "alert"} + + url = "https://shuffler.io/functions/sendmail" + headers = {"Authorization": "Bearer %s" % apikey} + return requests.post(url, headers=headers, json=data).text + + def send_email_smtp( + self, smtp_host, recipient, subject, body, smtp_port, attachments="", username="", password="", ssl_verify="True", body_type="html" + ): + if type(smtp_port) == str: + try: + smtp_port = int(smtp_port) + except ValueError: + return "SMTP port needs to be a number (Current: %s)" % smtp_port + + try: + s = smtplib.SMTP(host=smtp_host, port=smtp_port) + except socket.gaierror as e: + return f"Bad SMTP host or port: {e}" + + # This is not how it should work.. + # Port 465 & 587 = TLS. Sometimes 25. + if ssl_verify == "false" or ssl_verify == "False": + pass + else: + s.starttls() + + if len(username) > 1 or len(password) > 1: + try: + s.login(username, password) + except smtplib.SMTPAuthenticationError as e: + return { + "success": False, + "reason": f"Bad username or password: {e}" + } + + if body_type == "" or len(body_type) < 3: + body_type = "html" + + # setup the parameters of the message + msg = MIMEMultipart() + msg["From"] = username + msg["To"] = recipient + msg["Subject"] = subject + msg.attach(MIMEText(body, body_type)) + + # Read the attachments + attachment_count = 0 + try: + if attachments != None and len(attachments) > 0: + print("Got attachments: %s" % attachments) + attachmentsplit = attachments.split(",") + + #attachments = parse_list(attachments, splitter=",") + #print("Got attachments2: %s" % attachmentsplit) + print("Before loop") + files = [] + for file_id in attachmentsplit: + print(f"Looping {file_id}") + file_id = file_id.strip() + new_file = self.get_file(file_id) + print(f"New file: {new_file}") + try: + part = MIMEApplication( + new_file["data"], + Name=new_file["filename"], + ) + part["Content-Disposition"] = f"attachment; filename=\"{new_file['filename']}\"" + msg.attach(part) + attachment_count += 1 + except Exception as e: + print(f"[WARNING] Failed to attach {file_id}: {e}") + + + #files.append(new_file) + + #return files + #data["attachments"] = files + except Exception as e: + self.logger.info(f"Error in attachment parsing for email: {e}") + + + try: + s.send_message(msg) + except smtplib.SMTPDataError as e: + return { + "success": False, + "reason": f"Failed to send mail: {e}" + } + + self.logger.info("Successfully sent email with subject %s to %s" % (subject, recipient)) + return { + "success": True, + "reason": "Email sent to %s!" % recipient, + "attachments": attachment_count + } + + def get_emails_imap( + self, + username, + password, + imap_server, + foldername, + amount, + unread, + fields, + include_raw_body, + include_attachment_data, + upload_email_shuffle, + upload_attachments_shuffle, + ssl_verify="True", + mark_as_read="False", + ): + def path_to_dict(path, value=None): + def pack(parts): + return ( + {parts[0]: pack(parts[1:]) if len(parts) > 1 else value} + if len(parts) > 1 + else {parts[0]: value} + ) + + return pack(path.split(".")) + + def merge(d1, d2): + for k in d2: + if k in d1 and isinstance(d1[k], dict) and isinstance(d2[k], dict): + merge(d1[k], d2[k]) + else: + d1[k] = d2[k] + + #if isinstance(mark_as_read, str): + # if str(mark_as_read).lower() == "true": + # mark_as_read = True + # else: + # mark_as_read = False + + if type(amount) == str: + try: + amount = int(amount) + except ValueError: + return { + "success": False, + "reason": "Amount needs to be a number, not %s" % amount, + } + + try: + email = imaplib.IMAP4_SSL(imap_server) + except ConnectionRefusedError as error: + try: + email = imaplib.IMAP4(imap_server) + + if ssl_verify == "false" or ssl_verify == "False" or ssl_verify == False: + pass + else: + email.starttls() + except socket.gaierror as error: + return { + "success": False, + "reason": "Can't connect to IMAP server %s: %s" % (imap_server, error), + } + except socket.gaierror as error: + return { + "success": False, + "reason": "Can't connect to IMAP server %s: %s" % (imap_server, error), + } + + try: + email.login(username, password) + except imaplib.IMAP4.error as error: + return { + "success": False, + "reason": "Failed to log into %s: %s" % (username, error), + } + + email.select(foldername) + unread = True if unread.lower().strip() == "true" else False + + try: + # IMAP search queries, e.g. "seen" or "read" + # https://www.rebex.net/secure-mail.net/features/imap-search.aspx + mode = "(UNSEEN)" if unread else "ALL" + thistype, data = email.search(None, mode) + except imaplib.IMAP4.error as error: + return { + "success": False, + "reason": "Couldn't find folder %s." % (foldername), + } + + email_ids = data[0] + id_list = email_ids.split() + if id_list == None: + return { + "success": False, + "reason": f"Couldn't retrieve email. Data: {data}", + } + + #try: + # self.logger.info(f"LIST: {id_list}") + #except TypeError: + # return { + # "success": False, + # "reason": "Error getting email. Data: %s" % data, + # } + + mark_as_read = True if str(mark_as_read).lower().strip() == "true" else False + include_raw_body = True if str(include_raw_body).lower().strip() == "true" else False + include_attachment_data = ( + True if str(include_attachment_data).lower().strip() == "true" else False + ) + upload_email_shuffle = ( + True if str(upload_email_shuffle).lower().strip() == "true" else False + ) + upload_attachments_shuffle = ( + True if str(upload_attachments_shuffle).lower().strip() == "true" else False + ) + + # Convert of mails in json + emails = [] + ep = eml_parser.EmlParser( + include_attachment_data=include_attachment_data + or upload_attachments_shuffle, + include_raw_body=include_raw_body, + ) + + if len(id_list) == 0: + return { + "success": True, + "messages": [], + } + + try: + amount = len(id_list) if len(id_list)= 2: + parsed_headers[splitheader[0].strip()] = splititem.join(splitheader[1:]).strip() else: self.logger.info("Skipping header %s with split %s cus only one item" % (header, splititem)) continue diff --git a/http/1.4.0/Dockerfile b/http/1.4.0/Dockerfile new file mode 100644 index 00000000..9bbc5110 --- /dev/null +++ b/http/1.4.0/Dockerfile @@ -0,0 +1,27 @@ +# Base our app image off of the WALKOFF App SDK image +FROM frikky/shuffle:app_sdk as base + +# We're going to stage away all of the bloat from the build tools so lets create a builder stage +FROM base as builder + +# Install all alpine build tools needed for our pip installs +RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-dev + +# Install all of our pip packages in a single directory that we can copy to our base image later +RUN mkdir /install +WORKDIR /install +COPY requirements.txt /requirements.txt +RUN pip install --prefix="/install" -r /requirements.txt + +# Switch back to our base image and copy in all of our built packages and source code +FROM base +COPY --from=builder /install /usr/local +COPY src /app +RUN apk add curl + +# Install any binary dependencies needed in our final image +# RUN apk --no-cache add --update my_binary_dependency + +# Finally, lets run our app! +WORKDIR /app +CMD python app.py --log-level DEBUG diff --git a/http/1.4.0/api.yaml b/http/1.4.0/api.yaml new file mode 100644 index 00000000..cad10c2c --- /dev/null +++ b/http/1.4.0/api.yaml @@ -0,0 +1,529 @@ +walkoff_version: 1.4.0 +app_version: 1.4.0 +name: http +description: HTTP app +tags: + - Testing + - HTTP +categories: + - Other + - HTTP +contact_info: + name: "@frikkylikeme" + url: https://github.com/frikky + email: "frikky@shuffler.io" +actions: + - name: GET + description: Runs a GET request towards the specified endpoint + parameters: + - name: url + description: The URL to get + multiline: false + example: "https://example.com" + required: true + schema: + type: string + - name: headers + description: Headers to use + multiline: true + required: false + example: "Content-Type: application/json" + schema: + type: string + - name: username + description: The username to use + multiline: false + required: false + example: "Username" + schema: + type: string + - name: password + description: The password to use + multiline: false + required: false + example: "*****" + schema: + type: string + - name: verify + description: Check certificate + multiline: false + options: + - false + - true + required: false + example: "false" + schema: + type: bool + - name: http_proxy + description: Add a HTTP proxy + multiline: false + required: false + example: "http://192.168.0.1:8080" + schema: + type: bool + - name: https_proxy + description: Add a HTTPS proxy + multiline: false + required: false + example: "http://192.168.0.1:8080" + schema: + type: bool + - name: timeout + description: Add a timeout for the request, in seconds + multiline: false + required: false + example: "10" + schema: + type: bool + - name: to_file + description: Makes the response into a file, and returns it as an ID + multiline: false + required: false + options: + - false + - true + example: "true" + schema: + type: bool + returns: + schema: + type: string + example: "404 NOT FOUND" + - name: POST + description: Runs a POST request towards the specified endpoint + parameters: + - name: url + description: The URL to post to + multiline: false + example: "https://example.com" + required: true + schema: + type: string + - name: body + description: The body to use + multiline: true + example: "{\n\t'json': 'blob'\n}" + required: false + schema: + type: string + - name: headers + description: Headers to use + multiline: true + required: false + example: "Content-Type: application/json" + schema: + type: string + - name: username + description: The username to use + multiline: false + required: false + example: "Username" + schema: + type: string + - name: password + description: The password to use + multiline: false + required: false + example: "*****" + schema: + type: string + - name: verify + description: Whether to check the certificate or not + multiline: false + required: false + options: + - false + - true + example: "false" + schema: + type: bool + - name: http_proxy + description: Add a HTTP proxy + multiline: false + required: false + example: "http://192.168.0.1:8080" + schema: + type: bool + - name: https_proxy + description: Add a HTTPS proxy + multiline: false + required: false + example: "http://192.168.0.1:8080" + schema: + type: bool + - name: timeout + description: Add a timeout for the request, in seconds + multiline: false + required: false + example: "10" + schema: + type: bool + returns: + schema: + type: string + example: "404 NOT FOUND" + - name: PATCH + description: Runs a PATCH request towards the specified endpoint + parameters: + - name: url + description: The URL to post to + multiline: false + example: "https://example.com" + required: true + schema: + type: string + - name: body + description: The body to use + multiline: true + example: "{\n\t'json': 'blob'\n}" + required: false + schema: + type: string + - name: headers + description: Headers to use + multiline: true + required: false + example: "Content-Type: application/json" + schema: + type: string + - name: username + description: The username to use + multiline: false + required: false + example: "Username" + schema: + type: string + - name: password + description: The password to use + multiline: false + required: false + example: "*****" + schema: + type: string + - name: verify + description: Whether to check the certificate or not + multiline: false + required: false + options: + - false + - true + example: "false" + schema: + type: bool + - name: http_proxy + description: Add a HTTP proxy + multiline: false + required: false + example: "http://192.168.0.1:8080" + schema: + type: bool + - name: https_proxy + description: Add a HTTPS proxy + multiline: false + required: false + example: "http://192.168.0.1:8080" + schema: + type: bool + - name: timeout + description: Add a timeout for the request, in seconds + multiline: false + required: false + example: "10" + schema: + type: bool + returns: + schema: + type: string + example: "404 NOT FOUND" + - name: PUT + description: Runs a PUT request towards the specified endpoint + parameters: + - name: url + description: The URL to PUT to + multiline: false + example: "https://example.com" + required: true + schema: + type: string + - name: body + description: The body to use + multiline: true + example: "{\n\t'json': 'blob'\n}" + required: false + schema: + type: string + - name: headers + description: Headers to use + multiline: true + required: false + example: "Content-Type: application/json" + schema: + type: string + - name: username + description: The username to use + multiline: false + required: false + example: "Username" + schema: + type: string + - name: password + description: The password to use + multiline: false + required: false + example: "*****" + schema: + type: string + - name: verify + description: Whether to check the certificate or not + multiline: false + required: false + options: + - false + - true + example: "false" + schema: + type: bool + - name: http_proxy + description: Add a HTTP proxy + multiline: false + required: false + example: "http://192.168.0.1:8080" + schema: + type: bool + - name: https_proxy + description: Add a HTTPS proxy + multiline: false + required: false + example: "http://192.168.0.1:8080" + schema: + type: bool + - name: timeout + description: Add a timeout for the request, in seconds + multiline: false + required: false + example: "10" + schema: + type: bool + returns: + schema: + type: string + example: "404 NOT FOUND" + - name: DELETE + description: Runs a DELETE request towards the specified endpoint + parameters: + - name: url + description: The URL to post to + multiline: false + example: "https://example.com" + required: true + schema: + type: string + - name: body + description: The body to use + multiline: true + example: "{\n\t'json': 'blob'\n}" + required: false + schema: + type: string + - name: headers + description: Headers to use + multiline: true + required: false + example: "Content-Type: application/json" + schema: + type: string + - name: username + description: The username to use + multiline: false + required: false + example: "Username" + schema: + type: string + - name: password + description: The password to use + multiline: false + required: false + example: "*****" + schema: + type: string + - name: verify + description: Whether to check the certificate or not + multiline: false + required: false + options: + - true + - false + example: "false" + schema: + type: bool + - name: http_proxy + description: Add a HTTP proxy + multiline: false + required: false + example: "http://192.168.0.1:8080" + schema: + type: bool + - name: https_proxy + description: Add a HTTPS proxy + multiline: false + required: false + example: "http://192.168.0.1:8080" + schema: + type: bool + - name: timeout + description: Add a timeout for the request, in seconds + multiline: false + required: false + example: "10" + schema: + type: bool + returns: + schema: + type: string + example: "404 NOT FOUND" + - name: HEAD + description: Runs a HEAD request towards the specified endpoint + parameters: + - name: url + description: The URL to HEAD to + multiline: false + example: "https://example.com" + required: true + schema: + type: string + - name: headers + description: Headers to use + multiline: true + required: false + example: "Content-Type: application/json" + schema: + type: string + - name: username + description: The username to use + multiline: false + required: false + example: "Username" + schema: + type: string + - name: password + description: The password to use + multiline: false + required: false + example: "*****" + schema: + type: string + - name: verify + description: Whether to check the certificate or not + multiline: false + required: false + options: + - false + - true + example: "false" + schema: + type: bool + - name: http_proxy + description: Add a HTTP proxy + multiline: false + required: false + example: "http://192.168.0.1:8080" + schema: + type: bool + - name: https_proxy + description: Add a HTTPS proxy + multiline: false + required: false + example: "http://192.168.0.1:8080" + schema: + type: bool + - name: timeout + description: Add a timeout for the request, in seconds + multiline: false + required: false + example: "10" + schema: + type: bool + returns: + schema: + type: string + example: "404 NOT FOUND" + - name: OPTIONS + description: Runs a OPTIONS request towards the specified endpoint + parameters: + - name: url + description: The URL to HEAD to + multiline: false + example: "https://example.com" + required: true + schema: + type: string + - name: headers + description: Headers to use + multiline: true + required: false + example: "Content-Type: application/json" + schema: + type: string + - name: username + description: The username to use + multiline: false + required: false + example: "Username" + schema: + type: string + - name: password + description: The password to use + multiline: false + required: false + example: "*****" + schema: + type: string + - name: verify + description: Whether to check the certificate or not + multiline: false + required: false + options: + - false + - true + example: "false" + schema: + type: bool + - name: http_proxy + description: Add a HTTP proxy + multiline: false + required: false + example: "http://192.168.0.1:8080" + schema: + type: bool + - name: https_proxy + description: Add a HTTPS proxy + multiline: false + required: false + example: "http://192.168.0.1:8080" + schema: + type: bool + - name: timeout + description: Add a timeout for the request, in seconds + multiline: false + required: false + example: "10" + schema: + type: bool + returns: + schema: + type: string + example: "404 NOT FOUND" + - name: curl + description: Run a curl command + parameters: + - name: statement + description: The curl command to run + multiline: true + example: "curl https://example.com" + required: true + schema: + type: string + returns: + schema: + type: string +large_image: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGQAAABjCAIAAADihTK7AAAABGdBTUEAALGPC/xhBQAAACBjSFJNAAB6JgAAgIQAAPoAAACA6AAAdTAAAOpgAAA6mAAAF3CculE8AAAABmJLR0QA/wD/AP+gvaeTAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAB3RJTUUH4wgeDyYG/VPJzQAAMI9JREFUeNrFvXecJWd1Jvyc81bVzd238+SomZFGYZRRQAkLIQQ2XuEFG4zBXsv2EtYG24vZ9WKC1wmEZbDXWnvx77Mx0ZhgsUgIS8gSoJxnNDnHzuHmW/We8/3xVtW9HUYJyXvV6l9P9e0Kz3vyed5zSVXxar4UEFUoDFN6cLIWHphq7Jlq7ploHJxpnaq0p5u2GSmIiAAiqIKgAMBEVAxoMO+t7M2sL2fOGshtHsiu6w0yHqcntKJE1HWFV+VFrx5YqhDVFKNGaJ86WXvkWOXJE9X9083Jum1ZVcAwPMOGDDGINIGYQABIKT6VFY1EFWCmYsDLCv5Zg9mLlxcvXVE4oz/buSKUiV4l0F4VsERVAUMEoBXJQ0fn7t47/dCRypHZVjOCZyjjk2+YiYigCSJdMBFIAQIAIsAJnBIBBFWySqHVllVR7c2Ysway160r/tS60ub+nDuJ1VcFslcYLFEAykQADk83v7l9/M590/smmpEgF3DGYyJSgqp7dIdDDIsCcDoIAjiWMkIXap3Hd0ADsKKNSNtWe7N80XD2Z7f0vWFDT9bjVwOyVwysbqV79lT1S0+N3rV7eqIR5TzOBx4xRcmViIhASgQih0hiq6AOmhgydN7ewXXRAwDOWkWi9UisYEt/5q1nlt+6pbcv6znIzCuE2CsDlpUYpl1jtdsfPHHX3ql6KKWs5xtWkDg0KX5aItIEgdg2EUgJ7jhUGQATlsTotKLiECaiRiiNSNb0eO/c2veOs/uLgVGFAj+5+f9JwUrvY7oe/tWPj3356dFKW3uzHjOJOrGgLv1agBQTOWlKMCCOn7tLoLoRSnUx1c4lUWNCM0KtbTf1Zd574cBbNpW7V/T/DVjp5b+9Y/zWfzt6cLrRk/M9Jok1i4mgMUBIkXLCpGBnygECE6AAxWChI4NIQHaKuxC45P7Tg+njOMjqoTQjuX5t8XcuGT6jP+t++bKV8uWD5ZAarbQ+8f0Dd+ycygcm4xtRxPeeyFS39oFjJ6cgZUrMF8U/USJOFCMao+FQ7gKx+6f07tN/0nzICDrbsqWA33/BwHvOHQRgFeZl4fVywFKFQpnoB3snf//O/UcrYV/eV4UARKwEAhGTxFqTPn4aOhEI7GJIYgUiRaQQ9xUjEQeZhskw+cweEzOYSGKfm7wvuSl3do3h7eClqoYQis617A1rix9/7fKhvPfyrP5LBks0jgz+8odHPvNvRz3PFAIORTWVDRDxaWWKmcAsSm1rm1YtOGO4mOH+rF/OmnLW9GaMb4gJqmhYnWvrXNtON2W2bRsWqvAMsoYCJiKSVJYAOKFGilH6DarqtHKqEa3tCf70muUXjuRehny9NLAcUs1QPvx/93z9mbH+fEDMoqkrAkBMpESaKJIzWERkmKxSPZKWIO/z2t7MucO5bSO5zf2ZVaWgP+flfbO0IEPnWnaiER2ei3ZOt3ZMtHZPt8cbVhU5nzOGANjuh0geSd1/Gh9TVY9Qi4SBP7hi+D9sKr/UQOwlgOVEd6rW/rV/2vngkdnBQsaqxv6JEgUkQuL+EiEjNtSKtBpJzjfnDudft67nqjWlLQOZwHD3+Z1j7ZKO2PUtfpqZln1movnA8fqDJ5tHqxETih4zkdVEvFLEEsDUnV+VCVZ0thn91oX9771wWDRZ5VcQLIfUqUrrV768Y/totT8fhAJiIpAkuCww6kRkDDWt1tq6vCd40+byz27pO284l/p8q85rIg1FE1FKbq7riCYwUFfEVAujB0+17jhQfXi02bIoBcxQ2yVNi7+nGfpU3d6yrfy7lwzHJuSVAstp36m55i998dndE61y3kQCYtbEeoOBGDpSQJl8plBoNrTreoN3njf0c1v7BvOBe2yRxL4tupDqPNREl44k40pGkn4C2D7Z/Ore6j3H6i2rPT5LIkpQ0i6gEjFTVTAw3ghvObfvw5cOW1HmF8brhcFySE1UW+/8x2d2jTXLeS9SAMTMnWiTQMQOOmYippmmlHPeL1849O5tQ2WXdiyqoijm4aXqIikKRV3c5TPFochpFEUBURCUCQDtnG79n+fm7jteD5hyHkUSw+8UcB5UzqED4/Xwvdv6Pnjx8Ivxjy8AlvtlrR394j8+88SxSrkQRKJEHGscJXEmwMRCMIbaVhuhvumsvv96xfK15ZyDafG6LXBGqiDSk9XwIz8an2qK71EzwuvX5H7rgr5I4PFC67bgdBKXvgDoD443/vLZ2f1zYV9gREXVPaSqxqghsfeqysBEI/qdiwZ/bVv/C+L1fGA5lWGm3/inHd/ZMT5YyIRxhJCE2pyEUkRExIZmmzJY8H7/mpVvOWsQQCRqFtkDpyBMaIRRYIxxcYZqI5Rfvvvk9ql2T2AE8AxNt+yHLijfcnZfqo+VdlQKGCCrtNjxpzWPWiife3bm6wdqWUMBwUonnlDtIOUwBDDTtJ++ZtmbNpSeHy/G6V8iapg+fe+Bf3l2bLAQhKLJcyfpXleua5gm6tHlq4vffMeWt5w1aEVV4S3yzJKEPN/YM/uXT0w6PXJY3Pbk1Pap1kjB8z3K+RwYWlbwbn+u8sREkykODmbb8j8enTlSDQ3NDxcQS5bziQWff+/C/j9+zUDO44Yl33CcZnW+iNLAkKiUMR/98diOiaYhktNr2mnBctnMd3eMfvb+IwOFTGS1k5NR/Gdx7stERBP18D9dNPyF/7hpVU/W/S0ttfJMaFn70R+e+uC9J96wocdjsqqG8NR445/2VgfzfqRJkQ8gIp/4tqfnIhGCWtVVxaDo89vvmbzneM0QRLH40QxBAat4/er83147uK7Xn4mQMUxJQkFMzA6l+C59pkjxu/efmmlGlBifFwuWq0wdnar/t+/syQdenEXESJEmkqUEJgJophl99HWrP/66tQSyunRyb0WZMFaPfvm7R29/euqnN/WcN5ST2HLp3zw7173miFMZKmZ4+0x497EGU5zGvHtzMefzf3+88n/2zLlAf/GjEWAIkei6UvA3Vw9cNpKZDtX3WGOQQEScQAYiAUo+H5gL//ChMSIIlkZrCbA0Niv6ke/snmpI4MUxOiUVy1QHXS2q0rZ/duPaWy5ebgVEZFz0sBT6JyqtX7zj8JNjraG895YzepAUM58cazx0qtGTNdpVb0h8COU8/trBhlsDUVqW965clgk8/vzuxm3bl8bL/dNjEkUp4D+/vP+nVmWn2up7zEnQ4taGOXZWFhjIeXccrP/znhlDZJeSriXAElFD9IVHj9+7d7o371lRV5sDdQe7scbPtsJP3bju7ecOR6KGQbEvdz9Il/bpWK39njuPHqmEeZ9WFv3XriwATjDxz/tqAuLUlDCBSZlgGMSFwOyZix4fbxHU2anXLw+s6mDOfPlg83PPzTHpfEOjlKDHBFHyDP/RpeXrV2Vn2mo8BjMTJ8+gnYVR9AT8mSemTtZCXsp4LQRLFEx0fKZ56w8OlXK+lfjiScUtKTARmDDVjD5x/dqfO2coEvWYUl2baYYnKi0m1kT/m5H+57uPH5wNB/L+XFuvWl3IeRyJGtKJRvTjk41iYCS5CBETc7LmygwQ3Xm8hSR2v2gwuzznNSIdytIX9te/dKBmCKksiGLfXJiE6+C4r8afvKTvNcNBJYTPpAwX8hEhkS0CkPFouim3Pjq+pEdcLFlKhD+958BErR0Y7gosKF0yBTxDE/XofZctf/eFyzpIqRqmwzONzz4yWvCNC7KtKhP+4IHRx0cbAzkTivqGr1uVR+zp6f7j9cmWBKZT3lJOFxvMrKCCx49PhtMtcU6w4PMFA34jUlEMZMxf7aw9NNZwjkwBJqq05fZdVaJOIZdAPtEfXdq7vsfUBcbEFj7ReCYiYraKvizfeah2/9EqL3K488ByFuSRwzPffna0nHfxZ0eU4moU4BuabkQ3bi5/5NrVabHUihqiE5XWe7598KYzyr1ZDyBReExf3zX1lZ0zwwUvUrQFK4retuEsANck/bcTDc+QWyRwnHsTEXO85gB8xkTTPjHZdjcJ4LIhPyktIGf4T56pTTQjp31W9YLBbCW0n3hqlkld6kMEAXoD/48uKWc8cpF10iNSmleGJc/w556cCq3yfOM7DywCiepnfnDQ2Z1umUp/MEyNUFf2BH/2xg2apMGiMEwT9dbPfHnPpSuLF68oujiQCccr7T96cLycNVZhiFoW5wzmSoFnRZloshHtmArzHgsSe8sduVpQhn9oPEzv+Nw+vzegSKGKDNN4Uz77XJVIVeMm2ge2lu4/1f7M9jlDlCaSVrGhx//wtlLNqknCh0SgYxkToBjwM5OtO/bPEqHbcs3rgDPh/n1TP9w/3ZP1rKZNhKSnkCDWsvonN64fyAfi2nsKIjRC+yvfOjjXsh++ckVyx0qQP35wdKYlgRc/sFW9eFkWSQXqmcnWVNMGJtWEuK8VO/UuS5o1tH06bFprGKoYzpn1RdOyyoRItTfgfz3ReuBUy1koUWQNf+Cswt/srv/jvmoawRqCVXrDqsLPrM3OhuozJ9dNFSi29Hmf//65mWYkTB3h4q6lA4DPP3TMVQ+65SppV8FnmqpH77pg+Or1fZFIEiUoQf/r9w//4FDl1y8e7s95AriO9ANHa9/dX+nPsZVYALOe2TaUSS/3xHhbnAYS4lycujqoqR0FAqaTDXugEhEQKQBs7fPbogxyxiFj+H/vqbesdXm1Am9YlbtwwL/tueo9JxqpB2CCqH7g7NLKgtfWTsstNl2JJuY93j0dfu9wlZIKRwcsZ62eODr3wP7pUsZbKsogAjUjWV3OfOiqVaJqiBFnefTZB09+bcfUmUPZt2/t18S+ta3e9ug4E6xAQQpuWhopeBt7MwA8IlU8Pdn0DIlCQOJa+UktQVS7vxTaiPTpqSjueIDOLnsKWFVRiUQyBntnom8dbjjdEaWs4beszTLRnz5b3T3TdAi6P+4NvPdtLTasMif2mByVoJPFBYa+umvWmYsYAufvXB3mQ9/Y8ZUnR/vzvtU0LkRaXPeNmaiHt/70hreft6y7B6eqeyfrRJT1aHVvNtXX0MqBmaZvTAq2KDIGK0u+y5dE9Uilra4QNs88Lt15jlSLHoZznis6tKycqEdMHeUQqE+6Mu/H/W1F20anmhopSgZDOS/VbRf6feDH04+NtwoMayVNrUVURFWVoHNN+7evX3H5yoKrkXipdIxVW/fumSwGpttfprUQJqq27fkrim89Z1jULQgUcUK/ebCQ6kvnbpi2DOQXPbVzTjEC63qC+QXRBf2tJV6apPMZw+tL/uI3RAKTUAAynllbjIk4kSZVkuR1y5bC4+MtpCQnpPro5AgCfGt/5fKVBUrV0Onk93ZOjFbagfPnRPMrRsRMrcj++muWe8yakjoAj8nriFjnWZlwmlpHxx7xwrZyt9s97df8sy7xBm9e7SwJxIk8ptRau8j+vIHMlcuCSgTmpN/bRT8RaN7nH5+oTzQi5zc8JDnHnc+NGsOdllznOzGh1rZbRwpv2NyvqoZdTovJeutfdk8HxtRDe+WanrOH80lFCfcdnts3HWY918WIWxoC/PSGUjljREEkPz7ZOFiRrElbQS9QCidCy8rVyzIr854rYDw12dw5K1nTSU2Y0LJ69Yi/ouAn1lPvPt6YbZNC+wO6fmVGlBKODt6+Mf/AqXbcJHehB5TiHygwNFaP7j9Wu3lTr6h6zlodnW48daySD4ykUt71YqJGaH9+20jgmUjUIwjUEP3VQ6duffDkYCEzF9o7fmGL0zHDVA+jj9x36uBsO+cbMJhdf4yzvrl+TbGcccvAt++oPDzaKmWMMhkmYmJ6vtaUIUy15JMX0VvXepFqQLRjJvqfT1f7M5yaDvee956Zf/9WX+JwgR4ZD798sNEXcCi6PN+7tS/jqh2quHAgc16/v30yzDGJKClApBozVAAYxn1Hqjdv6iWieInu3zc1VQv7i8ESpS9C28pI0X/z1gF3bVEYopOV1jd2Tm3oz4ZKW4Zzl64spqL42KnGWD1a05MRKDGY2TC3BOt7g+G8k2U0rdRCXZE3gWc0zgXx/JLlEYj4VM2mgr+l1x/OcdHjjncn5Ix5dCKMVD0iqwD0hpWZ759o9QU82ZKvHGx+si+jiaIZohtXZZ8Yb+cMYtPl3Jrze6I5j58eb041ov6c5yw1Htg/yadhmBhCtW1fu753qBCIanqirz07OdmIPKZqy24bzhmmdHnvO1yJRK1qpLAK93PLSjnLaXw43bTTLStEoaoFWcULfoUKVZysW6cpAPoz7BFFop33CDzCkZrdOxumq72lN+jLcDXSgkcPjrUPVyMmiKOmANcszw5mjZU05kwqTEQgBIZG6/aZsQZcL6YR2mdPVDIeL1FIczZYceOWgbR5x0zNyN6xazKfNJ2cWKmqqwQ9fKKe9bibkcBEVml5wUfiTyaaUo/UcNq6eVHtS0MYa4kibnb0Z7jow85LzpzY6uOTbXdOBXoD3lDyWqIeUy2Su4833XEiiNJA1jt/0G9EGufy82NMAqzqY6MNOG+4f7x2Yq4VeOw4AQtebavDpcxla3pdaO2K6I8cq+ybauR9DkULPp89nEtV4+hs+/Bs20HfHQiI6kAujmMBjDeipKj/YvvnqvCIZtvSjOL7LBgqeLwghBbAI3pmOnRxgluzs3pNJFAga8wDo+22tdy1RJcNZ6Jk1bohICJRBIa2TzQBZQA7T1VrbWs6jJTuVaJmaM9eVujP+67T7W7s7r0zri4aWowU/dW9QXrh7RONSls86nTS3d8QMJLz0jNPNqVbIl4k48AQqpFWI3Ho+4b6A47md8ZUETAdqthGZFPTsqnXdwYka3CoanfORg5Hp3rnD/gln0N3n2n5PHmkgPlIJZpqCAPYNVYFll5gIrStXrK6hER9PKa2tQ8fq+R8VtW21TW9mbzvpSZ2+1hzyYI/EfqynWh7vGFftEglKMSUPp1rO7AAUMFfWNJUwGdMtuRY3aYH1+ZN3pBbnlD00YkQiSaq6uqit7poml3Flg5jCfAY083o0FyLAeyfqHu8dANRFb6h85cXkeQrAPZPNo/OtB1nPxLZUM4gKbEC2Dfd9Bgp0zg9jyHqDUx6eLZtKb2dhBX0gi8iRIqGRbLq6M8Y1YWwM6ER6ZFqB6yhnCkHsQvymZ6dCpHUXQUwxJt6/VDU0TYpSSHdrbvYbf9Mm0MrJ+daPtNirIgQifZmvfX9OQCcJJLPjtbrbWuS/GutC5wAJkRij1Xaydk69GSB+ky9WZNCONsWg2QjxcJQ/vleVnW6JSlYRS8m+C4Ww6NdklX0MZDhUFSBDNORmp0LJUGEAGzq8RPB6lrjJN8T1UOzbZ6qh5PVtjG8VIBFodWhoj/sUrDkJLvGawCc+TJMy3s6CdpUQybrkWc6OyXi1VEwIWsIcaFDq22hJFJ24fKLcYcEWEU16rw3Y5aGmQgnap2OCUBD2Th29RjTbTlWs531BNYUjXFv1aTXN19Uj861earWTsRkibAhsjJSCgJjnBN0Vu/gZMs35FgXHmMw3zHbs62oFjqRmd9vATyGT52Va1lNLWgM54uLHgAKu2Qp6/HigEcBA5puW9c2cOLTn2GJ928gtHq0GqErd1+eNzmPbPcaa0okARON1kJvotpqhuI7T79gjQiR6HAxQEIGYoKqTtRD38RbRnIel7MdyZppSiSa8TCvxgMACJgCr6NvLuFIm9uuBveCzGsmYpJWV2Gkxyd2dMMuyAjwGbOhuuzNodWfcXzN+C3jTZvKIICeDGc9brQtI+1JJaBBDWGqEXmVprWq/iLBUlUCqaI/5ydHQIS5lj0622pGYhWRatY3eb/j4ybq4XQjIiLRpCXHMIYiVcPkKI1EENXplm1EEjcqDAwxJQ3j5wHLkNZCTLc7xsgj1CPJmHmdGOfyJhsSKYKk0pA1XAslwwCoHumRmu0+c8HjvEe1VpwzdquGe/BmpJ7bmkWgpdrgEGg556ELyUjkzKFcqDBEFsj5JushlZeCT5esKPRkfUnWzBELrKKc9fx4SwBFIht6/FJgAhM3OZnZsRBoKYPQLVmVtgy4JwYA9GXovAG/5HfSw+QSKAcQEbDn3jmSM+f1e72BUUU1ksEsxRzFGErKGRJHfUrEJf1OQEvgNSMR1a6qPLS78KAIvJTIDwD9ee8rv3Bmd/FeVYB4O+FVa4pXr+05jWtTEdfZIyb69GuH5v/2hct+3acC4DMBuGwwuOyq4PRX1DQ3um559rrlma5fqiNUJU9HHjkLRTH3rZvKC1hRz4r79YKrJAzIRXaE4NKLpK4IdBd2QcbOh36eXHBcYjacZo4L6H8vKoDornsp8ekZZtRNglOgO5ij+b9F3KmiBc4p7msrROElFCr3Jk1aQukFVBbey2K6V1edGM9PnovfqZp2AV5iFL/UFZ/HzHVryeJ3akpMAACIpMFhzAnv2nkAA3jZuAo7LzBKVEIBNMPOHxBhuh7+7ncPNEU9ZlFkPPrUjevKucC1MB46Vrn1odFS2nZMWKai6M2aP75qJO97gIZiP/7w5HhDMj7HjVpOm4ZEnFJ2Fr6YUAn1plXZn1mTjwQe49Hx1uf31EvBwqTHKnp8/r3zinmPHU3w/lOtLx6o9/gMoNKW1wx6v7ylKBrTXK3YtpVY/1S7NdEJl2Hysh6ZOIjuaBYSagMRpuohumh+nuEnTtTG6mHGM04ya6GWc/Efz7bk3kNz/fnAxqyDuHtqgb6817bI+1CFz2bHZHvPdFjIeM5+ctJcdR3p07lFjzHZ0gsHAsCxqGi8qQ+Ot/szHHUFX0RoC5bnnG2Pjcyxmn1orN0fsKhONmVNngHWREqbFvVIieJtBt3kcHfIN/BKOc909TY60gs4fuZ0I+wGsZTxVpcztdDmA6NAqKh2OfJlxWAo72d9J1NMMd2blCggNG1cYGJCX4YLPhc9t5VHCcScyBeDCbQUXoaoLdKf6RibULTk84JCDQMNwVCWHd/ZiWnTasnnvEcq2jS6pphs6FCAUAu11raMVPvm2XirWvCYh4rZTMJ4SZFC0un0mE7Ndbg+TtSHC34rih1NI5SZZgesnoANwYqKqIVYVVcpFdGmjYPJVNMjkUjUioioqFpRtWJFrWgksCJWVNINUNrpvGa6crNqpFYXvQ0IRXsDJuLUqE+2rBWJBJGSKEaSEog711TL1kPhlPqdksABKKzoYM7jgYLveoXxhslu1KAe02il1YyieOuEKoCNA9lIxAXfVmS81k7/pC9negITiSQUCU1PGFqEVhILSlkX1MQWIbb67gjENTtJZGHa5HD2u9xYwyo6S5C8AxCBE8DEF+hkU6CqIlYkQ7qmFKCrunCyFjUiS12s7/hHcRZNhwqGyzl/sBhEVpYwqQqfeaIanpxto+t+tg7nKeaWQxTHO9Vu9GS8wZyJbLJkXWFwZLURdR6rEBgrXbY0lXnXG046xCIqIt3VJUPalbmjZZcuWljVlflY0VzXb6wuBqSq7Uj6M7y6GLco3OvgbLKUibXSuHbk4gZd25tlw7SynA07ZOR5ymgYlVa0d6IGx8omAnDuskJPhq2qQgk4ONOIF1OViVb3ZiKrHDMDnOooAaGVqUaUSBbKPrs5DQkq8XFRURGkTXQnaqKSeDuPqDdwETwBmG27KGShABqmNYXONrNKaMebkSFAtB3K+pKX9zhldALYNdVmQDpt/G4tVCZsLAcMYNNQwS7RAovNfCT61PEqEsOs0DXl3Ib+bDMUQH2mQ9POqMXOe/NAJnJaGPNr4lgwEp3rcgXljHFapvMkqeOtncqISGJBRESsiGHkvHjLOYCZtvAiqKwiy1hb7IB1qm5nmtaoQKVt9YKBuA7uHqptZfd0y2dIZ0tBZ4eBFc15vKGcYQBbhgvzttSkNw2IamDosSNzSHYVicAwXbmutxkKgMDQsZnWXCtMBfPsobwhgSROuStlGK/bVA2H8txtGZzGqVKCF1RS4yWqKgIVRFazhLIfN51UdS4UpgXioK1IBwJake+AdbASNUJh1chq1uhly7KxwXK139nW0UroM0kXVCqxuISiA1le6+R567JiMestIVyqqprzzXNj1eOzTRdgOFBu2NSf8UgFPmOs2j443UzNxtnD+XLWi0RIiURIHeaqirF6lL5tKOcZitPFLrwk2SEizvFBFFZgRUVFNbQoeCj6TrKoZWWyaSnR01hbFc1INpS8TBcbYdd0KKJQrUeyudffVPZVNaU+PHqqWWtbRrwwqQLGZjGSDeWgJ2MYwLqB/JpythXJ/B5JvMI+01Qt/NGBKSRWSRXnryicPZyvta3HVA/lmZP1VKqXl4KNfZlmJLF3TUBn0rFalNrUobzxSK0LGrr304g6qxWT8RO5UhWIhNb2ehQk5IZqqNWWkMKKM20xxmEk2/rjYglDFdgx1fJIATQj+alVeUMxjq5se/+xmkfosledaIugkZVtwzkAbEV9wxes6mmFktJP5gUQCsN0567J9DlFlYlvPmewFVmPKTD0xImqMyIiSsDlq4qR1cDAEBkmw2xYMgYuyHDqPJTzenwDVY/UIxgiQzDkqoDKIIYy1BAMlAlG1YdCdFmOU27MdFvqoXgQUiEVhjAUagtGLx4K3P0TYaoZHp4LCx5FIv0ZvnFtPjWyRDhSCZ8aa+ZMytKCSkdWVOEzLl6eBxBXhK8+Y+BLj59c0saLaDEwDx2ePTzdWNuXk4Th+h/OGfrLHx0/PN1g5u/vnZ5phOWc78Lo69aXPvWjk6MaAiQER0BuCZ6baDYjcYNi+rKGCccrYd7nlLff2TeQsq45JtggyXX6M3EHAaBDc+HxSrscdIghRKiHcv5gsKEnkzKufniisXuqPZg14/XwP53dO5z3HTlNREF018HKTNMOZDlciJQC2rJ2Rck7d6QAIE6jr9zQP1LKVNtRd6vVpewK9ZgnauE/Pz36oWvXqSozqaI369/2M2fsGm9kfRNajWJ6MgE4d6T42ZvWNSNh7sxjcBIqsTNGYOi/Xz40Wo8Cw9rZ7ULxNKPknxwnqXEHoGn1spGsJtuD1pa8j13SlzWUxG9EhGYkW/p8V/BkQKGriv4nLxvIGW5Gcv2anEIZ5HLXtrV37K/kDKyoyjykVNUjnQvl0g29pcBY532cJbrly898Z8dYOeenlj4lVxvmtuhwMbjrNy4sBjHZUBcFgvpyCy6v6muJ3gKAZHfo9w7Ovf+eE+UMR1YXyBRUGJhp2r++ad0bN/Za0Q754M1nDy0uw2vCOM16vH+y/vWnR4nIxYcuCg1jUwzMRyqysjhQ7Nw+gC4W8Et69vl/tbgnFKeK6SVSO2xd+pm8j6Ci8v9tn/YIIpJGxSlSBLQiXd3jv3Z1EQAzdZh/124aXNOXHa+FvlnYcE0sl/f5h46/7fyRnO/H2wWIPOhopTnTFCYMFvxyLjarzBirtmea1uOkVUUkqqWARgquCqyhlaOVkJcS0yW7rgoEjJXF+BKRyNFKe37dFJGV1T1+JmX9qhyrRqGgbWV9bxAkx90Ohu8fqjx6stYbeNZ25VvxvANlUDW0b1vT53TQEHkAiGBFe3P+TVuH/9cPjwwU/KgrF3P0LlHN+7x/ovF3Dx//wFXr3OYTVRBoumHf9sXnphr2TWf2/++bN4ubskJ8YKb5jq/tK2Y8Rz4zRLOhXLGq+OWbNzovUQ/1Hd8+VAnV43SAhibKn/4PB4fHmGnJWzeV/ueVI6Goz3RwNvqF7570TUITBeqRntmX+fsbl2uifUer4TvuPDFalxvW5P/iumWOuedWJrTy2ccnfeLu1MbJFikUJNC8obee1Zcu3LxmxDsvWdmT9aLu6LRTV9VItDdnbv/xsaPTDcc9YoJVPXO4+LHXr4Xq/YdmnxurUryVRy5b1XP9ht5KKwqYDIgIBY8OTLdmm9Zxyfpy3qa+TDsSj8BKDJASK1jBUAJIlRQEdbVxUZw/lE1vaudUsxZZQ3AdZENoW/mVc0ouFnV567f2VY7ORetL3scvH0qIe/Feki/smHp2tJH3yAUMiVipSwcYqDbt5SuL5y0rpDSOGCz38GcMFW44c7DajBZsRU0tl0c024g+cfe+tFzoMVvRm88Zef8VK45MN77wxJiLRd1mhd+8fLnPJCpQUdWAMVkLd082AbgluXBZLrRCgPND8UQBcalivGEAKlCJIskbPW8wSJf2qfFmnHGrEnSmaS8eyV6/pigqDDAw24q+vmeuaPBnVw0PJpQpV6Q7Mtf83GOTpYCjBCmoUNyAFoWQClTfvW0QSXkdWLSF7pYrVgfePJvVCVBVrWhfzv+/z03+01MnDFO8m4nJqn74urW/dMGyv3v05MGphvOWVvTskcI7zxuYqoUeQwUEall54mQtPflrVhQ8UtG4+t0pz1iIRXf1oRHZVUVvQzlQt0VV5OmxZsCudiiu3PihC/qZGHADluiLz00/N9H89LXD54/kbZLcuIX46P2js+3I49hIpaV3UgHUU6q15KLl+avX9jj+z0KwnHCdv6r3jVuHZhtht3B11wOsaE/G/MFdB/ZP1BxeFBNs8Bdv2XTZ6t7fv/uAS1BdiPxbV6xY35ept8WQqqpP9MjxKgA3h2bbcG5FwWu51nRXGbczrEIgFqSot+WikZzPJrJKwJFKeHCmlWGNrBhgvB6+a2vP+SN5qwDIEMbq4W2PTd567fI3biiHIu6BrajHfPuTY/ccqpYDE8V2XeapoYCgkchvXDximLr3Sy+ULAU+eN2GYmBkUV4dK6OIx1Rv2/d/Y2cjjLqyawoMf+WdW5uhfGfnmEnC0d6s/4fXr2mGLsOTnEfbR+uT9bYhikRLGe+i5flGS0y8bWfplyNdX7OqkNhgPHKyPtMIScHQ2Va4tT/4zYuGEvutAD7xo9H/ctHAu8/pj0RdZdWKekw/Ola59ZGJ/ixHViievuLyGuvkyzDm2vbqNaXXre8Vnbf1YR5YTrg2DxffdcnK6UbozReujnsWLWW8p49Xf/tbu5jIqmiy2yrnm394+9bdY43RSst1wiKRq9eVP3jF8rFqaIh8xmi1/cjxmiYV/Rs29Eq8vMkImaTy4JAioBnJipJ3yfIckjLWfUeqpICgHalH+NQ1I67IBYUheuDI3FUrc++/cCgScYetwjAdmG5+4O5jQbytJM1uJBYoBZFaUQ/021esWEwkWChZDBLFf7luwxkD+UZol6xzQTWyMpD3v/Hs+Mfv2usxu+aFK0gUAvOBK1dyUityHuA3r1jx1q39Y9W2zwTVf90/TUlX/eo1pVUlvxWKwwsST0FB4hYYWm3ba1fnS4EXiXqMiXr7sZONgsdWdLYlf3rNirMGC5Go21Qiqmf0Z96+td+KeOzGi6ghTNTDX/3u4UoryrALT0VVRSUpf6sCHnimbn9xW/95ywq20wk+DViOY1nOeR994xnNtl3cjErjkUh0qBD89YPH/+zeA4bJipseCVUEnjdUzKTBIhOJ0q03bbhufc/JSrucNT88XJlpRoZjTbxxY0+1FXkxP05UJSmQgCCiEjBu3tyLpNlx39HqWDUMDMYb4SdeO3zTxl4rSPWAiZYXM6JkOB7taohmmtF7vnPo0Eyz6HNo47IkqZCCNOZwsaLesmf0B7952cqUJf98YAFwZvvGrSM/f9HyqVp7aWVUdc5xKO//+X1H/uRfD3jMzpHRIla7Cy0zhv/2Zze9bl1puh6OVcN79s8kD4+fP3ug4LOosCqE4tKUiqoYRbVpX7M8d/6yQjo09jt754h0sh598rUjv3TuYCgyf2xZZyqpG4YzXgvf9e1DO0YbvYFpWxs/h8QwxfVzBSBtG33s2lU9GQPo4oxy6YkhzpF97KYtm4by9ZbtNnILTK8VHSz4tz1w5MPf2e2asjbJHBcIrChyvvn8zZveds7ATCP86rPjgLqF2TyYe+PG3pmG9ZgIQukeTVUFQpFbzh8ikFUwYe9U896Dc3mD265f8Z5tg6n9nne5xLZ6TPummm/7xr7tY7VylkIrLCBRdlEVACUIoOKRTtXCWy4euXp9uXtD5guD5dgPvTn/L37uHGKyqksm7ileQwX/Hx49+e4v7Ziotd3zL06RXT/KN+bTb9z4mTeuf/BI5e690+xuFfS+1ywreBzZJG6AQsUnzDai160tXbOuKG4wLvRTD55YVfS+evP6t2zpj7q0r/slCqvqMd13aPZt/7zv2GxYznAUKatSQpzUxA+qikeYaURXrin9zpUrreppd+Z87GMfOw1eZEVXlrNDxeBfnh3NB95SdGZKyLwoZcyusdpdOye2jpTW9GUJagUL1ocSob9wRena9T1PnqhcuKLkMQt0IO8T6V17pss5AwUre0yhikf0v960vi/nuXLYwenGeC383I3rVpYy6TyJBQroKCpM+NzDJ//bvcdVNe9TJB22UcKYivuCjgc+nPf+7ubNvdlOZWEJTHQJIei83A194s49n73/0HApG9kO+yLZ7xmXNN3ux0Ykovq+K9d84OpVgfGsKmGJmfbJmCqxQsn8LAD49W/v+/auqVLWqBAIbaufvWntzVsH0wS4FUZZ38NpJtyl06/2TdU/dt/xfztc6c8ZgETUjW9wkeL8DraKwFr9wn/cdMmqnuefn/VCk9ncvDHC+776zNefHhsqBWEXdzMdspNKmVuT6UZ0/srSR65ff9WGPgBWhIgWLJdoOscU6b032+En7zt6vGIDQy0rb95cftu5wwtmIi85z9Cquo3zrUg+/+TY7Y+OVUPbmzGukUEKxLP/OnUBJBvJqy37uTevf9OZAy84fPmFZ/6530civ/qlp+/cOTFUPA1endkI8A1X21ZU33z20HuvXL11pIg4Clw8haB7YZxPmmdGRZVPv9SutejFG7b1jt1Tf/3I2PaxRjnrttbFGHWvfUJWgwEINNOM/viGNe/YNrKkUr9ksJDIfCO0v/blZ7+3a8JxI55HvpBQTGcatpDx3rx14BcvXn7BypJzU1YVp0FNk9Ft6T8XK0VimzU9QzOy39s78/dPTTx+spIxXPA56sjQQmqJK9e7uQSzjfDj1699z0XLXgxSLxasdIXbkfznrz5zx/bxwWLQXfZagJej3JCbeiuYa0bZwFy+vuet5w5fs7G/r4srHrvNtIFDS9T10VUXoPn71PdNNr67Z/qO3VO7JxqB4WLAoouq1d17z+A2RSK02gjtJ29Y987zR1786POXMAHX4WVFfvubO7/42ImBQqDz1y1tcKS23/3omjq1tg0Vq8qZy9eWr91YvnBVcU05h9O0OJ6n99GKot0TjQePVO4/NPfUyfpsyxZ8zvnkmFmLtj2kZ4yTTcPUCIWgn75pw5vPGnxJQ+Jf2mzlmDIP/Pm9Bz997/6cz75nurtBne8JeJocMUTE1LJSD0WBvlywcTB3zkjhzJH8+v7c8p6gL+sVMp6bDZUsjzRCqbXtRD06PtvaN9l8bqz+3Hj96GyrEWpgqBAYz4V1mlKt5rWHNaUzAAA8pplGtLIn+PM3b7x0dc9LHaf/kqd2azLf/I7to7/37Z2zzagn67+gSnYfMUwghKKtSNsWCvUNFTKmGPjFDOd8N7WbItW21VrbVtu21pZGJFbhEWU8zph4WN7z9Ic6TYSYHQtSTNWjq9b33PqmM1b0ZF7GBw+8zOH57kp7x6q/882dDx6aKef9tHC6GC93qPNz8hY3f8+FqlZhgcgJg6omzQpmRwCIybwJySQpLWv3brKuWTPz9/EYRq0tIvKrly7/3WvWeMz/TvPgF+AVWrnt3gO3//hoM5LerJe0lGI8Ohz0rhEkCyBLh9clY4c7aowF7MEln6774KJHceM5ZxrR5sHc/7h+7bUb+3UeC//fCyx0BUFPHZv9w7v2PnBgOh94OZ9TyGi+QKXh2Hz2ftq6T1mLHasHINnIoKcDbMlfOC79XCvKGLzzomW/9drVbnrTT/JJPD/xp6Mkg3JV9UuPHf+rBw7vn2gUMibrLQ1Zt1ZinpQhDRziEmUsdV3TGYF0F5IjK4ibWJzeSWf/jNbaVkWv2VD+4NWrt62Ix3z+hB/A88p87k4qYjP18B8ePvqPj584PNXI+SYXGGjM7scCEXNskYV4JZh1JAzoAot0Xp1JOxIXj3SzqtWWZeCS1aVbLlv5+s0DOM0k7P9nYLlX6l+mau2vPnnya0+c2DVaY6J8YDw351ahiyDrpm0soSLzLJiz6NS944aST8pqRVJv20LAV6wrv+ui5a/b1I8u3/2KPOAr/Flh2vUhWI3Qfn/n+DefOfXw4dnJWtszlPM938RD9TpWuws7nX9kwa2m6Dnj5iZUtK00QmHCmr7sdRv7bj5v+PyVPfHivXIffPWqgLUYMgCHphr/umv8nj0TzxyvTNbaAmQ8znjGfUxM6vU699RBrYNnqopWJBRtW4msBoZX92UvWdN7w5bBK9eXSxkTXx2vMEyvIlgxZMknMKSh37HpxpNHZx85PP3syerRmeZMI2xZAOCYTZmQ/5AaecQU3IQtZIjyAQ8W/I2D+W0rS5es7jl3RU8pE/MXrSgRXiml+3cFK325ds2CKfon51qHJmr7Jur7J+pHZ5pjc63persRaWjVuqiTySPOGipkeLiYGenJrOnPbxzMbRoqrClnS9nOlv9kJupPEBS8uNf/DxrLmMA1V9rHAAAAJXRFWHRkYXRlOmNyZWF0ZQAyMDE5LTA4LTMwVDE1OjM4OjA2LTA0OjAwvtdu7gAAACV0RVh0ZGF0ZTptb2RpZnkAMjAxOS0wOC0zMFQxNTozODowNi0wNDowMM+K1lIAAAAASUVORK5CYII= diff --git a/servicenow/1.0.0/requirements.txt b/http/1.4.0/requirements.txt similarity index 51% rename from servicenow/1.0.0/requirements.txt rename to http/1.4.0/requirements.txt index fd7d3e06..ae3e5391 100644 --- a/servicenow/1.0.0/requirements.txt +++ b/http/1.4.0/requirements.txt @@ -1 +1,2 @@ +uncurl==0.0.10 requests==2.25.1 \ No newline at end of file diff --git a/http/1.4.0/src/app.py b/http/1.4.0/src/app.py new file mode 100755 index 00000000..ff2ec91b --- /dev/null +++ b/http/1.4.0/src/app.py @@ -0,0 +1,450 @@ +import time +import json +import ast +import random +import socket +import uncurl +import asyncio +import requests +import subprocess + +from walkoff_app_sdk.app_base import AppBase + +class HTTP(AppBase): + __version__ = "1.3.0" + app_name = "http" + + def __init__(self, redis, logger, console_logger=None): + print("INIT") + """ + Each app should have this __init__ to set up Redis and logging. + :param redis: + :param logger: + :param console_logger: + """ + super().__init__(redis, logger, console_logger) + + # This is dangerously fun :) + # Do we care about arbitrary code execution here? + # Probably not huh + def curl(self, statement): + process = subprocess.Popen(statement, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, shell=True) + stdout = process.communicate() + item = "" + if len(stdout[0]) > 0: + print("Succesfully ran bash!") + item = stdout[0] + else: + print("FAILED to run bash!") + item = stdout[1] + + try: + ret = item.decode("utf-8") + return ret + except: + return item + + return item + + def splitheaders(self, headers): + parsed_headers = {} + if headers: + split_headers = headers.split("\n") + self.logger.info(split_headers) + for header in split_headers: + if ":" in header: + splititem = ":" + elif "=" in header: + splititem = "=" + else: + self.logger.info("Skipping header %s as its invalid" % header) + continue + + splitheader = header.split(splititem) + if len(splitheader) >= 2: + parsed_headers[splitheader[0].strip()] = splititem.join(splitheader[1:]).strip() + else: + self.logger.info("Skipping header %s with split %s cus only one item" % (header, splititem)) + continue + + return parsed_headers + + def checkverify(self, verify): + if str(verify).lower().strip() == "false": + return False + elif verify == None: + return False + elif verify: + return True + elif not verify: + return False + else: + return True + + def checkbody(self, body): + # Indicates json + if isinstance(body, str): + if body.strip().startswith("{"): + body = json.dumps(ast.literal_eval(body)) + + + # Not sure if loading is necessary + # Seemed to work with plain string into data=body too, and not parsed json=body + #try: + # body = json.loads(body) + #except json.decoder.JSONDecodeError as e: + # return body + + return body + else: + return body + + if isinstance(body, dict) or isinstance(body, list): + try: + body = json.dumps(body) + except: + return body + + return body + + def fix_url(self, url): + # Random bugs seen by users + if "hhttp" in url: + url = url.replace("hhttp", "http") + + if "http:/" in url and not "http://" in url: + url = url.replace("http:/", "http://", -1) + if "https:/" in url and not "https://" in url: + url = url.replace("https:/", "https://", -1) + if "http:///" in url: + url = url.replace("http:///", "http://", -1) + if "https:///" in url: + url = url.replace("https:///", "https://", -1) + if not "http://" in url and not "http" in url: + url = f"http://{url}" + + return url + + def return_file(self, requestdata): + filedata = { + "filename": "response.txt", + "data": requestdata, + } + fileret = self.set_files([filedata]) + if len(fileret) == 1: + return {"success": True, "file_id": fileret[0]} + + return fileret + + def prepare_response(self, request): + try: + parsedheaders = {} + for key, value in request.headers.items(): + parsedheaders[key] = value + + cookies = {} + if request.cookies: + for key, value in request.cookies.items(): + cookies[key] = value + + + jsondata = request.text + try: + jsondata = json.loads(jsondata) + except: + pass + + parseddata = { + "status": request.status_code, + "body": jsondata, + "url": request.url, + "headers": parsedheaders, + "cookies":cookies, + "success": True, + } + + return json.dumps(parseddata) + except Exception as e: + print(f"[WARNING] Failed in request: {e}") + return request.text + + def GET(self, url, headers="", username="", password="", verify=True, http_proxy="", https_proxy="", timeout=5, to_file=False): + url = self.fix_url(url) + + parsed_headers = self.splitheaders(headers) + parsed_headers["User-Agent"] = "Shuffle Automation" + verify = self.checkverify(verify) + proxies = None + if http_proxy: + proxies["http"] = http_proxy + if https_proxy: + proxies["https"] = https_proxy + + auth=None + if username or password: + # Shouldn't be used if authorization headers exist + if "Authorization" in parsed_headers: + #print("Found authorization - skipping username & pw") + pass + else: + auth = requests.auth.HTTPBasicAuth(username, password) + + if not timeout: + timeout = 5 + if timeout: + timeout = int(timeout) + + if to_file == "true": + to_file = True + else: + to_file = False + + request = requests.get(url, headers=parsed_headers, auth=auth, verify=verify, proxies=proxies, timeout=timeout) + if not to_file: + return self.prepare_response(request) + + return self.return_file(request.text) + + def POST(self, url, headers="", body="", username="", password="", verify=True, http_proxy="", https_proxy="", timeout=5, to_file=False): + url = self.fix_url(url) + + parsed_headers = self.splitheaders(headers) + parsed_headers["User-Agent"] = "Shuffle Automation" + verify = self.checkverify(verify) + body = self.checkbody(body) + proxies = None + if http_proxy: + proxies["http"] = http_proxy + if https_proxy: + proxies["https"] = https_proxy + + auth=None + if username or password: + # Shouldn't be used if authorization headers exist + if "Authorization" in parsed_headers: + #print("Found authorization - skipping username & pw") + pass + else: + auth = requests.auth.HTTPBasicAuth(username, password) + + if not timeout: + timeout = 5 + if timeout: + timeout = int(timeout) + + if to_file == "true": + to_file = True + else: + to_file = False + + request = requests.post(url, headers=parsed_headers, auth=auth, data=body, verify=verify, proxies=proxies, timeout=timeout) + if not to_file: + return self.prepare_response(request) + + return self.return_file(request.text) + + def PUT(self, url, headers="", body="", username="", password="", verify=True, http_proxy="", https_proxy="", timeout=5, to_file=False): + url = self.fix_url(url) + + parsed_headers = self.splitheaders(headers) + parsed_headers["User-Agent"] = "Shuffle Automation" + verify = self.checkverify(verify) + body = self.checkbody(body) + proxies = None + if http_proxy: + proxies["http"] = http_proxy + if https_proxy: + proxies["https"] = https_proxy + + + auth=None + if username or password: + # Shouldn't be used if authorization headers exist + if "Authorization" in parsed_headers: + #print("Found authorization - skipping username & pw") + pass + else: + auth = requests.auth.HTTPBasicAuth(username, password) + + if not timeout: + timeout = 5 + if timeout: + timeout = int(timeout) + + if to_file == "true": + to_file = True + else: + to_file = False + + request = requests.put(url, headers=parsed_headers, auth=auth, data=body, verify=verify, proxies=proxies, timeout=timeout) + if not to_file: + return self.prepare_response(request) + + return self.return_file(request.text) + + def PATCH(self, url, headers="", body="", username="", password="", verify=True, http_proxy="", https_proxy="", timeout=5, to_file=False): + url = self.fix_url(url) + + parsed_headers = self.splitheaders(headers) + parsed_headers["User-Agent"] = "Shuffle Automation" + verify = self.checkverify(verify) + body = self.checkbody(body) + proxies = None + if http_proxy: + proxies["http"] = http_proxy + if https_proxy: + proxies["https"] = https_proxy + + auth=None + if username or password: + # Shouldn't be used if authorization headers exist + if "Authorization" in parsed_headers: + #print("Found authorization - skipping username & pw") + pass + else: + auth = requests.auth.HTTPBasicAuth(username, password) + + if to_file == "true": + to_file = True + else: + to_file = False + + request = requests.patch(url, headers=parsed_headers, data=body, auth=auth, verify=verify, proxies=proxies, timeout=timeout) + if not to_file: + return self.prepare_response(request) + + return self.return_file(request.text) + + def DELETE(self, url, headers="", body="", username="", password="", verify=True, http_proxy="", https_proxy="", timeout=5, to_file=False): + url = self.fix_url(url) + + parsed_headers = self.splitheaders(headers) + parsed_headers["User-Agent"] = "Shuffle Automation" + verify = self.checkverify(verify) + body = self.checkbody(body) + proxies = None + if http_proxy: + proxies["http"] = http_proxy + if https_proxy: + proxies["https"] = https_proxy + + auth=None + if username or password: + # Shouldn't be used if authorization headers exist + if "Authorization" in parsed_headers: + #print("Found authorization - skipping username & pw") + pass + else: + auth = requests.auth.HTTPBasicAuth(username, password) + + if not timeout: + timeout = 5 + if timeout: + timeout = int(timeout) + + if to_file == "true": + to_file = True + else: + to_file = False + + request = requests.delete(url, headers=parsed_headers, data=body, auth=auth, verify=verify, proxies=proxies, timeout=timeout) + if not to_file: + return self.prepare_response(request) + + return self.return_file(request.text) + + def HEAD(self, url, headers="", body="", username="", password="", verify=True, http_proxy="", https_proxy="", timeout=5, to_file=False): + url = self.fix_url(url) + + parsed_headers = self.splitheaders(headers) + parsed_headers["User-Agent"] = "Shuffle Automation" + verify = self.checkverify(verify) + body = self.checkbody(body) + proxies = None + if http_proxy: + proxies["http"] = http_proxy + if https_proxy: + proxies["https"] = https_proxy + + auth=None + if username or password: + # Shouldn't be used if authorization headers exist + if "Authorization" in parsed_headers: + #print("Found authorization - skipping username & pw") + pass + else: + auth = requests.auth.HTTPBasicAuth(username, password) + + if not timeout: + timeout = 5 + if timeout: + timeout = int(timeout) + + if to_file == "true": + to_file = True + else: + to_file = False + + request = requests.head(url, headers=parsed_headers, auth=auth, verify=verify, proxies=proxies, timeout=timeout) + if not to_file: + return self.prepare_response(request) + + return self.return_file(request.text) + + def OPTIONS(self, url, headers="", body="", username="", password="", verify=True, http_proxy="", https_proxy="", timeout=5, to_file=False): + url = self.fix_url(url) + + parsed_headers = self.splitheaders(headers) + parsed_headers["User-Agent"] = "Shuffle Automation" + verify = self.checkverify(verify) + body = self.checkbody(body) + proxies = None + if http_proxy: + proxies["http"] = http_proxy + if https_proxy: + proxies["https"] = https_proxy + + auth=None + if username or password: + # Shouldn't be used if authorization headers exist + if "Authorization" in parsed_headers: + #print("Found authorization - skipping username & pw") + pass + else: + auth = requests.auth.HTTPBasicAuth(username, password) + + if not timeout: + timeout = 5 + + if timeout: + timeout = int(timeout) + + if to_file == "true": + to_file = True + else: + to_file = False + + request = requests.options(url, headers=parsed_headers, auth=auth, verify=verify, proxies=proxies, timeout=timeout) + if not to_file: + return self.prepare_response(request) + + return self.return_file(request.text) + + +# Run the actual thing after we've checked params +def run(request): + print("Starting cloud!") + action = request.get_json() + print(action) + print(type(action)) + authorization_key = action.get("authorization") + current_execution_id = action.get("execution_id") + + if action and "name" in action and "app_name" in action: + HTTP.run(action) + return f'Attempting to execute function {action["name"]} in app {action["app_name"]}' + else: + return f'Invalid action' + +if __name__ == "__main__": + HTTP.run() diff --git a/outlook-exchange/1.0.0/src/app.py b/outlook-exchange/1.0.0/src/app.py index 43673a10..38d8b2cd 100644 --- a/outlook-exchange/1.0.0/src/app.py +++ b/outlook-exchange/1.0.0/src/app.py @@ -198,14 +198,13 @@ def send_email( account=account, subject=subject, body=body, - to_recipients=[ - Mailbox(email_address=address) for address in recipient.split(", ") - ], - cc_recipients=[ - Mailbox(email_address=address) for address in ccrecipient.split(", ") - ], + to_recipients=[] ) + for address in recipient.split(", "): + address = address.strip() + m.to_recipients.append(Mailbox(email_address=address)) + file_uids = str(attachments).split() if len(file_uids) > 0: for file_uid in file_uids: diff --git a/servicenow/1.0.0/api.yaml b/servicenow/1.0.0/api.yaml deleted file mode 100644 index 4fa62bef..00000000 --- a/servicenow/1.0.0/api.yaml +++ /dev/null @@ -1,146 +0,0 @@ -walkoff_version: 1.0.0 -app_version: 1.0.0 -name: servicenow -description: servicenow app -tags: - - tickets -categories: - - tickets -contact_info: - name: "@frikkylikeme" - url: https://github.com/frikky - email: "frikky@shuffler.io" -authentication: - required: true - parameters: - - name: url - description: The url your instance is at - multiline: false - example: "test.service-now.com" - required: true - schema: - type: string - - name: username - description: The user to authenticate with - multiline: false - example: "username12345" - required: true - schema: - type: string - - name: password - description: The password for the user to authenticate with - multiline: false - example: "pw1234" - required: true - schema: - type: string -actions: - - name: get_ticket - description: Get ticket ids - parameters: - - name: table_name - description: The type to get. Empty as default - multiline: false - example: "incident" - required: true - schema: - type: string - - name: sys_id - description: The ID to get from the table - multiline: false - example: "INC123456" - required: true - schema: - type: string - - name: number - description: The number to get instead of record_id - multiline: false - example: "20" - required: false - schema: - type: string - returns: - schema: - type: string - - name: create_ticket - description: Create a ticket - parameters: - - name: table_name - description: The table to create the ticket in - multiline: false - example: "incident" - required: true - schema: - type: string - - name: body - description: The body of the ticket - multiline: true - example: "{'short_description':'Unable to connect to office wifi','assignment_group':'287ebd7da9fe198100f92cc8d1d2154e','urgency':'2','impact':'2'}" - required: true - schema: - type: string - - name: file_id - description: Optional file to attach - multiline: false - example: "ca0c88a6-626e-4235-896f-ca18c96fd48e" - required: false - schema: - type: string - returns: - schema: - type: string - - name: update_ticket - description: Update a ticket - parameters: - - name: table_name - description: The table to create the ticket in - multiline: false - example: "incident" - required: true - schema: - type: string - - name: sys_id - description: The ticket to edit - multiline: false - example: "incident" - required: true - schema: - type: string - - name: body - description: JSON data of the data to replace - multiline: true - example: "{'short_description':'Unable to connect to office wifi','assignment_group':'287ebd7da9fe198100f92cc8d1d2154e','urgency':'2','impact':'2'}" - required: true - schema: - type: string - - name: file_id - description: Optional file to attach - multiline: false - example: "ca0c88a6-626e-4235-896f-ca18c96fd48e" - required: false - schema: - type: string - returns: - schema: - type: string - - name: list_table - description: Get ticket ids - parameters: - - name: table_name - description: The type to get. Empty as default - multiline: false - example: "incident" - required: true - schema: - type: string - - name: limit - description: The limit of items to get - multiline: false - example: "1" - required: false - schema: - type: string - returns: - schema: - type: string -large_image: data:image/jpg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4QAqRXhpZgAASUkqAAgAAAABADEBAgAHAAAAGgAAAAAAAABHb29nbGUAAP/iC/hJQ0NfUFJPRklMRQABAQAAC+gAAAAAAgAAAG1udHJSR0IgWFlaIAfZAAMAGwAVACQAH2Fjc3AAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAD21gABAAAAANMtAAAAACn4Pd6v8lWueEL65MqDOQ0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEGRlc2MAAAFEAAAAeWJYWVoAAAHAAAAAFGJUUkMAAAHUAAAIDGRtZGQAAAngAAAAiGdYWVoAAApoAAAAFGdUUkMAAAHUAAAIDGx1bWkAAAp8AAAAFG1lYXMAAAqQAAAAJGJrcHQAAAq0AAAAFHJYWVoAAArIAAAAFHJUUkMAAAHUAAAIDHRlY2gAAArcAAAADHZ1ZWQAAAroAAAAh3d0cHQAAAtwAAAAFGNwcnQAAAuEAAAAN2NoYWQAAAu8AAAALGRlc2MAAAAAAAAAH3NSR0IgSUVDNjE5NjYtMi0xIGJsYWNrIHNjYWxlZAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABYWVogAAAAAAAAJKAAAA+EAAC2z2N1cnYAAAAAAAAEAAAAAAUACgAPABQAGQAeACMAKAAtADIANwA7AEAARQBKAE8AVABZAF4AYwBoAG0AcgB3AHwAgQCGAIsAkACVAJoAnwCkAKkArgCyALcAvADBAMYAywDQANUA2wDgAOUA6wDwAPYA+wEBAQcBDQETARkBHwElASsBMgE4AT4BRQFMAVIBWQFgAWcBbgF1AXwBgwGLAZIBmgGhAakBsQG5AcEByQHRAdkB4QHpAfIB+gIDAgwCFAIdAiYCLwI4AkECSwJUAl0CZwJxAnoChAKOApgCogKsArYCwQLLAtUC4ALrAvUDAAMLAxYDIQMtAzgDQwNPA1oDZgNyA34DigOWA6IDrgO6A8cD0wPgA+wD+QQGBBMEIAQtBDsESARVBGMEcQR+BIwEmgSoBLYExATTBOEE8AT+BQ0FHAUrBToFSQVYBWcFdwWGBZYFpgW1BcUF1QXlBfYGBgYWBicGNwZIBlkGagZ7BowGnQavBsAG0QbjBvUHBwcZBysHPQdPB2EHdAeGB5kHrAe/B9IH5Qf4CAsIHwgyCEYIWghuCIIIlgiqCL4I0gjnCPsJEAklCToJTwlkCXkJjwmkCboJzwnlCfsKEQonCj0KVApqCoEKmAquCsUK3ArzCwsLIgs5C1ELaQuAC5gLsAvIC+EL+QwSDCoMQwxcDHUMjgynDMAM2QzzDQ0NJg1ADVoNdA2ODakNww3eDfgOEw4uDkkOZA5/DpsOtg7SDu4PCQ8lD0EPXg96D5YPsw/PD+wQCRAmEEMQYRB+EJsQuRDXEPURExExEU8RbRGMEaoRyRHoEgcSJhJFEmQShBKjEsMS4xMDEyMTQxNjE4MTpBPFE+UUBhQnFEkUahSLFK0UzhTwFRIVNBVWFXgVmxW9FeAWAxYmFkkWbBaPFrIW1hb6Fx0XQRdlF4kXrhfSF/cYGxhAGGUYihivGNUY+hkgGUUZaxmRGbcZ3RoEGioaURp3Gp4axRrsGxQbOxtjG4obshvaHAIcKhxSHHscoxzMHPUdHh1HHXAdmR3DHeweFh5AHmoelB6+HukfEx8+H2kflB+/H+ogFSBBIGwgmCDEIPAhHCFIIXUhoSHOIfsiJyJVIoIiryLdIwojOCNmI5QjwiPwJB8kTSR8JKsk2iUJJTglaCWXJccl9yYnJlcmhya3JugnGCdJJ3onqyfcKA0oPyhxKKIo1CkGKTgpaymdKdAqAio1KmgqmyrPKwIrNitpK50r0SwFLDksbiyiLNctDC1BLXYtqy3hLhYuTC6CLrcu7i8kL1ovkS/HL/4wNTBsMKQw2zESMUoxgjG6MfIyKjJjMpsy1DMNM0YzfzO4M/E0KzRlNJ402DUTNU01hzXCNf02NzZyNq426TckN2A3nDfXOBQ4UDiMOMg5BTlCOX85vDn5OjY6dDqyOu87LTtrO6o76DwnPGU8pDzjPSI9YT2hPeA+ID5gPqA+4D8hP2E/oj/iQCNAZECmQOdBKUFqQaxB7kIwQnJCtUL3QzpDfUPARANER0SKRM5FEkVVRZpF3kYiRmdGq0bwRzVHe0fASAVIS0iRSNdJHUljSalJ8Eo3Sn1KxEsMS1NLmkviTCpMcky6TQJNSk2TTdxOJU5uTrdPAE9JT5NP3VAnUHFQu1EGUVBRm1HmUjFSfFLHUxNTX1OqU/ZUQlSPVNtVKFV1VcJWD1ZcVqlW91dEV5JX4FgvWH1Yy1kaWWlZuFoHWlZaplr1W0VblVvlXDVchlzWXSddeF3JXhpebF69Xw9fYV+zYAVgV2CqYPxhT2GiYfViSWKcYvBjQ2OXY+tkQGSUZOllPWWSZedmPWaSZuhnPWeTZ+loP2iWaOxpQ2maafFqSGqfavdrT2una/9sV2yvbQhtYG25bhJua27Ebx5veG/RcCtwhnDgcTpxlXHwcktypnMBc11zuHQUdHB0zHUodYV14XY+dpt2+HdWd7N4EXhueMx5KnmJeed6RnqlewR7Y3vCfCF8gXzhfUF9oX4BfmJ+wn8jf4R/5YBHgKiBCoFrgc2CMIKSgvSDV4O6hB2EgITjhUeFq4YOhnKG14c7h5+IBIhpiM6JM4mZif6KZIrKizCLlov8jGOMyo0xjZiN/45mjs6PNo+ekAaQbpDWkT+RqJIRknqS45NNk7aUIJSKlPSVX5XJljSWn5cKl3WX4JhMmLiZJJmQmfyaaJrVm0Kbr5wcnImc951kndKeQJ6unx2fi5/6oGmg2KFHobaiJqKWowajdqPmpFakx6U4pammGqaLpv2nbqfgqFKoxKk3qamqHKqPqwKrdavprFys0K1ErbiuLa6hrxavi7AAsHWw6rFgsdayS7LCszizrrQltJy1E7WKtgG2ebbwt2i34LhZuNG5SrnCuju6tbsuu6e8IbybvRW9j74KvoS+/796v/XAcMDswWfB48JfwtvDWMPUxFHEzsVLxcjGRsbDx0HHv8g9yLzJOsm5yjjKt8s2y7bMNcy1zTXNtc42zrbPN8+40DnQutE80b7SP9LB00TTxtRJ1MvVTtXR1lXW2Ndc1+DYZNjo2WzZ8dp22vvbgNwF3IrdEN2W3hzeot8p36/gNuC94UThzOJT4tvjY+Pr5HPk/OWE5g3mlucf56noMui86Ubp0Opb6uXrcOv77IbtEe2c7ijutO9A78zwWPDl8XLx//KM8xnzp/Q09ML1UPXe9m32+/eK+Bn4qPk4+cf6V/rn+3f8B/yY/Sn9uv5L/tz/bf//ZGVzYwAAAAAAAAAuSUVDIDYxOTY2LTItMSBEZWZhdWx0IFJHQiBDb2xvdXIgU3BhY2UgLSBzUkdCAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAAAAAAFAAAAAAAABtZWFzAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJYWVogAAAAAAAAAxYAAAMzAAACpFhZWiAAAAAAAABvogAAOPUAAAOQc2lnIAAAAABDUlQgZGVzYwAAAAAAAAAtUmVmZXJlbmNlIFZpZXdpbmcgQ29uZGl0aW9uIGluIElFQyA2MTk2Ni0yLTEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFhZWiAAAAAAAAD21gABAAAAANMtdGV4dAAAAABDb3B5cmlnaHQgSW50ZXJuYXRpb25hbCBDb2xvciBDb25zb3J0aXVtLCAyMDA5AABzZjMyAAAAAAABDEQAAAXf///zJgAAB5QAAP2P///7of///aIAAAPbAADAdf/bAEMAAgEBAgEBAgICAgICAgIDBQMDAwMDBgQEAwUHBgcHBwYHBwgJCwkICAoIBwcKDQoKCwwMDAwHCQ4PDQwOCwwMDP/bAEMBAgICAwMDBgMDBgwIBwgMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDP/AABEIAGQAZAMBIgACEQEDEQH/xAAcAAEAAgIDAQAAAAAAAAAAAAAACAkFCgIEBwb/xABAEAAABgECBAQCBQYPAAAAAAAAAQIDBAUGBxEICRIhEzFBUQoUFRYiM2FSYnaBgrQjJCkyNDg5QkNxcnN0g7X/xAAaAQEAAgMBAAAAAAAAAAAAAAAAAgMBBAUG/8QAJBEAAwABBAIBBQEAAAAAAAAAAAECAwQREhMhMUEFFEJRYdH/2gAMAwEAAhEDEQA/AK3wAB1DyYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAdymxyxyN1xFdXWFitpPUtMSK5IUgt9iMyQRmRb+pjpi1L4U7vxMaskfl9V4X74sRuuM7luHH2Wo/ZWbi+k2V5xeSaukxbJrmzh/wBIhwamRJkR+2/220INSO35REMbkmMWeG3LtbcVtjUWLH3sSdFcjSG/9TbhEov1kNgfjv55Gk3Lr1uuMEpcGscsy/5hE7JCrDZrY7D7rSFJN19STN582zbM9kmRJNJGoj7FnNTMI0n5+XLtcyimqFxL8mJTdLMmMITaY3asEe8dTiTMlNKV0kpJKNC23CVsStjKruftrwbf2UveYvel8GucMrR4JeZPFN+spLmxYT5uxILr6C/aSkyEjeXXwgwdY7SdlOVwykUlHI+UYr3Pu5ksiI1+J+U23uRGn+8pRb9iMjk3qdx76a6D5OrGXpNg/KrTJl9iohEtiAZf4ZmSkpJReqU77eR7H2C821cZW7OBm1rm+vFPJr2ViyozkKQ4y+24y80ey23EGhaD/FJ9y/WMhZ4TdUkH5qbS3MKMRkXjSa95lrc/IupSST39O/cWVapaQYBx76PlaVj0OTJktLTW3TTfRIiul2Np7ciUad+y21+RHuWx7GJl8JqHOahyUr3TPJFkrOsdgScKsTeURuR7au6VQpCjPv8Aa6IyzV67r/EYWfdejc+nZJ1XKfVL4Nf6trJNzNRGhxpMyS7v0Mx2VOuL2Lc9kpIzPsRn2L0HK2pZtBL+Xnw5kCR0krwpTC2HNj8j6VkR7H77C1P4X3hJk3XEDnmqV3AXHLBI541BQ6kyU1Yv95W35zTKCQf++Y+7+Ki4W0vUWn+s8CMZqr1rxa6Wgu5tL6n4iz/yWT6N/d1JCbyrnxN5aR9PaU9U2HXGRsKdrqe3sWkK6FLiQXpCEq8+kzQkyI9jLt59xjnG1MuKQtKkLQZpUlRbGky7GRl6GNgLTxRcm3kQfSjyUQs3kUx2BoM+lbl9abE0gy9TZ62yP82MY1/nHFvOKW44t51Zmpbiz3U4o+5qM/UzPcz/ABMSi+W5DPh61Pny1ucQABM1wLUfhT/6y+rX6LQv3xYquEoOV1zLZfLN1GyzIYeHxswXlNWzWmw9ZKglGJt43evqJtzq33222L339BDJLctIv01qMqqvRkuecf8AKt6wf86F/wCdFFp3wvf9nPc/p1Y/u8MUqcbfE+9xncUuXanSKZrHnsrfZeVXtyjkpjeHHaY2Jw0pNW5Nb/zS89hJzln88iy5cHD3LwGHpxBy1qXdyLo5r12uEpJutso8PoJlZbF4W++/fq8uwruG4UovwZonO7b8eT0bl42kWboPMYjqSl+Bk1q1I280LVKUtJn+wpJ/qFc+rGK2uBalXlTfNuMW0Wa98wT3ZThmtSvEIz80rI+oleRkY9G4YuMe04b8+ubBuB9J0uQvLenVvj+GZLNalIcbXsey09Rp7lspPn5FtkdYeP8AyvVXKEzU0+Iw4kVX8UYk00exdaTv2JTr6FGZ+/SSS9iGJipttLwzg48WXHnukt1RI3lPYjb0ujF/YTWnma68s0O1qXCMvFJDXQ46kvyVK6Ukfr0H7D2Pko8Y8TTrmxao4CuWlOOarzJCYX2iJsrSH1KQovxcbKQn8TSgRDf5q2USNJpNQVDWxclcaOOzbxF+ExHQZbdaY+x7OEXlsrpI9j27bCOel+pNtpBqdj+XUklce7xqzj20N8zMzJ5l1LiTM/Xcy2P3Iz9xicTbp18lugnJj1Fai1t/hsvcWOS4lyvuBzWHNsZiN1siwk2F+20SyL5u7sXCQgyL2N5aD2LySg/YfB8tvOKDmr8q3Eq7UMjyKVWrjU+RE6r+Efn1khp1t5Rnue7qW2HVe5OqL1FTvM752eR8yXSShwx7DYmFVFXa/S0tMe1XNOxcS2pDST3ab6UoNa1eu5mny6e+A5WfN3v+WPEzCBExWNmdNlrkaScN+zVB+SkNEpBupUTbm/W2aUmWxfdpPftsMdNcf6ehetjt2/HbYlP8UnxZfWXVDCdGKyUaouNsfWS7QhR7HKfJTcVtRe6GfFX/AN6TFTI93z3jePWHiB1Bz/N8Hx3MJGfWv0iuFYLUSa9tLbrTUdp9JE8lCEKZL7Ck7/LpMy7mPFMhsmrnILCZHiMwGJkp19qK193FQtalJaT+agjJJfgkhfjnitjQ1GTst2dMAATKAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD//2Q== diff --git a/servicenow/1.0.0/src/app.py b/servicenow/1.0.0/src/app.py deleted file mode 100755 index d85dc832..00000000 --- a/servicenow/1.0.0/src/app.py +++ /dev/null @@ -1,204 +0,0 @@ -import time -import json -import random -import socket -import asyncio -import requests - -from walkoff_app_sdk.app_base import AppBase - -class Servicenow(AppBase): - __version__ = "1.0.0" - app_name = "servicenow" - - def __init__(self, redis, logger, console_logger=None): - """ - Each app should have this __init__ to set up Redis and logging. - :param redis: - :param logger: - :param console_logger: - """ - super().__init__(redis, logger, console_logger) - - def send_request(self, url, username, password, path, method='get', body=None, params=None, headers=None, json=None, files=None): - body = body if body is not None else {} - params = params if params is not None else {} - - url = '{}{}'.format(url, path) - print("HEADERS: %s" % headers) - if not headers and files == None: - headers = { - 'Accept': 'application/json', - 'Content-Type': 'application/json' - } - - if files: - # Not supported in v2 - url = url.replace('v2', 'v1') - #{'file': ('report.csv', 'some,data,to,send\nanother,row,to,send\n')} - #file_entry = file['id'] - #file_name = file['name'] - try: - #shutil.copy(demisto.getFilePath(file_entry)['path'], file_name) - #with open(file_name, 'rb') as f: - #files = {'file': f} - - try: - res = requests.request(method, url, headers=headers, params=params, data=body, files=files, json=json, auth=(username, password)) - except requests.exceptions.ReadTimeout as e: - return "Readtimeout: %s" % e - except requests.exceptions.ConnectionError as e: - return "ConnectionError: %s" % e - - #shutil.rmtree(demisto.getFilePath(file_entry)['name'], ignore_errors=True) - except Exception as e: - return 'Failed to upload file - ' + str(e) - else: - try: - res = requests.request(method, url, headers=headers, data=json.dumps(body) if body else {}, json=json, params=params, auth=(username, password)) - except requests.exceptions.ReadTimeout as e: - return "Readtimeout: %s" % e - except requests.exceptions.ConnectionError as e: - return "ConnectionError: %s" % e - - try: - obj = res.json() - except Exception as e: - if not res.content: - return '' - return 'Error parsing reply - {} - {}'.format(res.content, str(e)) - - if 'error' in obj: - message = obj.get('error', {}).get('message') - details = obj.get('error', {}).get('detail') - if message == 'No Record found': - return { - # Return an empty results array - 'result': [] - } - return 'ServiceNow Error: {}, details: {}'.format(message, details) - - if res.status_code < 200 or res.status_code >= 300: - return 'Got status code {} with url {} with body {} with headers {}'.format(str(res.status_code), url, str(res.content), str(res.headers)) - - #print("RES: %s" % res) - #print("TEXT: %s" % res.text) - return res.text - - def get_ticket(self, url, username, password, table_name, sys_id, number=None): - path = None - query_params = {} # type: Dict - if sys_id: - path = "/api/now/v1/table/%s/%s" % (table_name, sys_id) - elif number: - path = '/api/now/v1/table/%s' % table_name - query_params = { - 'number': number - } - else: - # Only in cases where the table is of type ticket - return 'servicenow-get-ticket requires either ticket ID or ticket number' - - print("PATH: %s" % path) - return self.send_request(url, username, password, path, 'get', params=query_params) - - def list_table(self, url, username, password, table_name, limit=1): - query_params = { - "sysparm_limit": limit, - } - - #path = '/table/%s' % table_name - path = "/api/now/v1/table/%s" % table_name - - return self.send_request(url, username, password, path, 'get', params=query_params) - - def create_ticket(self, url, username, password, table_name, body, file_id=""): - if not isinstance(body, list) and not isinstance(body, object) and not isinstance(body, dict): - try: - data = json.loads(body) - except json.decoder.JSONDecodeError as e: - return {"success": False, "reason": e} - else: - data = body - - - path = "/api/now/v1/table/%s" % table_name - query_params = {} - base_request = self.send_request(url, username, password, path, 'post', params=query_params, json=data) - - if file_id: - tmp_file = self.get_file(file_id) - files = {'file': (tmp_file["filename"], tmp_file["data"])} - - try: - parsed_return = json.loads(base_request) - except: - print("[INFO] Failed parsed_return loading") - return base_request - - ticket_id = parsed_return["result"]["sys_id"] - params = { - "file_name": tmp_file["filename"], - "table_name": table_name, - "table_sys_id": ticket_id, - } - - filepath = "/api/now/v1/attachment/file" - file_request = self.send_request(url, username, password, filepath, 'post', params=params, files=files, headers={}) - print(file_request) - - return base_request - - def update_ticket(self, url, username, password, table_name, sys_id, body, file_id=""): - if not isinstance(body, list) and not isinstance(body, object) and not isinstance(body, dict): - try: - data = json.loads(body) - except json.decoder.JSONDecodeError as e: - return {"success": False, "reason": e} - else: - data = body - - - path = "/api/now/v1/table/%s/%s" % (table_name, sys_id) - query_params = {} - base_request = self.send_request(url, username, password, path, 'patch', params=query_params, json=data) - - if file_id: - tmp_file = self.get_file(file_id) - files = {'file': (tmp_file["filename"], tmp_file["data"])} - - try: - parsed_return = json.loads(base_request) - except: - print("[INFO] Failed parsed_return loading") - return base_request - - ticket_id = parsed_return["result"]["sys_id"] - params = { - "file_name": tmp_file["filename"], - "table_name": table_name, - "table_sys_id": ticket_id, - } - - filepath = "/api/now/v1/attachment/file" - file_request = self.send_request(url, username, password, filepath, '', params=params, files=files, headers={}) - print(file_request) - - return base_request - -# Run the actual thing after we've checked params -def run(request): - action = request.get_json() - print(action) - print(type(action)) - authorization_key = action.get("authorization") - current_execution_id = action.get("execution_id") - - if action and "name" in action and "app_name" in action: - Servicenow.run(action) - return f'Attempting to execute function {action["name"]} in app {action["app_name"]}' - else: - return f'Invalid action' - -if __name__ == "__main__": - Servicenow.run() diff --git a/shuffle-ai/1.0.0/api.yaml b/shuffle-ai/1.0.0/api.yaml index 40e0832c..ae0ae960 100644 --- a/shuffle-ai/1.0.0/api.yaml +++ b/shuffle-ai/1.0.0/api.yaml @@ -11,6 +11,67 @@ contact_info: url: https://shuffler.io email: support@shuffler.io actions: + - name: autoformat_text + description: Input ANY kind of data in the format you want, and the format you want it in. Default is a business-y email. Uses ShuffleGPT, which is based on OpenAI and our own model. + parameters: + - name: apikey + description: Your https://shuffler.io apikey + required: true + multiline: false + example: "" + schema: + type: string + - name: text + description: The text you want to be converted (ANY format) + required: true + multiline: true + example: "Bad IPs are 1.2.3.4 and there's no good way to format this. JSON works too!" + schema: + type: string + - name: formatting + description: The format to use. + required: false + multiline: true + example: "Make it work as a ticket we can put in service now that is human readable for security analysts" + schema: + type: string + returns: + schema: + type: string + - name: generate_report + description: Input ANY kind of data in the format you want, and it will make an HTML report for you. This can be downloaded from the File location. + parameters: + - name: apikey + description: Your https://shuffler.io apikey + required: true + multiline: false + example: "" + schema: + type: string + - name: input_data + description: The text you want to be converted (ANY format) + required: true + multiline: true + example: "Bad IPs are 1.2.3.4 and there's no good way to format this. JSON works too!" + schema: + type: string + - name: report_title + description: The report title to be used in the report + required: true + multiline: true + example: "Statistics for October" + schema: + type: string + - name: report_name + description: The name of the HTML file + required: false + multiline: true + example: "statistics.html" + schema: + type: string + returns: + schema: + type: string - name: extract_text_from_pdf description: Returns text from a pdf parameters: diff --git a/shuffle-ai/1.0.0/src/app.py b/shuffle-ai/1.0.0/src/app.py index 655f84bd..4a76c673 100644 --- a/shuffle-ai/1.0.0/src/app.py +++ b/shuffle-ai/1.0.0/src/app.py @@ -13,6 +13,89 @@ class Tools(AppBase): def __init__(self, redis, logger, console_logger=None): super().__init__(redis, logger, console_logger) + def autoformat_text(self, apikey, text, formatting="auto"): + headers = { + "Authorization": "Bearer %s" % apikey, + } + + if not formatting: + formatting = "auto" + + output_formatting= "Format the following data to be a good email that can be sent to customers. Don't make it too business sounding." + if formatting != "auto": + output_formatting = formatting + + ret = requests.post( + "https://shuffler.io/api/v1/conversation", + json={ + "query": text, + "formatting": output_formatting, + "output_format": "formatting" + }, + headers=headers, + ) + + if ret.status_code != 200: + print(ret.text) + return { + "success": False, + "reason": "Status code for auto-formatter is not 200" + } + + return ret.text + + def generate_report(self, apikey, input_data, report_title, report_name="generated_report.html"): + headers = { + "Authorization": "Bearer %s" % apikey, + } + + if not report_name: + report_name = "generated_report.html" + + if "." in report_name and not ".html" in report_name: + report_name = report_name.split(".")[0] + + if not "html" in report_name: + report_name = report_name + ".html" + + report_name = report_name.replace(" ", "_", -1) + + if not formatting: + formatting = "auto" + + output_formatting= "Format the following text into an HTML report with relevant graphs and tables. Title of the report should be {report_title}." + ret = requests.post( + "https://shuffler.io/api/v1/conversation", + json={ + "query": text, + "formatting": output_formatting, + "output_format": "formatting" + }, + headers=headers, + ) + + if ret.status_code != 200: + print(ret.text) + return { + "success": False, + "reason": "Status code for auto-formatter is not 200" + } + + # Make it into a shuffle file with self.set_files() + new_file = { + "name": report_name, + "data": ret.text, + } + + retdata = self.set_files([new_file]) + if retdata["success"]: + return retdata + + return { + "success": False, + "reason": "Failed to upload file" + } + def extract_text_from_pdf(self, file_id): def extract_pdf_text(pdf_path): @@ -105,7 +188,7 @@ def extract_text_from_image(self, file_id): image = Image.open(temp.name) image = image.resize((500,300)) custom_config = r'-l eng --oem 3 --psm 6' - text = pytesseract.image_to_string(image,config=custom_config + text = pytesseract.image_to_string(image,config=custom_config) data = { "success": True, diff --git a/shuffle-ai/1.0.0/upload.sh b/shuffle-ai/1.0.0/upload.sh new file mode 100755 index 00000000..33f84bac --- /dev/null +++ b/shuffle-ai/1.0.0/upload.sh @@ -0,0 +1,6 @@ + +gcloud run deploy shuffle-ai-1-0-0 \ + --region=europe-west2 \ + --max-instances=3 \ + --set-env-vars=SHUFFLE_APP_EXPOSED_PORT=8080,SHUFFLE_SWARM_CONFIG=run,SHUFFLE_LOGS_DISABLED=true --source=./ \ + --timeout=1800s diff --git a/shuffle-subflow/1.0.0/src/app.py b/shuffle-subflow/1.0.0/src/app.py index 2fc46527..82253179 100644 --- a/shuffle-subflow/1.0.0/src/app.py +++ b/shuffle-subflow/1.0.0/src/app.py @@ -172,6 +172,12 @@ def run_subflow(self, user_apikey, workflow, argument, source_workflow="", sourc else: print("No startnode") + if len(self.full_execution["execution_id"]) > 0 and self.full_execution["execution_id"] != source_execution: + params["source_execution"] = self.full_execution["execution_id"] + + if len(self.full_execution["authorization"]) > 0 and self.full_execution["authorization"] != source_auth: + params["source_auth"] = self.full_execution["authorization"] + if len(str(backend_url)) > 0: url = "%s/api/v1/workflows/%s/execute" % (backend_url, workflow) print("[INFO] Changed URL to %s for this execution" % url) diff --git a/shuffle-subflow/1.1.0/api.yaml b/shuffle-subflow/1.1.0/api.yaml index 4148c19a..0c64bda1 100644 --- a/shuffle-subflow/1.1.0/api.yaml +++ b/shuffle-subflow/1.1.0/api.yaml @@ -1,4 +1,4 @@ -app_version: 1.0.0 +app_version: 1.1.0 name: Shuffle Subflow description: The Shuffle Subflow app tags: diff --git a/shuffle-subflow/1.1.0/src/app.py b/shuffle-subflow/1.1.0/src/app.py index f1fa452a..5c4b4d8a 100644 --- a/shuffle-subflow/1.1.0/src/app.py +++ b/shuffle-subflow/1.1.0/src/app.py @@ -51,11 +51,11 @@ def run_userinput(self, user_apikey, sms="", email="", subflow="", information=" url = backend_url print("Found backend url: %s" % url) - print("AUTH: %s" % self.full_execution["authorization"]) + #print("AUTH: %s" % self.full_execution["authorization"]) #if len(information): # print("Should run arg: %s", information) - if len(subflow): + if len(subflow) > 0: #print("Should run subflow: %s", subflow) # Missing startnode (user input trigger) @@ -67,7 +67,8 @@ def run_userinput(self, user_apikey, sms="", email="", subflow="", information=" print("Should change port to 3001.") if "appspot.com" in frontend_url: frontend_url = "https://shuffler.io" - + if "run.app" in frontend_url: + frontend_url = "https://shuffler.io" for item in subflows: # In case of URL being passed, and not just ID @@ -79,10 +80,10 @@ def run_userinput(self, user_apikey, sms="", email="", subflow="", information=" argument = json.dumps({ "information": information, "parent_workflow": self.full_execution["workflow"]["id"], - "frontend_continue": "%s/workflows/%s/run?authorization=%s&reference_execution=%s&answer=true" % (frontend_url, self.full_execution["workflow"]["id"], self.full_execution["authorization"], self.full_execution["execution_id"]), - "frontend_abort": "%s/workflows/%s/run?authorization=%s&reference_execution=%s&answer=false" % (frontend_url, self.full_execution["workflow"]["id"], self.full_execution["authorization"], self.full_execution["execution_id"]), - "api_continue": "%s/api/v1/workflows/%s/execute?authorization=%s&reference_execution=%s&answer=true" % (frontend_url, self.full_execution["workflow"]["id"], self.full_execution["authorization"], self.full_execution["execution_id"]), - "api_abort": "%s/api/v1/workflows/%s/execute?authorization=%s&reference_execution=%s&answer=false" % (frontend_url, self.full_execution["workflow"]["id"], self.full_execution["authorization"], self.full_execution["execution_id"]), + "frontend_continue": "%s/workflows/%s/run?authorization=%s&reference_execution=%s&answer=true&source_node=%s" % (frontend_url, self.full_execution["workflow"]["id"], self.full_execution["authorization"], self.full_execution["execution_id"], source_node), + "frontend_abort": "%s/workflows/%s/run?authorization=%s&reference_execution=%s&answer=false&source_node=%s" % (frontend_url, self.full_execution["workflow"]["id"], self.full_execution["authorization"], self.full_execution["execution_id"], source_node), + "api_continue": "%s/api/v1/workflows/%s/execute?authorization=%s&reference_execution=%s&answer=true&source_node=%s" % (frontend_url, self.full_execution["workflow"]["id"], self.full_execution["authorization"], self.full_execution["execution_id"], source_node), + "api_abort": "%s/api/v1/workflows/%s/execute?authorization=%s&reference_execution=%s&answer=false&source_node=%s" % (frontend_url, self.full_execution["workflow"]["id"], self.full_execution["authorization"], self.full_execution["execution_id"], source_node), }) ret = self.run_subflow(user_apikey, item, argument, source_workflow=self.full_execution["workflow"]["id"], source_execution=self.full_execution["execution_id"], source_auth=self.full_execution["authorization"], startnode=startnode, backend_url=backend_url, source_node=source_node) @@ -106,7 +107,7 @@ def run_userinput(self, user_apikey, sms="", email="", subflow="", information=" print("Should run email with targets: %s", jsondata["targets"]) - ret = requests.post("%s/api/v1/functions/sendmail" % url, json=jsondata, headers=headers) + ret = requests.post("%s/api/v1/functions/sendmail" % url, json=jsondata, headers=headers, verify=False, proxies=self.proxy_config) if ret.status_code != 200: print("Failed sending email. Data: %s" % ret.text) result["email"] = False @@ -131,7 +132,7 @@ def run_userinput(self, user_apikey, sms="", email="", subflow="", information=" print("Should send sms with targets: %s", jsondata["numbers"]) - ret = requests.post("%s/api/v1/functions/sendsms" % url, json=jsondata, headers=headers) + ret = requests.post("%s/api/v1/functions/sendsms" % url, json=jsondata, headers=headers, verify=False, proxies=self.proxy_config) if ret.status_code != 200: print("Failed sending email. Data: %s" % ret.text) result["sms"] = False @@ -142,7 +143,7 @@ def run_userinput(self, user_apikey, sms="", email="", subflow="", information=" return json.dumps(result) - def run_subflow(self, user_apikey, workflow, argument, source_workflow="", source_execution="", source_node="", source_auth="", startnode="", backend_url=""): + def run_subflow(self, user_apikey, workflow, argument, source_workflow="", source_execution="", source_node="", source_auth="", startnode="", backend_url="", check_result=""): #print("STARTNODE: %s" % startnode) url = "%s/api/v1/workflows/%s/execute" % (self.url, workflow) if len(self.base_url) > 0: @@ -174,6 +175,12 @@ def run_subflow(self, user_apikey, workflow, argument, source_workflow="", sourc else: print("No startnode") + if len(self.full_execution["execution_id"]) > 0 and self.full_execution["execution_id"] != source_execution: + params["source_execution"] = self.full_execution["execution_id"] + + if len(self.full_execution["authorization"]) > 0 and self.full_execution["authorization"] != source_auth: + params["source_auth"] = self.full_execution["authorization"] + if len(str(backend_url)) > 0: url = "%s/api/v1/workflows/%s/execute" % (backend_url, workflow) print("[INFO] Changed URL to %s for this execution" % url) @@ -184,7 +191,7 @@ def run_subflow(self, user_apikey, workflow, argument, source_workflow="", sourc } if len(str(argument)) == 0: - ret = requests.post(url, headers=headers, params=params) + ret = requests.post(url, headers=headers, params=params, verify=False, proxies=self.proxy_config) else: if not isinstance(argument, list) and not isinstance(argument, dict): try: @@ -194,14 +201,14 @@ def run_subflow(self, user_apikey, workflow, argument, source_workflow="", sourc #print(f"ARG: {argument}") try: - ret = requests.post(url, headers=headers, params=params, json=argument) + ret = requests.post(url, headers=headers, params=params, json=argument, verify=False, proxies=self.proxy_config) print(f"Successfully sent argument of length {len(str(argument))} as JSON") except: try: - ret = requests.post(url, headers=headers, json=argument, params=params) + ret = requests.post(url, headers=headers, json=argument, params=params, verify=False, proxies=self.proxy_config) print("Successfully sent as JSON (2)") except: - ret = requests.post(url, headers=headers, data=argument, params=params) + ret = requests.post(url, headers=headers, data=argument, params=params, verify=False, proxies=self.proxy_config) print("Successfully sent as data (3)") print("Status: %d" % ret.status_code) diff --git a/shuffle-tools/1.2.0/api.yaml b/shuffle-tools/1.2.0/api.yaml index 50f19a25..cfa1a55b 100644 --- a/shuffle-tools/1.2.0/api.yaml +++ b/shuffle-tools/1.2.0/api.yaml @@ -1,7 +1,7 @@ --- app_version: 1.2.0 name: Shuffle Tools -description: A tool app for Shuffle. Gives access to most missing features along with Liquid. +description: A tool app for Shuffle. Gives access to most missing features along with Liquid. tags: - Testing - Shuffle @@ -25,6 +25,16 @@ actions: returns: schema: type: string + - name: execute_python + description: Runs python with the data input. Any prints will be returned. + parameters: + - name: code + description: The code to run. Can be a file ID from within Shuffle. + required: true + multiline: true + example: print("hello world") + schema: + type: string - name: check_cache_contains description: Checks Shuffle cache whether a user-provided key contains a value. Returns ALL the values previously appended. parameters: @@ -85,6 +95,19 @@ actions: returns: schema: type: string + - name: delete_cache_value + description: Delete a value saved to your organization in Shuffle + parameters: + - name: key + description: The key to delete + required: true + multiline: false + example: "timestamp" + schema: + type: string + returns: + schema: + type: string - name: send_sms_shuffle description: Send an SMS from Shuffle parameters: @@ -158,22 +181,22 @@ actions: skip_multicheck: true parameters: - name: input_list - description: The list to check + description: The list to filter from. Don't use .# into this. required: true multiline: false example: '[{"data": "1.2.3.4"}, {"data": "1.2.3.5"}]' schema: type: string - name: field - description: The field to check - required: false + description: The field to check in the input list + required: true multiline: false example: "data" schema: type: string - name: check description: Type of check - required: true + required: false example: "equals" options: - equals @@ -191,15 +214,15 @@ actions: schema: type: string - name: value - description: The value to check with - required: false + description: The value to compare with + required: false multiline: false example: "1.2.3.4" schema: type: string - name: opposite description: Whether to add or to NOT add - required: true + required: false options: - False - True @@ -245,7 +268,7 @@ actions: # schema: # type: string - name: parse_ioc - description: Parse IOC's based on https://github.com/fhightower/ioc-finder + description: "Parse IOC's based on https://github.com/fhightower/ioc-finder. Specify input type to optimize speed: domains, urls, email_addresses, ipv4s, ipv4_cidrs, md5s, sha256s, sha1s, cves and more.." parameters: - name: input_string description: The string to check @@ -258,7 +281,7 @@ actions: description: The string to check required: false multiline: false - example: "md5s" + example: "domains,urls,email_addresses,ipv4s,ipv4_cidrs,ipv6s,md5s,sha256s,sha1s,cves" schema: type: string returns: @@ -901,6 +924,7 @@ actions: options: - encode - decode + - "to image" schema: type: string - name: get_timestamp @@ -968,16 +992,7 @@ actions: example: '{"data": "Hello world"}' schema: type: string - - name: execute_python - description: Runs python with the data input. Any prints will be returned. - parameters: - - name: code - description: The code to run. Can be a file ID from within Shuffle. - required: true - multiline: true - example: print("hello world") - schema: - type: string + - name: run_math_operation description: Takes a math input and gives you the result parameters: @@ -1111,6 +1126,26 @@ actions: example: 'ls -la' schema: type: string + #- name: parse_ioc_new + # description: Parse IOC's based on https://github.com/fhightower/ioc-finder + # parameters: + # - name: input_string + # description: The string to check + # required: true + # multiline: true + # example: "123ijq192.168.3.6kljqwiejs8 https://shuffler.io" + # schema: + # type: string + # - name: input_type + # description: The string to check + # required: false + # multiline: false + # example: "md5s" + # schema: + # type: string + # returns: + # schema: + # type: string large_image: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAK4AAACuCAYAAACvDDbuAAAABGdBTUEAALGPC/xhBQAAACBjSFJNAAB6JgAAgIQAAPoAAACA6AAAdTAAAOpgAAA6mAAAF3CculE8AAAABmJLR0QA/wD/AP+gvaeTAAAAB3RJTUUH5AgXDjM6hEZGWwAAD+lJREFUeNrtXb/vJTcRH7/v3iVBCqRBiCAQAtHwq4AWRElHwX8AoqbmXwDRpiH/QyQkGoogUSAhKIKUAE1IdSRSREhQQk7c3XtD8X55vePxjNfe3bk3H+nu+96uPf54POtnj8fe8OQX30JwOIxhtzYBh6MGOsPF0z9p2iWwpd8LjX6W5vWUYaiqlBuvLT5b5TQDPlRwmMSAABBg+kCer+XuAeQf4tL9tAxJ/hIfZGSm8rhyEfjytfxr9FeSX+KjvVfipNVpWlaPNhsAEPCS7Ao8FYnRlbO4ksLnjiSQvIanv4FNjwJ5pXIlMq6MQpIqqPnQKQKbjuPDtZlG55o6UHXWtVncZZTbbNBVB1P5dJYguCbJJ1WjOG8PVOioSm5HPrVt1rwuyN+K+PSZnNV1M/MmEFubfFjjU9tmK9XBJ2cOk3DDdZiEG67DJOrGuA7HyvAe12ESAxa73KPrN1z8gUikCCdvcD5NXnpQpA8nNhh9m5Yn4ZMrV8dHV/8a/dRA0x419a3lI9GBtM2GcrGYFXRNUU5TyluTOpdXwqeUt6YOpby9DUTLZylOcRlzdBTf2yV3ZBFOmKSHQh5KpjSSSpqG4s6VkUubqw8W8knTSnWk0Y+2jF5tlmuDUloJn6T8gRVcEpJ+3srChHSNt8RJsq4p+S41LC13KTcu/RJt1pLPKY1Pzhwm4YbrMAk3XIdJTMe4aeCjJhBVk0YiQ1MWZHhLgmO5QNVWfKRlavlIIQnurQmcnaMjSbBxhtMwYUxODpLcl2tUhvPlNE6VkiuoFVLXKT6ZfBjxRIIzOSlgWpLSB8uZ0g3BjeVDlFGEos0mfKKL7CQrY2ES7pM2i/OX22w4/sWReEhEnUOTxx3a+FrawQGZh04/rWe6oJBKo5zT4zLjPHE9ZHym5YzToogzfQcmfLgOhuLF/Sjm2izVDyXnrKtcmmmdaKumf+RyCw5Xn7OmzQaJF0fiEZG6BjXpYUYaSVkaPrXeHe4eVaZEr3Prqrmmrbc2T8lrmOMjn5xJHeJLYkk+PfzNTxOflrwF0EeHbU0Zt2wsW+PTkncB7g5zmMSwzUfS4eDhPa7DJK5jXGorsnZxonbRIbeAoOUjkUvlp+qxFp9YNuWL0nBqsVCkqUsrHQnuX+Nx5/qcJDI0kWgtJh7ihYCN8aG+13DqOXlbWUfD+fN0AUEmp3RcUWlVEwCynb5ssYLnxHViJT6ULCykb8EnzUfpqBWfVAdcnt5tprGhIe10WnjHpB2FtMPWcpM66yXyOad4Lz4Srq34SHhwZfRos1w9Y/jkzGESvj3dYRLe4zpMwg3XYRJuuA6T4M/Hzfk/OGd9OP2HOE2f8wtBlCebJrkfp+Gc3AGmiSiuaVlpwkmajL4osPUm9FMqIzBOJolfjGuzEtdUwWl53Dm7Eh9pzIdps+FiYJyi1N+Rvs/6OLCQBul8Ip8R08ik3EwhLZz1Wv8XmU7ZZqX7OT2gUIB2oaRBm+2ovDm5nM+ulEeiD8yka8UnJ1PCP82r9YWW8iCU5XO8W/PhPmvllNKW7lEyszsgNKuzkspJFZFL15uPtIweq7A1xiKpz1J8tGXP+dE53/fJmcMk6hcgJO8XqokEKi5uYzTG29LqSev95JqyKsoOOxjNpKQBD7VFc5GBJRsi+NQHkkv6+7m/UxTufwLCCy+CbAruyOLDdwEf/uf6vbbNJukzlogZC6wMdhAcM7ohHPawe/GrcO+HPwe4u782G7sIAE9++0vYv/YKwO6usfCaka0etgwXAGB3D8JznwIYnlmbiW0M92FbQy0d+MmZ3Xo5JDDcvuXJ2ZYqtyUuTwuM6nSXctcufHCOZqkjPScXhbIcdeD0XUpfKyNNy8nlyhuozLkM8XxR6pjm7tc4Fdx620I7lWq10JCm0ZanWoBwm3FsBe1WznpadbTg4A9PI2xx7FUKHopQjg7TKqNnpbioIUcFUGUsy1CS8fFYBYdJuOE6TMIN12ESgyiKiwO1bQOJe1w+6p42Etmhwmi6kLZXfC2G9IUj2vulY2wIPrv4onRhIXcRqS0DiWxkhF0uIb37wG22LRCSuVCyekC2GSXj9CG3YyT+krWh+KPAhkTvgGDKqbqnWbBwY+2Pnm3Wy4aMRYc1MuPDvp0skwgAh8PaJGbh5k4kx0f/hce/ewnw/QenXQCTFJDfQy45PzFNn5NHsoPy/u6gzE+nObzz91P9Z+6kWAm2zg6bDMoq8OQxHN78Axze/htAaB1EbQhhdzyfgRqIGoCxoUIjhDuA3ZDpcR0W4C3nMInbNVw7v4oOAsehArVFPL0uOjMM+DlM+pk7t7/BDuwcJsM6gcM7WweOX05nFCHNi12ASRfLo3QaX9O0GWTylOTnZIMwf4YPPTlD4iMm7aZwAGOUf3Rf48wjHNzVOMkKFA8pp0RHZ1mjdihs5R61PWbsWlphgs/E5gptNvFfSLY8QPk7dVbh+UNg8qfnJsZ8Bo0hzF0Y2Nqvc0s+Vbs5YL5OLfPRcorT2hvjtuxyHWZhzHCX6AMcFtB2B0RvtKZqqe6OEYz1uA7HEbdruN7ZmsZtGq4brXnQhlsbLFkDrY9mC9giH41/dSlONfeEIBcgss7nXopInPdkYN95J3XD1bMgkJUNFOxsDNLgyiynhYyX5dnAhnLyhzmO4V7IO8+xyZEgx5UqvJ41rOUTdhBOr2w6KjZc+B1FBkLGVUoAABQEcmPu6rPPw73v/gh2n/wMANYEhAd4/NqvYf/Wn5pEyPW2IUrOzQWSHyHdkEJgN8D97/0Edp/7GgDu9fnDDvD9t+HRqy8BPvxQ9i6xEXUEuPcMDF//Puw+/aVqDewfvA77f/zx9M40e7jNeNw5CDu4++K34e4r36kWcXj3TYDfvwz8D79ml1clDPuxx9FhuUik0rblVihFWLX+7ZFEXE2ioLBNg9fUSRopVsOjJbioskZlDuyAvmflpOWsOUNu/cBQ8jW/1A0np11RG+GjwG36cQHqFWnBcG4Axgx37d/I1uXXcvCnx6BXoQXf3mOAzvVpooJzaOcWdKBH1fZ07dCsFZpNgmfZbaOJ2dxnpwkNFC3C9MBcGxo0OugxwV8LWKm5lg9sFQdszKGhLAla2dCuduuOZcypx+UXdk0OK5e/hXKNTc4cjiPGhtvTX1njI6Z2+vbuKtaKspLooXdkXs1u5yUR7/LdROMsraSSIfTa6pqWodE9Mvla6sCI8d7uUMEXIEzjdg3XYRr2osOePIbDR+9BGO7re78QAD/+AODwpK5sBDg6dGyGAtL1sYnLGDe3+2BNTNycYQf7B2/Aw5d/XB9HejjA4YN3jgHUNQ132MOTv/wG9v98A+CgFBCO/+FH/wJ89PBaSY1OULZzQyQL2skayVwg/7Dk3Ky2IlcEgEcfw/7dt+YJnRP1f9jDoz+/AvM0FU4c1u8mes59e+ZXDhXmPE+tForD+lH73Q6EluiozfaldnzWQUWQzdprPk87lg44nkTKN+DT/10S7lW4VYz8wWucOTAPtl5e4mgfjmu0/b3HdZiEG67DJNxwbxlGhwkAuZeXAJS3Qpfemq7dds1tS5dsbc6dAyQpS5uGe+lKrJLSGUqlCb2GcwUuCxBzt71T2/g7t9mQniofv0yjWOtMYdSLM6Sy0pd5iLdFSQtUyiJtRnjmGOdhqq5bo5WzUXAYzns2Lu2tjaqb0WaTHRBrR9cvEVG4VF3WkLsGnzXqohzjbk3dt4hG/jDDxy8BLL5y5miBZi1wa9vT14dJ0o2qft6/1GhQZ1SV9uJxd3cQ7j+XD7RJ40JK38/XAPKz4ly+OG+KwOTDwn0uDSKEZ58/vgH+hmHLcA97uPvCN+G5H/wMoCaQ/KkAAtzdg/DCZ9cmsipsGS4ce5u7z38DYHhmbTL2YfjBH28DOM80s+MoxllVvfkwKudSbiL0dB0NTya2iGpNYmIzl+/EdexjQ8PEGE4FhdPHMAlbLhcsdWaPnfDEAxQJnbx53TEPJ51j3N7CrEfbSNt+arzXt57X2RBx94LsUGHOGRQtF7Fa8HFQQOabJmc5XQ8b8iAbh0mYNFzvdefD+nRhyPowqWitc2VbRyutGCF18+ilU2mEXWX51zFuKbqlZ/RLy0gixzagiS6sgL2hghuwAywarsMBxgzXO9u2sBzZWHwHRLwrQ5rWYQBIfuwCKnZJEpvEYSg9dRoncnejtdxFbBRLqFQzr5fSudH3nDmOaH26yHIwNcZ1NIZNmwWArYU1Fg8HDLB/7wH879VfAey2Rd0a9g/+2ubUyZUOdAz//umXjT136GPd2cDNnM9bC4Pd1gbOx3WsDh/jOkzCDddhEpcjmKiFhvGLQwDitJNrYTz05H7MS+N56hiq0mbYCfeIj2STb2s+cSJEOrguJ4fScaneOW7kOWZJm4VCmaPFg8wKgcSGuLpzR49Rerm8vIRaaECgvyB1Tbl9qOZoMiykHeVhVoZKwW9N+CSJuPwsH4YY12aTa5TxYyZPpsxSDG/Rhgp1lyxUnK/7UMFhEm64DpNIlnzTAdXcsJml8rdO1yt/K+R45EJUluS9zHaWITuQJb9rsVT+HvuKe+RvhdIIcE3ey4Rj+VDBYRJuuA6TcMN1mMT15SWMZ5h10Oc86+dr50s14QWch7rEh5PHef+psgsyqB0iI2e+hE+pDlpvvkQ/uVUMDfdSnTq12TA58injFUdOMPB5AeiALtHcUrstXrqSINnaoVjxyE5ra1ZipHMsTV2kMiQ8NDw7tdmqQ4WtzNEd9uBjXIdJuOE6TMLoy0sct46KHndNS6d2pW5tp+rW+Jw5rVl2qpP5Oqrcnr52w9RMgbfA8db5tAsp8DGuwyTaGW6DB7ppn9CCzxKnvKz9Kz7j/prUi0cwqQLQDBtvrp5uvMc/Wf00oFAT5FjscbcwMloCt1LPWvTUT41sH+M6TMIN12ESw3UPd8gPtrh7JeTyXvZGn0KD0jSlMms5Sfhw92vkUvXT5tPWt3WbSfjMsSFl3ujlJdy+4xkjnFze+PWrNWXWclqaT6t82vq2bjMJnzk2pMzrQwWHSbjhOkzCDdchxpZchpezwySQvHhiyVMLevPRctXwqeWmfcv5GaVTGKRy557YIHnhpETeoCl05grhbPlL89HK1vCp5darvZbgo+XEwYcKDpNww3WYxC6/U5PY5oun66MzPHH8L05PpqHKghn+TpjyictkZQLPh4u6yeknvXeWU+JD6TDHJ/cbn93Bi8nnDKdJm8EG2+zIZwBudlbjUOYOpj1frClPwyf3OZuXuaEx3lgWZixKxIfZ911rvJO65PRFVmZjbYY+VHDYhBuuwyTccB0mcdkB0cr5z70pW/pm7Bo+LesgqUsrPjVye9WXkqld8FiizRCi6LBWjmTRPGGG/JZ5ejvoa1ai1qwvlWarbeZDBYdJuOE6TKKP4W7xJdFb4+R8ZvH5P852gxhpwOZ9AAAAJXRFWHRkYXRlOmNyZWF0ZQAyMDIwLTA4LTIzVDE0OjUyOjAwKzAyOjAwetRgVgAAACV0RVh0ZGF0ZTptb2RpZnkAMjAyMC0wOC0yM1QxNDo1MTo1OCswMjowMJuxI+oAAAAASUVORK5CYII= # yamllint disable-line rule:line-length diff --git a/shuffle-tools/1.2.0/src/app.py b/shuffle-tools/1.2.0/src/app.py index 13cb7b4b..03019ed7 100644 --- a/shuffle-tools/1.2.0/src/app.py +++ b/shuffle-tools/1.2.0/src/app.py @@ -1,4 +1,4 @@ -import asyncio +import hmac import datetime import json import time @@ -13,8 +13,6 @@ import hashlib from io import StringIO from contextlib import redirect_stdout -from liquid import Liquid -import liquid import random import string @@ -36,6 +34,8 @@ import struct import paramiko +import concurrent.futures +import multiprocessing from walkoff_app_sdk.app_base import AppBase @@ -69,6 +69,30 @@ def base64_conversion(self, string, operation): encoded_string = str(encoded_bytes, "utf-8") return encoded_string + elif operation == "to image": + # Decode the base64 into an image and upload it as a file + decoded_bytes = base64.b64decode(string) + + # Make the bytes into unicode escaped bytes + # UnicodeDecodeError - 'utf-8' codec can't decode byte 0x89 in position 0: invalid start byte + try: + decoded_bytes = str(decoded_bytes, "utf-8") + except: + pass + + filename = "base64_image.png" + file = { + "filename": filename, + "data": decoded_bytes, + } + + fileret = self.set_files([file]) + value = {"success": True, "filename": filename, "file_id": fileret} + if len(fileret) == 1: + value = {"success": True, "filename": filename, "file_id": fileret[0]} + + return value + elif operation == "decode": try: decoded_bytes = base64.b64decode(string) @@ -145,7 +169,7 @@ def send_sms_shuffle(self, apikey, phone_numbers, body): url = "https://shuffler.io/api/v1/functions/sendsms" headers = {"Authorization": "Bearer %s" % apikey} - return requests.post(url, headers=headers, json=data).text + return requests.post(url, headers=headers, json=data, verify=False).text # This is an email function of Shuffle def send_email_shuffle(self, apikey, recipients, subject, body, attachments=""): @@ -181,7 +205,7 @@ def send_email_shuffle(self, apikey, recipients, subject, body, attachments=""): url = "https://shuffler.io/api/v1/functions/sendmail" headers = {"Authorization": "Bearer %s" % apikey} - return requests.post(url, headers=headers, json=data).text + return requests.post(url, headers=headers, json=data, verify=False).text def repeat_back_to_me(self, call): return call @@ -251,69 +275,16 @@ def parse(data): return "Invalid input" return return_value - # https://github.com/fhightower/ioc-finder - def parse_ioc(self, input_string, input_type="all"): - #if len(input_string) > 2500000 and (input_type == "" or input_type == "all"): - # return { - # "success": False, - # "reason": "Data too large (%d). Please reduce it below 2.5 Megabytes to use this action or specify the input type" % len(input_string) - # } - - # https://github.com/fhightower/ioc-finder/blob/6ff92a73a60e9233bf09b530ccafae4b4415b08a/ioc_finder/ioc_finder.py#L433 - ioc_types = ["domains", "urls", "email_addresses", "ipv6s", "ipv4s", "ipv4_cidrs", "md5s", "sha256s", "sha1s", "cves"] - input_string = str(input_string) - if input_type == "": - input_type = "all" - else: - input_type = input_type.split(",") - for item in input_type: - item = item.strip() - - ioc_types = input_type - - iocs = find_iocs(input_string, included_ioc_types=ioc_types) - newarray = [] - for key, value in iocs.items(): - if input_type != "all": - if key not in input_type: - continue - - if len(value) > 0: - for item in value: - # If in here: attack techniques. Shouldn't be 3 levels so no - # recursion necessary - if isinstance(value, dict): - for subkey, subvalue in value.items(): - if len(subvalue) > 0: - for subitem in subvalue: - data = { - "data": subitem, - "data_type": "%s_%s" % (key[:-1], subkey), - } - if data not in newarray: - newarray.append(data) - else: - data = {"data": item, "data_type": key[:-1]} - if data not in newarray: - newarray.append(data) - - # Reformatting IP - for item in newarray: - if "ip" in item["data_type"]: - item["data_type"] = "ip" - try: - item["is_private_ip"] = ipaddress.ip_address(item["data"]).is_private - except: - self.logger.info("Error parsing %s" % item["data"]) - + def parse_list(self, items, splitter="\n"): + # Check if it's already a list first try: - newarray = json.dumps(newarray) - except json.decoder.JSONDecodeError as e: - return "Failed to parse IOC's: %s" % e + newlist = json.loads(items) + if isinstance(newlist, list): + return newlist - return newarray + except Exception as e: + self.logger.info("[WARNING] Parse error - fallback: %s" % e) - def parse_list(self, items, splitter="\n"): if splitter == "": splitter = "\n" @@ -507,7 +478,7 @@ def regex_replace( return re.sub(regex, replace_string, input_data) def execute_python(self, code): - self.logger.info(f"Python code {len(code)} {code}. If uuid, we'll try to download and use the file.") + self.logger.info(f"Python code {len(code)}. If uuid, we'll try to download and use the file.") if len(code) == 36 and "-" in code: filedata = self.get_file(code) @@ -536,6 +507,14 @@ def custom_print(*args, **kwargs): # Add globals in it too globals_copy = globals().copy() globals_copy["print"] = custom_print + + # Add self to globals_copy + for key, value in locals().copy().items(): + if key not in globals_copy: + globals_copy[key] = value + + globals_copy["self"] = self + exec(code, globals_copy) s = f.getvalue() @@ -549,13 +528,19 @@ def custom_print(*args, **kwargs): try: return { "success": True, - "message": s.strip(), + "message": json.loads(s.strip()), } except Exception as e: - return { - "success": True, - "message": s, - } + try: + return { + "success": True, + "message": s.strip(), + } + except Exception as e: + return { + "success": True, + "message": s, + } except Exception as e: return { @@ -679,7 +664,7 @@ def filter_list(self, input_list, field, check, value, opposite): self.logger.info("Checklist and tmp: %s - %s" % (checklist, tmp)) found = False for subcheck in checklist: - subcheck = subcheck.strip() + subcheck = str(subcheck).strip() #ext.lower().strip() == value.lower().strip() if type(tmp) == list and subcheck in tmp: @@ -690,8 +675,17 @@ def filter_list(self, input_list, field, check, value, opposite): new_list.append(item) found = True break + elif type(tmp) == int and str(tmp) == subcheck: + new_list.append(item) + found = True + break else: - print("Nothing matching") + if str(tmp) == str(subcheck): + new_list.append(item) + found = True + break + else: + print("Nothing matching") if not found: failed_list.append(item) @@ -762,14 +756,62 @@ def filter_list(self, input_list, field, check, value, opposite): # CONTAINS FIND FOR LIST AND IN FOR STR elif check == "larger than": - if int(tmp) > int(value): - new_list.append(item) - else: + list_set = False + try: + if str(tmp).isdigit() and str(value).isdigit(): + if int(tmp) > int(value): + new_list.append(item) + list_set = True + except AttributeError as e: + self.logger.info("FAILED CHECKING LARGER THAN: %s" % e) + pass + + try: + value = len(json.loads(value)) + except Exception as e: + self.logger.info(f"[WARNING] Failed to convert destination to list: {e}") + + try: + # Check if it's a list in autocast and if so, check the length + if len(json.loads(tmp)) > int(value): + new_list.append(item) + list_set = True + except Exception as e: + self.logger.info(f"[WARNING] Failed to check if larger than as list: {e}") + + if not list_set: failed_list.append(item) elif check == "less than": - if int(tmp) < int(value): - new_list.append(item) - else: + # Old + #if int(tmp) < int(value): + # new_list.append(item) + #else: + # failed_list.append(item) + + list_set = False + try: + if str(tmp).isdigit() and str(value).isdigit(): + if int(tmp) < int(value): + new_list.append(item) + list_set = True + except AttributeError as e: + self.logger.info("FAILED CHECKING LARGER THAN: %s" % e) + pass + + try: + value = len(json.loads(value)) + except Exception as e: + self.logger.info(f"[WARNING] Failed to convert destination to list: {e}") + + try: + # Check if it's a list in autocast and if so, check the length + if len(json.loads(tmp)) < int(value): + new_list.append(item) + list_set = True + except Exception as e: + self.logger.info(f"[WARNING] Failed to check if larger than as list: {e}") + + if not list_set: failed_list.append(item) elif check == "in cache key": @@ -912,6 +954,7 @@ def get_file_meta(self, file_id): "%s/api/v1/files/%s?execution_id=%s" % (self.url, file_id, self.current_execution_id), headers=headers, + verify=False, ) self.logger.info(f"RET: {ret}") @@ -928,6 +971,7 @@ def delete_file(self, file_id): "%s/api/v1/files/%s?execution_id=%s" % (self.url, file_id, self.current_execution_id), headers=headers, + verify=False, ) return ret.text @@ -1456,7 +1500,7 @@ def merge_lists(self, list_one, list_two, set_field="", sort_key_list_one="", so return list_one def merge_json_objects(self, list_one, list_two, set_field="", sort_key_list_one="", sort_key_list_two=""): - self.merge_lists(self, list_one, list_two, set_field="", sort_key_list_one="", sort_key_list_two="") + return self.merge_lists(list_one, list_two, set_field=set_field, sort_key_list_one=sort_key_list_one, sort_key_list_two=sort_key_list_two) def fix_json(self, json_data): try: @@ -1705,7 +1749,7 @@ def check_cache_contains(self, key, value, append): else: append = False - get_response = requests.post(url, json=data) + get_response = requests.post(url, json=data, verify=False) try: allvalues = get_response.json() try: @@ -1720,7 +1764,7 @@ def check_cache_contains(self, key, value, append): data["value"] = json.dumps(new_value) set_url = "%s/api/v1/orgs/%s/set_cache" % (self.url, org_id) - set_response = requests.post(set_url, json=data) + set_response = requests.post(set_url, json=data, verify=False) try: allvalues = set_response.json() #allvalues["key"] = key @@ -1753,37 +1797,47 @@ def check_cache_contains(self, key, value, append): if allvalues["value"] == None or allvalues["value"] == "null": allvalues["value"] = "[]" + allvalues["value"] = str(allvalues["value"]) + try: parsedvalue = json.loads(allvalues["value"]) except json.decoder.JSONDecodeError as e: - parsedvalue = [] + parsedvalue = [str(allvalues["value"])] + except Exception as e: + print("Error parsing JSON - overriding: %s" % e) + parsedvalue = [str(allvalues["value"])] - #return parsedvalue + print("In ELSE2: '%s'" % parsedvalue) - for item in parsedvalue: - #return "%s %s" % (item, value) - if item == value: - if not append: - return { - "success": True, - "found": True, - "reason": "Found and not appending!", - "key": key, - "search": value, - "value": json.loads(allvalues["value"]), - } - else: - return { - "success": True, - "found": True, - "reason": "Found, was appending, but item already exists", - "key": key, - "search": value, - "value": json.loads(allvalues["value"]), - } - - # Lol - break + try: + for item in parsedvalue: + #return "%s %s" % (item, value) + if item == value: + if not append: + return { + "success": True, + "found": True, + "reason": "Found and not appending!", + "key": key, + "search": value, + "value": json.loads(allvalues["value"]), + } + else: + return { + "success": True, + "found": True, + "reason": "Found, was appending, but item already exists", + "key": key, + "search": value, + "value": json.loads(allvalues["value"]), + } + + # Lol + break + except Exception as e: + print("Error in check_cache_contains: %s" % e) + parsedvalue = [str(parsedvalue)] + append = True if not append: return { @@ -1795,24 +1849,15 @@ def check_cache_contains(self, key, value, append): "value": json.loads(allvalues["value"]), } - #parsedvalue = json.loads(allvalues["value"]) - #if parsedvalue == None: - # parsedvalue = [] - - #return parsedvalue new_value = parsedvalue if new_value == None: new_value = [value] new_value.append(value) - - #return new_value - data["value"] = json.dumps(new_value) - #return allvalues set_url = "%s/api/v1/orgs/%s/set_cache" % (self.url, org_id) - response = requests.post(set_url, json=data) + response = requests.post(set_url, json=data, verify=False) exception = "" try: allvalues = response.json() @@ -1843,10 +1888,11 @@ def check_cache_contains(self, key, value, append): #return allvalues except Exception as e: + print("[ERROR] Failed to handle cache contains: %s" % e) return { "success": False, "key": key, - "reason": f"Failed to get cache: {e}", + "reason": f"Failed to handle cache contains. Is the original value a list?: {e}", "search": value, "found": False, } @@ -1884,7 +1930,7 @@ def change_cache_subkey(self, key, subkey, value, overwrite): "value": value, } - response = requests.post(url, json=data) + response = requests.post(url, json=data, verify=False) try: allvalues = response.json() allvalues["key"] = key @@ -1904,6 +1950,9 @@ def change_cache_subkey(self, key, subkey, value, overwrite): self.logger.info("Value couldn't be parsed") return response.text + def delete_cache_value(self, key): + return self.delete_cache(key) + def get_cache_value(self, key): org_id = self.full_execution["workflow"]["execution_org"]["id"] url = "%s/api/v1/orgs/%s/get_cache" % (self.url, org_id) @@ -1915,14 +1964,14 @@ def get_cache_value(self, key): "key": key, } - value = requests.post(url, json=data) + value = requests.post(url, json=data, verify=False) try: allvalues = value.json() - self.logger.info("VAL1: ", allvalues) + #self.logger.info("VAL1: ", allvalues) allvalues["key"] = key - self.logger.info("VAL2: ", allvalues) + #self.logger.info("VAL2: ", allvalues) - if allvalues["success"] == True: + if allvalues["success"] == True and len(allvalues["value"]) > 0: allvalues["found"] = True else: allvalues["success"] = True @@ -1964,7 +2013,7 @@ def set_cache_value(self, key, value): "value": value, } - response = requests.post(url, json=data) + response = requests.post(url, json=data, verify=False) try: allvalues = response.json() allvalues["key"] = key @@ -2091,7 +2140,7 @@ def run_oauth_request(self, url, jwt): data = "grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion=%s" % jwt - return requests.post(url, data=data, headers=headers).text + return requests.post(url, data=data, headers=headers, verify=False).text # Based on https://google-auth.readthedocs.io/en/master/reference/google.auth.crypt.html def get_jwt_from_file(self, file_id, jwt_audience, scopes, complete_request=True): @@ -2386,7 +2435,7 @@ def run_ssh_command(self, host, port, user_name, private_key_file_id, password, except Exception as e: return {"success":"false","message":str(e)} else: - print("AUTH WITH PASSWORD") + #print("AUTH WITH PASSWORD") try: ssh_client.connect(hostname=host,username=user_name,port=port, password=str(password)) except Exception as e: @@ -2399,6 +2448,213 @@ def run_ssh_command(self, host, port, user_name, private_key_file_id, password, return {"success":"true","output": stdout.read().decode(errors='ignore')} + def parse_ioc(self, input_string, input_type="all"): + ioc_types = ["domains", "urls", "email_addresses", "ipv4s", "ipv4_cidrs", "md5s", "sha256s", "sha1s", "cves"] + + # Remember overriding ioc types we care about + if input_type == "" or input_type == "all": + input_type = "all" + else: + input_type = input_type.split(",") + for item in input_type: + item = item.strip() + + ioc_types = input_type + + iocs = find_iocs(str(input_string), included_ioc_types=ioc_types) + newarray = [] + for key, value in iocs.items(): + if input_type != "all": + if key not in input_type: + continue + + if len(value) > 0: + for item in value: + # If in here: attack techniques. Shouldn't be 3 levels so no + # recursion necessary + if isinstance(value, dict): + for subkey, subvalue in value.items(): + if len(subvalue) > 0: + for subitem in subvalue: + data = { + "data": subitem, + "data_type": "%s_%s" % (key[:-1], subkey), + } + if data not in newarray: + newarray.append(data) + else: + data = {"data": item, "data_type": key[:-1]} + if data not in newarray: + newarray.append(data) + + # Reformatting IP + for item in newarray: + if "ip" in item["data_type"]: + item["data_type"] = "ip" + try: + item["is_private_ip"] = ipaddress.ip_address(item["data"]).is_private + except: + self.logger.info("Error parsing %s" % item["data"]) + + try: + newarray = json.dumps(newarray) + except json.decoder.JSONDecodeError as e: + return "Failed to parse IOC's: %s" % e + + return newarray + + + def split_text(self, text): + # Split text into chunks of 10kb. Add each 10k to array + # In case e.g. 1.2.3.4 lands exactly on 20k boundary, it may be useful to overlap here. + # (just shitty code to reduce chance of issues) while still going fast + arr_one = [] + max_len = 5000 + current_string = "" + overlaps = 100 + + for i in range(0, len(text)): + current_string += text[i] + if len(current_string) > max_len: + # Appending just in case even with overlaps + if len(text) > i+overlaps: + current_string += text[i+1:i+overlaps] + else: + current_string += text[i+1:] + + arr_one.append(current_string) + current_string = "" + + if len(current_string) > 0: + arr_one.append(current_string) + + return arr_one + + def _format_result(self, result): + final_result = {} + + for res in result: + for key,val in res.items(): + if key in final_result: + if isinstance(val, list) and len(val) > 0: + for i in val: + final_result[key].append(i) + elif isinstance(val, dict): + #print(key,":::",val) + if key in final_result: + if isinstance(val, dict): + for k,v in val.items(): + #print("k:",k,"v:",v) + val[k].append(v) + #print(val) + #final_result[key].append([i for i in val if len(val) > 0]) + else: + final_result[key] = val + + return final_result + + # See function for how it works~: parse_ioc_new(..) + def _with_concurency(self, array_of_strings, ioc_types): + results = [] + #start = time.perf_counter() + + # Workers dont matter..? + # What can we use instead? + print("Strings:", len(array_of_strings)) + + workers = 4 + with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor: + # Submit the find_iocs function for each string in the array + futures = [executor.submit( + find_iocs, + text=string, + included_ioc_types=ioc_types, + ) for string in array_of_strings] + + # Wait for all tasks to complete + concurrent.futures.wait(futures) + + # Retrieve the results if needed + results = [future.result() for future in futures] + + #print("Total time taken:", time.perf_counter()-start) + return self._format_result(results) + + # FIXME: Make this good and actually faster than normal + # For now: Concurrency doesn't make it faster due to GIL in python. + # May need to offload this to an executable or something + def parse_ioc_new(self, input_string, input_type="all"): + if input_type == "": + input_type = "all" + + ioc_types = ["domains", "urls", "email_addresses", "ipv4s", "ipv4_cidrs", "md5s", "sha256s", "sha1s", "cves"] + + if input_type == "" or input_type == "all": + ioc_types = ioc_types + else: + input_type = input_type.split(",") + for item in input_type: + item = item.strip() + + ioc_types = input_type + + input_string = str(input_string) + + if len(input_string) > 10000: + iocs = self._with_concurency(self.split_text(input_string), ioc_types=ioc_types) + else: + iocs = find_iocs(input_string, included_ioc_types=ioc_types) + + newarray = [] + for key, value in iocs.items(): + if input_type != "all": + if key not in input_type: + continue + + if len(value) == 0: + continue + + for item in value: + # If in here: attack techniques. Shouldn't be 3 levels so no + # recursion necessary + if isinstance(value, dict): + for subkey, subvalue in value.items(): + if len(subvalue) == 0: + continue + + for subitem in subvalue: + data = { + "data": subitem, + "data_type": "%s_%s" % (key[:-1], subkey), + } + + if data not in newarray: + newarray.append(data) + else: + data = {"data": item, "data_type": key[:-1]} + if data not in newarray: + newarray.append(data) + + # Reformatting IP + i = -1 + for item in newarray: + i += 1 + if "ip" not in item["data_type"]: + continue + + newarray[i]["data_type"] = "ip" + try: + newarray[i]["is_private_ip"] = ipaddress.ip_address(item["data"]).is_private + except Exception as e: + print("Error parsing %s: %s" % (item["data"], e)) + + try: + newarray = json.dumps(newarray) + except json.decoder.JSONDecodeError as e: + return "Failed to parse IOC's: %s" % e + + return newarray + if __name__ == "__main__": Tools.run() diff --git a/shuffle-tools/1.2.0/src/concurrency.py b/shuffle-tools/1.2.0/src/concurrency.py new file mode 100644 index 00000000..420d1686 --- /dev/null +++ b/shuffle-tools/1.2.0/src/concurrency.py @@ -0,0 +1,201 @@ +import time +import json +import ipaddress +import concurrent.futures +from functools import partial +from ioc_finder import find_iocs + +class Test(): + def split_text(self, text): + # Split text into chunks of 10kb. Add each 10k to array + # In case e.g. 1.2.3.4 lands exactly on 20k boundary, it may be useful to overlap here. + # (just shitty code to reduce chance of issues) while still going fast + + arr_one = [] + max_len = 2500 + current_string = "" + overlaps = 100 + + + for i in range(0, len(text)): + current_string += text[i] + if len(current_string) > max_len: + # Appending just in case even with overlaps + if len(text) > i+overlaps: + current_string += text[i+1:i+overlaps] + else: + current_string += text[i+1:] + + arr_one.append(current_string) + current_string = "" + + if len(current_string) > 0: + arr_one.append(current_string) + + #print("DATA:", arr_one) + print("Strings:", len(arr_one)) + #exit() + + return arr_one + + def _format_result(self, result): + final_result = {} + + for res in result: + for key, val in res.items(): + if key in final_result: + if isinstance(val, list) and len(val) > 0: + for i in val: + final_result[key].append(i) + elif isinstance(val, dict): + #print(key,":::",val) + if key in final_result: + if isinstance(val, dict): + for k,v in val.items(): + #print("k:",k,"v:",v) + val[k].append(v) + #print(val) + #final_result[key].append([i for i in val if len(val) > 0]) + else: + final_result[key] = val + + return final_result + + def worker_function(self, inputdata): + return find_iocs(inputdata["data"], included_ioc_types=inputdata["ioc_types"]) + + def _with_concurency(self, array_of_strings, ioc_types): + results = [] + #start = time.perf_counter() + + # Workers dont matter..? + # What can we use instead? + + results = [] + workers = 4 + with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor: + # Submit the find_iocs function for each string in the array + futures = [executor.submit( + find_iocs, + text=string, + included_ioc_types=ioc_types, + ) for string in array_of_strings] + + # Wait for all tasks to complete + concurrent.futures.wait(futures) + + # Retrieve the results if needed + results = [future.result() for future in futures] + + return self._format_result(results) + + def parse_ioc_new(self, input_string, input_type="all"): + if input_type == "": + input_type = "all" + + #ioc_types = ["domains", "urls", "email_addresses", "ipv6s", "ipv4s", "ipv4_cidrs", "md5s", "sha256s", "sha1s", "cves"] + ioc_types = ["domains", "urls", "email_addresses", "ipv4s", "ipv4_cidrs", "md5s", "sha256s", "sha1s", "cves"] + + # urls = 10.4 -> 9.1 + # emails = 10.4 -> 9.48 + # ipv6s = 10.4 -> 7.37 + # ipv4 cidrs = 10.4 -> 10.44 + + if input_type == "" or input_type == "all": + ioc_types = ioc_types + else: + input_type = input_type.split(",") + for item in input_type: + item = item.strip() + + ioc_types = input_type + + input_string = str(input_string) + if len(input_string) > 10000: + iocs = self._with_concurency(self.split_text(input_string), ioc_types=ioc_types) + else: + iocs = find_iocs(input_string, included_ioc_types=ioc_types) + + newarray = [] + for key, value in iocs.items(): + if input_type != "all": + if key not in input_type: + continue + + if len(value) == 0: + continue + + for item in value: + # If in here: attack techniques. Shouldn't be 3 levels so no + # recursion necessary + if isinstance(value, dict): + for subkey, subvalue in value.items(): + if len(subvalue) == 0: + continue + + for subitem in subvalue: + data = { + "data": subitem, + "data_type": "%s_%s" % (key[:-1], subkey), + } + + if data not in newarray: + newarray.append(data) + else: + data = {"data": item, "data_type": key[:-1]} + if data not in newarray: + newarray.append(data) + + # Reformatting IP + i = -1 + for item in newarray: + i += 1 + if "ip" not in item["data_type"]: + continue + + newarray[i]["data_type"] = "ip" + try: + newarray[i]["is_private_ip"] = ipaddress.ip_address(item["data"]).is_private + except Exception as e: + print("Error parsing %s: %s" % (item["data"], e)) + + try: + newarray = json.dumps(newarray) + except json.decoder.JSONDecodeError as e: + return "Failed to parse IOC's: %s" % e + + return newarray + +# Make it not run this for multithreads +if __name__ == "__main__": + + input_string = "" + with open("testdata.txt", "r") as f: + input_string = f.read() + + try: + json_data = json.loads(input_string) + # If array, loop + if isinstance(json_data, list): + cnt = 0 + start = time.perf_counter() + for item in json_data: + cnt += 1 + classdata = Test() + + ret = classdata.parse_ioc_new(item) + #print("OUTPUT1: ", ret) + + #if cnt == 5: + # break + + print("Total time taken:", time.perf_counter()-start) + else: + classdata = Test() + ret = classdata.parse_ioc_new(input_string) + print("OUTPUT2: ", ret) + except Exception as e: + classdata = Test() + ret = classdata.parse_ioc_new(json_data) + print("OUTPUT3: ", ret) +