From d5d138929c0d50dd9c05776c70c2a587bea58e25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Ra=C5=BAniewski?= Date: Sun, 6 Mar 2022 19:37:48 +0100 Subject: [PATCH] Improve docs with latest changes (#134) * Add varios scripts to improve ansible workflow * Fix text * Fix var * Update settings * Changed references of 0.0.x to master * Start docker containers on reboot (#128) * Changed script to down and timeout only for one second, added restart always * Added tool terraform-do-uashield (#122) * added tool terraform-do-uashield * added readme * Changed script to restart always and don't wait too much on docker compose down Co-authored-by: Roman Demachkovych Co-authored-by: adam * Create README.MD Co-authored-by: Denys Butenko Co-authored-by: vtinkerer <72277127+vtinkerer@users.noreply.github.com> Co-authored-by: rdemachkovych Co-authored-by: Roman Demachkovych --- Dockerfile | 2 + README-en.md | 4 +- README.md | 4 +- pwd-docker-compose.yml | 6 +-- tools/README.MD | 7 +++ tools/ansible/README.md | 8 ++++ tools/ansible/ansible.cfg | 5 +++ tools/ansible/create-vm.sh | 18 ++++++++ tools/ansible/roles/stats/tasks/main.yml | 4 +- tools/ansible/roles/uashield/tasks/main.yml | 44 ++++++++++++++----- tools/ansible/stats.yaml | 2 + tools/ansible/update-hosts.sh | 14 ++++++ tools/ansible/update.yaml | 10 +++++ tools/ansible/vars/vars.yaml | 2 + tools/azure/Readme.md | 2 +- tools/azure/azure-custom-data-script.sh | 6 ++- tools/helm/README.md | 2 +- tools/helm/values.yaml | 2 +- tools/terraform-do-uashield/README.md | 36 +++++++++++++++ tools/terraform-do-uashield/provider.tf | 16 +++++++ .../terraform-do-uashield/scripts/uashield.sh | 39 ++++++++++++++++ tools/terraform-do-uashield/uashield.tf | 33 ++++++++++++++ tools/terraform-do-uashield/variables.tf | 14 ++++++ 23 files changed, 256 insertions(+), 24 deletions(-) create mode 100644 tools/README.MD create mode 100644 tools/ansible/ansible.cfg create mode 100755 tools/ansible/create-vm.sh create mode 100755 tools/ansible/update-hosts.sh create mode 100644 tools/ansible/update.yaml create mode 100644 tools/terraform-do-uashield/README.md create mode 100644 tools/terraform-do-uashield/provider.tf create mode 100644 tools/terraform-do-uashield/scripts/uashield.sh create mode 100644 tools/terraform-do-uashield/uashield.tf create mode 100644 tools/terraform-do-uashield/variables.tf diff --git a/Dockerfile b/Dockerfile index 37d062d..399a3e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,8 @@ COPY tsconfig.headless.json ./tsconfig.json RUN yarn build:headless + +# Optimizes the build, so no NODE_MODULES included in image. Don't remove this FROM node:16.9.0-alpine WORKDIR /code diff --git a/README-en.md b/README-en.md index ffafc85..a305de2 100644 --- a/README-en.md +++ b/README-en.md @@ -45,7 +45,7 @@ Targets being attacked are changed automatically and are downloaded from the con Or use [pre-built image](https://github.com/opengs/uashield/pkgs/container/uashield): ```bash -docker run -d ghcr.io/opengs/uashield:0.0.x 512 true +docker run -d ghcr.io/opengs/uashield:master 512 true ``` ## Docker-compose version @@ -67,7 +67,7 @@ docker run -d ghcr.io/opengs/uashield:0.0.x 512 true ## Deploy with Play With Docker - free instance for 4 hours -[![Try in PWD](https://raw.githubusercontent.com/play-with-docker/stacks/master/assets/images/button.png)](https://labs.play-with-docker.com/?stack=https://raw.githubusercontent.com/opengs/uashield/0.0.x/pwd-docker-compose.yml) +[![Try in PWD](https://raw.githubusercontent.com/play-with-docker/stacks/master/assets/images/button.png)](https://labs.play-with-docker.com/?stack=https://raw.githubusercontent.com/opengs/uashield/master/pwd-docker-compose.yml) ## Donations Donations will be used to fund our operations: diff --git a/README.md b/README.md index d585818..f4145b4 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ Або за допомогою вже [зібраного імежду](https://github.com/opengs/uashield/pkgs/container/uashield): ```bash -docker run -d ghcr.io/opengs/uashield:0.0.x 512 true +docker run -d ghcr.io/opengs/uashield:master 512 true ``` ## Docker-compose версія @@ -67,7 +67,7 @@ docker run -d ghcr.io/opengs/uashield:0.0.x 512 true ## Деплой на Play With Docker - безкоштовний інстанс на 4 години -[![Try in PWD](https://raw.githubusercontent.com/play-with-docker/stacks/master/assets/images/button.png)](https://labs.play-with-docker.com/?stack=https://raw.githubusercontent.com/opengs/uashield/0.0.x/pwd-docker-compose.yml) +[![Try in PWD](https://raw.githubusercontent.com/play-with-docker/stacks/master/assets/images/button.png)](https://labs.play-with-docker.com/?stack=https://raw.githubusercontent.com/opengs/uashield/master/pwd-docker-compose.yml) ## Пожертвування Пожертвування будуть використовуватися виключно для цілей програми: diff --git a/pwd-docker-compose.yml b/pwd-docker-compose.yml index 384baf9..1f72796 100644 --- a/pwd-docker-compose.yml +++ b/pwd-docker-compose.yml @@ -1,11 +1,11 @@ version: '3' services: uashield: - image: ghcr.io/opengs/uashield:0.0.x + image: ghcr.io/opengs/uashield:master restart: always environment: - WORKERS: '512' + WORKERS: '256' USEPROXY: 'false' deploy: - replicas: 6 + replicas: 3 diff --git a/tools/README.MD b/tools/README.MD new file mode 100644 index 0000000..cf935e8 --- /dev/null +++ b/tools/README.MD @@ -0,0 +1,7 @@ +# COMMUNITY CREATED + +Fortunately or unfortunately these tools are created by community. + +They can be not up to date, but we are trying our best. + +If you see any issue - make an issue or contact us via Discord. diff --git a/tools/ansible/README.md b/tools/ansible/README.md index 8104f5a..6a0eae0 100644 --- a/tools/ansible/README.md +++ b/tools/ansible/README.md @@ -52,6 +52,10 @@ Optional step, can be skipped ansible-playbook -u root stats.yaml -i hosts +### 8. Update to the latest version and restart containers + + ansible-playbook -u root update.yaml -i hosts + ## UA ### 1. Інсталюйте ansible на свою машину #### Mac OS @@ -97,3 +101,7 @@ ssh-agent bash -c "ssh-add /path/to/keys/*.pem" ### 7. Перевірка статистики успішних запитів ansible-playbook -u root stats.yaml -i hosts + +### 8. Оновити до останньої версії та перезавантажити + + ansible-playbook -u root update.yaml -i hosts diff --git a/tools/ansible/ansible.cfg b/tools/ansible/ansible.cfg new file mode 100644 index 0000000..5c03493 --- /dev/null +++ b/tools/ansible/ansible.cfg @@ -0,0 +1,5 @@ +[defaults] +deprecation_warnings=False +host_key_checking=False +strategy=linear ; default +; strategy=free ; do not wait for complete all hosts diff --git a/tools/ansible/create-vm.sh b/tools/ansible/create-vm.sh new file mode 100755 index 0000000..1eab4ed --- /dev/null +++ b/tools/ansible/create-vm.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Usage example +# ./create-vm.sh eastus2 + +# Variables +INPUT=$1 +LOCATION="${INPUT:-'norwayeast'}" +RESOURCE_GROUP='STOPWAR' +USER='azureuser' +IMAGE='UbuntuLTS' +for i in {1..4} +do + NAME=uashield-$LOCATION-unit-$i + echo "$NAME : Initializing vm..." + az vm create --admin-user=$USER --location=$LOCATION --name $NAME --resource-group $RESOURCE_GROUP --image $IMAGE --generate-ssh-keys +done + +bash ./update-hosts.sh diff --git a/tools/ansible/roles/stats/tasks/main.yml b/tools/ansible/roles/stats/tasks/main.yml index 3cd06d1..9c3ed57 100644 --- a/tools/ansible/roles/stats/tasks/main.yml +++ b/tools/ansible/roles/stats/tasks/main.yml @@ -1,6 +1,6 @@ - name: Get stats - shell: echo $(docker logs uashield | grep '| 200' -c) + shell: echo $(docker ps -q | xargs -L 1 docker logs | grep '| 200' -c) register: echo_content - debug: - msg: "Hits {{ echo_content.stdout }} requests." + msg: "Hit {{ echo_content.stdout }} requests." diff --git a/tools/ansible/roles/uashield/tasks/main.yml b/tools/ansible/roles/uashield/tasks/main.yml index e17cffd..ae35d6d 100644 --- a/tools/ansible/roles/uashield/tasks/main.yml +++ b/tools/ansible/roles/uashield/tasks/main.yml @@ -1,30 +1,52 @@ --- -- name: Clone uashield repo +- name: Check if repo is exist + stat: + path: "{{uashield_work_dir}}" + failed_when: false + changed_when: false + register: check_uashield_repo + +- name: Clone repo git: repo: "{{uashield_git_url}}" dest: "{{uashield_work_dir}}" accept_hostkey: true force: true version: "master" + when: not check_uashield_repo.stat.exists -- name: Stop all containers - shell: "docker-compose down" +- name: Pull Latest Changes + shell: "git pull" args: chdir: "{{uashield_work_dir}}" + when: check_uashield_repo.stat.exists + +- name: Get Running Docker Containers + shell: "docker ps -aq" + register: docker_info + +- name: Stop Running Docker Containers + shell: "docker stop $(docker ps -aq)" + when: docker_info.stdout + +- name: Remove Docker Containers + shell: "docker rm $(docker ps -aq)" + when: docker_info.stdout -- name: Remove old uashield images +- name: Remove Old Docker Image shell: "docker rmi -f uashield" -- name: Run uashield - shell: "docker-compose up -d" +- name: Build Docker Image + shell: "docker build . -t uashield" args: chdir: "{{uashield_work_dir}}" -- name: Check docker started - shell: "docker ps" - args: - chdir: "{{uashield_work_dir}}" +- name: Run Instance + shell: "docker run -d uashield {{ uashield_threads }} {{ uashield_proxy }}" + +- name: Check Docker is started + command: "docker ps" register: output - ansible.builtin.debug: - var: output + var: output.stdout_lines diff --git a/tools/ansible/stats.yaml b/tools/ansible/stats.yaml index 42a49be..82c33b9 100644 --- a/tools/ansible/stats.yaml +++ b/tools/ansible/stats.yaml @@ -1,5 +1,7 @@ --- - hosts: servers + strategy: linear + gather_facts: no become: yes become_user: root vars_files: diff --git a/tools/ansible/update-hosts.sh b/tools/ansible/update-hosts.sh new file mode 100755 index 0000000..ce68d26 --- /dev/null +++ b/tools/ansible/update-hosts.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +PUBLIC_IP_ADDRESSES="$(az vm list-ip-addresses --query '[*].virtualMachine.network.publicIpAddresses[0].ipAddress' | sed '1d;$d' | cut -d'"' -f2)" +for IP in $PUBLIC_IP_ADDRESSES +do + EXIST="$(cat hosts | awk '{print $1}' | grep $IP)" + if [ "$EXIST" == "$IP" ] + then + echo "[Skipping] $IP already exist" + else + echo $IP >> hosts + echo "[Changed] $IP is added to hosts" + fi +done diff --git a/tools/ansible/update.yaml b/tools/ansible/update.yaml new file mode 100644 index 0000000..4a563a5 --- /dev/null +++ b/tools/ansible/update.yaml @@ -0,0 +1,10 @@ +--- +- hosts: servers + become: yes + become_user: root + vars_files: + - vars/vars.yaml + roles: + - uashield + handlers: + - include: handlers/main.yml diff --git a/tools/ansible/vars/vars.yaml b/tools/ansible/vars/vars.yaml index c28a41f..ec4b48b 100644 --- a/tools/ansible/vars/vars.yaml +++ b/tools/ansible/vars/vars.yaml @@ -12,3 +12,5 @@ docker_pip_executable: pip3 uashield_git_url: "https://github.com/opengs/uashield" uashield_work_dir: /root/uashield/ +uashield_threads: 2500 +uashield_proxy: true diff --git a/tools/azure/Readme.md b/tools/azure/Readme.md index cc918e0..6a67030 100644 --- a/tools/azure/Readme.md +++ b/tools/azure/Readme.md @@ -30,7 +30,7 @@ PLEASE SHARE WITH YOUR FRIENDS AND ANYONE WHO WANTS TO HELP BUT DOESN'T KNOW ![](https://github.com/opengs/uashield/blob/master/tools/azure/images/7.jpg) ![](https://github.com/opengs/uashield/blob/master/tools/azure/images/8.jpg) -8. Go to "Advanced" tab (above) and paste the script [https://github.com/opengs/uashield/blob/0.0.x/tools/azure/azure-custom-data-script.sh](https://github.com/opengs/uashield/blob/0.0.x/tools/azure/azure-custom-data-script.sh) (just copy it as a text) to "Custom data". +8. Go to "Advanced" tab (above) and paste the script [https://github.com/opengs/uashield/blob/master/tools/azure/azure-custom-data-script.sh](https://github.com/opengs/uashield/blob/master/tools/azure/azure-custom-data-script.sh) (just copy it as a text) to "Custom data". ![](https://github.com/opengs/uashield/blob/master/tools/azure/images/9.jpg) 9. Go to "Disk" tab and change "OS disk type" from "Premium SSD" to "Standart SSD". ![](https://github.com/opengs/uashield/blob/master/tools/azure/images/10.jpg) diff --git a/tools/azure/azure-custom-data-script.sh b/tools/azure/azure-custom-data-script.sh index 3027f5d..8b66789 100644 --- a/tools/azure/azure-custom-data-script.sh +++ b/tools/azure/azure-custom-data-script.sh @@ -22,6 +22,7 @@ version: \"3.3\" services: worker: image: ghcr.io/opengs/uashield:latest + restart: always command: - \"7500\" - \"true\"" >> /home/docker-compose.yaml @@ -32,5 +33,8 @@ cd /home/ sudo docker-compose pull && sudo docker-compose up -d --scale worker=$(grep -c ^processor /proc/cpuinfo) -sudo echo "*/30 * * * * cd /home/ && sudo docker-compose down && sudo docker-compose pull && sudo docker-compose up -d --scale worker=$(grep -c ^processor /proc/cpuinfo)" >> /home/cronjob +sudo echo "*/30 * * * * cd /home/ && sudo docker-compose down -t 1 && sudo docker-compose pull && sudo docker-compose up -d --scale worker=$(grep -c ^processor /proc/cpuinfo)" >> /home/cronjob + +# restart:always should do the job to run container on startup, but the hard restart is good here to avoid problems +sudo echo "@reboot cd /home/ && sudo docker-compose down -t 1 && sudo docker-compose pull && sudo docker-compose up -d --scale worker=$(grep -c ^processor /proc/cpuinfo)" >> /home/cronjob crontab /home/cronjob diff --git a/tools/helm/README.md b/tools/helm/README.md index 136ce1c..a093abc 100644 --- a/tools/helm/README.md +++ b/tools/helm/README.md @@ -33,7 +33,7 @@ helm upgrade --install \ | fullnameOverride | string | `""` | | | image.pullPolicy | string | `"IfNotPresent"` | | | image.repository | string | `"ghcr.io/opengs/uashield"` | | -| image.tag | string | `"0.0.x"` | | +| image.tag | string | `"master"` | | | imagePullSecrets | list | `[]` | | | nameOverride | string | `""` | | | nodeSelector | object | `{}` | | diff --git a/tools/helm/values.yaml b/tools/helm/values.yaml index e98f580..603c392 100644 --- a/tools/helm/values.yaml +++ b/tools/helm/values.yaml @@ -15,7 +15,7 @@ image: repository: ghcr.io/opengs/uashield pullPolicy: IfNotPresent # We can find a tag here: https://github.com/opengs/uashield/pkgs/container/uashield - tag: "0.0.x" + tag: "master" resources: # We usually recommend not to specify default resources and to leave this as a conscious diff --git a/tools/terraform-do-uashield/README.md b/tools/terraform-do-uashield/README.md new file mode 100644 index 0000000..ec1d9ca --- /dev/null +++ b/tools/terraform-do-uashield/README.md @@ -0,0 +1,36 @@ +[![DigitalOcean Referral Badge](https://web-platforms.sfo2.cdn.digitaloceanspaces.com/WWW/Badge%201.svg)](https://www.digitalocean.com/?refcode=4e29ef6429c9&utm_campaign=Referral_Invite&utm_medium=Referral_Program&utm_source=badge) + +[Get $100 to try DigitalOcean, link for sing in above](https://try.digitalocean.com/freetrialoffer/) + + +## Requirements +- [Instal terraform](https://www.terraform.io/downloads) +- [Add SSH key](https://docs.digitalocean.com/products/droplets/how-to/add-ssh-keys/to-account/) to DO with name `ssh` + + +## Export environment vars +Extra logs +``` +export TF_LOG=INFO +``` +## Create and export DO PAT +- [DO PAT](https://docs.digitalocean.com/reference/api/create-personal-access-token/) +``` +export DO_PAT="" +``` +## Init terraform +``` +terraform init +``` + +``` +terraform apply \ + -var "do_token=${DO_PAT}" \ + -var "pvt_key=$HOME/.ssh/id_ed25519" +``` + +``` +terraform destroy \ + -var "do_token=${DO_PAT}" \ + -var "pvt_key=$HOME/.ssh/id_ed25519" +``` \ No newline at end of file diff --git a/tools/terraform-do-uashield/provider.tf b/tools/terraform-do-uashield/provider.tf new file mode 100644 index 0000000..306321b --- /dev/null +++ b/tools/terraform-do-uashield/provider.tf @@ -0,0 +1,16 @@ +terraform { + required_providers { + digitalocean = { + source = "digitalocean/digitalocean" + version = "~> 2.0" + } + } +} + +provider "digitalocean" { + token = var.do_token +} + +data "digitalocean_ssh_key" "ssh" { + name = "drvdo" +} diff --git a/tools/terraform-do-uashield/scripts/uashield.sh b/tools/terraform-do-uashield/scripts/uashield.sh new file mode 100644 index 0000000..cedca7e --- /dev/null +++ b/tools/terraform-do-uashield/scripts/uashield.sh @@ -0,0 +1,39 @@ +#!/bin/sh + +sudo apt-get install -y \ + ca-certificates \ + curl \ + gnupg \ + lsb-release \ + wget + +curl -sSL https://repos.insights.digitalocean.com/install.sh | sudo bash + +wget -O - https://get.docker.com/ | bash + +sudo systemctl enable docker.service +sudo systemctl start docker.service + +mkdir -p ~/.docker/cli-plugins/ +curl -SL https://github.com/docker/compose/releases/download/v2.2.3/docker-compose-linux-x86_64 -o ~/.docker/cli-plugins/docker-compose +chmod +x ~/.docker/cli-plugins/docker-compose +sudo chown $USER /var/run/docker.sock + +sudo echo " +version: \"3.3\" +services: + worker: + image: ghcr.io/opengs/uashield:latest + restart: always + command: + - \"7500\" + - \"true\"" >> /home/docker-compose.yaml + +sudo apt install -y docker-compose + +cd /home/ + +sudo docker-compose pull && sudo docker-compose up -d --scale worker=$(grep -c ^processor /proc/cpuinfo) + +sudo echo "*/30 * * * * cd /home/ && sudo docker-compose down -t 1 && sudo docker-compose pull && sudo docker-compose up -d --scale worker=$(grep -c ^processor /proc/cpuinfo)" >> /home/cronjob +crontab /home/cronjob \ No newline at end of file diff --git a/tools/terraform-do-uashield/uashield.tf b/tools/terraform-do-uashield/uashield.tf new file mode 100644 index 0000000..2bc5034 --- /dev/null +++ b/tools/terraform-do-uashield/uashield.tf @@ -0,0 +1,33 @@ +resource "digitalocean_droplet" "uashield" { + count = var.droplet_instance_number + image = "ubuntu-20-04-x64" + name = "uashield-${count.index}" + region = "nyc3" + size = var.droplet_instance_size + monitoring = true + + ssh_keys = [ + data.digitalocean_ssh_key.ssh.id + ] + + connection { + host = self.ipv4_address + user = "root" + type = "ssh" + private_key = file(var.pvt_key) + timeout = "2m" + } + + provisioner "file" { + source = "scripts/uashield.sh" + destination = "/opt/uashield.sh" + } + + provisioner "remote-exec" { + inline = [ + "chmod +x /opt/uashield.sh", + "/opt/uashield.sh", + ] + } +} + diff --git a/tools/terraform-do-uashield/variables.tf b/tools/terraform-do-uashield/variables.tf new file mode 100644 index 0000000..cb98468 --- /dev/null +++ b/tools/terraform-do-uashield/variables.tf @@ -0,0 +1,14 @@ +variable "droplet_instance_number" { + type = number + description = "Digital Ocean droplet instace number" + default = 1 +} + +variable "droplet_instance_size" { + type = string + description = "Digital Ocean droplet instace size" + default = "s-1vcpu-1gb" +} + +variable "do_token" {} +variable "pvt_key" {}