From 169ccfb526671f3f1de36ab1a2e0c1523d7042a6 Mon Sep 17 00:00:00 2001 From: Martin Lupa <80256770+MartinLupa@users.noreply.github.com> Date: Fri, 10 May 2024 12:10:05 +0200 Subject: [PATCH 01/25] Update readme.md --- infrastructure/readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infrastructure/readme.md b/infrastructure/readme.md index 7f3d356..0a0c3f9 100644 --- a/infrastructure/readme.md +++ b/infrastructure/readme.md @@ -70,7 +70,7 @@ These environment variables should be set in `secrets-production` or `secrets-st #### 3. Using `bootstrap.sh` to Deploy Infrastructure 1. **Prepare Environment Variables:** - Set the necessary environment variables as the script checks these before proceeding. - - Configure `secrets` or `secrets.staging` files to load environment-specific variables. the files must be saved in the same directory as `bootstrap.sh` + - Configure `secrets-production` or `secrets-staging` files to load environment-specific variables. the files must be saved in the same directory as `bootstrap.sh` 1. **Run the Script:** - Execute `bash bootstrap.sh production` or `bash bootstrap.sh staging` depending on the target environment. - The script will initialize Terraform, validate configurations, and apply them to create the infrastructure. It also handles SSH key generation if SSH keys are missing/not provided. @@ -87,4 +87,4 @@ These environment variables should be set in `secrets-production` or `secrets-st - Ensure all environment variables and secrets are correctly set as the script will verify these before proceeding. 2. **Execute the Script:** - Run `./destroy.sh production` or `./destroy.sh staging`. - - This script will remove all the deployed resources and delete the associated Terraform workspace. \ No newline at end of file + - This script will remove all the deployed resources and delete the associated Terraform workspace. From 534c8850f5aed9b9286c682bad633ad745257664 Mon Sep 17 00:00:00 2001 From: Mathias Gleitze Date: Mon, 20 May 2024 09:46:25 +0200 Subject: [PATCH 02/25] Added a 'tests' workflow --- .github/workflows/tests.yml | 46 +++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..8ba2042 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,46 @@ +name: Run Tests +on: + workflow_run: + workflows: ["Format Check"] + types: + - completed + +jobs: + run-tests: + runs-on: ubuntu-latest + services: + app: + image: csharp-minitwit:latest + options: --health-cmd='curl -f http://localhost:5000/public || exit 1' --health-interval=10s --health-retries=3 --health-start-period=30s --health-timeout=10s + ports: + - 5000:5000 + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Build Docker image + run: docker build -t csharp-minitwit:latest ./csharp-minitwit + + - name: Wait for the service to be healthy + run: | + until [ "$(docker inspect --format='{{.State.Health.Status}}' app)" == "healthy" ]; do + echo "Waiting for Docker service to be healthy..." + sleep 5 + done + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.x' + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + + - name: Run Python tests + env: + BASE_URL: http://localhost:5000 + run: | + cd ./Tests + pytest minitwit_sim_api_test.py + pytest refactored_minitwit_tests.py From b62631348918692084fd2d4a694d33a165466255 Mon Sep 17 00:00:00 2001 From: Mathias Gleitze Date: Mon, 20 May 2024 09:57:38 +0200 Subject: [PATCH 03/25] Trigger build --- .github/workflows/tests.yml | 36 +++++++++++++++++++----------------- infrastructure/secrets | 0 2 files changed, 19 insertions(+), 17 deletions(-) create mode 100644 infrastructure/secrets diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8ba2042..044ea8c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,19 +1,16 @@ name: Run Tests +# on: +# workflow_run: +# workflows: ["Format Check"] +# types: +# - completed on: - workflow_run: - workflows: ["Format Check"] - types: - - completed + pull_request: + types: [opened, reopened, edited, synchronize, ready_for_review] jobs: run-tests: runs-on: ubuntu-latest - services: - app: - image: csharp-minitwit:latest - options: --health-cmd='curl -f http://localhost:5000/public || exit 1' --health-interval=10s --health-retries=3 --health-start-period=30s --health-timeout=10s - ports: - - 5000:5000 steps: - name: Checkout repo uses: actions/checkout@v4 @@ -21,26 +18,31 @@ jobs: - name: Build Docker image run: docker build -t csharp-minitwit:latest ./csharp-minitwit - - name: Wait for the service to be healthy + - name: Run Docker container run: | - until [ "$(docker inspect --format='{{.State.Health.Status}}' app)" == "healthy" ]; do - echo "Waiting for Docker service to be healthy..." - sleep 5 - done + docker run -d --name csharp-minitwit-container -p 5000:5000 csharp-minitwit:latest - name: Set up Python uses: actions/setup-python@v4 with: python-version: '3.x' + - name: Wait for the service to be up + run: + sleep 10 + - name: Install Python dependencies run: | python -m pip install --upgrade pip + # pip install -r ./Tests/requirements.txt - name: Run Python tests - env: - BASE_URL: http://localhost:5000 run: | cd ./Tests pytest minitwit_sim_api_test.py pytest refactored_minitwit_tests.py + + - name: Clean up + run: | + docker stop csharp-minitwit-container + docker rm csharp-minitwit-container diff --git a/infrastructure/secrets b/infrastructure/secrets new file mode 100644 index 0000000..e69de29 From 148bbac3950ca84bd8a5ab247a1997120c16e89c Mon Sep 17 00:00:00 2001 From: Mathias Gleitze Date: Mon, 20 May 2024 10:29:01 +0200 Subject: [PATCH 04/25] Test tests --- .github/workflows/quality.yml | 3 ++- .github/workflows/tests.yml | 9 ++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 134ed7f..a8b534e 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -1,7 +1,8 @@ name: Format check on: pull_request: - types: [opened, reopened, edited, synchronize, ready_for_review] + # types: [opened, reopened, edited, synchronize, ready_for_review] + types: [opened] jobs: dotnet-format: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 044ea8c..b54afcd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,6 +15,10 @@ jobs: - name: Checkout repo uses: actions/checkout@v4 + - name: Wait for docker + run: + sleep 5 + - name: Build Docker image run: docker build -t csharp-minitwit:latest ./csharp-minitwit @@ -38,9 +42,8 @@ jobs: - name: Run Python tests run: | - cd ./Tests - pytest minitwit_sim_api_test.py - pytest refactored_minitwit_tests.py + pytest Tests/minitwit_sim_api_test.py + pytest Tests/refactored_minitwit_tests.py - name: Clean up run: | From 5fff26ee909acf3783119ff4ee38455ba73950b4 Mon Sep 17 00:00:00 2001 From: Mathias Gleitze Date: Mon, 20 May 2024 10:31:32 +0200 Subject: [PATCH 05/25] Trigger build --- .github/workflows/tests.yml | 1 - infrastructure/secrets | 0 2 files changed, 1 deletion(-) delete mode 100644 infrastructure/secrets diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b54afcd..20aea3f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -38,7 +38,6 @@ jobs: - name: Install Python dependencies run: | python -m pip install --upgrade pip - # pip install -r ./Tests/requirements.txt - name: Run Python tests run: | diff --git a/infrastructure/secrets b/infrastructure/secrets deleted file mode 100644 index e69de29..0000000 From a681601ad7a3897e7c5982b952c9488839ab7baa Mon Sep 17 00:00:00 2001 From: Mathias Gleitze Date: Mon, 20 May 2024 10:34:15 +0200 Subject: [PATCH 06/25] Install pytest --- .github/workflows/tests.yml | 11 +++++------ csharp-minitwit/Tests/requirements.txt | 1 + 2 files changed, 6 insertions(+), 6 deletions(-) create mode 100644 csharp-minitwit/Tests/requirements.txt diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 20aea3f..4a8c1f8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -31,18 +31,17 @@ jobs: with: python-version: '3.x' - - name: Wait for the service to be up - run: - sleep 10 - - name: Install Python dependencies run: | + cd ./csharp-minitwit/Tests python -m pip install --upgrade pip + pip install -r requirements.txt - name: Run Python tests run: | - pytest Tests/minitwit_sim_api_test.py - pytest Tests/refactored_minitwit_tests.py + cd ./csharp-minitwit/Tests + pytest minitwit_sim_api_test.py + pytest refactored_minitwit_tests.py - name: Clean up run: | diff --git a/csharp-minitwit/Tests/requirements.txt b/csharp-minitwit/Tests/requirements.txt new file mode 100644 index 0000000..55b033e --- /dev/null +++ b/csharp-minitwit/Tests/requirements.txt @@ -0,0 +1 @@ +pytest \ No newline at end of file From f115eb94553331825546863cc234d45d85383399 Mon Sep 17 00:00:00 2001 From: Mathias Gleitze Date: Mon, 20 May 2024 10:49:21 +0200 Subject: [PATCH 07/25] Added requirement 'requests' --- csharp-minitwit/Tests/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/csharp-minitwit/Tests/requirements.txt b/csharp-minitwit/Tests/requirements.txt index 55b033e..16d3b77 100644 --- a/csharp-minitwit/Tests/requirements.txt +++ b/csharp-minitwit/Tests/requirements.txt @@ -1 +1,2 @@ -pytest \ No newline at end of file +pytest +requests \ No newline at end of file From db01adde70098694daf0ff4126023a7cba245927 Mon Sep 17 00:00:00 2001 From: Mathias Gleitze Date: Mon, 20 May 2024 10:55:04 +0200 Subject: [PATCH 08/25] Check if application is up --- .github/workflows/tests.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4a8c1f8..707239a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -37,6 +37,14 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt + - name: Check Public Endpoint + run: | + status_code=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:5000/public) + if [ "$status_code" -ne 200 ]; then + echo "Error: Expected status code 200, but received $status_code" + exit 1 + fi + - name: Run Python tests run: | cd ./csharp-minitwit/Tests From df129d4a0b201a990f6bb0ebd159482064e51899 Mon Sep 17 00:00:00 2001 From: Mathias Gleitze Date: Mon, 20 May 2024 10:58:44 +0200 Subject: [PATCH 09/25] Check docker logs --- .github/workflows/tests.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 707239a..a0c8b93 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -45,6 +45,10 @@ jobs: exit 1 fi + - name: Print Docker Logs + if: ${{ failure() }} + run: docker logs csharp-minitwit-container + - name: Run Python tests run: | cd ./csharp-minitwit/Tests From f50aef473ad09803c49e8ece0626e449ec7f5398 Mon Sep 17 00:00:00 2001 From: Mathias Gleitze Date: Mon, 20 May 2024 11:11:51 +0200 Subject: [PATCH 10/25] Test access --- .github/workflows/tests.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a0c8b93..b10863d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -39,11 +39,8 @@ jobs: - name: Check Public Endpoint run: | - status_code=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:5000/public) - if [ "$status_code" -ne 200 ]; then - echo "Error: Expected status code 200, but received $status_code" - exit 1 - fi + response=$(curl -v http://localhost:5000/public) + echo "Response: $response" - name: Print Docker Logs if: ${{ failure() }} From 3ba6bb05e8381639dd0c0c8c7619f629ee829c2b Mon Sep 17 00:00:00 2001 From: Mathias Gleitze Date: Mon, 20 May 2024 11:21:48 +0200 Subject: [PATCH 11/25] Attempt using service. --- .github/workflows/tests.yml | 44 ++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b10863d..2cd6248 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,22 +9,39 @@ on: types: [opened, reopened, edited, synchronize, ready_for_review] jobs: + build-image: + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Build Docker image + run: docker build -t csharp-minitwit:latest ./csharp-minitwit + + - name: Wait 5 seconds + run: sleep 5 + run-tests: + needs: build-image runs-on: ubuntu-latest + services: + csharp-minitwit: + image: csharp-minitwit:latest + ports: + - 5000:5000 + options: --detach steps: - name: Checkout repo uses: actions/checkout@v4 - - name: Wait for docker - run: - sleep 5 - - - name: Build Docker image - run: docker build -t csharp-minitwit:latest ./csharp-minitwit - - - name: Run Docker container + - name: Check Public Endpoint run: | - docker run -d --name csharp-minitwit-container -p 5000:5000 csharp-minitwit:latest + response=$(curl -v http://localhost:5000/public) + echo "Response: $response" + + - name: Print Docker Logs + if: ${{ failure() }} + run: docker logs csharp-minitwit-container - name: Set up Python uses: actions/setup-python@v4 @@ -37,15 +54,6 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt - - name: Check Public Endpoint - run: | - response=$(curl -v http://localhost:5000/public) - echo "Response: $response" - - - name: Print Docker Logs - if: ${{ failure() }} - run: docker logs csharp-minitwit-container - - name: Run Python tests run: | cd ./csharp-minitwit/Tests From 85ae67a1c91d07ae7d56b6572a98d6db95114260 Mon Sep 17 00:00:00 2001 From: Mathias Gleitze Date: Mon, 20 May 2024 11:46:09 +0200 Subject: [PATCH 12/25] Another test --- .github/workflows/tests.yml | 46 ++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2cd6248..cdf86cc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,39 +9,22 @@ on: types: [opened, reopened, edited, synchronize, ready_for_review] jobs: - build-image: - runs-on: ubuntu-latest - steps: - - name: Checkout repo - uses: actions/checkout@v4 - - - name: Build Docker image - run: docker build -t csharp-minitwit:latest ./csharp-minitwit - - - name: Wait 5 seconds - run: sleep 5 - run-tests: - needs: build-image runs-on: ubuntu-latest - services: - csharp-minitwit: - image: csharp-minitwit:latest - ports: - - 5000:5000 - options: --detach steps: - name: Checkout repo uses: actions/checkout@v4 - - name: Check Public Endpoint - run: | - response=$(curl -v http://localhost:5000/public) - echo "Response: $response" + - name: Wait for docker + run: + sleep 5 + + - name: Build Docker image + run: docker build -t csharp-minitwit:latest ./csharp-minitwit - - name: Print Docker Logs - if: ${{ failure() }} - run: docker logs csharp-minitwit-container + - name: Run Docker container + run: | + docker run -d --name csharp-minitwit -p 5000:8080 csharp-minitwit:latest - name: Set up Python uses: actions/setup-python@v4 @@ -54,6 +37,17 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt + - name: Check Docker Logs + run: docker logs csharp-minitwit + + - name: Check Public Endpoint + run: | + status_code=$(curl -v -s -o /dev/null -w "%{http_code}" http://localhost:5000/public) + if [ "$status_code" -ne 200 ]; then + echo "Error: Expected status code 200, but received $status_code" + exit 1 + fi + - name: Run Python tests run: | cd ./csharp-minitwit/Tests From a06f6aff55a186eb1666b2d1ab7ca38a940e3f23 Mon Sep 17 00:00:00 2001 From: Mathias Gleitze Date: Mon, 20 May 2024 12:54:16 +0200 Subject: [PATCH 13/25] Changed database behavior --- .github/workflows/tests.yml | 4 ++-- csharp-minitwit/Databases/volume/minitwit.db | Bin 40960 -> 0 bytes csharp-minitwit/Dockerfile | 1 - csharp-minitwit/Program.cs | 13 +++++++++++++ csharp-minitwit/csharp-minitwit.csproj | 4 ++++ 5 files changed, 19 insertions(+), 3 deletions(-) delete mode 100644 csharp-minitwit/Databases/volume/minitwit.db diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cdf86cc..75cddd8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -42,9 +42,9 @@ jobs: - name: Check Public Endpoint run: | - status_code=$(curl -v -s -o /dev/null -w "%{http_code}" http://localhost:5000/public) + status_code=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:5000/public) if [ "$status_code" -ne 200 ]; then - echo "Error: Expected status code 200, but received $status_code" + echo "Error: $status_code" exit 1 fi diff --git a/csharp-minitwit/Databases/volume/minitwit.db b/csharp-minitwit/Databases/volume/minitwit.db deleted file mode 100644 index 267df0abce64d8ea7060563146aa29638ca2b38f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 40960 zcmeI*&2QRf90zc}5D4k0CruHWqG+DALs8oVLenJDGSwj_WdV~INTJeXnZ&%b7z1`f z(py?>haIN<8QWo({S(`H$7zRMcIYlWZK}4*9y@jdjAYrN>r#Cm8El^C@$<7k&toEE z*(q#io3_r^%$}jye3E*bVi@Wk&r=k|kk=G>`6)ye0{#awrw5kLTV$xY+OHArB^3^< z)b$^@Zz6w0zIrP#_2ty<@OLB&2?7v+00bZa0SLTl0<~}`x;R50hnwv^{V-Ej4c)S| zx~^*bcEjwc%{?z;Dy2wbQQ|Z5x>V+caWw@#FY|&|U*K;Dr)>zay~$8CIZGc;_;G4x zt7Z1}p4xAi1_|ST4*xBVU*gw13FGHFal9skm=+2}6C?^fF3LcV$qR#!ieqCq`LULw~wrYt@$S?X5QpdF>X(wX8Hq5t-si@@?Dz8M!EJND43Ki@aRQX8BZJE)*4! zWiusI8qU0pGc*|?`jG6S#aHD<|TR*e~-g!Tb=NEPM ztLmO+>%!T}WNZA=T1%_fb>eZ_@>${F2B$aGoHuexmfjQibY77%8?w9ALnDD#q_m_+ za!M-j!oFMghC6q8oguU?WhFAwQeq(`u1k@r*t7R1Lebe-`m<@*-0ymS9voKpv&qQt zfc%_Gxu?B>c?ozj?uMH@vpF4;mw@vlst+3GD4Z7}c4;+5B7-QC@(0IA9}R{w#>8@t zMB(2DBYm_L2u5$-q$_R@S=|;nX;Uq|yRWx*^^uPgL;W>Uc%!Wu`Yp0%?6|~+qSMp# zvxwWKGgO>cU}$ZB1bIE}R2c4(bB?Z)f%lJxmQ&5ypmUBL-O!q?bILk>wV_#!ky0Xc zT&6?Or6u}l-rX-%mD0Ipy{Fktvu)jPTDI9c8k@-s@A=p|XM_H<%?vqui&B|<%ba6z zPD2AY{iKg7IU$0}yx0C69R>#Is6JxVk@`rZP z`C}U)FPi(2A`=n>AOHafKmY;|fB*y_009U<00LJ|-~*bKq6_hbJMm zKDTQc;C@{WqWnC#hgT|nI@SojCMa;*NXaXx0TtF+lr|#Z|9EY z_a1bzN`6f)Jxs0FcPrK8-KBJ8V{^Z-Q9sB&e$-Zu*6xb)-E=8u*H>3NbdZY9A1^xh zMI_v}3nOOptvm5@V`W2Me$=>YCS+r+VmwT*ROBc2LebhfG9MmxdymDkG2c(;4ZZsD zeEUv+qqClRlF+&Zg+z4E|GA$i?sx7vnIS;{0uX=z1Rwwb2tWV=5P$##AaKP5uCeTF z*nc?3HHMv`z2~R|CTVt>c79)Ao%4V4&_9Yiq7MlI5P$##AOHafKmY;|fB*y_0D(78 zV3MX;#yS5FaGz7$FWmRs*JJ?+0uX=z1Rwwb2tWV=5P$##AOL|^C@>ocQgk9P$=VH_ z_b$}))gzwV(options => options.UseSqlite(connectionString)); } diff --git a/csharp-minitwit/csharp-minitwit.csproj b/csharp-minitwit/csharp-minitwit.csproj index ee1defd..1a8402e 100644 --- a/csharp-minitwit/csharp-minitwit.csproj +++ b/csharp-minitwit/csharp-minitwit.csproj @@ -34,4 +34,8 @@ + + + + From a3ef062680fd4d8acd1acd1e961792da69412b9f Mon Sep 17 00:00:00 2001 From: Mathias Gleitze Date: Mon, 20 May 2024 13:01:17 +0200 Subject: [PATCH 14/25] Changed workflows back to normal workflow --- .github/workflows/quality.yml | 3 +-- .github/workflows/tests.yml | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index a8b534e..134ed7f 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -1,8 +1,7 @@ name: Format check on: pull_request: - # types: [opened, reopened, edited, synchronize, ready_for_review] - types: [opened] + types: [opened, reopened, edited, synchronize, ready_for_review] jobs: dotnet-format: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 75cddd8..1f39c31 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -56,5 +56,5 @@ jobs: - name: Clean up run: | - docker stop csharp-minitwit-container - docker rm csharp-minitwit-container + docker stop csharp-minitwit + docker rm csharp-minitwit From caf5a985d09626d6e10a9f1b6284300823889480 Mon Sep 17 00:00:00 2001 From: Mathias Gleitze Date: Mon, 20 May 2024 13:17:00 +0200 Subject: [PATCH 15/25] Removed unused files and nuget packages --- csharp-minitwit/csharp-minitwit.csproj | 1 - infrastructure/archive/create_droplet.rb | 58 --------------- .../archive/preproduction_vagrantfile | 66 ----------------- infrastructure/archive/remote_files/deploy.sh | 6 -- .../archive/remote_files/docker-compose.yml | 29 -------- .../archive/remote_files/prometheus.yml | 29 -------- .../remote_files_preproduction/deploy.sh | 6 -- .../docker-compose.yml | 29 -------- .../remote_files_preproduction/prometheus.yml | 45 ------------ infrastructure/archive/vagrantfile | 73 ------------------- 10 files changed, 342 deletions(-) delete mode 100644 infrastructure/archive/create_droplet.rb delete mode 100644 infrastructure/archive/preproduction_vagrantfile delete mode 100644 infrastructure/archive/remote_files/deploy.sh delete mode 100644 infrastructure/archive/remote_files/docker-compose.yml delete mode 100644 infrastructure/archive/remote_files/prometheus.yml delete mode 100644 infrastructure/archive/remote_files_preproduction/deploy.sh delete mode 100644 infrastructure/archive/remote_files_preproduction/docker-compose.yml delete mode 100644 infrastructure/archive/remote_files_preproduction/prometheus.yml delete mode 100644 infrastructure/archive/vagrantfile diff --git a/csharp-minitwit/csharp-minitwit.csproj b/csharp-minitwit/csharp-minitwit.csproj index 1a8402e..339d25b 100644 --- a/csharp-minitwit/csharp-minitwit.csproj +++ b/csharp-minitwit/csharp-minitwit.csproj @@ -15,7 +15,6 @@ - runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/infrastructure/archive/create_droplet.rb b/infrastructure/archive/create_droplet.rb deleted file mode 100644 index d26192d..0000000 --- a/infrastructure/archive/create_droplet.rb +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/ruby -# Not able to load the enviroment files up to the server. -# To create a droplet, make sure to have a access token stored in the .env-file. Here reguired through the gem dotenv. -require 'dotenv' -Dotenv.load - -require 'droplet_kit' -ssh_key_id= ENV["SSH_KEY_ID"] -token= ENV["PROVISION_TOKEN"] -docker_user= ENV["DOCKER_USERNAME"] -docker_pw= ENV["DOCKER_PASSWORD"] -client = DropletKit::Client.new(access_token: token) - -user_data_script= <<-SCRIPT - #!/bin/bash\n - echo 'export DOCKER_USERNAME=#{docker_user}' >> ~/.bash_profile\n - echo 'export DOCKER_PASSWORD=#{docker_pw}' >> ~/.bash_profile\n - sudo apt-get update\n - sudo killall apt apt-get\n - sudo rm /var/lib/dpkg/lock-frontend\n - sudo apt-get install -y docker.io docker-compose-v2\n - sudo systemctl status docker\n - docker run --rm hello-world\n - ufw allow 5000\n - ufw allow 22/tcp\n - echo 'Droplet setup done'\n - echo 'csharp_minitwit will later be accessible at http://$(hostname -I | awk '{print $1}'):5000'\n - SCRIPT - -cloud_config = <<-CLOUD_CONFIG -users: - - name: root - ssh-authorized-keys: - - #{ssh_key_id} - -write_files: - - path: /etc/environment - content: | - DOCKER_USERNAME=#{docker_user} - DOCKER_PASSWORD=#{docker_pw} - -runcmd: - - apt-get update - - killall apt apt-get || true - - rm /var/lib/dpkg/lock-frontend || true - - apt-get install -y docker.io docker-compose-v2 - - systemctl status docker - - docker run --rm hello-world - - ufw allow 5000 - - ufw allow 22/tcp - - echo 'Droplet setup done' - - echo 'csharp_minitwit will later be accessible at http://$(hostname -I | awk \'{print $1}\'):5000' -CLOUD_CONFIG - -droplet = DropletKit::Droplet.new(name: 'csharp-minitwit', region: 'fra1', size: 's-1vcpu-1gb', image: 'ubuntu-22-04-x64', ssh_keys:ENV["SSH_KEY_ID"],timeout: 120, user_data: cloud_config, monitoring: true, tags: ['csharp-minitwit']) - -client.droplets.create(droplet) - diff --git a/infrastructure/archive/preproduction_vagrantfile b/infrastructure/archive/preproduction_vagrantfile deleted file mode 100644 index 9b94021..0000000 --- a/infrastructure/archive/preproduction_vagrantfile +++ /dev/null @@ -1,66 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -Vagrant.configure("2") do |config| - config.vm.box = 'digital_ocean' - config.vm.box_url = "https://github.com/devopsgroup-io/vagrant-digitalocean/raw/master/box/digital_ocean.box" - config.ssh.private_key_path = '~/.ssh/preproduction' - - config.vm.synced_folder "remote_files_preproduction", "/csharp-minitwit", type: "rsync" - config.vm.synced_folder '.', '/vagrant', disabled: true - - config.vm.define "csharp-minitwit", primary: true do |server| - - server.vm.provider :digital_ocean do |provider| - provider.ssh_key_name = "preproduction" - provider.token = ENV["PROVISION_TOKEN"] - provider.image = 'ubuntu-22-04-x64' - provider.region = 'fra1' - provider.size = 's-1vcpu-1gb' - end - - server.vm.hostname = "csharp-vagrant-preproduction-server" - - server.vm.provision "shell", inline: 'echo "export DOCKER_USERNAME=' + "'" + ENV["DOCKER_USERNAME"] + "'" + '" >> ~/.bash_profile' - server.vm.provision "shell", inline: 'echo "export DOCKER_PASSWORD=' + "'" + ENV["DOCKER_PASSWORD"] + "'" + '" >> ~/.bash_profile' - - server.vm.provision "shell", inline: <<-SHELL - - sleep 20 - - sudo apt-get update - - # The following address an issue in DO's Ubuntu images, which still contain a lock file - # sudo killall apt apt-get - # sudo rm /var/lib/dpkg/lock-frontend - - # Install docker and docker compose - sudo apt-get install -y docker.io docker-compose-v2 - - sudo systemctl status docker - # sudo usermod -aG docker ${USER} - - echo -e "\nVerifying that docker works ...\n" - docker run --rm hello-world - docker rmi hello-world - - echo -e "\nOpening port for preproduction minitwit ...\n" - ufw allow 5000 && \ - ufw allow 22/tcp - - echo ". $HOME/.bashrc" >> $HOME/.bash_profile - - echo -e "\nConfiguring credentials as environment variables...\n" - - source $HOME/.bash_profile - - echo -e "\nSelecting Minitwit Folder as default folder when you ssh into the server...\n" - echo "cd /csharp-minitwit" >> ~/.bash_profile - - chmod +x /csharp-minitwit/deploy.sh - - echo -e "\nVagrant setup done ..." - echo -e "/csharp-minitwit will later be accessible at http://$(hostname -I | awk '{print $1}'):5000" - SHELL - end -end \ No newline at end of file diff --git a/infrastructure/archive/remote_files/deploy.sh b/infrastructure/archive/remote_files/deploy.sh deleted file mode 100644 index b6baf92..0000000 --- a/infrastructure/archive/remote_files/deploy.sh +++ /dev/null @@ -1,6 +0,0 @@ -source ~/.bash_profile - -cd /csharp-minitwit - -docker compose -f docker-compose.yml pull -docker compose -f docker-compose.yml up -d \ No newline at end of file diff --git a/infrastructure/archive/remote_files/docker-compose.yml b/infrastructure/archive/remote_files/docker-compose.yml deleted file mode 100644 index b6cd57b..0000000 --- a/infrastructure/archive/remote_files/docker-compose.yml +++ /dev/null @@ -1,29 +0,0 @@ -version: "3.8" - -networks: - main: - -services: - minitwitimage: - image: ${DOCKER_USERNAME}/csharptwitimage - container_name: csharp-minitwit - networks: - - main - ports: - - "5000:8080" - environment: - - ConnectionStrings__DefaultConnection=${ConnectionStrings__DefaultConnection} - - prometheus: - image: prom/prometheus - container_name: prometheus - volumes: - - ./prometheus.yml:/etc/prometheus/prometheus.yml - ports: - - "9090:9090" - networks: - - main - # This extra host is the one being scrapped by Prometheus - # It is required so the scrapping endpoint is mapped to the name of the app's Docker image name and not to the containers changing ID - extra_hosts: - - "csharp-minitwit:host-gateway" diff --git a/infrastructure/archive/remote_files/prometheus.yml b/infrastructure/archive/remote_files/prometheus.yml deleted file mode 100644 index 4496375..0000000 --- a/infrastructure/archive/remote_files/prometheus.yml +++ /dev/null @@ -1,29 +0,0 @@ -global: - scrape_interval: 15s - evaluation_interval: 15s - -# Alertmanager configuration -alerting: - alertmanagers: - - static_configs: - - targets: - # - alertmanager:9093 - -# Load rules once and periodically evaluate them according to the global 'evaluation_interval'. -rule_files: - # - "first_rules.yml" - # - "second_rules.yml" - -scrape_configs: - - job_name: "prometheus" - - # metrics_path defaults to '/metrics' - # scheme defaults to 'http'. - - static_configs: - - targets: ["localhost:9090"] - - - job_name: 'csharp-minitwit' - scrape_interval: 5s - static_configs: - - targets: ["csharp-minitwit:5000"] ## Port being scrapped, it is to "csharp-minitwit:host-gateway" extra host inside the corresponding docker-compose \ No newline at end of file diff --git a/infrastructure/archive/remote_files_preproduction/deploy.sh b/infrastructure/archive/remote_files_preproduction/deploy.sh deleted file mode 100644 index b6baf92..0000000 --- a/infrastructure/archive/remote_files_preproduction/deploy.sh +++ /dev/null @@ -1,6 +0,0 @@ -source ~/.bash_profile - -cd /csharp-minitwit - -docker compose -f docker-compose.yml pull -docker compose -f docker-compose.yml up -d \ No newline at end of file diff --git a/infrastructure/archive/remote_files_preproduction/docker-compose.yml b/infrastructure/archive/remote_files_preproduction/docker-compose.yml deleted file mode 100644 index 03c5c04..0000000 --- a/infrastructure/archive/remote_files_preproduction/docker-compose.yml +++ /dev/null @@ -1,29 +0,0 @@ -version: "3.3" - -networks: - main: - -services: - minitwitimage: - image: ${DOCKER_USERNAME}/csharptwit-preproduction - container_name: csharp-preproduction-minitwit - networks: - - main - ports: - - "5000:8080" - environment: - - ConnectionStrings__DefaultConnection=${ConnectionStrings__DefaultConnection} - - prometheus: - image: prom/prometheus - container_name: prometheus - volumes: - - ./prometheus.yml:/etc/prometheus/prometheus.yml - ports: - - "9090:9090" - networks: - - main - # This extra host is the one being scrapped by Prometheus - # It is required so the scrapping endpoint is mapped to the name of the app's Docker image name and not to the containers changing ID - extra_hosts: - - "csharp-minitwit:host-gateway" \ No newline at end of file diff --git a/infrastructure/archive/remote_files_preproduction/prometheus.yml b/infrastructure/archive/remote_files_preproduction/prometheus.yml deleted file mode 100644 index ea0525c..0000000 --- a/infrastructure/archive/remote_files_preproduction/prometheus.yml +++ /dev/null @@ -1,45 +0,0 @@ -global: - scrape_interval: 15s - evaluation_interval: 15s - -# Alertmanager configuration -alerting: - alertmanagers: - - static_configs: - - targets: - # - alertmanager:9093 - -# Load rules once and periodically evaluate them according to the global 'evaluation_interval'. -rule_files: - # - "first_rules.yml" - # - "second_rules.yml" - -scrape_configs: - # - job_name: "prometheus" - - # # metrics_path defaults to '/metrics' - # # scheme defaults to 'http'. - - # static_configs: - # - targets: ["localhost:9090"] - - # Create a job for Docker Swarm containers. - - job_name: "dockerswarm" - dockerswarm_sd_configs: - - host: unix:///var/run/docker.sock - role: tasks - relabel_configs: - # Fetch metrics on port 9323. - - source_labels: [__meta_dockerswarm_node_address] - target_label: __address__ - replacement: $1:9323 - # Set hostname as instance label - - source_labels: [__meta_dockerswarm_node_hostname] - target_label: instance - - - job_name: "csharp-minitwit" - scrape_interval: 5s - static_configs: - - targets: ["app:8080"] ## Port being scrapped, it is to "csharp-minitwit:host-gateway" extra host inside the corresponding docker-compose - labels: - group: "csharp-apps" \ No newline at end of file diff --git a/infrastructure/archive/vagrantfile b/infrastructure/archive/vagrantfile deleted file mode 100644 index bbba1ea..0000000 --- a/infrastructure/archive/vagrantfile +++ /dev/null @@ -1,73 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -Vagrant.configure("2") do |config| - config.vm.box = 'digital_ocean' - config.vm.box_url = "https://github.com/devopsgroup-io/vagrant-digitalocean/raw/master/box/digital_ocean.box" - config.ssh.private_key_path = '~/.ssh/do_ssh_key' - - config.vm.synced_folder "remote_files", "/csharp-minitwit", type: "rsync" - config.vm.synced_folder '.', '/vagrant', disabled: true - - config.vm.define "csharp-minitwit", primary: true do |server| - - server.vm.provider :digital_ocean do |provider| - provider.ssh_key_name = "do_ssh_key" - provider.token = ENV["PROVISION_TOKEN"] - provider.image = 'ubuntu-22-04-x64' - provider.region = 'fra1' - provider.size = 's-1vcpu-1gb' - provider.monitoring = true - end - - server.vm.hostname = "csharp-vagrant-server" - - server.vm.provision "shell", inline: 'echo "export DOCKER_USERNAME=' + "'" + ENV["DOCKER_USERNAME"] + "'" + '" >> ~/.bash_profile' - server.vm.provision "shell", inline: 'echo "export DOCKER_PASSWORD=' + "'" + ENV["DOCKER_PASSWORD"] + "'" + '" >> ~/.bash_profile' - - server.vm.provision "shell", inline: <<-SHELL - - sleep 20 - - sudo apt-get update - - # The following address an issue in DO's Ubuntu images, which still contain a lock file - # sudo killall apt apt-get - # sudo rm /var/lib/dpkg/lock-frontend - - # Install docker and docker compose - sudo apt-get install -y docker.io docker-compose-v2 - - sudo systemctl status docker - # sudo usermod -aG docker ${USER} - - echo -e "\nVerifying that docker works ...\n" - docker run --rm hello-world - docker rmi hello-world - - echo -e "\nOpening port for minitwit ...\n" - ufw allow 5000 && \ - ufw allow 22/tcp - - echo ". $HOME/.bashrc" >> $HOME/.bash_profile - - echo -e "\nConfiguring credentials as environment variables...\n" - - source $HOME/.bash_profile - - echo -e "\nSelecting Minitwit Folder as default folder when you ssh into the server...\n" - echo "cd /csharp-minitwit" >> ~/.bash_profile - - chmod +x /csharp-minitwit/deploy.sh - - echo -e "\nVagrant setup done ..." - echo -e "csharp-minitwit will later be accessible at http://$(hostname -I | awk '{print $1}'):5000" - SHELL - end - end - - #export DOCKER_USERNAME=your_docker_username - #export DOCKER_PASSWORD=your_docker_password - #export PROVISION_TOKEN=your_do_token - # rm .vagrant - From 3ee6152c2aebe70af626aca4f4ef3cfa95bffc48 Mon Sep 17 00:00:00 2001 From: Mathias Gleitze Date: Wed, 22 May 2024 18:29:02 +0200 Subject: [PATCH 16/25] added secrets to .gitignore Add submodule and testing synchronization Testing synchronization Added initial workflow and changed .gitignore Report folder setup Testing report workflow Changes to report workflow Changes to report workflow Changes to workflow file Set working directory Print folder and file locations Getting there... Added a write permission Didn't work... Trying with deploy key Trying with submodules push Trying with github token Trying again with deploy key Trying with image for pushing to submodule Trying with image for pushing to submodule v2 Trying with image for pushing to submodule v2 again... Trying with submodule --update Trying with submodule --update Trial and error... Trial and error... Trial and error... Trial and error... Trial and error... Trial and error... Trial and error... Trial and error... Trial and error... Trial and error... Trial and error... Trial and error... Trial and error... Trial and error... removed test file removed test file Update develop branch Update develop branch Update develop branch Update develop branch Update develop branch Update develop branch Pushing to code repository using ssh... Pushing to code repository using ssh... Create pull request Create pull request Create pull request Create pull request Create pull request Create pull request Create pull request with ssh? Create pull request with ssh? Create pull request with ssh? Create pull request with ssh? Create pull request with ssh? Create pull request with ssh? Create pull request with evans workflow Create pull request with evans workflow Create pull request with evans workflow Create pull request with evans workflow Create pull request with evans workflow Create and push to branch using ssh Create and push to branch using ssh --- .github/workflows/report.yml | 76 ++++++++++++++++++++++++++++++++++++ .gitignore | 2 + .gitmodules | 3 ++ report | 1 + 4 files changed, 82 insertions(+) create mode 100644 .github/workflows/report.yml create mode 100644 .gitmodules create mode 160000 report diff --git a/.github/workflows/report.yml b/.github/workflows/report.yml new file mode 100644 index 0000000..4bacec3 --- /dev/null +++ b/.github/workflows/report.yml @@ -0,0 +1,76 @@ +name: Build Report + +on: + workflow_dispatch: + +permissions: + contents: write + +jobs: + build_latex: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: recursive + ssh-key: ${{ secrets.DEPLOY_KEY_FOR_REPORT_PRIVATE }} + + - name: Update submodules + run: | + git submodule update --init --recursive --remote + + - name: Set up LaTeX + uses: xu-cheng/latex-action@v3 + with: + working_directory: report + root_file: main.tex + + - name: Move main.pdf + run: | + cd report + mv -f main.pdf ./build + + - name: Setup SSH for pushing to report repository + run: | + mkdir -p ~/.ssh + echo "${{ secrets.DEPLOY_KEY_FOR_REPORT_PRIVATE }}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + ssh-keyscan github.com >> ~/.ssh/known_hosts + + - name: Commit and push main.pdf + run: | + cd report + + git config user.name "github-actions" + git config user.email "github-actions@github.com" + + git submodule update --remote + + git add build/main.pdf + git commit -m "Update report PDF" + + git remote set-url origin git@github.com:DevopsGroupC/report.git + GIT_SSH_COMMAND='ssh -i ~/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' git push -f origin main + + - name: Setup SSH for pushing to main repository + run: | + mkdir -p ~/.ssh + echo "${{ secrets.DEPLOY_KEY_FOR_MINITWIT_PRIVATE }}" > ~/.ssh/id_minitwit_rsa + chmod 600 ~/.ssh/id_minitwit_rsa + ssh-keyscan github.com >> ~/.ssh/known_hosts + + - name: Update submodule reference and push + run: | + git config user.name "github-actions" + git config user.email "github-actions@github.com" + + git fetch origin + git reset --hard origin/develop + + git submodule update --remote + + git checkout -b update-report-pdf + git add report + git commit -m "Update submodule reference to latest commit" + GIT_SSH_COMMAND='ssh -i ~/.ssh/id_minitwit_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' git push -f --set-upstream origin update-report-pdf \ No newline at end of file diff --git a/.gitignore b/.gitignore index 72c9c66..5ed5669 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ __pycache__/ .vagrant .Python build/ +!report/build/ develop-eggs/ dist/ downloads/ @@ -637,3 +638,4 @@ infrastructure/.terraform/* infrastructure/temp/* infrastructure/.terraform* infrastructure/grafana/ca_cert +infrastructure/secrets diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..2662f0a --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "report"] + path = report + url = https://github.com/DevopsGroupC/report diff --git a/report b/report new file mode 160000 index 0000000..0885296 --- /dev/null +++ b/report @@ -0,0 +1 @@ +Subproject commit 08852964b184a25200f580dec4293422c87cbfa8 From 189bf2514e940b288a2f92737e0d563667862959 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 22 May 2024 19:51:25 +0000 Subject: [PATCH 17/25] Update submodule reference to latest commit --- report | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/report b/report index 0885296..72614c4 160000 --- a/report +++ b/report @@ -1 +1 @@ -Subproject commit 08852964b184a25200f580dec4293422c87cbfa8 +Subproject commit 72614c42dcc9c66ee9182e3f4477263bcdcd0ee5 From 923c6741f260124dba6f97024b6ee8d9f7a54059 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 22 May 2024 20:00:57 +0000 Subject: [PATCH 18/25] Update submodule reference to latest commit --- report | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/report b/report index 72614c4..a990d11 160000 --- a/report +++ b/report @@ -1 +1 @@ -Subproject commit 72614c42dcc9c66ee9182e3f4477263bcdcd0ee5 +Subproject commit a990d11e0ba3d88adea6e18407a9b66c7788350a From 134a84e58efc914e3cfaf798282ba87d6f8f1f3c Mon Sep 17 00:00:00 2001 From: Mathias Gleitze Date: Thu, 23 May 2024 10:51:55 +0200 Subject: [PATCH 19/25] Removed vagrant related files --- vagrant.txt | 98 ----------------------------------------------------- 1 file changed, 98 deletions(-) delete mode 100644 vagrant.txt diff --git a/vagrant.txt b/vagrant.txt deleted file mode 100644 index 14a81c5..0000000 --- a/vagrant.txt +++ /dev/null @@ -1,98 +0,0 @@ -// REPLACE '********' where needed: - - -export DIGITAL_OCEAN_TOKEN='********' - -export DIGITALOCEAN_PRIVATE_NETWORKING=true -export DROPLETS_API="https://api.digitalocean.com/v2/droplets" -export BEARER_AUTH_TOKEN="Authorization: Bearer $DIGITAL_OCEAN_TOKEN" -export JSON_CONTENT="Content-Type: application/json" - - -CONFIG='{"name":"preproduction-swarm-manager","tags":["preproduction"], "region":"fra1", - "size":"s-1vcpu-1gb", "image":"docker-20-04", - "ssh_keys":["********'"]}' - -SWARM_MANAGER_ID=$(curl -X POST "$DROPLETS_API" -d "$CONFIG"\ - -H "$BEARER_AUTH_TOKEN" -H "$JSON_CONTENT"\ - | jq -r .droplet.id ) && sleep 5 && echo $SWARM_MANAGER_ID - - - -export JQFILTER='.droplets | .[] | select (.name == "preproduction-swarm-manager") - | .networks.v4 | .[]| select (.type == "public") | .ip_address' - - -SWARM_MANAGER_IP=$(curl -s GET "$DROPLETS_API"\ - -H "$BEARER_AUTH_TOKEN" -H "$JSON_CONTENT"\ - | jq -r "$JQFILTER") && echo "SWARM_MANAGER_IP=$SWARM_MANAGER_IP" - -// Creating worker nodes -// Worker1 -WORKER1_ID=$(curl -X POST "$DROPLETS_API"\ - -d'{"name":"preproduction-worker1","tags":["preproduction"],"region":"fra1", - "size":"s-1vcpu-1gb","image":"docker-20-04", - "ssh_keys":["********"]}'\ - -H "$BEARER_AUTH_TOKEN" -H "$JSON_CONTENT"\ - | jq -r .droplet.id )\ - && sleep 3 && echo $WORKER1_ID - -export JQFILTER='.droplets | .[] | select (.name == "preproduction-worker1") | .networks.v4 | .[]| select (.type == "public") | .ip_address' - - -WORKER1_IP=$(curl -s GET "$DROPLETS_API"\ - -H "$BEARER_AUTH_TOKEN" -H "$JSON_CONTENT"\ - | jq -r "$JQFILTER")\ - && echo "WORKER1_IP=$WORKER1_IP" - -// Worker2 -WORKER2_ID=$(curl -X POST "$DROPLETS_API"\ - -d'{"name":"preproduction-worker2","tags":["preproduction"],"region":"fra1", - "size":"s-1vcpu-1gb","image":"docker-20-04", - "ssh_keys":["********"]}'\ - -H "$BEARER_AUTH_TOKEN" -H "$JSON_CONTENT"\ - | jq -r .droplet.id )\ - && sleep 3 && echo $WORKER2_ID - -export JQFILTER='.droplets | .[] | select (.name == "preproduction-worker2") | .networks.v4 | .[]| select (.type == "public") | .ip_address' - - -WORKER2_IP=$(curl -s GET "$DROPLETS_API"\ - -H "$BEARER_AUTH_TOKEN" -H "$JSON_CONTENT"\ - | jq -r "$JQFILTER")\ - && echo "WORKER2_IP=$WORKER2_IP" - -// Openning ports on the firewall -ssh root@$SWARM_MANAGER_IP "ufw allow 22/tcp && ufw allow 2376/tcp &&\ -ufw allow 2377/tcp && ufw allow 7946/tcp && ufw allow 7946/udp &&\ -ufw allow 4789/udp && ufw reload && ufw --force enable &&\ -systemctl restart docker" - -ssh root@$WORKER1_IP "ufw allow 22/tcp && ufw allow 2376/tcp &&\ -ufw allow 2377/tcp && ufw allow 7946/tcp && ufw allow 7946/udp &&\ -ufw allow 4789/udp && ufw reload && ufw --force enable &&\ -systemctl restart docker" - -ssh root@$WORKER2_IP "ufw allow 22/tcp && ufw allow 2376/tcp &&\ -ufw allow 2377/tcp && ufw allow 7946/tcp && ufw allow 7946/udp &&\ -ufw allow 4789/udp && ufw reload && ufw --force enable &&\ -systemctl restart docker" - -ssh root@$SWARM_MANAGER_IP "docker swarm init --advertise-addr $SWARM_MANAGER_IP" - -WORKER_TOKEN=`ssh root@$SWARM_MANAGER_IP "docker swarm join-token worker -q"` -REMOTE_JOIN_CMD="docker swarm join --token $WORKER_TOKEN $SWARM_MANAGER_IP:2377" - -// Adding worker1 and worker2 to the swarm as workers -ssh root@$WORKER1_IP "$REMOTE_JOIN_CMD" -ssh root@$WORKER2_IP "$REMOTE_JOIN_CMD" - -// Check availability -ssh root@$SWARM_MANAGER_IP "docker node ls" - -export DOCKER_PASSWORD=dockerhub -export DOCKER_USERNAME=aswr -export ConnectionStrings__DefaultConnection='********' - - -docker service create -p 5000:8080 -e ConnectionStrings__DefaultConnection={$ConnectionStrings__DefaultConnection} --name minitwit aswr/csharptwit-preproduction \ No newline at end of file From 8b2d87eb707ecdf8fe5a0089bbd80d641cef13d1 Mon Sep 17 00:00:00 2001 From: Mathias Gleitze Date: Thu, 23 May 2024 11:32:03 +0200 Subject: [PATCH 20/25] Updated README.md to reflect the current state of the system --- README.md | 121 ++++++++++++++++++++---------------------------------- 1 file changed, 45 insertions(+), 76 deletions(-) diff --git a/README.md b/README.md index 139b749..40eb9b6 100644 --- a/README.md +++ b/README.md @@ -10,96 +10,64 @@
  • About The Project
  • - Running the project + Running the project
  • -
  • Acknowledgments
  • +
  • How to contribute
  • +
  • Acknowledgments
  • ## About The Project This project is part of the course [DevOps, Software Evolution and Software Maintenance, MSc (Spring 2024)](https://learnit.itu.dk/local/coursebase/view.php?ciid=1391) -The project currently consists of two folders: -- **itu-minitwit**, which is a small Twitter copy made in Python. +The project currently consists of three main folders: - **csharp-minitwit**, which is a ported version of itu-minitwit, made in C#. +- **infrastructure**, Containing shell and Terraform scripts for provisioning. +- **report**, Containing a report written in LaTex, generated as a pdf in the build folder. -### Built With -- .Net 8 -- ASP.NET -- SQLite -### Prerequisites - * A modern Python, i.e., version >= 3.10, - - There are various ways of installing it, either via the system's package manager (`apt search python3`) - - manually from [python.org](https://www.python.org/downloads/) - - via [pyenv](https://github.com/pyenv/pyenv) - - via [Anaconda](https://www.anaconda.com/products/individual) - * The Python dependencies from `itu-minitwit`: - - [Flask](https://flask.palletsprojects.com/en/3.0.x/) >= version 3.0.0 - - [Werkzeug](https://palletsprojects.com/p/werkzeug/) >= version 3.0.0 - - [Jinja2](https://palletsprojects.com/p/jinja/) >= version 3.0.0 - * A current C compiler, e.g., `gcc` - - Likely it is already part of your Linux installation. - - You can check that, for example with: - ```bash - gcc --version - ``` - - Since the `flag_tool` is a C program, you have to compile it again: - * `flag_tool` includes `sqlite3.h`, which is likely not yet installed on your system. - * It can be installed (together with the required shared library) via: - ```bash - sudo apt install libsqlite3-dev - ``` - * To work with the database, you likely need [`sqlite3`](https://sqlite.org/index.html) ((`apt search sqlite3`)) - - To inspect the database file while refactoring, you might want to use a tool to "look into" your database file, such as the [DB Browser for SQLite](https://sqlitebrowser.org/), which you can install via: - ```bash - sudo apt install sqlitebrowser - ``` +### The application +The 'csharp-minitwit' application is a miniature version of X (formerly known as Twitter). +### Prerequisites +* Linux. It is also possible to work on the project using MacOS, Windows has a few Git issues with the 'infrastructure/secrets:Zone.Identifier' file. * The [.NET 8.0 SDK](https://dotnet.microsoft.com/en-us/download/dotnet/8.0). There are multiple options for downloading depending on your operating system. If on linux, we recommend the [scripted install](https://learn.microsoft.com/en-us/dotnet/core/install/linux-scripted-manual#scripted-install). +* SQLite, multiple guides can be found online on how to install, this highly depends on which operating system is used. +* [Docker](https://docs.docker.com/engine/install/) +* Python with the pytest and request pip-packages installed (only needed for testing). -## Usage +## Running the project +This project can be run in two ways: -### itu-minitwit -This project can be run using a terminal. -cd into the correct folder: -```sh -cd itu-minitwit -``` -Run using `Python`: +### Run using docker (the prefered method): +Run using `docker` - runs only the app: ```sh -python minitwit.py +docker build -t csharp-minitwit . +docker run -p 5000:8080 csharp-minitwit ``` +At this point, the application can be accessed using the link provided in the terminal (http://localhost:5000). -#### itu-minitwit tests -Tests for `itu-minitwit` can be run by opening a new terminal and cd into correct folder: -```sh -cd itu-minitwit -``` -And running the tests using `Pytest`: +Run using `docker-compose` - runs multiple development services, such as Prometheus, Grafana, etc.: ```sh -pytest minitwit_tests.py +docker-compose up ``` -At this point, the application can be accessed using the link provided in the terminal (http://127.0.0.1:5000). -### csharp-minitwit -This project can be run using a terminal. +### Run using the dotnet runtime directly: +This firstly requires a few changes to the code. The default connection string points to a folder which will be generated in the docker container normally. +Change the connection string in 'appsettings.Development.json', from 'Data Source=/app/Databases/volume/minitwit.db' to 'Data Source=./Databases/volume/minitwit.db' ***DO NOT COMMIT THIS***. + cd into the correct folder: ```sh cd csharp-minitwit @@ -108,17 +76,8 @@ Run using `dotnet`: ```sh dotnet run ``` -Run using `docker` - runs only the app: -```sh -docker build -t csharp-minitwit . -docker run -p 5000:8080 csharp-minitwit -``` -At this point, the application can be accessed using the link provided in the terminal (http://localhost:5000). Furthermore, API documentation can be accessed at http://localhost:5000/swagger. -Run using `docker-compose` - runs multiple development services, such as Prometheus, Grafana, etc.: -```sh -docker-compose up -``` + ### Monitoring | Service | Endpoint | @@ -128,16 +87,26 @@ docker-compose up | Grafana | http://localhost:3000 | -#### csharp-minitwit tests -Tests for `csharp-minitwit` can be run by opening a new terminal and cd into the test folder: +### Testing +When developing APIs locally, Swagger is setup at http://localhost:5000/swagger/index.html for easy manual testing. + +Unit tests for `csharp-minitwit` can be run by opening a new terminal and cd into the test folder: ```sh cd csharp-minitwit/Tests ``` And running the tests using Pytest: ```sh pytest refactored_minitwit_tests.py +pytest minitwit_sim_api_test.py ``` +## How to contribute +For this repository we try to follow the [GitFlow](https://www.atlassian.com/git/tutorials/comparing-workflows/gitflow-workflow) workflow: +* Create an issue +* Create a branch using the GitHub issue tracker to ensure correct naming (remember to prefix the branch name with feature/{issue_name}) +* Develop feature +* Create a pull request + ## Acknowledgments -Shout out to Chatgpt for help with debugging. +Shout out to [ChatGPT](https://chatgpt.com/) for help with debugging. From 6a6ff1d3b174ef2ece89249555bb30c1d6bab63e Mon Sep 17 00:00:00 2001 From: Mathias Gleitze <95019126+Mathias-Gleitze@users.noreply.github.com> Date: Thu, 23 May 2024 11:40:39 +0200 Subject: [PATCH 21/25] Update report.yml --- .github/workflows/report.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/report.yml b/.github/workflows/report.yml index 4bacec3..61975a1 100644 --- a/.github/workflows/report.yml +++ b/.github/workflows/report.yml @@ -45,7 +45,7 @@ jobs: git config user.name "github-actions" git config user.email "github-actions@github.com" - git submodule update --remote + # git submodule update --remote git add build/main.pdf git commit -m "Update report PDF" @@ -73,4 +73,4 @@ jobs: git checkout -b update-report-pdf git add report git commit -m "Update submodule reference to latest commit" - GIT_SSH_COMMAND='ssh -i ~/.ssh/id_minitwit_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' git push -f --set-upstream origin update-report-pdf \ No newline at end of file + GIT_SSH_COMMAND='ssh -i ~/.ssh/id_minitwit_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' git push -f --set-upstream origin update-report-pdf From acfd2259e332273ae72acff1caccd0a53674c682 Mon Sep 17 00:00:00 2001 From: Mathias Gleitze <95019126+Mathias-Gleitze@users.noreply.github.com> Date: Thu, 23 May 2024 11:49:15 +0200 Subject: [PATCH 22/25] Update report.yml --- .github/workflows/report.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/report.yml b/.github/workflows/report.yml index 61975a1..b905a35 100644 --- a/.github/workflows/report.yml +++ b/.github/workflows/report.yml @@ -45,11 +45,20 @@ jobs: git config user.name "github-actions" git config user.email "github-actions@github.com" - # git submodule update --remote + Echo "git status:" + git status + + git submodule update --remote --rebase + Echo "git status after submodule update:" + git status + git add build/main.pdf git commit -m "Update report PDF" + Echo "git status after adding commit with new pdf:" + git status + git remote set-url origin git@github.com:DevopsGroupC/report.git GIT_SSH_COMMAND='ssh -i ~/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' git push -f origin main From ecc915f3241055e67d6071ef580cdedb81b4b7ca Mon Sep 17 00:00:00 2001 From: Mathias Gleitze <95019126+Mathias-Gleitze@users.noreply.github.com> Date: Thu, 23 May 2024 11:52:58 +0200 Subject: [PATCH 23/25] Update report.yml --- .github/workflows/report.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/report.yml b/.github/workflows/report.yml index b905a35..e95711e 100644 --- a/.github/workflows/report.yml +++ b/.github/workflows/report.yml @@ -45,18 +45,18 @@ jobs: git config user.name "github-actions" git config user.email "github-actions@github.com" - Echo "git status:" + echo "git status:" git status git submodule update --remote --rebase - Echo "git status after submodule update:" + echo "git status after submodule update:" git status git add build/main.pdf git commit -m "Update report PDF" - Echo "git status after adding commit with new pdf:" + echo "git status after adding commit with new pdf:" git status git remote set-url origin git@github.com:DevopsGroupC/report.git From 5fc3d7a232a09694169319a9d3681e986ae8ac3d Mon Sep 17 00:00:00 2001 From: Mathias Gleitze <95019126+Mathias-Gleitze@users.noreply.github.com> Date: Thu, 23 May 2024 11:59:48 +0200 Subject: [PATCH 24/25] Update report.yml --- .github/workflows/report.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/report.yml b/.github/workflows/report.yml index e95711e..ad096ae 100644 --- a/.github/workflows/report.yml +++ b/.github/workflows/report.yml @@ -47,10 +47,11 @@ jobs: echo "git status:" git status - - git submodule update --remote --rebase - echo "git status after submodule update:" + git checkout main + git pull origin main + + echo "git status after pull:" git status git add build/main.pdf From dfe9d2f102cf95237e012fc637eef0f7af3672b1 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 23 May 2024 10:02:30 +0000 Subject: [PATCH 25/25] Update submodule reference to latest commit --- report | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/report b/report index a990d11..0947c61 160000 --- a/report +++ b/report @@ -1 +1 @@ -Subproject commit a990d11e0ba3d88adea6e18407a9b66c7788350a +Subproject commit 0947c614690f7c2e756f9787f184b03e4679247f