From 41455d52d61f44501b76b5e10d24c96a9ad0bf0e Mon Sep 17 00:00:00 2001
From: Rub21 <rub2106@gmail.com>
Date: Wed, 10 Jul 2024 18:36:20 -0500
Subject: [PATCH] Update s3 tiler - cache staging

Add resource limits for tiler - cache staging

Update script to evaluate resource usage

Rename file rm_tegola_ps

Remove time ffor tiler-cache

Update script

Evalute cpu usage tiler-cache
---
 images/tiler-server/Dockerfile       | 14 +++++++------
 images/tiler-server/cache_cleaner.sh |  2 +-
 images/tiler-server/rm_ps.sh         | 27 -------------------------
 images/tiler-server/rm_tegola_ps.sh  | 30 ++++++++++++++++++++++++++++
 images/tiler-server/seed-by-diffs.sh |  2 +-
 values.staging.template.yaml         |  8 ++++----
 6 files changed, 44 insertions(+), 39 deletions(-)
 delete mode 100755 images/tiler-server/rm_ps.sh
 create mode 100755 images/tiler-server/rm_tegola_ps.sh

diff --git a/images/tiler-server/Dockerfile b/images/tiler-server/Dockerfile
index bd8cbcb0..e8817b3f 100644
--- a/images/tiler-server/Dockerfile
+++ b/images/tiler-server/Dockerfile
@@ -14,7 +14,8 @@ RUN apk --no-cache add \
     git \
     postgresql-client \
     coreutils \
-    jq
+    jq \
+    procps
 
 RUN pip install --upgrade pip && pip3 install awscli mercantile
 
@@ -24,21 +25,22 @@ RUN rm -rf /var/cache/apk/* \
 
 RUN ln -s /opt/tegola /usr/bin/tegola
 
-COPY ./config  /opt/config/
-COPY build_config.py  /opt/
+COPY ./config /opt/config/
+COPY build_config.py /opt/
 RUN mkdir /opt/tegola_config
 RUN cd /opt/ && python build_config.py \
     --template config/config.template.toml \
     --providers config/providers \
     --output /opt/tegola_config/config.toml
 
-# Copy config and  exec files
+# Copy config and exec files
 COPY ./tile2bounds.py .
 COPY ./start.sh .
 COPY ./expire-watcher.sh .
 COPY ./seed-by-diffs.sh .
 COPY ./tile_cache_downloader.sh .
 COPY ./cache_cleaner.sh .
-COPY ./rm_ps.sh .
+COPY ./rm_tegola_ps.sh .
+
 ENTRYPOINT ["/bin/bash", "-c"]
-CMD ./start.sh & ./tile_cache_downloader.sh & ./expire-watcher.sh
\ No newline at end of file
+CMD ./start.sh & ./tile_cache_downloader.sh & ./expire-watcher.sh
diff --git a/images/tiler-server/cache_cleaner.sh b/images/tiler-server/cache_cleaner.sh
index 68bb3a16..9ea8c42d 100755
--- a/images/tiler-server/cache_cleaner.sh
+++ b/images/tiler-server/cache_cleaner.sh
@@ -3,5 +3,5 @@ flag=true
 while "$flag" = true; do
   pg_isready -h $POSTGRES_HOST -p 5432 >/dev/null 2>&2 || continue
   flag=false
-  ./tile_cache_downloader.sh & ./expire-watcher.sh & ./rm_ps.sh
+  ./tile_cache_downloader.sh & ./expire-watcher.sh & ./rm_tegola_ps.sh
 done
diff --git a/images/tiler-server/rm_ps.sh b/images/tiler-server/rm_ps.sh
deleted file mode 100755
index 7465a4c8..00000000
--- a/images/tiler-server/rm_ps.sh
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/bin/bash
-set -e
-
-PROCESS_NAME="tegola"
-MAX_RUNTIME_SECONDS=300 
-SLEEP_INTERVAL=60
-
-calculate_elapsed_seconds() {
-    local start_time=$1
-    local current_time=$(date +%s)
-    echo $((current_time - start_time))
-}
-
-if [[ -n "${KILL_PROCESS}" && "${KILL_PROCESS}" == "manually" ]]; then
-    while true; do
-        for pid in $(pgrep -f ${PROCESS_NAME}); do
-            start_time=$(stat -c %Y /proc/$pid)
-            elapsed_seconds=$(calculate_elapsed_seconds $start_time)
-            if [[ $elapsed_seconds -gt $MAX_RUNTIME_SECONDS ]]; then
-                echo "The process ${PROCESS_NAME} with PID $pid has been running for $elapsed_seconds seconds"
-                kill -9 $pid
-                aws s3 rm s3://${TILER_CACHE_BUCKET}/mnt/data/osm/ --recursive
-            fi
-        done
-        sleep $SLEEP_INTERVAL
-    done
-fi
\ No newline at end of file
diff --git a/images/tiler-server/rm_tegola_ps.sh b/images/tiler-server/rm_tegola_ps.sh
new file mode 100755
index 00000000..fcdb37a5
--- /dev/null
+++ b/images/tiler-server/rm_tegola_ps.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+set -e
+PROCESS_NAME="tegola"
+SLEEP_INTERVAL=20
+CPU_USAGE_THRESHOLD=70
+
+check_total_cpu_usage() {
+    local total_cpu_usage=0
+    for cpu in $(ps -eo comm,pcpu | grep ${PROCESS_NAME} | awk '{print $2}'); do
+        total_cpu_usage=$(echo "$total_cpu_usage + $cpu" | bc)
+    done
+    echo $total_cpu_usage
+}
+
+while true; do
+    total_cpu_usage=$(check_total_cpu_usage)
+    total_cpu_usage=${total_cpu_usage%.*}
+    echo "Total cpu usage: $total_cpu_usage"
+    if [[ $total_cpu_usage -gt $CPU_USAGE_THRESHOLD ]]; then
+        echo "Total CPU usage of ${PROCESS_NAME} processes is ${total_cpu_usage}%, which is greater than the threshold of ${CPU_USAGE_THRESHOLD}%."
+        
+        echo "Terminating all ${PROCESS_NAME} processes..."
+        killall -9 ${PROCESS_NAME}
+
+        echo "Manually clearing S3 cache..."
+        aws s3 rm s3://${TILER_CACHE_BUCKET}/mnt/data/osm/ --recursive
+    fi
+
+    sleep $SLEEP_INTERVAL
+done
\ No newline at end of file
diff --git a/images/tiler-server/seed-by-diffs.sh b/images/tiler-server/seed-by-diffs.sh
index f74e64d7..869c5b47 100755
--- a/images/tiler-server/seed-by-diffs.sh
+++ b/images/tiler-server/seed-by-diffs.sh
@@ -30,7 +30,7 @@ fi
 
 for f in $imp_list; do
     # echo "Purge tiles from...$f"
-    ( set -x ; time tegola cache purge tile-list $f \
+    ( set -x ; tegola cache purge tile-list $f \
         --config=/opt/tegola_config/config.toml \
         --format="/zxy" \
         --min-zoom=0 \
diff --git a/values.staging.template.yaml b/values.staging.template.yaml
index 67c234ec..39a7465b 100644
--- a/values.staging.template.yaml
+++ b/values.staging.template.yaml
@@ -529,13 +529,13 @@ osm-seed:
       replicaCount: 1
       command: './cache_cleaner.sh'
       resources:
-        enabled: false
+        enabled: true
         requests:
           memory: "2Gi"
-          cpu: "1"
+          cpu: "500m"
         limits:
-          memory: "10Gi"
-          cpu: "2"
+          memory: "2Gi"
+          cpu: "800m"
       env:
         KILL_PROCESS: manually
         MAX_NUM_PS: 4