Update s3 tiler - cache staging

Add resource limits for tiler - cache staging

Update script to evaluate resource usage

Rename file rm_tegola_ps

Remove time ffor tiler-cache

Update script

Evalute cpu usage tiler-cache

images/tiler-server/Dockerfile

@@ -14,7 +14,8 @@ RUN apk --no-cache add \
     git \
     postgresql-client \
     coreutils \
-    jq
+    jq \
+    procps
 
 RUN pip install --upgrade pip && pip3 install awscli mercantile
 
@@ -24,21 +25,22 @@ RUN rm -rf /var/cache/apk/* \
 
 RUN ln -s /opt/tegola /usr/bin/tegola
 
-COPY ./config  /opt/config/
-COPY build_config.py  /opt/
+COPY ./config /opt/config/
+COPY build_config.py /opt/
 RUN mkdir /opt/tegola_config
 RUN cd /opt/ && python build_config.py \
     --template config/config.template.toml \
     --providers config/providers \
     --output /opt/tegola_config/config.toml
 
-# Copy config and  exec files
+# Copy config and exec files
 COPY ./tile2bounds.py .
 COPY ./start.sh .
 COPY ./expire-watcher.sh .
 COPY ./seed-by-diffs.sh .
 COPY ./tile_cache_downloader.sh .
 COPY ./cache_cleaner.sh .
-COPY ./rm_ps.sh .
+COPY ./rm_tegola_ps.sh .
+
 ENTRYPOINT ["/bin/bash", "-c"]
-CMD ./start.sh & ./tile_cache_downloader.sh & ./expire-watcher.sh
+CMD ./start.sh & ./tile_cache_downloader.sh & ./expire-watcher.sh

images/tiler-server/cache_cleaner.sh

@@ -3,5 +3,5 @@ flag=true
 while "$flag" = true; do
   pg_isready -h $POSTGRES_HOST -p 5432 >/dev/null 2>&2 || continue
   flag=false
-  ./tile_cache_downloader.sh & ./expire-watcher.sh & ./rm_ps.sh
+  ./tile_cache_downloader.sh & ./expire-watcher.sh & ./rm_tegola_ps.sh
 done

images/tiler-server/rm_tegola_ps.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+set -e
+PROCESS_NAME="tegola"
+SLEEP_INTERVAL=20
+CPU_USAGE_THRESHOLD=70
+
+check_total_cpu_usage() {
+    local total_cpu_usage=0
+    for cpu in $(ps -eo comm,pcpu | grep ${PROCESS_NAME} | awk '{print $2}'); do
+        total_cpu_usage=$(echo "$total_cpu_usage + $cpu" | bc)
+    done
+    echo $total_cpu_usage
+}
+
+while true; do
+    total_cpu_usage=$(check_total_cpu_usage)
+    total_cpu_usage=${total_cpu_usage%.*}
+    echo "Total cpu usage: $total_cpu_usage"
+    if [[ $total_cpu_usage -gt $CPU_USAGE_THRESHOLD ]]; then
+        echo "Total CPU usage of ${PROCESS_NAME} processes is ${total_cpu_usage}%, which is greater than the threshold of ${CPU_USAGE_THRESHOLD}%."
+
+        echo "Terminating all ${PROCESS_NAME} processes..."
+        killall -9 ${PROCESS_NAME}
+
+        echo "Manually clearing S3 cache..."
+        aws s3 rm s3://${TILER_CACHE_BUCKET}/mnt/data/osm/ --recursive
+    fi
+
+    sleep $SLEEP_INTERVAL
+done

images/tiler-server/seed-by-diffs.sh

@@ -30,7 +30,7 @@ fi
 
 for f in $imp_list; do
     # echo "Purge tiles from...$f"
-    ( set -x ; time tegola cache purge tile-list $f \
+    ( set -x ; tegola cache purge tile-list $f \
         --config=/opt/tegola_config/config.toml \
         --format="/zxy" \
         --min-zoom=0 \

values.staging.template.yaml

@@ -529,13 +529,13 @@ osm-seed:
       replicaCount: 1
       command: './cache_cleaner.sh'
       resources:
-        enabled: false
+        enabled: true
         requests:
           memory: "2Gi"
-          cpu: "1"
+          cpu: "500m"
         limits:
-          memory: "10Gi"
-          cpu: "2"
+          memory: "2Gi"
+          cpu: "800m"
       env:
         KILL_PROCESS: manually
         MAX_NUM_PS: 4