Release

Dockerfile

@@ -2,7 +2,7 @@ FROM bitnami/python
 
 # Install goaccess tool
 RUN apt-get update && \
- apt-get install nano goaccess -y
+ apt-get install vim libncursesw5-dev libmaxminddb-dev -y
 
 # Install aws-cli
 RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
@@ -13,13 +13,19 @@ RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2
 # Verify the installation
 RUN aws --version
 
+RUN curl "https://tar.goaccess.io/goaccess-1.5.5.tar.gz" -o "goaccess-1.5.5.tar.gz" && \
+ tar -xzvf goaccess-1.5.5.tar.gz && \
+ cd goaccess-1.5.5/ && \
+ ./configure --enable-utf8 --enable-geoip=mmdb --with-openssl && \ 
+ make && \
+ make install
+
 # Install python dependencies
 COPY requirements.txt /tmp/
 RUN pip3 install -r /tmp/requirements.txt
 
 # Create the directory structure for the metrics
-RUN mkdir -p /app/metrics/goaccess_outputs/filtered_by_status_code && \
- mkdir -p /app/metrics/rawmetrics && \
+RUN mkdir -p /app/metrics/rawmetrics/filtered_by_status_code && \
  mkdir -p /app/metrics/goaccess-metrics && \
  mkdir -p /app/metrics/prometheus-metrics && \
  mkdir -p /app/ui && \
@@ -35,4 +41,4 @@ COPY metrics_prom.py /app/
 COPY create_index.py /app/
 
 COPY dbip-country-lite-2024.mmdb /app/
-COPY goaccess.conf /etc/goaccess/
+COPY goaccess.conf /usr/local/etc/goaccess/goaccess.conf 

create_index.py

@@ -3,7 +3,6 @@
 cluster_id = os.getenv("CLUSTER_ID")
 # Configuration
 folder_path = '/app/metrics'
-s3_path=f'https://s3.amazonaws.com/metrics.oscar.grycap.net/{cluster_id}/'
 assets_base_url = 'https://s3.amazonaws.com/metrics.oscar.grycap.net/assets' # Local path to assets
 
 OUT_PATH="/app/ui/"
@@ -33,7 +32,7 @@
  <div class="mt-48 page-full-wrap relative ">
  <input type="hidden" value="creator-page" id="page-type">
  <img class="display-image m-auto" data-src="{assets_base_url}/images/logo.png" src="{assets_base_url}/images/logo.png" alt="OSCAR metrics"/>
- <h2 class="page-title page-text-color page-text-font mt-24 text-center text-fs-18">OSCAR-ai4eosc metrics index</h2>
+ <h2 class="page-title page-text-color page-text-font mt-24 text-center text-fs-18">{cluster_id} metrics index"</h2>
 
  <div class="mt-24">
 """
@@ -66,23 +65,26 @@ def get_icon(file_name):
  else:
  return f"{assets_base_url}/images/file.png"
 
-def generate_html(out_file, dir_path):
+def generate_html(out_file, dir_path, out_url=''):
  # Generate HTML content
  html_content = html_header
 
  for i, file_name in enumerate(os.listdir(dir_path)):
  file_path = os.path.join(dir_path, file_name)
  if os.path.isfile(file_path) or os.path.isdir(file_path):
- #file_url = file_path.replace("\\", "/")
  if os.path.isdir(file_path):
  relative_url=file_name+".html"
- generate_html(OUT_PATH+relative_url, file_path)
- file_url = s3_path+relative_url
+ generate_html(OUT_PATH+relative_url, file_path, out_url+file_name+"/")
+ file_url = relative_url
  else:
- file_url = s3_path+file_name
+ file_url = out_url+file_name
  icon = get_icon(file_name)
  if "dashboard" in file_name:
  file_name = "GoAccess Dashboard"
+ if "goaccess-metrics" in file_name:
+ file_name = "GoAccess Metrics"
+ if "prometheus-metrics" in file_name:
+ file_name = "Prometheus Metrics" 
  file_entry = html_file_entry_template.format(url=file_url, id=i, icon=icon, filename=file_name)
  html_content += file_entry
 

extract_goaccess_metrics.sh

@@ -1,57 +1,45 @@
 # Container paths
 OUTPUTS_PATH="/app/metrics/rawmetrics"
 FILTERED_PATH="$OUTPUTS_PATH/filtered_by_status_code"
-OUTPUT_FILE="output"
-
+OUTPUT_FILE="output_OSCAR_goaccess"
 # Path to the readonly volume with the cluster's ingress logs
 CLUSTER_LOGS_DIR="/var/log/clusterlogs"
 LOCAL_LOGS_DIR="/var/log/ingresslogs"
 OSCAR_LOGS_DIR="$LOCAL_LOGS_DIR/oscar"
 
-FULL_REPORT_FILE="$OSCAR_LOGS_DIR/oscar.log"
+HISTORY_LOGS="$OSCAR_LOGS_DIR/oscar.log"
+LATEST_LOGS="$OSCAR_LOGS_DIR/latest_oscar.log"
 mkdir -p $OSCAR_LOGS_DIR
 
 # Log format for goaccess
-LOG_FORMAT='%^ %^ %^ %h - - [%d:%t] %~ %m %U %^ %s %b %R %u %^'
+LOG_FORMAT='%^ %^ %^ %h - - [%d:%t] %~ %m %U %^ %s %^ %R %^ %^'
 
-metrics(){
- oscar_logfile="oscar_execlog_"
+addLog(){
  ingress_logfile=$1
- metric_date=$(echo $ingress_logfile | awk -F. '{print $3}')
-
- if [ -z $metric_date ]; then
- metric_date="current_"$(date +"%s")
- fi
-
- # Filter the ingress logs to get the OSCAR ones
- oscar_logfile=$oscar_logfile$metric_date.log
- cat $ingress_logfile | grep -a 'oscar-oscar' | grep -a '/job\|/run' > $OSCAR_LOGS_DIR/$oscar_logfile
- # Ensure the filtered log file is created and not empty 
- if [ ! -f "$OSCAR_LOGS_DIR/$oscar_logfile" ] || [ ! -s "$OSCAR_LOGS_DIR/$oscar_logfile" ]; then
- echo "[*] Warning: Failed to create log file or no OSCAR execution logs found for $OSCAR_LOGS_DIR/$oscar_logfile"
- return
- fi
-
- cat $OSCAR_LOGS_DIR/$oscar_logfile | tee -a $FULL_REPORT_FILE >/dev/null
+ cat $ingress_logfile | grep -a 'oscar-oscar' | grep -a '/job\|/run' | tee -a $HISTORY_LOGS >/dev/null
+}
 
- geo_err=$( { cat "$OSCAR_LOGS_DIR/$oscar_logfile" | goaccess - --log-format="${LOG_FORMAT}" -o "${OUTPUTS_PATH}/${OUTPUT_FILE}_${oscar_logfile}.json" --json-pretty-print; } 2>&1 )
- if [ ! -f "${OUTPUTS_PATH}/${OUTPUT_FILE}_${oscar_logfile}.json" ]; then
- echo "[*] Warning: Couldn't process file $oscar_logfile"
+metrics(){
+ LOG_FILE=$1
+ filename=`basename "$LOG_FILE"`
+ geo_err=$( { goaccess "${LOG_FILE}" --log-format="${LOG_FORMAT}" -o "${OUTPUTS_PATH}/${filename}_full.json" --json-pretty-print; } 2>&1 )
+ if [[ $filename == "latest"* ]]; then
+ python3 goaccess_metric_parser.py -f "${OUTPUTS_PATH}/${filename}_full.json" -g 0
  else
- python3 goaccess_metric_parser.py -f "${OUTPUTS_PATH}/${OUTPUT_FILE}_${oscar_logfile}.json" -g 0
+ python3 goaccess_metric_parser.py -f "${OUTPUTS_PATH}/${filename}_full.json" -g 0 -u
  fi
-
+ 
  status_codes=('200' '204' '404' '500')
  init="t"
 
- out="${FILTERED_PATH}/${OUTPUT_FILE}_${oscar_logfile}"
+ out="${FILTERED_PATH}/${filename}"
 
  for code in "${status_codes[@]}"; do
- code_logs=$(cat $OSCAR_LOGS_DIR/$oscar_logfile | grep -e 'HTTP/[0-9].[0-9]" '${code}' ')
+ code_logs=$(cat $LOG_FILE| grep -e 'HTTP/[0-9].[0-9]" '${code}' ')
  if [ ! -z "$code_logs" ]; then
- app_err=$( { cat $OSCAR_LOGS_DIR/$oscar_logfile | grep -e 'HTTP/[0-9].[0-9]" '${code}' ' | goaccess - -o "${out}_f${code}.json" --json-pretty-print --log-format="${LOG_FORMAT}" --enable-panel=REQUESTS --enable-panel=STATUS_CODES; } 2>&1 )
+ app_err=$( { cat $LOG_FILE | grep -e 'HTTP/[0-9].[0-9]" '${code}' ' | goaccess - -o "${out}_f${code}.json" --json-pretty-print --log-format="${LOG_FORMAT}"; } 2>&1 )
  if [ ! -f "${out}_f${code}.json" ]; then
- echo "[*] Warning: Couldn't process file $oscar_logfile for status code '$code'"
+ echo "[*] Warning: Couldn't process file $LOG_FILE for status code '$code'"
  else
  if [ $init == 't' ]; then
  python3 goaccess_metric_parser.py -f "${out}_f${code}.json" -p $code
@@ -73,6 +61,8 @@ do
  for logfile in "$LOCAL_LOGS_DIR/$log/controller/"*;
  do
  if [[ $logfile == *".gz" ]]; then
+ # upload a backup of the ingress logs to s3
+ aws s3 cp $logfile s3://metrics.oscar.grycap.net/"${CLUSTER_ID}"/ingresslogs/
  # unzip all log files
  gzip -d $logfile
  fi
@@ -84,13 +74,22 @@ done
 for logfile in "$LOCAL_LOGS_DIR/$log/controller/"*;
 do
  if [[ $logfile == *".log"* ]]; then
- metrics $logfile
+ if [[ $logfile == *".log" ]]; then
+ aws s3 cp $logfile s3://metrics.oscar.grycap.net/"${CLUSTER_ID}"/ingresslogs/
+ cat $logfile | grep -a 'oscar-oscar' | grep -a '/job\|/run' | tee -a $LATEST_LOGS >/dev/null
+ metrics $LATEST_LOGS
+ else
+ addLog $logfile
+ fi
  fi
 done
 
 # Generate the html file
-if [ ! -f "${FULL_REPORT_FILE}" ] || [ ! -s "${FULL_REPORT_FILE}" ]; then
- echo "Error: Failed to create html report."
- exit 1
+if [ ! -f "${HISTORY_LOGS}" ] || [ ! -s "${HISTORY_LOGS}" ]; then
+ goaccess "${LATEST_LOGS}" --log-format="${LOG_FORMAT}" -o "/app/metrics/dashboard.html"
+else
+ metrics $HISTORY_LOGS
+
+ cat $LATEST_LOGS | tee -a $HISTORY_LOGS >/dev/null
+ goaccess "${HISTORY_LOGS}" --log-format="${LOG_FORMAT}" -o "/app/metrics/dashboard.html"
 fi
-goaccess "${FULL_REPORT_FILE}" --log-format="${LOG_FORMAT}" -o "/app/metrics/dashboard.html"

goaccess.conf

@@ -154,11 +154,11 @@ hl-header true
 # The parser will only parse the value next to `html-prefs` (single line)
 # It allows the ability to customize each panel plot. See example below.
 #
-#html-prefs {"theme":"bright","perPage":5,"layout":"horizontal","showTables":true,"visitors":{"plot":{"chartType":"bar"}}}
+html-prefs {"theme":"dark purple","perPage":5,"layout":"horizontal","showTables":true,"requests":{"title":"Inference requests","plot":{"chartType":"bar"}},"hosts":{"title":"Reached countries","plot":{"chartType":"null"}}}
 
 # Set HTML report page title and header.
 #
-#html-report-title My Awesome Web Stats
+#html-report-title OSCAR Metrics
 
 # Format JSON output using tabs and newlines.
 #
@@ -449,7 +449,7 @@ double-decode false
 # Enable parsing/displaying the given panel.
 #
 #enable-panel VISITORS
-#enable-panel REQUESTS
+enable-panel REQUESTS
 #enable-panel REQUESTS_STATIC
 #enable-panel NOT_FOUND
 #enable-panel HOSTS
@@ -499,19 +499,20 @@ crawlers-only false
 
 # Ignore parsing and displaying the given panel.
 #
-#ignore-panel VISITORS
+ignore-panel VISITORS
 #ignore-panel REQUESTS
-#ignore-panel REQUESTS_STATIC
-#ignore-panel NOT_FOUND
+ignore-panel REQUESTS_STATIC
+ignore-panel NOT_FOUND
 #ignore-panel HOSTS
-#ignore-panel OS
-#ignore-panel BROWSERS
-#ignore-panel VISIT_TIMES
+ignore-panel OS
+ignore-panel ASN
+ignore-panel BROWSERS
+ignore-panel VISIT_TIMES
 #ignore-panel VIRTUAL_HOSTS
 ignore-panel REFERRERS
-#ignore-panel REFERRING_SITES
+ignore-panel REFERRING_SITES
 ignore-panel KEYPHRASES
-#ignore-panel STATUS_CODES
+ignore-panel STATUS_CODES
 #ignore-panel REMOTE_USER
 #ignore-panel CACHE_STATUS
 #ignore-panel GEO_LOCATION

goaccess_metric_parser.py

@@ -26,16 +26,20 @@
 
 with open(args.file_path, 'r') as rawfile:
  metrics = json.loads(rawfile.read())
- START_DATE = metrics["general"]["start_date"]
- END_DATE = metrics["general"]["end_date"]
+ try:
+ START_DATE = metrics["general"]["start_date"]
+ END_DATE = metrics["general"]["end_date"]
+ except:
+ START_DATE = metrics["general"]["date_time"]
+ END_DATE = metrics["general"]["date_time"]
+
 
 """
  > Countries reached 
  > Output format: {continent, country, total_visits, unique_visits, start_date, end_date}
 """
-def parse_geolocation_info():
-
- with open(f'{OUTPUT_PATH}/{TIMESTAMP}_geolocation_metrics.csv', 'w', newline='') as gfile:
+def parse_geolocation_info(write_type):
+ with open(f'{OUTPUT_PATH}/geolocation_metrics.csv', write_type, newline='') as gfile:
  writer = csv.writer(gfile)
  fields = ["continent", "country", "total_visits", "unique_visits", "start_metric_date", "end_metric_date"]
  writer.writerow(fields)
@@ -80,22 +84,22 @@ def parse_requests_info(status_code, write_type):
  exec_count+=sum_requests
 
  if create_count != 0:
- with open(f'{OUTPUT_PATH}/{TIMESTAMP}_created_apps_metrics.csv', write_type, newline='') as cfile:
+ with open(f'{OUTPUT_PATH}/created_apps_metrics.csv', write_type, newline='') as cfile:
  writer = csv.writer(cfile)
  if write_type == "w": writer.writerow(["application_count", "status_code", "start_metric_date", "end_metric_date"])
  writer.writerow([create_count, status_code, START_DATE, END_DATE])
 
  cfile.close()
 
  if exec_count != 0:
- with open(f'{OUTPUT_PATH}/{TIMESTAMP}_total_inference_metrics.csv', write_type, newline='') as efile:
+ with open(f'{OUTPUT_PATH}/total_inference_metrics.csv', write_type, newline='') as efile:
  writer = csv.writer(efile)
  if write_type == "w": writer.writerow(["inference_count", "status_code", "start_metric_date", "end_metric_date"])
  writer.writerow([exec_count, status_code, START_DATE, END_DATE])
 
  efile.close()
 
- with open(f'{OUTPUT_PATH}/{TIMESTAMP}_services_inference_metrics.csv', write_type, newline='') as sfile:
+ with open(f'{OUTPUT_PATH}/services_inference_metrics.csv', write_type, newline='') as sfile:
  writer = csv.writer(sfile)
  if write_type == "w": writer.writerow(["service_name", "exec_type", "status_code", "inference_count" , "start_metric_date", "end_metric_date"])
  for k in inference.keys():
@@ -105,17 +109,11 @@ def parse_requests_info(status_code, write_type):
  sfile.close() 
 
 
-if args.general:
- parse_geolocation_info()
-elif args.partial:
- if args.use_existing:
- parse_requests_info(args.status_code, "a")
- else:
- parse_requests_info(args.status_code, "w")
-
-
-
-
-
-
+wr="w"
+if args.use_existing:
+ wr="a"
 
+if args.general:
+ parse_geolocation_info(wr)
+if args.partial:
+ parse_requests_info(args.status_code, wr)

metrics_prom.py

@@ -1,13 +1,15 @@
-from datetime import date
 import time
 import requests
 import json
 import csv
 import argparse
+from datetime import datetime, timedelta
 from oscar_python.client import Client
 
 QUERY_ENDPOINT = "/api/v1/query?query="
 TIME = "5d"
+END_DATE = datetime.today()
+START_DATE = END_DATE - timedelta(days=5)
 
 parser = argparse.ArgumentParser(description="Command-line to retreive Prometheus metrics from OSCAR", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 
@@ -55,12 +57,12 @@ def query(cpu_usage_query):
  return json.loads(response.text)
 
 def generate_file_name():
- return f"/app/metrics/prometheus-metrics/metric-{str(int(time.time()))}.csv"
+ return f"/app/metrics/prometheus-metrics/metric-{END_DATE}.csv"
 
 def extract_metrics(cluster_services):
  with open(generate_file_name(), 'w', newline='') as file:
  writer = csv.writer(file)
- fields = ["service_name", "pod_name", "cpu_usage_seconds", "vo"]
+ fields = ["service_name", "pod_name", "cpu_usage_seconds", "vo", "start_date", "end_date"]
  writer.writerow(fields)
 
  for svc in cluster_services:
@@ -80,7 +82,7 @@ def extract_metrics(cluster_services):
  pod_name = m["metric"]["pod"]
  if k in pod_name:
  value = m["value"][1]
- writer.writerow([svc_name,pod_name, value, svc_vo])
+ writer.writerow([svc_name,pod_name, value, svc_vo, START_DATE, END_DATE])
  else:
  if "expose" in svc_name:
  cpu_usage_query = get_exposed_query(svc_name)
@@ -92,7 +94,7 @@ def extract_metrics(cluster_services):
  for m in metrics:
  #pod_name = m["metric"]["pod"]
  value = m["value"][1]
- writer.writerow([svc_name,svc_name, value, svc_vo])
+ writer.writerow([svc_name,svc_name, value, svc_vo, START_DATE, END_DATE])
 
 ######## MAIN ########## 
 if __name__ == "__main__":