Changed output format

extract_logs_metrics.sh

@@ -47,10 +47,10 @@ metrics(){
  echo "[*] Warning: Couldn't process file $LOG_FILE for status code '$code'"
  else
  if [ $init == 't' ]; then
- python3 logs_metric_parser.py -f "${out}_f${code}.json" -p $code
+ python3 logs_metric_parser.py -f "${out}_f${code}.json" -a $code
  init="f"
  else
- python3 logs_metric_parser.py -f "${out}_f${code}.json" -p $code -u
+ python3 logs_metric_parser.py -f "${out}_f${code}.json" -a $code -u
  fi
  fi
  fi
@@ -93,14 +93,18 @@ done
 # Generate the html file
 if [ ! -f "${HISTORY_LOGS_INFERENCE}" ] || [ ! -s "${HISTORY_LOGS_INFERENCE}" ]; then
  goaccess "${LATEST_LOGS_INFERENCE}" --log-format="${LOG_FORMAT}" -o "/app/metrics/dashboard.html"
+ python3 logs_metric_parser.py -f $LATEST_LOGS_INFERENCE -i 0
 else
  metrics $HISTORY_LOGS_INFERENCE
 
  cat $LATEST_LOGS_INFERENCE | tee -a $HISTORY_LOGS_INFERENCE >/dev/null
+ python3 logs_metric_parser.py -f $HISTORY_LOGS_INFERENCE -i 0 -u
  goaccess "${HISTORY_LOGS_INFERENCE}" --log-format="${LOG_FORMAT}" -o "/app/metrics/dashboard.html"
+
 fi
 
 if [ ! -f "${HISTORY_LOGS_CREATE}" ] || [ ! -s "${HISTORY_LOGS_CREATE}" ]; then
- python3 logs_metric_parser.py -f $LATEST_LOGS_CREATE -c
+ python3 logs_metric_parser.py -f $LATEST_LOGS_CREATE -c 0
 else
- python3 logs_metric_parser.py -f $HISTORY_LOGS_CREATE -c -u
+ python3 logs_metric_parser.py -f $HISTORY_LOGS_CREATE -c 0 -u
+fi

logs_metric_parser.py

@@ -17,10 +17,11 @@
 
 parser = argparse.ArgumentParser(description="Command-line to retreive Prometheus metrics from OSCAR", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 parser.add_argument("-f", "--file_path", type=str, help="Logfile path/name")
-parser.add_argument("-g", "--general", action="store_true", help="Complete logfile")
+parser.add_argument("-g", "--geolocation", action="store_true", help="Using as input complete generated GoAccess json")
 parser.add_argument("-u", "--use_existing", action="store_true", required=False, help="Use existing output file")
-parser.add_argument("-p", "--partial", action="store_true", help="Filtered by status code logfile")
+parser.add_argument("-i", "--inference", action="store_true", help="Using as input inference log format file")
 parser.add_argument("-c", "--create", action="store_true", help="Using as input created services log format file")
+parser.add_argument("-a", "--goaccess", action="store_true", help="Using as input generated GoAccess json filtered")
 parser.add_argument("status_code", type=int, help="Complete logfile")
 
 
@@ -56,49 +57,67 @@ def parse_geolocation_info(write_type):
  gfile.close()
 
 """ 
- > Number of AI applications (created services -> POST requests to /system/services)
- > Output format: {num_created, start_date, end_date}
-
  > Processed inference executions (POST requests to /run or /job)
- > Output format: {service, executions, type, successfull, failed, start_date, end_date}
+ > Output format: {service_name, status_code, total_visits, unique_visits, start_date, end_date}
 """
 
-def parse_inference_info(status_code, write_type):
-
- inference = dict()
+def parse_inference_goaccess(status_code, write_type):
  requests = metrics["requests"]["data"]
- exec_count = 0
-
- for r in requests:
- if r["method"] == "POST":
+
+ with open(f'{OUTPUT_PATH}/total_inference_metrics.csv', write_type, newline='') as efile:
+ writer = csv.writer(efile)
+ if write_type == "w": writer.writerow(["service_name", "status_code", "total_visits", "unique_visits", "start_metric_date", "end_metric_date"])
+ for r in requests:
  path = r["data"]
  if RUN_PATH in path or JOB_PATH in path:
- sum_requests = r["hits"]["count"]
+ total_visits = r["hits"]["count"]
+ unique_visits = r["visitors"]["count"]
  split_path = split(path)
- service = split_path[1]
- if service in inference.keys():
- inference[service].append({"exec_type": split_path[0], "status_code": status_code, "count": sum_requests})
- else:
- inference[service] = [{"exec_type": split_path[0], "status_code": status_code, "count": sum_requests}]
- exec_count+=sum_requests
-
- if exec_count != 0:
- with open(f'{OUTPUT_PATH}/total_inference_metrics.csv', write_type, newline='') as efile:
- writer = csv.writer(efile)
- if write_type == "w": writer.writerow(["inference_count", "status_code", "start_metric_date", "end_metric_date"])
- writer.writerow([exec_count, status_code, START_DATE, END_DATE])
-
- with open(f'{OUTPUT_PATH}/services_inference_metrics.csv', write_type, newline='') as sfile:
- writer = csv.writer(sfile)
- if write_type == "w": writer.writerow(["service_name", "exec_type", "status_code", "inference_count" , "start_metric_date", "end_metric_date"])
- for k in inference.keys():
- for item in inference[k]:
- writer.writerow([k, item["exec_type"], item["status_code"], item["count"], START_DATE, END_DATE]) 
-
-def parse_create_info(write_type):
+ service_name = split_path[1]
+
+ writer.writerow([service_name, status_code, total_visits, unique_visits, START_DATE, END_DATE])
+
+"""
+ > Info about all inference executions
+ > Output format: {service_name, exec_type, status_code, owner_uid, request_date, request_time}
+"""
+def parse_inference_log(write_type):
+ with open(f'{OUTPUT_PATH}/services_inference_metrics.csv', write_type, newline='') as sfile:
+ writer = csv.writer(cfile)
+ if write_type == "w": write.writerow(["service_name", "exec_type", "status_code", "owner_uid", "request_date", "request_time"])
+ with open(args.file_path, 'r') as rawfile:
+ for log in rawfile:
+ pattern = re.compile(
+ r'\[GIN-EXECUTIONS-LOGGER\]\s' # Literal text "[GIN-EXECUTIONS-LOGGER]"
+ r'(?P<date>\d{4}/\d{2}/\d{2})\s-\s(?P<time>\d{2}:\d{2}:\d{2})\s\|\s' # Date and time
+ r'(?P<status>\d{3})\s\|\s' # HTTP status code
+ r'(?P<latency>[\d.]+s)\s\|\s' # Latency (time taken)
+ r'(?P<client_ip>\d{1,3}(?:\.\d{1,3}){3})\s\|\s' # Client IP
+ r'(?P<method>[A-Z]+)\s+\|\s' # HTTP method (e.g., POST)
+ r'(?P<path>\S+)\s\|\s' # Request path
+ r'(?P<user>\S+)' # User (e.g., <uid>@egi.eu)
+ )
+ match = pattern.search(log)
+ if match:
+ path = match.group('path')
+ split_path = split(path)
+ exec_type = split_path[0]
+ service_name = split_path[1]
+
+ status_code = match.group('status')
+ request_date = match.group('date')
+ request_time = match.group('time')
+ owner_uid = match.group('user')
+ writer.writerow([service_name, exec_type, status_code, owner_uid, request_date, request_time]) 
+
+"""
+ > Number of AI applications (created services -> POST requests to /system/services)
+ > Output format: {service_name, owner_uid, creation_date, creation_time}
+"""
+def parse_create_log(write_type):
  with open(f'{OUTPUT_PATH}/created_services_info.csv', write_type, newline='') as cfile:
  writer = csv.writer(cfile)
- if write_type == "w": writer.writerow(["service_name", "owner_uid", "creation_time"])
+ if write_type == "w": writer.writerow(["service_name", "owner_uid", "creation_date", "creation_time"])
  with open(args.file_path, 'r') as rawfile:
  for log in rawfile:
  pattern = re.compile(
@@ -113,18 +132,21 @@ def parse_create_info(write_type):
  )
  match = pattern.search(log)
  if match:
- creation_time = match.group(date)
- service_name = match.group(service_name)
- owner_uid = match.group(user)
- writer.writerow([service_name, owner_uid, creation_time])
+ creation_date = match.group('date')
+ creation_time = match.group('time')
+ service_name = match.group('service_name')
+ owner_uid = match.group('user')
+ writer.writerow([service_name, owner_uid, creation_date, creation_time])
 
 wr="w"
 if args.use_existing:
  wr="a"
 
 if args.create:
- parse_create_info(wr)
-if args.general:
+ parse_create_log(wr)
+if args.inference:
+ parse_inference_log(wr)
+if args.geolocation:
  parse_geolocation_info(wr)
-if args.partial:
- parse_inference_info(args.status_code, wr)
+if args.goaccess:
+ parse_inference_goaccess(args.status_code, wr)