add simple 'exec' liveness probe

DataONEorg · Apr 30, 2024 · aedcb98 · aedcb98
1 parent 939360d
commit aedcb98
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 26 deletions.
diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -18,7 +18,8 @@ RUN apt update && apt -y install \
     nano
 
 #Add a user & group with id=1001 and name=d1indexer
-RUN groupadd -g 1000 d1indexer && useradd -u 1000 -g 1000 d1indexer
+RUN groupadd -g 1000 d1indexer && useradd -u 1000 -g 1000 d1indexer \
+    && touch ./livenessprobe
 
 # The most recently built jar file is copied from the maven build directory to this dir by maven, so that
 # it can be copied to the image.
@@ -28,6 +29,7 @@ COPY ./docker/entrypoint.sh .
 # Change the ownership of the jar and sh files
 RUN chown d1indexer dataone-index-worker-${TAG}-shaded.jar
 RUN chown d1indexer entrypoint.sh
+RUN chown d1indexer livenessprobe
 
 #Run Container as d1indexer
 USER 1000

diff --git a/helm/templates/deployment.yaml b/helm/templates/deployment.yaml
@@ -25,8 +25,7 @@ spec:
         {{- toYaml . | nindent 8 }}
       {{- end }}
       {{- if .Values.affinity }}
-      affinity: {{- include "idxhelpers.tplvalues.render" (dict "value" .Values.affinity "context"
-      $) | nindent 8 }}
+      affinity: {{- include "idxhelpers.tplvalues.render" (dict "value" .Values.affinity "context" $) | nindent 8 }}
       {{- end }}
       {{- if .Values.nodeSelector }}
       nodeSelector: {{- include "idxhelpers.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }}
@@ -52,15 +51,14 @@ spec:
             {{- toYaml .Values.securityContext | nindent 12 }}
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
-          # livenessProbe:
-          #   httpGet:
-          #     path: /
-          #     port: http
-          # readinessProbe:
-          #   httpGet:
-          #     path: /
-          #     port: http
-          #     initialDelaySeconds:
+          livenessProbe:
+            exec:
+              command:
+                - /bin/sh
+                - -c
+                - test $(($(date +%s) - $(stat -f %m /var/lib/dataone-indexer/livenessprobe))) -lt 20
+            initialDelaySeconds: 20
+            periodSeconds: 10
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
           volumeMounts:
@@ -130,18 +128,6 @@ spec:
               | grep -c "<schema name=\"dataone") == 1 ]]; do
               echo waiting for Solr Schema to be accessible at http://{{ $solrHost }}:
               {{- $solrPort }}$URI; sleep 1; done;
-      {{- with .Values.nodeSelector }}
-      nodeSelector:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.affinity }}
-      affinity:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.tolerations }}
-      tolerations:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
       volumes:
         - name: {{ .Release.Name }}-config-volume
           configMap:

diff --git a/src/main/java/org/dataone/cn/indexer/IndexWorker.java b/src/main/java/org/dataone/cn/indexer/IndexWorker.java
@@ -2,12 +2,18 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.FileTime;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.concurrent.ScheduledExecutorService;
 
 import javax.xml.parsers.ParserConfigurationException;
 import javax.xml.xpath.XPathExpressionException;
@@ -93,7 +99,7 @@ public class IndexWorker {
     private String specifiedThreadNumberStr = null;
     private int specifiedThreadNumber = 0;
     private ExecutorService executor = null;
-
+    private ScheduledExecutorService scheduler;
 
     /**
      * Commandline main for the IndexWorker to be started.
@@ -222,9 +228,25 @@ public IndexWorker(Boolean initialize) throws IOException, TimeoutException, Ser
             initIndexParsers();
             ObjectManager.getInstance();
             OntologyModelService.getInstance();
+            startLivenessProbe();
         }
     }
-
+
+    private void startLivenessProbe() {
+        scheduler = Executors.newScheduledThreadPool(1);
+        Runnable task = () -> {
+            Path path = Paths.get("./livenessprobe");
+            try {
+                Files.createFile(path);
+                Files.setLastModifiedTime(path, FileTime.fromMillis(System.currentTimeMillis()));
+                logger.info("IndexWorker.startLivenessProbe - starting livenessProbe");
+            } catch (IOException e) {
+                logger.error("IndexWorker.startLivenessProbe - failed to touch file: " + path, e);
+            }
+        };
+        scheduler.scheduleAtFixedRate(task, 0, 10, TimeUnit.SECONDS);
+    }
+
     /**
      * Initialize the RabbitMQ service
      * @throws IOException