fix: configure systemd priorities (#1384)

<!-- Please make sure there is an issue that this PR is correlated to. -->
Fixes RVT-4031
## Changes

<!-- If there are frontend changes, please include screenshots. -->

packages/services/cluster/src/workflows/server/install/install_scripts/files/node_exporter.sh

@@ -35,6 +35,11 @@ ExecStart=/usr/bin/node_exporter --collector.disable-defaults --collector.cpu --
 Restart=always
 RestartSec=2
 
+# Medium CPU priority
+Nice=-10
+# Standard service
+CPUSchedulingPolicy=other
+
 [Install]
 WantedBy=multi-user.target
 EOF

packages/services/cluster/src/workflows/server/install/install_scripts/files/ok_server.sh

@@ -25,6 +25,13 @@ Type=oneshot
 ExecStart=/usr/bin/ok_server.sh
 Type=simple
 
+# High CPU priority
+CPUSchedulingPriority=90
+# Real time service
+CPUSchedulingPolicy=fifo
+# Prevent killing from system OOM
+OOMScoreAdjust=-800
+
 [Install]
 WantedBy=multi-user.target
 EOF

packages/services/cluster/src/workflows/server/install/install_scripts/files/pegboard_configure.sh

@@ -338,6 +338,19 @@ ConditionPathExists=/etc/rivet-client/
 ExecStart=/usr/local/bin/rivet-client -c /etc/rivet-client/config.json
 Restart=always
 RestartSec=2
+
+# High CPU priority
+CPUSchedulingPriority=90
+# Real time service
+CPUSchedulingPolicy=fifo
+# Prevent killing from system OOM
+OOMScoreAdjust=-1000
+# Kill main process, not children
+KillMode=process
+# Increase limit of file watches
+LimitNOFILE=65536
+# Increase max process limits
+LimitNPROC=infinity
 TasksMax=infinity
 
 [Install]

packages/services/cluster/src/workflows/server/install/install_scripts/files/rivet_create_hook.sh

@@ -16,6 +16,13 @@ RestartSec=1s
 ExecStart=/usr/bin/rivet_hook.sh
 ExecStartPost=/bin/touch /var/tmp/rivet_hook.completed
 
+# High CPU priority
+CPUSchedulingPriority=90
+# Real time service
+CPUSchedulingPolicy=fifo
+# Prevent killing from system OOM
+OOMScoreAdjust=-800
+
 [Install]
 WantedBy=multi-user.target
 EOF

packages/services/cluster/src/workflows/server/install/install_scripts/files/rivet_fetch_tls.sh

@@ -75,6 +75,13 @@ OnCalendar=*:0
 RandomizedDelaySec=60
 Unit=rivet_fetch_tls.service
 
+# High CPU priority
+CPUSchedulingPriority=90
+# Real time service
+CPUSchedulingPolicy=fifo
+# Prevent killing from system OOM
+OOMScoreAdjust=-800
+
 [Install]
 WantedBy=timers.target
 EOF

packages/services/cluster/src/workflows/server/install/install_scripts/files/traefik_instance.sh

@@ -40,6 +40,13 @@ AmbientCapabilities=CAP_NET_BIND_SERVICE
 Restart=always
 RestartSec=2
 
+# High CPU priority
+CPUSchedulingPriority=85
+# Real time service
+CPUSchedulingPolicy=fifo
+# Prevent killing from system OOM
+OOMScoreAdjust=-900
+
 [Install]
 WantedBy=multi-user.target
 EOF

packages/services/cluster/src/workflows/server/install/install_scripts/files/traffic_server_configure.sh

@@ -38,6 +38,11 @@ ExecStart=/usr/bin/docker run --rm --name trafficserver \
 	"__IMAGE__"
 ExecStop=/usr/bin/docker stop trafficserver
 
+# Medium CPU priority
+CPUSchedulingPriority=60
+# Real time service
+CPUSchedulingPolicy=fifo
+
 [Install]
 WantedBy=multi-user.target
 EOF

packages/services/cluster/src/workflows/server/install/install_scripts/files/vector_configure.sh

@@ -34,6 +34,13 @@ AmbientCapabilities=CAP_NET_BIND_SERVICE
 StartLimitInterval=10
 StartLimitBurst=5
 
+# Medium CPU priority
+CPUSchedulingPriority=60
+# Real time service
+CPUSchedulingPolicy=fifo
+# Prevent killing from system OOM
+OOMScoreAdjust=-500
+
 [Install]
 WantedBy=multi-user.target
 EOF