Merge pull request #2626 from xrmx/methane-worker-graceful-stop

core/uwsgi: graceful stop worker when max_requests/reload_on_*

core/loop.c

@@ -60,6 +60,9 @@ void *uwsgi_get_loop(char *name) {
 
 void simple_loop() {
 	uwsgi_loop_cores_run(simple_loop_run);
+	// Other threads may still run. Make sure they will stop.
+	uwsgi.workers[uwsgi.mywid].manage_next_request = 0;
+
 	if (uwsgi.workers[uwsgi.mywid].shutdown_sockets)
 		uwsgi_shutdown_all_sockets();
 }

core/uwsgi.c

@@ -1255,6 +1255,10 @@ void wait_for_threads() {
 			if (ret) {
 				uwsgi_log("pthread_join() = %d\n", ret);
 			}
+			else {
+				// uwsgi_worker_is_busy() should not consider this thread as busy.
+				uwsgi.workers[uwsgi.mywid].cores[i].in_request = 0;
+			}
 		}
 	}
 
@@ -1319,14 +1323,12 @@ void end_me(int signum) {
 }
 
 static void simple_goodbye_cruel_world(const char *reason) {
-
-	if (uwsgi.threads > 1 && !uwsgi_instance_is_dying) {
-		wait_for_threads();
-	}
-
+	int prev = uwsgi.workers[uwsgi.mywid].manage_next_request;
 	uwsgi.workers[uwsgi.mywid].manage_next_request = 0;
-	uwsgi_log("...The work of process %d is done (%s). Seeya!\n", getpid(), (reason != NULL ? reason : "no reason given"));
-	exit(0);
+	if (prev) {
+		// Avoid showing same message from all threads.
+		uwsgi_log("...The work of process %d is done (%s). Seeya!\n", getpid(), (reason != NULL ? reason : "no reason given"));
+	}
 }
 
 void goodbye_cruel_world(const char *reason_fmt, ...) {
@@ -3721,6 +3723,11 @@ void uwsgi_ignition() {
 		}
 	}
 
+	// main thread waits other threads.
+	if (uwsgi.threads > 1) {
+		wait_for_threads();
+	}
+
 	// end of the process...
 	end_me(0);
 }

tests/threads_atexit.py

@@ -0,0 +1,44 @@
+# https://github.com/unbit/uwsgi/pull/2615
+# atexit should be called when reached max-requests.
+#
+# Start this app:
+#
+#   $ ./uwsgi --http-socket :8000 --master -L --wsgi-file=tests/threads_atexit.py \
+#       --workers 1 --threads 32 --max-requests 40 --min-worker-lifetime 6 --lazy-apps
+#
+# Access to this app with hey[1]:
+#
+#   # Do http access for 5 minutes with 32 concurrency
+#   $ ./hey -c 32 -z 5m 'http://127.0.0.1:8000/'
+#
+# Search how many stamp files:
+#
+#   $ ls uwsgi_worker*.txt | wc -l
+#   39  # should be 0
+#
+# [1] https://github.com/rakyll/hey
+
+import atexit
+import os
+import sys
+import time
+
+
+pid = os.getpid()
+stamp_file = f"./uwsgi_worker{pid}.txt"
+
+
+with open(stamp_file, "w") as f:
+    print(time.time(), file=f)
+
+
+@atexit.register
+def on_finish_worker():
+    print(f"removing {stamp_file}", file=sys.stderr)
+    os.remove(stamp_file)
+
+
+def application(env, start_response):
+    time.sleep(1)
+    start_response('200 OK', [('Content-Type', 'text/html')])
+    return [b"Hello World"]

tests/threads_heavy.py

@@ -0,0 +1,29 @@
+# https://github.com/unbit/uwsgi/pull/2615
+# CPU heavy application in multi threaded uWSGI doesn't shutdown gracefully.
+#
+# $ ./uwsgi \
+#     --wsgi-file=threads_heavy.py --master --http-socket=:8000 \
+#     --workers=4 --threads=8 --max-requests=20 --min-worker-lifetime=6 -L  \
+#     --worker-reload-mercy=20 2>&1 | tee uwsgi.log
+#
+# $ hey -c 16 -z 3m 'http://127.0.0.1:8000/'
+#
+# $ grep MERCY uwsgi.log
+# Tue Mar 19 14:01:59 2024 - worker 1 (pid: 62113) is taking too much time to die...NO MERCY !!!
+# Tue Mar 19 14:02:23 2024 - worker 2 (pid: 62218) is taking too much time to die...NO MERCY !!!
+# ...
+#
+# This was caused by pthread_cancel() is called from non-main thread.
+
+def fibonacci(n):
+    if n <= 1:
+        return n
+    return fibonacci(n - 1) + fibonacci(n - 2)
+
+
+def application(env, start_response):
+    start_response('200 OK', [('Content-Type', 'text/html')])
+    n = 24
+    r = fibonacci(n)
+    s = f"F({n}) = {r}"
+    return [s.encode()]