Fix thread-pool join threads in multiple vCPUs

Signed-off-by: Coldwings <[email protected]>
Coldwings · Nov 2, 2023 · c22467e · c22467e
1 parent 248b926
commit c22467e
Show file tree

Hide file tree

Showing 6 changed files with 38 additions and 32 deletions.
diff --git a/.github/workflows/ci.linux.arm.yml b/.github/workflows/ci.linux.arm.yml
@@ -2,9 +2,9 @@ name: Linux ARM
 
 on:
   push:
-    branches: [ "main" ]
+    branches: [ "main", "release/*" ]
   pull_request:
-    branches: [ "main" ]
+    branches: [ "main", "release/*" ]
 
 jobs:
   centos8-gcc921-epoll-release:

diff --git a/.github/workflows/ci.linux.x86.yml b/.github/workflows/ci.linux.x86.yml
@@ -2,9 +2,9 @@ name: Linux x86
 
 on:
   push:
-    branches: [ "main" ]
+    branches: [ "main", "release/*" ]
   pull_request:
-    branches: [ "main" ]
+    branches: [ "main", "release/*" ]
 
 jobs:
   centos8-gcc921-epoll-release:

diff --git a/.github/workflows/ci.macos.arm.yml b/.github/workflows/ci.macos.arm.yml
@@ -2,9 +2,9 @@ name: macOS ARM
 
 on:
   push:
-    branches: [ "main" ]
+    branches: [ "main", "release/*" ]
   pull_request:
-    branches: [ "main" ]
+    branches: [ "main", "release/*" ]
 
 jobs:
   macOS-clang-debug:

diff --git a/.github/workflows/ci.macos.yml b/.github/workflows/ci.macos.yml
@@ -2,9 +2,9 @@ name: macOS
 
 on:
   push:
-    branches: [ "main" ]
+    branches: [ "main", "release/*" ]
   pull_request:
-    branches: [ "main" ]
+    branches: [ "main", "release/*" ]
 
 jobs:
   macOS-12-Monterey-debug:

diff --git a/common/identity-pool.cpp b/common/identity-pool.cpp
@@ -49,8 +49,8 @@ void IdentityPoolBase::put(void* obj)
             m_mtx.lock();
         }
         --m_refcnt;
+        m_cvar.notify_all();
     }
-    m_cvar.notify_all();
     assert(m_size <= m_capacity);
 }
 

diff --git a/thread/thread-pool.cpp b/thread/thread-pool.cpp
@@ -29,8 +29,8 @@ namespace photon
             pCtrl->joining = false;
             pCtrl->start = start;
             pCtrl->arg = arg;
+            pCtrl->cvar.notify_one();
         }
-        pCtrl->cvar.notify_one();
         return pCtrl;
     }
     void* ThreadPoolBase::stub(void* arg)
@@ -41,37 +41,41 @@ namespace photon
         thread_yield_to(th);
         while(true)
         {
-            {
-                SCOPED_LOCK(ctrl.m_mtx);
-                while (!ctrl.start)                     // wait for `create()` to give me
-                    ctrl.cvar.wait(ctrl.m_mtx);           // thread_entry and argument
+            photon::locker<decltype(ctrl.m_mtx)> lock(ctrl.m_mtx);
+            while (!ctrl.start)                     // wait for `create()` to give me
+                ctrl.cvar.wait(ctrl.m_mtx);           // thread_entry and argument
 
-                if (ctrl.start == &stub)
-                    break;
-                ((partial_thread*) CURRENT)->tls = nullptr;
-            }
+            if (ctrl.start == &stub)
+                break;
+
+            ((partial_thread*) CURRENT)->tls = nullptr;
+            lock.unlock();
             ctrl.start(ctrl.arg);
             deallocate_tls();
-            {
-                SCOPED_LOCK(ctrl.m_mtx);
-                if (ctrl.joining) {
-                    assert(ctrl.joinable);
-                    ctrl.cvar.notify_all();
-                } else if (ctrl.joinable) {
-                    ctrl.joining = true;
-                    ctrl.cvar.wait(ctrl.m_mtx);
-                }
-                ctrl.joinable = false;
-                ctrl.joining = false;
-                ctrl.start = nullptr;
+            lock.lock();
+            bool should_put = !ctrl.joining;
+            if (ctrl.joining) {
+                assert(ctrl.joinable);
+                ctrl.cvar.notify_all();
+            } else if (ctrl.joinable) {
+                ctrl.joining = true;
+                ctrl.cvar.wait(ctrl.m_mtx);
+            }
+            ctrl.joinable = false;
+            ctrl.joining = false;
+            ctrl.start = nullptr;
+            if (should_put) {
+                lock.unlock();
+                // if has no other joiner waiting
+                // collect it into pool
+                ctrl.pool->put(&ctrl);
             }
-            ctrl.pool->put(&ctrl);
         }
         return nullptr;
     }
     void ThreadPoolBase::join(TPControl* pCtrl)
     {
-        SCOPED_LOCK(pCtrl->m_mtx);
+        photon::locker<decltype(pCtrl->m_mtx)> lock(pCtrl->m_mtx);
         if (!pCtrl->joinable)
             LOG_ERROR_RETURN(EINVAL, , "thread is not joinable");
         if (!pCtrl->start)
@@ -84,6 +88,8 @@ namespace photon
         } else {
             pCtrl->joining = true;
             pCtrl->cvar.wait(pCtrl->m_mtx);
+            lock.unlock();
+            pCtrl->pool->put(pCtrl);
         }
     }
     int ThreadPoolBase::ctor(ThreadPoolBase* pool, TPControl** out)