refactor: bulk request (#22)

TencentBlueKing · Nov 4, 2023 · 9e20da8 · 9e20da8
1 parent 7025b21
commit 9e20da8
Show file tree

Hide file tree

Showing 8 changed files with 22 additions and 14 deletions.
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-0.4.5
+0.4.6
diff --git a/bk_resource/base.py b/bk_resource/base.py
@@ -292,22 +292,26 @@ def bulk_request(self, request_data_iterable=None, ignore_exceptions=False):
         """
         基于多线程的批量并发请求
         """
+
+        # 预检查
         if not isinstance(request_data_iterable, (list, tuple)):
             raise TypeError("'request_data_iterable' object is not iterable")
 
-        pool = ThreadPool(processes=get_processes())
-        futures = []
-
+        # 模块引入，放在文件头可能导致 django 未完全初始化异常
         from blueapps.utils.request_provider import get_local_request
 
+        # 初始化
+        futures = []
         _request = get_local_request()
 
-        for request_data in request_data_iterable:
-            futures.append(pool.apply_async(self.request, args=(request_data,), kwds={"_request": _request}))
-
-        pool.close()
-        pool.join()
+        # 线程池
+        with ThreadPool(processes=get_processes()) as pool:
+            for request_data in request_data_iterable:
+                futures.append(pool.apply_async(self.request, args=(request_data,), kwds={"_request": _request}))
+            pool.close()
+            pool.join()
 
+        # 获取结果
         results = []
         exceptions = []
         for future in futures:

diff --git a/bk_resource/tools.py b/bk_resource/tools.py
@@ -221,7 +221,7 @@ def get_processes() -> int:
     # 容器限制
     # 先给默认值避免后续比较报错
     container_cpu = cpu_count
-    with ignored(Exception):
+    with ignored(Exception, log_exception=False):
         with open("/sys/fs/cgroup/cpu/cpu.cfs_quota_us", "r") as f:
             cfs_quota_us = float(f.read().strip())
         with open("/sys/fs/cgroup/cpu/cpu.cfs_period_us", "r") as f:

diff --git a/readme.md b/readme.md
@@ -1,7 +1,7 @@
 ![logo.png](assests/logo.png)
 
 [![License](https://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat)](https://github.com/TencentBlueKing/bk-resource/blob/main/LICENSE.txt)
-[![Release Version](https://img.shields.io/badge/release-0.4.5-brightgreen.svg)](https://github.com/TencentBlueKing/bk-resource/releases)
+[![Release Version](https://img.shields.io/badge/release-0.4.6-brightgreen.svg)](https://github.com/TencentBlueKing/bk-resource/releases)
 [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](https://github.com/TencentBlueKing/bk-resource/pulls)
 [![codecov](https://codecov.io/gh/TencentBlueKing/bk-resource/branch/main/graph/badge.svg)](https://codecov.io/gh/TencentBlueKing/bk-resource)
 [![Unittest Py3](https://github.com/TencentBlueKing/bk-resource/actions/workflows/unittest.yml/badge.svg)](https://github.com/TencentBlueKing/bk-resource/actions/workflows/unittest.yml)

diff --git a/readme_en.md b/readme_en.md
@@ -1,7 +1,7 @@
 ![logo.png](assests/logo.png)
 
 [![License](https://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat)](https://github.com/TencentBlueKing/bk-resource/blob/main/LICENSE.txt)
-[![Release Version](https://img.shields.io/badge/release-0.4.5-brightgreen.svg)](https://github.com/TencentBlueKing/bk-resource/releases)
+[![Release Version](https://img.shields.io/badge/release-0.4.6-brightgreen.svg)](https://github.com/TencentBlueKing/bk-resource/releases)
 [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](https://github.com/TencentBlueKing/bk-resource/pulls)
 [![codecov](https://codecov.io/gh/TencentBlueKing/bk-resource/branch/main/graph/badge.svg)](https://codecov.io/gh/TencentBlueKing/bk-resource)
 [![Unittest Py3](https://github.com/TencentBlueKing/bk-resource/actions/workflows/unittest.yml/badge.svg)](https://github.com/TencentBlueKing/bk-resource/actions/workflows/unittest.yml)

diff --git a/release.md b/release.md
@@ -1,5 +1,9 @@
 # 版本日志
 
+## 0.4.6
+
+- 优化并发请求中线程池的回收
+
 ## 0.4.5
 
 - 支持配置并发线程数，容器内使用容器CPU限制作为线程数，非容器环境使用CPU数量

diff --git a/setup.py b/setup.py
@@ -20,7 +20,7 @@
 
 setup(
     name="bk_resource",
-    version="0.4.5",
+    version="0.4.6",
     author="blueking",
     url="https://bk.tencent.com",
     author_email="[email protected]",

diff --git a/template/{{cookiecutter.project_name}}/requirements.txt b/template/{{cookiecutter.project_name}}/requirements.txt
@@ -6,7 +6,7 @@
 # 各自环境所需的不同依赖, 通过 sites/${env}/deploy/requirements_env.txt得到
 -r requirements_env.txt
 
-bk_resource==0.4.5
+bk_resource==0.4.6
 
 # web server
 gunicorn==19.6.0