From 1de666408fbeb6c02cc5069a143aa799f1a25e1f Mon Sep 17 00:00:00 2001
From: dcd <1151627903@qq.com>
Date: Mon, 26 Aug 2024 16:23:31 +0800
Subject: [PATCH] =?UTF-8?q?feat:=20=E5=90=91cmdb=E5=90=8C=E6=AD=A5?=
=?UTF-8?q?=E4=BA=91=E5=8C=BA=E5=9F=9F=E6=9C=8D=E5=8A=A1=E5=95=86=20(close?=
=?UTF-8?q?d=20#2386)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
apps/node_man/constants.py | 29 ++++-
apps/node_man/handlers/cloud.py | 8 +-
apps/node_man/handlers/cmdb.py | 18 ++-
.../commands/sync_all_isp_to_cmdb.py | 19 +++
...ate_isp_and_accesspoint_regionid_cityid.py | 52 ++++++++
apps/node_man/models.py | 4 +-
apps/node_man/periodic_tasks/__init__.py | 1 +
.../periodic_tasks/sync_all_isp_to_cmdb.py | 60 +++++++++
.../tests/test_handlers/test_cloud.py | 39 ++++++
.../test_sync_all_isp_to_cmdb.py | 40 ++++++
env/__init__.py | 3 +
.../kubernetes/helm/bk-nodeman/README.md | 121 +++++++++---------
.../templates/configmaps/env-configmap.yaml | 1 +
.../kubernetes/helm/bk-nodeman/values.yaml | 2 +
14 files changed, 324 insertions(+), 73 deletions(-)
create mode 100644 apps/node_man/management/commands/sync_all_isp_to_cmdb.py
create mode 100644 apps/node_man/migrations/0084_update_isp_and_accesspoint_regionid_cityid.py
create mode 100644 apps/node_man/periodic_tasks/sync_all_isp_to_cmdb.py
create mode 100644 apps/node_man/tests/test_pericdic_tasks/test_sync_all_isp_to_cmdb.py
diff --git a/apps/node_man/constants.py b/apps/node_man/constants.py
index bee4b0e2b..f15d11c3c 100644
--- a/apps/node_man/constants.py
+++ b/apps/node_man/constants.py
@@ -59,6 +59,7 @@ class TimeUnit:
COLLECT_AUTO_TRIGGER_JOB_INTERVAL = 5 * TimeUnit.MINUTE
SYNC_CMDB_CLOUD_AREA_INTERVAL = 10 * TimeUnit.SECOND
SYNC_AGENT_STATUS_TASK_INTERVAL = 10 * TimeUnit.MINUTE
+SYNC_ISP_TO_CMDB_INTERVAL = 1 * TimeUnit.DAY
SYNC_PROC_STATUS_TASK_INTERVAL = settings.SYNC_PROC_STATUS_TASK_INTERVAL
SYNC_BIZ_TO_GRAY_SCOPE_LIST_INTERVAL = 30 * TimeUnit.MINUTE
@@ -75,12 +76,14 @@ class TimeUnit:
# 默认管控区域ID
DEFAULT_CLOUD = int(os.environ.get("DEFAULT_CLOUD", 0))
DEFAULT_CLOUD_NAME = os.environ.get("DEFAULT_CLOUD_NAME", _("直连区域"))
+# 未分配管控区域ID
+UNASSIGNED_CLOUD_ID = int(os.environ.get("BKAPP_UNASSIGNED_CLOUD_ID", 90000001))
# 自动选择接入点ID
DEFAULT_AP_ID = int(os.environ.get("DEFAULT_AP_ID", -1))
# 自动选择安装通道ID
-DEFAULT_INSTALL_CHANNEL_ID = int(os.environ.get("DEFAULT_INSTALL_CHANNEL_ID", -1))
+DEFAULT_INSTALL_CHANNEL_ID = int(os.environ.get("BKAPP_DEFAULT_INSTALL_CHANNEL_ID", -1))
# 自动选择的云区域ID
-AUTOMATIC_CHOICE_CLOUD_ID = int(os.environ.get("AUTOMATIC_CHOICE_CLOUD_ID", -1))
+AUTOMATIC_CHOICE_CLOUD_ID = int(os.environ.get("BKAPP_AUTOMATIC_CHOICE_CLOUD_ID", -1))
# 自动选择
AUTOMATIC_CHOICE = os.environ.get("AUTOMATIC_CHOICE", _("自动选择"))
# 默认安装通道
@@ -567,6 +570,7 @@ def _get_member__alias_map(cls) -> Dict[Enum, str]:
QUERY_CLOUD_LIMIT = 200
QUERY_HOST_SERVICE_TEMPLATE_LIMIT = 200
QUERY_MODULE_ID_THRESHOLD = 15
+UPDATE_CMDB_CLOUD_AREA_LIMIT = 50
VERSION_PATTERN = re.compile(r"[vV]?(\d+\.){1,5}\d+(-rc\d)?$")
# 语义化版本正则，参考：https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string
SEMANTIC_VERSION_PATTERN = re.compile(
@@ -603,6 +607,27 @@ def _get_member__alias_map(cls) -> Dict[Enum, str]:
MAX_HOST_IDS_LENGTH = 5000
# 操作系统对应账户名
OS_ACCOUNT = {"LINUX": LINUX_ACCOUNT, "WINDOWS": WINDOWS_ACCOUNT}
+# NODEMAN云服务商对应CMDB接口云服务商映射
+CMDB_CLOUD_VENDOR_MAP = {
+ "AWS": "1",
+ "TencentCloud": "2",
+ "GoogleCloud": "3",
+ "Azure": "4",
+ "PrivateCloud": "5",
+ "SalesForce": "6",
+ "OracleCloud": "7",
+ "IBMCloud": "8",
+ "AlibabaCloud": "9",
+ "ECloud": "10",
+ "UCloud": "11",
+ "MOS": "12",
+ "KSyun": "13",
+ "BaiduCloud": "14",
+ "HuaweiCloud": "15",
+ "capitalonline": "16",
+ "TencentPrivateCloud": "17",
+ "Zenlayer": "18",
+}
class ProxyFileFromType(Enum):
diff --git a/apps/node_man/handlers/cloud.py b/apps/node_man/handlers/cloud.py
index e015d38c2..3fcc82f7a 100644
--- a/apps/node_man/handlers/cloud.py
+++ b/apps/node_man/handlers/cloud.py
@@ -199,7 +199,8 @@ def create(self, params: dict, username: str):
"""
bk_cloud_name = params["bk_cloud_name"]
- bk_cloud_id = CmdbHandler.get_or_create_cloud(bk_cloud_name)
+ bk_cloud_vendor = const.CMDB_CLOUD_VENDOR_MAP.get(params["isp"])
+ bk_cloud_id = CmdbHandler.get_or_create_cloud(bk_cloud_name, bk_cloud_vendor=bk_cloud_vendor)
if bk_cloud_name == str(DEFAULT_CLOUD_NAME):
raise ValidationError(_("管控区域不可名为「直连区域」"))
@@ -236,8 +237,9 @@ def update(bk_cloud_id: int, bk_cloud_name: str, isp: str, ap_id: int):
if Cloud.objects.filter(bk_cloud_name=bk_cloud_name).exclude(bk_cloud_id=bk_cloud_id).exists():
raise ValidationError(_("管控区域名称不可重复"))
- # 向CMDB修改管控区域名称
- CmdbHandler.rename_cloud(bk_cloud_id, bk_cloud_name)
+ # 向CMDB修改管控区域名称以及云服务商
+ bk_cloud_vendor: str = const.CMDB_CLOUD_VENDOR_MAP.get(isp)
+ CmdbHandler.rename_cloud(bk_cloud_id, bk_cloud_name, bk_cloud_vendor=bk_cloud_vendor)
cloud.bk_cloud_name = bk_cloud_name
cloud.isp = isp
diff --git a/apps/node_man/handlers/cmdb.py b/apps/node_man/handlers/cmdb.py
index 298da62cd..3da5b5e31 100644
--- a/apps/node_man/handlers/cmdb.py
+++ b/apps/node_man/handlers/cmdb.py
@@ -323,12 +323,12 @@ def check_biz_permission(self, bk_biz_scope: list, action: str):
raise PermissionDeniedError(action_name=action, apply_url=apply_url, permission=apply_data)
@staticmethod
- def add_cloud(bk_cloud_name):
+ def add_cloud(bk_cloud_name: str, bk_cloud_vendor: str = None):
"""
新增管控区域
"""
# 增删改查CMDB操作以admin用户进行
- data = client_v2.cc.create_cloud_area({"bk_cloud_name": bk_cloud_name})
+ data = client_v2.cc.create_cloud_area({"bk_cloud_name": bk_cloud_name, "bk_cloud_vendor": bk_cloud_vendor})
return data.get("created", {}).get("id")
@staticmethod
@@ -364,20 +364,24 @@ def get_cloud(bk_cloud_name):
raise CloudNotExistError
@staticmethod
- def rename_cloud(bk_cloud_id, bk_cloud_name):
+ def rename_cloud(bk_cloud_id: int, bk_cloud_name: str, bk_cloud_vendor: str = None):
try:
# 增删改查CMDB操作以admin用户进行
- client_v2.cc.update_cloud_area({"bk_cloud_id": bk_cloud_id, "bk_cloud_name": bk_cloud_name})
+ client_v2.cc.update_cloud_area(
+ {"bk_cloud_id": bk_cloud_id, "bk_cloud_name": bk_cloud_name, "bk_cloud_vendor": bk_cloud_vendor}
+ )
except ComponentCallError as e:
logger.error("esb->call update_cloud_area error %s" % e.message)
- client_v2.cc.update_inst(bk_obj_id="plat", bk_inst_id=bk_cloud_id, bk_cloud_name=bk_cloud_name)
+ client_v2.cc.update_inst(
+ bk_obj_id="plat", bk_inst_id=bk_cloud_id, bk_cloud_name=bk_cloud_name, bk_cloud_vendor=bk_cloud_vendor
+ )
@classmethod
- def get_or_create_cloud(cls, bk_cloud_name):
+ def get_or_create_cloud(cls, bk_cloud_name: str, bk_cloud_vendor: str = None):
try:
return cls.get_cloud(bk_cloud_name)
except CloudNotExistError:
- return cls.add_cloud(bk_cloud_name)
+ return cls.add_cloud(bk_cloud_name, bk_cloud_vendor=bk_cloud_vendor)
def fetch_topo(self, bk_biz_id: int, with_biz_node: bool = False) -> List:
"""
diff --git a/apps/node_man/management/commands/sync_all_isp_to_cmdb.py b/apps/node_man/management/commands/sync_all_isp_to_cmdb.py
new file mode 100644
index 000000000..568c260b2
--- /dev/null
+++ b/apps/node_man/management/commands/sync_all_isp_to_cmdb.py
@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available.
+Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+
+from django.core.management.base import BaseCommand
+
+from apps.node_man.periodic_tasks import sync_all_isp_to_cmdb_periodic_task
+
+
+class Command(BaseCommand):
+ def handle(self, **kwargs):
+ sync_all_isp_to_cmdb_periodic_task()
diff --git a/apps/node_man/migrations/0084_update_isp_and_accesspoint_regionid_cityid.py b/apps/node_man/migrations/0084_update_isp_and_accesspoint_regionid_cityid.py
new file mode 100644
index 000000000..a77fd745c
--- /dev/null
+++ b/apps/node_man/migrations/0084_update_isp_and_accesspoint_regionid_cityid.py
@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available.
+Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+from django.db import migrations
+
+
+def update_isp_and_ap_region_city_id(apps, schema_editor):
+ """更新全局配置中的ISP和存量接入点的region_id和city_id"""
+ isp_list = [
+ {"isp": "PrivateCloud", "isp_name": "企业私有云"},
+ {"isp": "AWS", "isp_name": "亚马逊云"},
+ {"isp": "Azure", "isp_name": "微软云"},
+ {"isp": "GoogleCloud", "isp_name": "谷歌云"},
+ {"isp": "SalesForce", "isp_name": "SalesForce"},
+ {"isp": "OracleCloud", "isp_name": "Oracle Cloud"},
+ {"isp": "IBMCloud", "isp_name": "IBM Cloud"},
+ {"isp": "AlibabaCloud", "isp_name": "阿里云"},
+ {"isp": "TencentCloud", "isp_name": "腾讯云"},
+ {"isp": "ECloud", "isp_name": "中国电信"},
+ {"isp": "UCloud", "isp_name": "UCloud"},
+ {"isp": "MOS", "isp_name": "美团云"},
+ {"isp": "KSyun", "isp_name": "金山云"},
+ {"isp": "BaiduCloud", "isp_name": "百度云"},
+ {"isp": "HuaweiCloud", "isp_name": "华为云"},
+ {"isp": "capitalonline", "isp_name": "首都云"},
+ {"isp": "TencentPrivateCloud", "isp_name": "腾讯自研云"},
+ {"isp": "Zenlayer", "isp_name": "Zenlayer"},
+ ]
+ # 创建or更新ISP
+ GlobalSettings = apps.get_model("node_man", "GlobalSettings")
+ GlobalSettings.objects.update_or_create(defaults={"v_json": isp_list}, **{"key": "isp"})
+ # 更新存量接入点的region_id和city_id
+ AccessPoint = apps.get_model("node_man", "AccessPoint")
+ AccessPoint.objects.filter(region_id="test").update(region_id="default")
+ AccessPoint.objects.filter(city_id="test").update(city_id="default")
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("node_man", "0083_subscription_operate_info"),
+ ]
+
+ operations = [
+ migrations.RunPython(update_isp_and_ap_region_city_id),
+ ]
diff --git a/apps/node_man/models.py b/apps/node_man/models.py
index 8dc074de9..8fe2d6df4 100644
--- a/apps/node_man/models.py
+++ b/apps/node_man/models.py
@@ -170,6 +170,8 @@ class KeyEnum(Enum):
INSTALL_CHANNEL_ID_NETWORK_SEGMENT = "INSTALL_CHANNEL_ID_NETWORK_SEGMENT"
# 需要执行清理订阅的APP_CODE
NEED_CLEAN_SUBSCRIPTION_APP_CODE = "NEED_CLEAN_SUBSCRIPTION_APP_CODE"
+ # CMDB内置云区域IDS
+ CMDB_INTERNAL_CLOUD_IDS = "CMDB_INTERNAL_CLOUD_IDS"
key = models.CharField(_("键"), max_length=255, db_index=True, primary_key=True)
v_json = JSONField(_("值"))
@@ -187,7 +189,7 @@ def map_values(self, objs, source, target):
def fetch_isp(self):
isps = dict(GlobalSettings.objects.filter(key="isp").values_list("key", "v_json")).get("isp", [])
result = self.map_values(
- isps, lambda isp: isp["isp"], lambda isp: {"isp_name": isp["isp_name"], "isp_icon": isp["isp_icon"]}
+ isps, lambda isp: isp["isp"], lambda isp: {"isp_name": isp["isp_name"]}
)
return result
diff --git a/apps/node_man/periodic_tasks/__init__.py b/apps/node_man/periodic_tasks/__init__.py
index 6a0bdc9c6..eaffce09a 100644
--- a/apps/node_man/periodic_tasks/__init__.py
+++ b/apps/node_man/periodic_tasks/__init__.py
@@ -15,6 +15,7 @@
clean_subscription_record_info_periodic_task,
)
from .sync_agent_status_task import sync_agent_status_periodic_task # noqa
+from .sync_all_isp_to_cmdb import sync_all_isp_to_cmdb_periodic_task # noqa
from .sync_cmdb_cloud_area import sync_cmdb_cloud_area_periodic_task # noqa
from .sync_cmdb_host import sync_cmdb_host_periodic_task # noqa
from .sync_proc_status_task import sync_proc_status_periodic_task # noqa
diff --git a/apps/node_man/periodic_tasks/sync_all_isp_to_cmdb.py b/apps/node_man/periodic_tasks/sync_all_isp_to_cmdb.py
new file mode 100644
index 000000000..42f7674a6
--- /dev/null
+++ b/apps/node_man/periodic_tasks/sync_all_isp_to_cmdb.py
@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available.
+Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+import time
+from typing import Any, Dict, List
+
+from celery.task import periodic_task
+
+from apps.component.esbclient import client_v2
+from apps.exceptions import ComponentCallError
+from apps.node_man import constants
+from apps.node_man.models import Cloud, GlobalSettings
+from apps.utils.basic import chunk_lists
+from common.log import logger
+
+
+def sync_all_isp_to_cmdb(task_id):
+ logger.info(f"{task_id} | Start syncing cloud isp info.")
+ # CMDB内置云区域不更新,默认为直连区域与未分配管控区域，如有其他内置云区域通过GlobalSettings配置
+ cmdb_internal_cloud_ids = GlobalSettings.get_config(
+ key=GlobalSettings.KeyEnum.CMDB_INTERNAL_CLOUD_IDS.value,
+ default=[constants.DEFAULT_CLOUD, constants.UNASSIGNED_CLOUD_ID],
+ )
+ cloud_info: List[Dict[str, Any]] = list(Cloud.objects.values("bk_cloud_id", "isp"))
+ # 分片请求：一次五十条
+ for chunk_clouds in chunk_lists(cloud_info, constants.UPDATE_CMDB_CLOUD_AREA_LIMIT):
+ for cloud in chunk_clouds:
+ bk_cloud_id: int = cloud["bk_cloud_id"]
+ if bk_cloud_id in cmdb_internal_cloud_ids:
+ continue
+ bk_cloud_vendor: str = constants.CMDB_CLOUD_VENDOR_MAP.get(cloud["isp"])
+ try:
+ client_v2.cc.update_cloud_area({"bk_cloud_id": bk_cloud_id, "bk_cloud_vendor": bk_cloud_vendor})
+ except ComponentCallError as e:
+ logger.error("esb->call update_cloud_area error %s" % e.message)
+ client_v2.cc.update_inst(bk_obj_id="plat", bk_inst_id=bk_cloud_id, bk_cloud_vendor=bk_cloud_vendor)
+ # 休眠1秒避免一次性全量请求导致接口超频
+ time.sleep(1)
+
+ logger.info(f"{task_id} | Sync cloud isp info task complete.")
+
+
+@periodic_task(
+ queue="default",
+ options={"queue": "default"},
+ run_every=constants.SYNC_ISP_TO_CMDB_INTERVAL,
+)
+def sync_all_isp_to_cmdb_periodic_task():
+ """
+ 同步云服务商至CMDB
+ """
+ task_id = sync_all_isp_to_cmdb_periodic_task.request.id
+ sync_all_isp_to_cmdb(task_id)
diff --git a/apps/node_man/tests/test_handlers/test_cloud.py b/apps/node_man/tests/test_handlers/test_cloud.py
index ea9e9873a..f4eda3570 100644
--- a/apps/node_man/tests/test_handlers/test_cloud.py
+++ b/apps/node_man/tests/test_handlers/test_cloud.py
@@ -181,3 +181,42 @@ def test_list_cloud_name(self, *args, **kwargs):
cloud_info = CloudHandler().list_cloud_name()
self.assertEqual(len(cloud_info), 1)
+
+ @patch("apps.node_man.handlers.cmdb.client_v2", MockClient)
+ def test_cloud_create_and_sync_isp(self):
+ with patch("apps.node_man.handlers.cmdb.client_v2.cc.search_cloud_area") as search_cloud:
+ search_cloud.return_value = {"info": []}
+ with patch("apps.node_man.handlers.cmdb.client_v2.cc.create_cloud_area") as create_cloud:
+ create_cloud.return_value = {"created": {"id": 10000}}
+ CloudHandler().create(
+ {
+ "isp": ["TencentCloud", "AlibabaCloud", "AWS"][random.randint(0, 2)],
+ "ap_id": -1,
+ "bk_cloud_name": "".join(random.choice(DIGITS) for x in range(8)),
+ },
+ "admin",
+ )
+ call_args = create_cloud.call_args
+ bk_cloud_vendor_scope = [str(bk_cloud_vendor) for bk_cloud_vendor in range(1, 19)]
+ self.assertIn(call_args[0][0]["bk_cloud_vendor"], bk_cloud_vendor_scope)
+
+ @patch("apps.node_man.handlers.cmdb.client_v2", MockClient)
+ def test_update_cloud_and_isp(self):
+ kwarg = {
+ "isp": ["TencentCloud", "AlibabaCloud", "AWS"][random.randint(0, 2)],
+ "ap_id": -1,
+ "bk_cloud_name": "".join(random.choice(DIGITS) for x in range(8)),
+ }
+ cloud = CloudHandler().create(kwarg, "admin")
+
+ # 测试更新isp
+ bk_cloud_id = cloud["bk_cloud_id"]
+ kwarg["ap_id"] = 1
+ kwarg["bk_cloud_name"] = "cdtest"
+
+ with patch("apps.node_man.handlers.cmdb.client_v2.cc.update_cloud_area") as update_cloud:
+ update_cloud.return_value = {"result": True}
+ CloudHandler().update(bk_cloud_id, kwarg["bk_cloud_name"], kwarg["isp"], kwarg["ap_id"])
+ call_args = update_cloud.call_args
+ bk_cloud_vendor_scope = [str(bk_cloud_vendor) for bk_cloud_vendor in range(1, 19)]
+ self.assertIn(call_args[0][0]["bk_cloud_vendor"], bk_cloud_vendor_scope)
diff --git a/apps/node_man/tests/test_pericdic_tasks/test_sync_all_isp_to_cmdb.py b/apps/node_man/tests/test_pericdic_tasks/test_sync_all_isp_to_cmdb.py
new file mode 100644
index 000000000..8a91c5e7c
--- /dev/null
+++ b/apps/node_man/tests/test_pericdic_tasks/test_sync_all_isp_to_cmdb.py
@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available.
+Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+
+from unittest.mock import patch
+
+from django.test import TestCase
+
+from apps.node_man import models
+from apps.node_man.periodic_tasks.sync_all_isp_to_cmdb import (
+ sync_all_isp_to_cmdb_periodic_task,
+)
+from apps.node_man.tests.utils import MockClient, create_cloud_area
+
+
+class TestSyncAllIspToCmdb(TestCase):
+ @staticmethod
+ def init_db():
+ create_cloud_area(2)
+
+ @patch("apps.node_man.periodic_tasks.sync_all_isp_to_cmdb.client_v2", MockClient)
+ def test_sync_all_isp_to_cmdb(self):
+ self.init_db()
+ # 构造CMDB内置云区域ID
+ models.GlobalSettings.set_config(key=models.GlobalSettings.KeyEnum.CMDB_INTERNAL_CLOUD_IDS.value, value=[1])
+ models.Cloud.objects.filter(bk_cloud_id=2).update(isp="TencentCloud")
+ with patch("apps.node_man.periodic_tasks.sync_all_isp_to_cmdb.client_v2.cc.update_cloud_area") as update_cloud:
+ update_cloud.return_value = {"result": True}
+ sync_all_isp_to_cmdb_periodic_task()
+ call_args = update_cloud.call_args
+ bk_cloud_vendor_scope = [str(bk_cloud_vendor) for bk_cloud_vendor in range(1, 19)]
+ self.assertIn(call_args[0][0]["bk_cloud_vendor"], bk_cloud_vendor_scope)
+ self.assertNotIn(1, call_args[0][0])
diff --git a/env/__init__.py b/env/__init__.py
index f0cd3b853..d2bddf936 100644
--- a/env/__init__.py
+++ b/env/__init__.py
@@ -70,6 +70,8 @@
# 自动选择安装通道相关配置
"BKAPP_DEFAULT_INSTALL_CHANNEL_ID",
"BKAPP_AUTOMATIC_CHOICE_CLOUD_ID",
+ # 未分配管控区域ID
+ "BKAPP_UNASSIGNED_CLOUD_ID",
]
# ===============================================================================
@@ -96,6 +98,7 @@
)
BKAPP_DEFAULT_INSTALL_CHANNEL_ID = get_type_env(key="BKAPP_DEFAULT_INSTALL_CHANNEL_ID", default=-1, _type=int)
BKAPP_AUTOMATIC_CHOICE_CLOUD_ID = get_type_env(key="BKAPP_AUTOMATIC_CHOICE_CLOUD_ID", default=-1, _type=int)
+BKAPP_UNASSIGNED_CLOUD_ID = get_type_env(key="BKAPP_UNASSIGNED_CLOUD_ID", default=90000001, _type=int)
# ===============================================================================
# 日志
diff --git a/support-files/kubernetes/helm/bk-nodeman/README.md b/support-files/kubernetes/helm/bk-nodeman/README.md
index ce463e9cf..c26b7cfa3 100644
--- a/support-files/kubernetes/helm/bk-nodeman/README.md
+++ b/support-files/kubernetes/helm/bk-nodeman/README.md
@@ -291,67 +291,68 @@ externalRabbitMQ:
用于生成运行环境变量，具体参考：`support-files/kubernetes/helm/bk-nodeman/templates/configmaps/env-configmap.yaml`
-| 参数 | 描述 | 默认值 |
-|------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------|
-| `config.appCode` | app code | `bk_nodeman` |
-| `config.appSecret` | app secret | `""` |
-| `config.bkAppRunEnv` | 运行环境，ce / ee / ieod，影响 gse 端口等配置 | `ce` |
-| `config.bkAppEnableDHCP` | 是否开启动态主机配置协议适配，开启后支持动态主机场景，将安装 2.0 Agent，开启 AgentID 特性 | `false` |
-| `config.bkPaasMajorVersion` | 开发框架 PaaS 版本适配，目前仅支持 `3` | `3` |
-| `config.bkPaaSEnvironment` | 开发框架 PaaS 环境适配，目前仅支持 `prod` | `prod` |
-| `config.logType` | 日志类别，`DEFAULT`- `STDOUT` | `STDOUT` |
-| `config.logLevel` | 日志级别 | `INFO` |
-| `config.bkLogDir` | 日志所在目录，`config.logType=DEFAULT` 时有效 | `/data/bkee/logs/bknodeman` |
-| `config.bkCmdbResourcePoolBizId` | 蓝鲸配置平台相关配置，资源池 ID | `1` |
-| `config.defaultSupplierAccount` | 蓝鲸配置平台相关配置，企业账户 | `0` |
-| `config.jobVersion` | 蓝鲸作业平台相关配置，API 版本，可选项 `V2` `V3` | `V3` |
-| `config.bluekingBizId` | 蓝鲸作业平台相关配置，调用作业平台 API 所使用的业务集 ID | `9991001` |
-| `config.bkAppUseIam` | 蓝鲸权限中心相关配置，是否启用权限中心 | `true` |
-| `config.bkIamV3AppCode` | 蓝鲸权限中心相关配置，权限中心 AppCode | `bk_iam` |
-| `config.bkAppIamResourceApiHost` | 蓝鲸权限中心相关配置，权限中心拉取权限相关资源的访问地址，默认取 `{{ .Values.bkNodemanUrl }}` | `""` |
-| `config.bkAppBkNodeApiGateway` | 组件 API 接入地址，节点管理网关地址，用于覆盖 `bkComponentApiUrl` 访问节点管理
⚠️ 配置为 `{{ .Values.bkNodemanApiUrl }`} 由于 JWT 校验问题，会导致 Agent 安装步骤中「安装预制插件」失败 | `""` |
-| `config.bkAppBkGseApiGateway` | 管控平台 API 访问地址，用于覆盖 `bkComponentApiUrl` 访问管控平台 API | `""` |
-| `config.bkAppBackendHost` | 节点管理自身模块依赖，后台访问地址，渲染时为空默认取 `{{ .Values.bkNodemanApiUrl }}` | `""` |
-| `config.bkAppNodemanCallbackUrl` | 节点管理自身模块依赖，后台内网回调地址，渲染时为空取 `{{ .Values.bkNodemanUrl }}/backend` | `""` |
-| `config.bkAppNodemanOuterCallbackUrl` | 节点管理自身模块依赖，后台外网回调地址，渲染时为空取 `{{ .Values.bkNodemanUrl }}/backend` | `""` |
-| `config.gseVersion` | 蓝鲸管控平台版本，默认为 `V1`,可选：`V1` `V2` | `V1` |
-| `config.gseCertPath` | GSE 本地证书路径，渲染时为空默认取 `/data/bk{{ .Values.config.bkAppRunEnv }}/cert` | `""` |
-| `config.gseEnablePushEnvironFile` | 是否启用推送 GSE 环境变量文件，如果启用，将在 Agent `安装`/`重装`/`重载配置`/`灰度` 操作成功后，进行如下操作：
Windows：推送 `environ.sh` & `environ.bat` 到目标机器的 `GSE_ENVIRON_WIN_DIR` 路径
Linux：推送 `environ.sh` 到目标机器的 `GSE_ENVIRON_DIR` 路径 | `false` |
-| `config.gseEnvironDir` | GSE 环境变量目录 | `/etc/sysconfig/gse/bk` |
-| `config.gseEnvironWinDir` | GSE 环境变量目录（Windows） | `C:\\Windows\\System32\\config\\gse\\bk` |
-| `config.gseEnableSvrDisCovery` | 蓝鲸管控平台 Agent，AgentXXDir 仅在初次部署有效，后续可以在页面「全局配置」维护。是否启用 GSE 服务探测，默认为 `true` | `true` |
-| `config.bkAppGseZkHost` | 蓝鲸管控平台 Agent，zk hosts 信息，host:port，多个 hosts 以 `,` 分隔
⚠️ ZK hosts 将作为 Agent 配置，需要保证 Agent 可访问，所以不能使用 k8s service 信息 进行配置
如果 zk 通过 k8s 部署，建议通过 NodePort 等方式暴露服务，使用 NodeIP:NodePort 进行配置 | `127.0.0.1:2181` |
-| `config.bkAppGseZkAuth` | 蓝鲸管控平台 Agent，ZK 认证信息，用户名:密码 | `bkzk:zkpass` |
-| `config.bkAppGseAgentHome` | 蓝鲸管控平台 Agent，AgentXXDir 仅在初次部署有效，后续可以在页面「全局配置」维护。Linux Agent 安装目录 | `/usr/local/gse` |
-| `config.bkAppGseAgentLogDir` | 蓝鲸管控平台 Agent，AgentXXDir 仅在初次部署有效，后续可以在页面「全局配置」维护。Linux Agent 日志目录 | `/usr/log/gse` |
-| `config.bkAppGseAgentRunDir` | 蓝鲸管控平台 Agent，AgentXXDir 仅在初次部署有效，后续可以在页面「全局配置」维护。Linux Agent 运行目录 | `/usr/run/gse` |
-| `config.bkAppGseAgentDataDir` | 蓝鲸管控平台 Agent，AgentXXDir 仅在初次部署有效，后续可以在页面「全局配置」维护。Linux Agent 数据目录 | `/usr/data/gse` |
-| `config.bkAppGseWinAgentHome` | 蓝鲸管控平台 Agent，AgentXXDir 仅在初次部署有效，后续可以在页面「全局配置」维护。Windows Agent 安装目录 | `C:\\\\gse` |
-| `config.bkAppGseWinAgentLogDir` | 蓝鲸管控平台 Agent，AgentXXDir 仅在初次部署有效，后续可以在页面「全局配置」维护。Windows Agent 日志目录 | `C:\\\\gse\\\\logs` |
-| `config.bkAppGseWinAgentRunDir` | 蓝鲸管控平台 Agent，AgentXXDir 仅在初次部署有效，后续可以在页面「全局配置」维护。Windows Agent 运行目录 | `C:\\\\gse\\\\data` |
-| `config.bkAppGseWinAgentDataDir` | 蓝鲸管控平台 Agent，AgentXXDir 仅在初次部署有效，后续可以在页面「全局配置」维护。Windows Agent 数据目录 | `C:\\\\gse\\\\data` |
-| `config.storageType` | 存储，存储类型`FILE_SYSTEM` `BLUEKING_ARTIFACTORY` | `BLUEKING_ARTIFACTORY` |
-| `config.lanIp` | 存储，文件服务器内网IP，用于物理机文件分发，在 `storageType=FILE_SYSTEM` 时必须设置为有效中 | `127.0.0.1` |
-| `config.bkAppPublicPath` | 存储，文件存储目录 | `/data/bkee/public/bknodeman/` |
-| `config.bkRepoProject` | 存储，蓝鲸制品库项目 | `""` |
-| `config.bkRepoPassword` | 存储，蓝鲸制品库密码 | `""` |
-| `config.bkRepoUsername` | 存储，蓝鲸制品库用户 | `""` |
-| `config.bkRepoBucket` | 存储，蓝鲸制品库仓库 | `""` |
-| `config.bkRepoPublicBucket` | 存储，蓝鲸制品库公共仓库 | `""` |
-| `config.bkRepoPrivateBucket` | 存储，蓝鲸制品库私有仓库 | `""` |
-| `config.bkAppEnableOtelTrace` | 可观测，是否开启 Trace | `false` |
-| `config.bkAppOtelInstrumentDbApi` | 可观测，是否开启 DB 访问 trace（开启后 span 数量会明显增多） | `false` |
-| `config.bkAppOtelSampler` | 可观测，配置采样策略，可选值 `always_on`，`always_off`, `parentbased_always_on`,`parentbased_always_off`, `traceidratio`, `parentbased_traceidratio` | `parentbased_always_off` |
-| `config.bkAppOtelBkDataToken` | 可观测，监控上报配置项 | `""` |
-| `config.bkAppOtelGrpcUrl` | 可观测，监控上报配置项 | `""` |
-| `config.concurrentNumber` | 线程最大并发数 | `50` |
-| `config.bkAppNavOpenSourceUrl` | 导航栏开源社区地址 | `https://github.com/TencentBlueKing/bk-nodeman` |
+| 参数 | 描述 | 默认值 |
+|------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------|
+| `config.appCode` | app code | `bk_nodeman` |
+| `config.appSecret` | app secret | `""` |
+| `config.bkAppRunEnv` | 运行环境，ce / ee / ieod，影响 gse 端口等配置 | `ce` |
+| `config.bkAppEnableDHCP` | 是否开启动态主机配置协议适配，开启后支持动态主机场景，将安装 2.0 Agent，开启 AgentID 特性 | `false` |
+| `config.bkPaasMajorVersion` | 开发框架 PaaS 版本适配，目前仅支持 `3` | `3` |
+| `config.bkPaaSEnvironment` | 开发框架 PaaS 环境适配，目前仅支持 `prod` | `prod` |
+| `config.logType` | 日志类别，`DEFAULT`- `STDOUT` | `STDOUT` |
+| `config.logLevel` | 日志级别 | `INFO` |
+| `config.bkLogDir` | 日志所在目录，`config.logType=DEFAULT` 时有效 | `/data/bkee/logs/bknodeman` |
+| `config.bkCmdbResourcePoolBizId` | 蓝鲸配置平台相关配置，资源池 ID | `1` |
+| `config.defaultSupplierAccount` | 蓝鲸配置平台相关配置，企业账户 | `0` |
+| `config.jobVersion` | 蓝鲸作业平台相关配置，API 版本，可选项 `V2` `V3` | `V3` |
+| `config.bluekingBizId` | 蓝鲸作业平台相关配置，调用作业平台 API 所使用的业务集 ID | `9991001` |
+| `config.bkAppUseIam` | 蓝鲸权限中心相关配置，是否启用权限中心 | `true` |
+| `config.bkIamV3AppCode` | 蓝鲸权限中心相关配置，权限中心 AppCode | `bk_iam` |
+| `config.bkAppIamResourceApiHost` | 蓝鲸权限中心相关配置，权限中心拉取权限相关资源的访问地址，默认取 `{{ .Values.bkNodemanUrl }}` | `""` |
+| `config.bkAppBkNodeApiGateway` | 组件 API 接入地址，节点管理网关地址，用于覆盖 `bkComponentApiUrl` 访问节点管理
⚠️ 配置为 `{{ .Values.bkNodemanApiUrl }`} 由于 JWT 校验问题，会导致 Agent 安装步骤中「安装预制插件」失败 | `""` |
+| `config.bkAppBkGseApiGateway` | 管控平台 API 访问地址，用于覆盖 `bkComponentApiUrl` 访问管控平台 API | `""` |
+| `config.bkAppBackendHost` | 节点管理自身模块依赖，后台访问地址，渲染时为空默认取 `{{ .Values.bkNodemanApiUrl }}` | `""` |
+| `config.bkAppNodemanCallbackUrl` | 节点管理自身模块依赖，后台内网回调地址，渲染时为空取 `{{ .Values.bkNodemanUrl }}/backend` | `""` |
+| `config.bkAppNodemanOuterCallbackUrl` | 节点管理自身模块依赖，后台外网回调地址，渲染时为空取 `{{ .Values.bkNodemanUrl }}/backend` | `""` |
+| `config.gseVersion` | 蓝鲸管控平台版本，默认为 `V1`,可选：`V1` `V2` | `V1` |
+| `config.gseCertPath` | GSE 本地证书路径，渲染时为空默认取 `/data/bk{{ .Values.config.bkAppRunEnv }}/cert` | `""` |
+| `config.gseEnablePushEnvironFile` | 是否启用推送 GSE 环境变量文件，如果启用，将在 Agent `安装`/`重装`/`重载配置`/`灰度` 操作成功后，进行如下操作：
Windows：推送 `environ.sh` & `environ.bat` 到目标机器的 `GSE_ENVIRON_WIN_DIR` 路径
Linux：推送 `environ.sh` 到目标机器的 `GSE_ENVIRON_DIR` 路径 | `false` |
+| `config.gseEnvironDir` | GSE 环境变量目录 | `/etc/sysconfig/gse/bk` |
+| `config.gseEnvironWinDir` | GSE 环境变量目录（Windows） | `C:\\Windows\\System32\\config\\gse\\bk` |
+| `config.gseEnableSvrDisCovery` | 蓝鲸管控平台 Agent，AgentXXDir 仅在初次部署有效，后续可以在页面「全局配置」维护。是否启用 GSE 服务探测，默认为 `true` | `true` |
+| `config.bkAppGseZkHost` | 蓝鲸管控平台 Agent，zk hosts 信息，host:port，多个 hosts 以 `,` 分隔
⚠️ ZK hosts 将作为 Agent 配置，需要保证 Agent 可访问，所以不能使用 k8s service 信息 进行配置
如果 zk 通过 k8s 部署，建议通过 NodePort 等方式暴露服务，使用 NodeIP:NodePort 进行配置 | `127.0.0.1:2181` |
+| `config.bkAppGseZkAuth` | 蓝鲸管控平台 Agent，ZK 认证信息，用户名:密码 | `bkzk:zkpass` |
+| `config.bkAppGseAgentHome` | 蓝鲸管控平台 Agent，AgentXXDir 仅在初次部署有效，后续可以在页面「全局配置」维护。Linux Agent 安装目录 | `/usr/local/gse` |
+| `config.bkAppGseAgentLogDir` | 蓝鲸管控平台 Agent，AgentXXDir 仅在初次部署有效，后续可以在页面「全局配置」维护。Linux Agent 日志目录 | `/usr/log/gse` |
+| `config.bkAppGseAgentRunDir` | 蓝鲸管控平台 Agent，AgentXXDir 仅在初次部署有效，后续可以在页面「全局配置」维护。Linux Agent 运行目录 | `/usr/run/gse` |
+| `config.bkAppGseAgentDataDir` | 蓝鲸管控平台 Agent，AgentXXDir 仅在初次部署有效，后续可以在页面「全局配置」维护。Linux Agent 数据目录 | `/usr/data/gse` |
+| `config.bkAppGseWinAgentHome` | 蓝鲸管控平台 Agent，AgentXXDir 仅在初次部署有效，后续可以在页面「全局配置」维护。Windows Agent 安装目录 | `C:\\\\gse` |
+| `config.bkAppGseWinAgentLogDir` | 蓝鲸管控平台 Agent，AgentXXDir 仅在初次部署有效，后续可以在页面「全局配置」维护。Windows Agent 日志目录 | `C:\\\\gse\\\\logs` |
+| `config.bkAppGseWinAgentRunDir` | 蓝鲸管控平台 Agent，AgentXXDir 仅在初次部署有效，后续可以在页面「全局配置」维护。Windows Agent 运行目录 | `C:\\\\gse\\\\data` |
+| `config.bkAppGseWinAgentDataDir` | 蓝鲸管控平台 Agent，AgentXXDir 仅在初次部署有效，后续可以在页面「全局配置」维护。Windows Agent 数据目录 | `C:\\\\gse\\\\data` |
+| `config.storageType` | 存储，存储类型`FILE_SYSTEM` `BLUEKING_ARTIFACTORY` | `BLUEKING_ARTIFACTORY` |
+| `config.lanIp` | 存储，文件服务器内网IP，用于物理机文件分发，在 `storageType=FILE_SYSTEM` 时必须设置为有效中 | `127.0.0.1` |
+| `config.bkAppPublicPath` | 存储，文件存储目录 | `/data/bkee/public/bknodeman/` |
+| `config.bkRepoProject` | 存储，蓝鲸制品库项目 | `""` |
+| `config.bkRepoPassword` | 存储，蓝鲸制品库密码 | `""` |
+| `config.bkRepoUsername` | 存储，蓝鲸制品库用户 | `""` |
+| `config.bkRepoBucket` | 存储，蓝鲸制品库仓库 | `""` |
+| `config.bkRepoPublicBucket` | 存储，蓝鲸制品库公共仓库 | `""` |
+| `config.bkRepoPrivateBucket` | 存储，蓝鲸制品库私有仓库 | `""` |
+| `config.bkAppEnableOtelTrace` | 可观测，是否开启 Trace | `false` |
+| `config.bkAppOtelInstrumentDbApi` | 可观测，是否开启 DB 访问 trace（开启后 span 数量会明显增多） | `false` |
+| `config.bkAppOtelSampler` | 可观测，配置采样策略，可选值 `always_on`，`always_off`, `parentbased_always_on`,`parentbased_always_off`, `traceidratio`, `parentbased_traceidratio` | `parentbased_always_off` |
+| `config.bkAppOtelBkDataToken` | 可观测，监控上报配置项 | `""` |
+| `config.bkAppOtelGrpcUrl` | 可观测，监控上报配置项 | `""` |
+| `config.concurrentNumber` | 线程最大并发数 | `50` |
+| `config.bkAppNavOpenSourceUrl` | 导航栏开源社区地址 | `https://github.com/TencentBlueKing/bk-nodeman` |
| `config.bkAppNavHelperUrl` | 导航栏技术支持地址 | `https://wpa1.qq.com/KziXGWJs?_type=wpa&qidian=true` |
-| `config.bkAppSyncProcStatusTaskInterval` | 插件进程状态同步周期 | `20 * 60` |
-| `config.bkAppScriptHooks` | Agent安装前置脚本 | `""` |
-| `config.bkAppIEODActiveFirewallPolicyScriptInfo` | WINDOWS IEOD脚本内容 | `""` |
-| `config.bkAppDefaultInstallChannelId` | 自动选择安装通道ID | `-1` |
-| `config.bkAppAutomaticChoiceCloudId` | 自动选择安装通道对应云区域ID | `-1` |
+| `config.bkAppSyncProcStatusTaskInterval` | 插件进程状态同步周期 | `20 * 60` |
+| `config.bkAppScriptHooks` | Agent安装前置脚本 | `""` |
+| `config.bkAppIEODActiveFirewallPolicyScriptInfo` | WINDOWS IEOD脚本内容 | `""` |
+| `config.bkAppDefaultInstallChannelId` | 自动选择安装通道ID | `-1` |
+| `config.bkAppAutomaticChoiceCloudId` | 自动选择安装通道对应云区域ID | `-1` |
+| `config.bkAppUnassignedCloudId` | 未分配管控区域ID | `90000001` |
## 额外的环境变量
diff --git a/support-files/kubernetes/helm/bk-nodeman/templates/configmaps/env-configmap.yaml b/support-files/kubernetes/helm/bk-nodeman/templates/configmaps/env-configmap.yaml
index 648a257b7..b3e1ea915 100644
--- a/support-files/kubernetes/helm/bk-nodeman/templates/configmaps/env-configmap.yaml
+++ b/support-files/kubernetes/helm/bk-nodeman/templates/configmaps/env-configmap.yaml
@@ -134,3 +134,4 @@ data:
BKAPP_IEOD_ACTIVE_FIREWALL_POLICY_SCRIPT_INFO: '{{ .Values.config.bkAppIEODActiveFirewallPolicyScriptInfo }}'
BKAPP_DEFAULT_INSTALL_CHANNEL_ID: "{{ .Values.config.bkAppDefaultInstallChannelId}}"
BKAPP_AUTOMATIC_CHOICE_CLOUD_ID: "{{ .Values.config.bkAppAutomaticChoiceCloudId}}"
+ BKAPP_UNASSIGNED_CLOUD_ID: "{{ .Values.config.bkAppUnassignedCloudId}}"
diff --git a/support-files/kubernetes/helm/bk-nodeman/values.yaml b/support-files/kubernetes/helm/bk-nodeman/values.yaml
index 9da7e9303..ccc6cb742 100644
--- a/support-files/kubernetes/helm/bk-nodeman/values.yaml
+++ b/support-files/kubernetes/helm/bk-nodeman/values.yaml
@@ -473,6 +473,8 @@ config:
bkAppDefaultInstallChannelId: -1
## 自动选择安装通道对应云区域ID
bkAppAutomaticChoiceCloudId: -1
+ ## 未分配管控区域ID
+ bkAppUnassignedCloudId: 90000001
## --------------------------------------