feat: 向cmdb同步云区域服务商 (closed #2386)

apps/node_man/constants.py

@@ -59,6 +59,7 @@ class TimeUnit:
 COLLECT_AUTO_TRIGGER_JOB_INTERVAL = 5 * TimeUnit.MINUTE
 SYNC_CMDB_CLOUD_AREA_INTERVAL = 10 * TimeUnit.SECOND
 SYNC_AGENT_STATUS_TASK_INTERVAL = 10 * TimeUnit.MINUTE
+SYNC_ISP_TO_CMDB_INTERVAL = 1 * TimeUnit.DAY
 SYNC_PROC_STATUS_TASK_INTERVAL = settings.SYNC_PROC_STATUS_TASK_INTERVAL
 SYNC_BIZ_TO_GRAY_SCOPE_LIST_INTERVAL = 30 * TimeUnit.MINUTE
 
@@ -559,6 +560,7 @@ def _get_member__alias_map(cls) -> Dict[Enum, str]:
 QUERY_CLOUD_LIMIT = 200
 QUERY_HOST_SERVICE_TEMPLATE_LIMIT = 200
 QUERY_MODULE_ID_THRESHOLD = 15
+UPDATE_CMDB_CLOUD_AREA_LIMIT = 50
 VERSION_PATTERN = re.compile(r"[vV]?(\d+\.){1,5}\d+(-rc\d)?$")
 # 语义化版本正则，参考：https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string
 SEMANTIC_VERSION_PATTERN = re.compile(
@@ -595,6 +597,25 @@ def _get_member__alias_map(cls) -> Dict[Enum, str]:
 MAX_HOST_IDS_LENGTH = 5000
 # 操作系统对应账户名
 OS_ACCOUNT = {"LINUX": LINUX_ACCOUNT, "WINDOWS": WINDOWS_ACCOUNT}
+# NODEMAN云服务商对应CMDB接口云服务商映射
+CMDB_CLOUD_VENDOR_MAP = {
+    "Amazon": "1",
+    "Tencent": "2",
+    "Google": "3",
+    "MicroSoft": "4",
+    "PrivateCloud": "5",
+    "SalesForce": "6",
+    "Oracle": "7",
+    "IBM": "8",
+    "Aliyun": "9",
+    "ECloud": "10",
+    "UCloud": "11",
+    "MOS": "12",
+    "KSCLOUD": "13",
+    "baidu": "14",
+    "huawei": "15",
+    "capitalonline": "16",
+}
 
 
 class ProxyFileFromType(Enum):

apps/node_man/handlers/cloud.py

@@ -199,7 +199,8 @@ def create(self, params: dict, username: str):
         """
 
         bk_cloud_name = params["bk_cloud_name"]
-        bk_cloud_id = CmdbHandler.get_or_create_cloud(bk_cloud_name)
+        bk_cloud_vendor = const.CMDB_CLOUD_VENDOR_MAP.get(params["isp"])
+        bk_cloud_id = CmdbHandler.get_or_create_cloud(bk_cloud_name, bk_cloud_vendor=bk_cloud_vendor)
 
         if bk_cloud_name == str(DEFAULT_CLOUD_NAME):
             raise ValidationError(_("管控区域不可名为「直连区域」"))
@@ -236,8 +237,9 @@ def update(bk_cloud_id: int, bk_cloud_name: str, isp: str, ap_id: int):
         if Cloud.objects.filter(bk_cloud_name=bk_cloud_name).exclude(bk_cloud_id=bk_cloud_id).exists():
             raise ValidationError(_("管控区域名称不可重复"))
 
-        # 向CMDB修改管控区域名称
-        CmdbHandler.rename_cloud(bk_cloud_id, bk_cloud_name)
+        # 向CMDB修改管控区域名称以及云服务商
+        bk_cloud_vendor: str = const.CMDB_CLOUD_VENDOR_MAP.get(isp)
+        CmdbHandler.rename_cloud(bk_cloud_id, bk_cloud_name, bk_cloud_vendor=bk_cloud_vendor)
 
         cloud.bk_cloud_name = bk_cloud_name
         cloud.isp = isp

apps/node_man/handlers/cmdb.py

@@ -322,12 +322,12 @@ def check_biz_permission(self, bk_biz_scope: list, action: str):
         raise PermissionDeniedError(action_name=action, apply_url=apply_url, permission=apply_data)
 
     @staticmethod
-    def add_cloud(bk_cloud_name):
+    def add_cloud(bk_cloud_name: str, bk_cloud_vendor: str = None):
         """
         新增管控区域
         """
         # 增删改查CMDB操作以admin用户进行
-        data = client_v2.cc.create_cloud_area({"bk_cloud_name": bk_cloud_name})
+        data = client_v2.cc.create_cloud_area({"bk_cloud_name": bk_cloud_name, "bk_cloud_vendor": bk_cloud_vendor})
         return data.get("created", {}).get("id")
 
     @staticmethod
@@ -363,20 +363,24 @@ def get_cloud(bk_cloud_name):
         raise CloudNotExistError
 
     @staticmethod
-    def rename_cloud(bk_cloud_id, bk_cloud_name):
+    def rename_cloud(bk_cloud_id: int, bk_cloud_name: str, bk_cloud_vendor: str = None):
         try:
             # 增删改查CMDB操作以admin用户进行
-            client_v2.cc.update_cloud_area({"bk_cloud_id": bk_cloud_id, "bk_cloud_name": bk_cloud_name})
+            client_v2.cc.update_cloud_area(
+                {"bk_cloud_id": bk_cloud_id, "bk_cloud_name": bk_cloud_name, "bk_cloud_vendor": bk_cloud_vendor}
+            )
         except ComponentCallError as e:
             logger.error("esb->call update_cloud_area error %s" % e.message)
-            client_v2.cc.update_inst(bk_obj_id="plat", bk_inst_id=bk_cloud_id, bk_cloud_name=bk_cloud_name)
+            client_v2.cc.update_inst(
+                bk_obj_id="plat", bk_inst_id=bk_cloud_id, bk_cloud_name=bk_cloud_name, bk_cloud_vendor=bk_cloud_vendor
+            )
 
     @classmethod
-    def get_or_create_cloud(cls, bk_cloud_name):
+    def get_or_create_cloud(cls, bk_cloud_name: str, bk_cloud_vendor: str = None):
         try:
             return cls.get_cloud(bk_cloud_name)
         except CloudNotExistError:
-            return cls.add_cloud(bk_cloud_name)
+            return cls.add_cloud(bk_cloud_name, bk_cloud_vendor=bk_cloud_vendor)
 
     def fetch_topo(self, bk_biz_id: int, with_biz_node: bool = False) -> List:
         """

apps/node_man/management/commands/sync_all_isp_to_cmdb.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available.
+Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+
+from django.core.management.base import BaseCommand
+
+from apps.node_man.periodic_tasks import sync_all_isp_to_cmdb_periodic_task
+
+
+class Command(BaseCommand):
+    def handle(self, **kwargs):
+        sync_all_isp_to_cmdb_periodic_task()

apps/node_man/models.py

@@ -164,6 +164,8 @@ class KeyEnum(Enum):
         IP_CHOOSER_ENABLE_SHOW_REALTIME_AGENT_STATE = "IP_CHOOSER_ENABLE_SHOW_REALTIME_AGENT_STATE"
         # IP选择器详情接口实时展示agent状态业务白名单
         IP_CHOOSER_BIZ_WHITELIST = "IP_CHOOSER_BIZ_WHITELIST"
+        # CMDB内置云区域IDS
+        CMDB_INTERNAL_CLOUD_IDS = "CMDB_INTERNAL_CLOUD_IDS"
 
     key = models.CharField(_("键"), max_length=255, db_index=True, primary_key=True)
     v_json = JSONField(_("值"))

apps/node_man/periodic_tasks/__init__.py

@@ -15,6 +15,7 @@
     clean_subscription_record_info_periodic_task,
 )
 from .sync_agent_status_task import sync_agent_status_periodic_task  # noqa
+from .sync_all_isp_to_cmdb import sync_all_isp_to_cmdb_periodic_task  # noqa
 from .sync_cmdb_cloud_area import sync_cmdb_cloud_area_periodic_task  # noqa
 from .sync_cmdb_host import sync_cmdb_host_periodic_task  # noqa
 from .sync_proc_status_task import sync_proc_status_periodic_task  # noqa

apps/node_man/periodic_tasks/sync_all_isp_to_cmdb.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available.
+Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+
+import time
+from typing import Any, Dict, List
+
+from celery.task import periodic_task
+
+from apps.component.esbclient import client_v2
+from apps.exceptions import ComponentCallError
+from apps.node_man import constants
+from apps.node_man.models import Cloud, GlobalSettings
+from apps.utils.basic import chunk_lists
+from common.log import logger
+
+
+def sync_all_isp_to_cmdb(task_id):
+    logger.info(f"{task_id} | Start syncing cloud isp info.")
+    # CMDB内置云区域不更新
+    cmdb_internal_cloud_ids = GlobalSettings.get_config(
+        key=GlobalSettings.KeyEnum.CMDB_INTERNAL_CLOUD_IDS.value, default=[]
+    )
+    cloud_info: List[Dict[str, Any]] = list(Cloud.objects.values("bk_cloud_id", "isp"))
+    # 分片请求：一次五十条
+    for chunk_clouds in chunk_lists(cloud_info, constants.UPDATE_CMDB_CLOUD_AREA_LIMIT):
+        for cloud in chunk_clouds:
+            bk_cloud_id: int = cloud["bk_cloud_id"]
+            if bk_cloud_id in cmdb_internal_cloud_ids:
+                continue
+            bk_cloud_vendor: str = constants.CMDB_CLOUD_VENDOR_MAP.get(cloud["isp"])
+            try:
+                client_v2.cc.update_cloud_area({"bk_cloud_id": bk_cloud_id, "bk_cloud_vendor": bk_cloud_vendor})
+            except ComponentCallError as e:
+                logger.error("esb->call update_cloud_area error %s" % e.message)
+                client_v2.cc.update_inst(bk_obj_id="plat", bk_inst_id=bk_cloud_id, bk_cloud_vendor=bk_cloud_vendor)
+        # 休眠1秒避免一次性全量请求导致接口超频
+        time.sleep(1)
+
+    logger.info(f"{task_id} | Sync cloud isp info task complete.")
+
+
+@periodic_task(
+    queue="default",
+    options={"queue": "default"},
+    run_every=constants.SYNC_ISP_TO_CMDB_INTERVAL,
+)
+def sync_all_isp_to_cmdb_periodic_task():
+    """
+    同步云服务商至CMDB
+    """
+    task_id = sync_all_isp_to_cmdb_periodic_task.request.id
+    sync_all_isp_to_cmdb(task_id)

apps/node_man/tests/test_handlers/test_cloud.py

@@ -181,3 +181,42 @@ def test_list_cloud_name(self, *args, **kwargs):
 
         cloud_info = CloudHandler().list_cloud_name()
         self.assertEqual(len(cloud_info), 1)
+
+    @patch("apps.node_man.handlers.cmdb.client_v2", MockClient)
+    def test_cloud_create_and_sync_isp(self):
+        with patch("apps.node_man.handlers.cmdb.client_v2.cc.search_cloud_area") as search_cloud:
+            search_cloud.return_value = {"info": []}
+            with patch("apps.node_man.handlers.cmdb.client_v2.cc.create_cloud_area") as create_cloud:
+                create_cloud.return_value = {"created": {"id": 10000}}
+                CloudHandler().create(
+                    {
+                        "isp": ["Tencent", "Aliyun", "AWS"][random.randint(0, 2)],
+                        "ap_id": -1,
+                        "bk_cloud_name": "".join(random.choice(DIGITS) for x in range(8)),
+                    },
+                    "admin",
+                )
+                call_args = create_cloud.call_args
+                bk_cloud_vendor_scope = [str(bk_cloud_vendor) for bk_cloud_vendor in range(1, 17)]
+                self.assertIn(call_args[0][0]["bk_cloud_vendor"], bk_cloud_vendor_scope)
+
+    @patch("apps.node_man.handlers.cmdb.client_v2", MockClient)
+    def test_update_cloud_and_isp(self):
+        kwarg = {
+            "isp": ["Tencent", "Aliyun", "AWS"][random.randint(0, 2)],
+            "ap_id": -1,
+            "bk_cloud_name": "".join(random.choice(DIGITS) for x in range(8)),
+        }
+        cloud = CloudHandler().create(kwarg, "admin")
+
+        # 测试更新isp
+        bk_cloud_id = cloud["bk_cloud_id"]
+        kwarg["ap_id"] = 1
+        kwarg["bk_cloud_name"] = "cdtest"
+
+        with patch("apps.node_man.handlers.cmdb.client_v2.cc.update_cloud_area") as update_cloud:
+            update_cloud.return_value = {"result": True}
+            CloudHandler().update(bk_cloud_id, kwarg["bk_cloud_name"], kwarg["isp"], kwarg["ap_id"])
+            call_args = update_cloud.call_args
+            bk_cloud_vendor_scope = [str(bk_cloud_vendor) for bk_cloud_vendor in range(1, 17)]
+            self.assertIn(call_args[0][0]["bk_cloud_vendor"], bk_cloud_vendor_scope)

apps/node_man/tests/test_pericdic_tasks/test_sync_all_isp_to_cmdb.py

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available.
+Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+
+from unittest.mock import patch
+
+from django.test import TestCase
+
+from apps.node_man import models
+from apps.node_man.periodic_tasks.sync_all_isp_to_cmdb import (
+    sync_all_isp_to_cmdb_periodic_task,
+)
+from apps.node_man.tests.utils import MockClient, create_cloud_area
+
+
+class TestSyncAllIspToCmdb(TestCase):
+    @staticmethod
+    def init_db():
+        create_cloud_area(2)
+
+    @patch("apps.node_man.periodic_tasks.sync_all_isp_to_cmdb.client_v2", MockClient)
+    def test_sync_all_isp_to_cmdb(self):
+        self.init_db()
+        # 构造CMDB内置云区域ID
+        models.GlobalSettings.set_config(key=models.GlobalSettings.KeyEnum.CMDB_INTERNAL_CLOUD_IDS.value, value=[1])
+        models.Cloud.objects.filter(bk_cloud_id=2).update(isp="Tencent")
+        with patch("apps.node_man.periodic_tasks.sync_all_isp_to_cmdb.client_v2.cc.update_cloud_area") as update_cloud:
+            update_cloud.return_value = {"result": True}
+            sync_all_isp_to_cmdb_periodic_task()
+            call_args = update_cloud.call_args
+            bk_cloud_vendor_scope = [str(bk_cloud_vendor) for bk_cloud_vendor in range(1, 17)]
+            self.assertIn(call_args[0][0]["bk_cloud_vendor"], bk_cloud_vendor_scope)
+            self.assertNotIn(1, call_args[0][0])