feat: 向cmdb同步云区域服务商 (closed #2386)

apps/node_man/constants.py

@@ -59,6 +59,7 @@ class TimeUnit:
 COLLECT_AUTO_TRIGGER_JOB_INTERVAL = 5 * TimeUnit.MINUTE
 SYNC_CMDB_CLOUD_AREA_INTERVAL = 10 * TimeUnit.SECOND
 SYNC_AGENT_STATUS_TASK_INTERVAL = 10 * TimeUnit.MINUTE
+SYNC_ISP_TO_CMDB_INTERVAL = 1 * TimeUnit.DAY
 SYNC_PROC_STATUS_TASK_INTERVAL = settings.SYNC_PROC_STATUS_TASK_INTERVAL
 SYNC_BIZ_TO_GRAY_SCOPE_LIST_INTERVAL = 30 * TimeUnit.MINUTE
 
@@ -75,12 +76,14 @@ class TimeUnit:
 # 默认管控区域ID
 DEFAULT_CLOUD = int(os.environ.get("DEFAULT_CLOUD", 0))
 DEFAULT_CLOUD_NAME = os.environ.get("DEFAULT_CLOUD_NAME", _("直连区域"))
+# 未分配管控区域ID
+UNASSIGNED_CLOUD_ID = int(os.environ.get("BKAPP_UNASSIGNED_CLOUD_ID", 90000001))
 # 自动选择接入点ID
 DEFAULT_AP_ID = int(os.environ.get("DEFAULT_AP_ID", -1))
 # 自动选择安装通道ID
-DEFAULT_INSTALL_CHANNEL_ID = int(os.environ.get("DEFAULT_INSTALL_CHANNEL_ID", -1))
+DEFAULT_INSTALL_CHANNEL_ID = int(os.environ.get("BKAPP_DEFAULT_INSTALL_CHANNEL_ID", -1))
 # 自动选择的云区域ID
-AUTOMATIC_CHOICE_CLOUD_ID = int(os.environ.get("AUTOMATIC_CHOICE_CLOUD_ID", -1))
+AUTOMATIC_CHOICE_CLOUD_ID = int(os.environ.get("BKAPP_AUTOMATIC_CHOICE_CLOUD_ID", -1))
 # 自动选择
 AUTOMATIC_CHOICE = os.environ.get("AUTOMATIC_CHOICE", _("自动选择"))
 # 默认安装通道
@@ -567,6 +570,7 @@ def _get_member__alias_map(cls) -> Dict[Enum, str]:
 QUERY_CLOUD_LIMIT = 200
 QUERY_HOST_SERVICE_TEMPLATE_LIMIT = 200
 QUERY_MODULE_ID_THRESHOLD = 15
+UPDATE_CMDB_CLOUD_AREA_LIMIT = 50
 VERSION_PATTERN = re.compile(r"[vV]?(\d+\.){1,5}\d+(-rc\d)?$")
 # 语义化版本正则，参考：https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string
 SEMANTIC_VERSION_PATTERN = re.compile(
@@ -603,6 +607,27 @@ def _get_member__alias_map(cls) -> Dict[Enum, str]:
 MAX_HOST_IDS_LENGTH = 5000
 # 操作系统对应账户名
 OS_ACCOUNT = {"LINUX": LINUX_ACCOUNT, "WINDOWS": WINDOWS_ACCOUNT}
+# NODEMAN云服务商对应CMDB接口云服务商映射
+CMDB_CLOUD_VENDOR_MAP = {
+    "AWS": "1",
+    "TencentCloud": "2",
+    "GoogleCloud": "3",
+    "Azure": "4",
+    "PrivateCloud": "5",
+    "SalesForce": "6",
+    "OracleCloud": "7",
+    "IBMCloud": "8",
+    "AlibabaCloud": "9",
+    "ECloud": "10",
+    "UCloud": "11",
+    "MOS": "12",
+    "KSyun": "13",
+    "BaiduCloud": "14",
+    "HuaweiCloud": "15",
+    "capitalonline": "16",
+    "TencentPrivateCloud": "17",
+    "Zenlayer": "18",
+}
 
 
 class ProxyFileFromType(Enum):

apps/node_man/handlers/cloud.py

@@ -199,7 +199,8 @@ def create(self, params: dict, username: str):
         """
 
         bk_cloud_name = params["bk_cloud_name"]
-        bk_cloud_id = CmdbHandler.get_or_create_cloud(bk_cloud_name)
+        bk_cloud_vendor = const.CMDB_CLOUD_VENDOR_MAP.get(params["isp"])
+        bk_cloud_id = CmdbHandler.get_or_create_cloud(bk_cloud_name, bk_cloud_vendor=bk_cloud_vendor)
 
         if bk_cloud_name == str(DEFAULT_CLOUD_NAME):
             raise ValidationError(_("管控区域不可名为「直连区域」"))
@@ -236,8 +237,9 @@ def update(bk_cloud_id: int, bk_cloud_name: str, isp: str, ap_id: int):
         if Cloud.objects.filter(bk_cloud_name=bk_cloud_name).exclude(bk_cloud_id=bk_cloud_id).exists():
             raise ValidationError(_("管控区域名称不可重复"))
 
-        # 向CMDB修改管控区域名称
-        CmdbHandler.rename_cloud(bk_cloud_id, bk_cloud_name)
+        # 向CMDB修改管控区域名称以及云服务商
+        bk_cloud_vendor: str = const.CMDB_CLOUD_VENDOR_MAP.get(isp)
+        CmdbHandler.rename_cloud(bk_cloud_id, bk_cloud_name, bk_cloud_vendor=bk_cloud_vendor)
 
         cloud.bk_cloud_name = bk_cloud_name
         cloud.isp = isp

apps/node_man/handlers/cmdb.py

@@ -323,12 +323,12 @@ def check_biz_permission(self, bk_biz_scope: list, action: str):
         raise PermissionDeniedError(action_name=action, apply_url=apply_url, permission=apply_data)
 
     @staticmethod
-    def add_cloud(bk_cloud_name):
+    def add_cloud(bk_cloud_name: str, bk_cloud_vendor: str = None):
         """
         新增管控区域
         """
         # 增删改查CMDB操作以admin用户进行
-        data = client_v2.cc.create_cloud_area({"bk_cloud_name": bk_cloud_name})
+        data = client_v2.cc.create_cloud_area({"bk_cloud_name": bk_cloud_name, "bk_cloud_vendor": bk_cloud_vendor})
         return data.get("created", {}).get("id")
 
     @staticmethod
@@ -364,20 +364,24 @@ def get_cloud(bk_cloud_name):
         raise CloudNotExistError
 
     @staticmethod
-    def rename_cloud(bk_cloud_id, bk_cloud_name):
+    def rename_cloud(bk_cloud_id: int, bk_cloud_name: str, bk_cloud_vendor: str = None):
         try:
             # 增删改查CMDB操作以admin用户进行
-            client_v2.cc.update_cloud_area({"bk_cloud_id": bk_cloud_id, "bk_cloud_name": bk_cloud_name})
+            client_v2.cc.update_cloud_area(
+                {"bk_cloud_id": bk_cloud_id, "bk_cloud_name": bk_cloud_name, "bk_cloud_vendor": bk_cloud_vendor}
+            )
         except ComponentCallError as e:
             logger.error("esb->call update_cloud_area error %s" % e.message)
-            client_v2.cc.update_inst(bk_obj_id="plat", bk_inst_id=bk_cloud_id, bk_cloud_name=bk_cloud_name)
+            client_v2.cc.update_inst(
+                bk_obj_id="plat", bk_inst_id=bk_cloud_id, bk_cloud_name=bk_cloud_name, bk_cloud_vendor=bk_cloud_vendor
+            )
 
     @classmethod
-    def get_or_create_cloud(cls, bk_cloud_name):
+    def get_or_create_cloud(cls, bk_cloud_name: str, bk_cloud_vendor: str = None):
         try:
             return cls.get_cloud(bk_cloud_name)
         except CloudNotExistError:
-            return cls.add_cloud(bk_cloud_name)
+            return cls.add_cloud(bk_cloud_name, bk_cloud_vendor=bk_cloud_vendor)
 
     def fetch_topo(self, bk_biz_id: int, with_biz_node: bool = False) -> List:
         """

apps/node_man/management/commands/sync_all_isp_to_cmdb.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available.
+Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+
+from django.core.management.base import BaseCommand
+
+from apps.node_man.periodic_tasks import sync_all_isp_to_cmdb_periodic_task
+
+
+class Command(BaseCommand):
+    def handle(self, **kwargs):
+        sync_all_isp_to_cmdb_periodic_task()

apps/node_man/migrations/0084_update_isp_and_accesspoint_regionid_cityid.py

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available.
+Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+from django.db import migrations
+
+
+def update_isp_and_ap_region_city_id(apps, schema_editor):
+    """更新全局配置中的ISP和存量接入点的region_id和city_id"""
+    isp_list = [
+        {"isp": "PrivateCloud", "isp_name": "企业私有云"},
+        {"isp": "AWS", "isp_name": "亚马逊云"},
+        {"isp": "Azure", "isp_name": "微软云"},
+        {"isp": "GoogleCloud", "isp_name": "谷歌云"},
+        {"isp": "SalesForce", "isp_name": "SalesForce"},
+        {"isp": "OracleCloud", "isp_name": "Oracle Cloud"},
+        {"isp": "IBMCloud", "isp_name": "IBM Cloud"},
+        {"isp": "AlibabaCloud", "isp_name": "阿里云"},
+        {"isp": "TencentCloud", "isp_name": "腾讯云"},
+        {"isp": "ECloud", "isp_name": "中国电信"},
+        {"isp": "UCloud", "isp_name": "UCloud"},
+        {"isp": "MOS", "isp_name": "美团云"},
+        {"isp": "KSyun", "isp_name": "金山云"},
+        {"isp": "BaiduCloud", "isp_name": "百度云"},
+        {"isp": "HuaweiCloud", "isp_name": "华为云"},
+        {"isp": "capitalonline", "isp_name": "首都云"},
+        {"isp": "TencentPrivateCloud", "isp_name": "腾讯自研云"},
+        {"isp": "Zenlayer", "isp_name": "Zenlayer"},
+    ]
+    # 创建or更新ISP
+    GlobalSettings = apps.get_model("node_man", "GlobalSettings")
+    GlobalSettings.objects.update_or_create(defaults={"v_json": isp_list}, **{"key": "isp"})
+    # 更新存量接入点的region_id和city_id
+    AccessPoint = apps.get_model("node_man", "AccessPoint")
+    AccessPoint.objects.filter(region_id="test").update(region_id="default")
+    AccessPoint.objects.filter(city_id="test").update(city_id="default")
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("node_man", "0083_subscription_operate_info"),
+    ]
+
+    operations = [
+        migrations.RunPython(update_isp_and_ap_region_city_id),
+    ]

apps/node_man/models.py

@@ -170,6 +170,8 @@ class KeyEnum(Enum):
         INSTALL_CHANNEL_ID_NETWORK_SEGMENT = "INSTALL_CHANNEL_ID_NETWORK_SEGMENT"
         # 需要执行清理订阅的APP_CODE
         NEED_CLEAN_SUBSCRIPTION_APP_CODE = "NEED_CLEAN_SUBSCRIPTION_APP_CODE"
+        # CMDB内置云区域IDS
+        CMDB_INTERNAL_CLOUD_IDS = "CMDB_INTERNAL_CLOUD_IDS"
 
     key = models.CharField(_("键"), max_length=255, db_index=True, primary_key=True)
     v_json = JSONField(_("值"))
@@ -187,7 +189,7 @@ def map_values(self, objs, source, target):
     def fetch_isp(self):
         isps = dict(GlobalSettings.objects.filter(key="isp").values_list("key", "v_json")).get("isp", [])
         result = self.map_values(
-            isps, lambda isp: isp["isp"], lambda isp: {"isp_name": isp["isp_name"], "isp_icon": isp["isp_icon"]}
+            isps, lambda isp: isp["isp"], lambda isp: {"isp_name": isp["isp_name"]}
         )
 
         return result

apps/node_man/periodic_tasks/__init__.py

@@ -15,6 +15,7 @@
     clean_subscription_record_info_periodic_task,
 )
 from .sync_agent_status_task import sync_agent_status_periodic_task  # noqa
+from .sync_all_isp_to_cmdb import sync_all_isp_to_cmdb_periodic_task  # noqa
 from .sync_cmdb_cloud_area import sync_cmdb_cloud_area_periodic_task  # noqa
 from .sync_cmdb_host import sync_cmdb_host_periodic_task  # noqa
 from .sync_proc_status_task import sync_proc_status_periodic_task  # noqa

apps/node_man/periodic_tasks/sync_all_isp_to_cmdb.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available.
+Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+import time
+from typing import Any, Dict, List
+
+from celery.task import periodic_task
+
+from apps.component.esbclient import client_v2
+from apps.exceptions import ComponentCallError
+from apps.node_man import constants
+from apps.node_man.models import Cloud, GlobalSettings
+from apps.utils.basic import chunk_lists
+from common.log import logger
+
+
+def sync_all_isp_to_cmdb(task_id):
+    logger.info(f"{task_id} | Start syncing cloud isp info.")
+    # CMDB内置云区域不更新,默认为直连区域与未分配管控区域，如有其他内置云区域通过GlobalSettings配置
+    cmdb_internal_cloud_ids = GlobalSettings.get_config(
+        key=GlobalSettings.KeyEnum.CMDB_INTERNAL_CLOUD_IDS.value,
+        default=[constants.DEFAULT_CLOUD, constants.UNASSIGNED_CLOUD_ID],
+    )
+    cloud_info: List[Dict[str, Any]] = list(Cloud.objects.values("bk_cloud_id", "isp"))
+    # 分片请求：一次五十条
+    for chunk_clouds in chunk_lists(cloud_info, constants.UPDATE_CMDB_CLOUD_AREA_LIMIT):
+        for cloud in chunk_clouds:
+            bk_cloud_id: int = cloud["bk_cloud_id"]
+            if bk_cloud_id in cmdb_internal_cloud_ids:
+                continue
+            bk_cloud_vendor: str = constants.CMDB_CLOUD_VENDOR_MAP.get(cloud["isp"])
+            try:
+                client_v2.cc.update_cloud_area({"bk_cloud_id": bk_cloud_id, "bk_cloud_vendor": bk_cloud_vendor})
+            except ComponentCallError as e:
+                logger.error("esb->call update_cloud_area error %s" % e.message)
+                client_v2.cc.update_inst(bk_obj_id="plat", bk_inst_id=bk_cloud_id, bk_cloud_vendor=bk_cloud_vendor)
+        # 休眠1秒避免一次性全量请求导致接口超频
+        time.sleep(1)
+
+    logger.info(f"{task_id} | Sync cloud isp info task complete.")
+
+
+@periodic_task(
+    queue="default",
+    options={"queue": "default"},
+    run_every=constants.SYNC_ISP_TO_CMDB_INTERVAL,
+)
+def sync_all_isp_to_cmdb_periodic_task():
+    """
+    同步云服务商至CMDB
+    """
+    task_id = sync_all_isp_to_cmdb_periodic_task.request.id
+    sync_all_isp_to_cmdb(task_id)

apps/node_man/tests/test_handlers/test_cloud.py

@@ -181,3 +181,42 @@ def test_list_cloud_name(self, *args, **kwargs):
 
         cloud_info = CloudHandler().list_cloud_name()
         self.assertEqual(len(cloud_info), 1)
+
+    @patch("apps.node_man.handlers.cmdb.client_v2", MockClient)
+    def test_cloud_create_and_sync_isp(self):
+        with patch("apps.node_man.handlers.cmdb.client_v2.cc.search_cloud_area") as search_cloud:
+            search_cloud.return_value = {"info": []}
+            with patch("apps.node_man.handlers.cmdb.client_v2.cc.create_cloud_area") as create_cloud:
+                create_cloud.return_value = {"created": {"id": 10000}}
+                CloudHandler().create(
+                    {
+                        "isp": ["TencentCloud", "AlibabaCloud", "AWS"][random.randint(0, 2)],
+                        "ap_id": -1,
+                        "bk_cloud_name": "".join(random.choice(DIGITS) for x in range(8)),
+                    },
+                    "admin",
+                )
+                call_args = create_cloud.call_args
+                bk_cloud_vendor_scope = [str(bk_cloud_vendor) for bk_cloud_vendor in range(1, 19)]
+                self.assertIn(call_args[0][0]["bk_cloud_vendor"], bk_cloud_vendor_scope)
+
+    @patch("apps.node_man.handlers.cmdb.client_v2", MockClient)
+    def test_update_cloud_and_isp(self):
+        kwarg = {
+            "isp": ["TencentCloud", "AlibabaCloud", "AWS"][random.randint(0, 2)],
+            "ap_id": -1,
+            "bk_cloud_name": "".join(random.choice(DIGITS) for x in range(8)),
+        }
+        cloud = CloudHandler().create(kwarg, "admin")
+
+        # 测试更新isp
+        bk_cloud_id = cloud["bk_cloud_id"]
+        kwarg["ap_id"] = 1
+        kwarg["bk_cloud_name"] = "cdtest"
+
+        with patch("apps.node_man.handlers.cmdb.client_v2.cc.update_cloud_area") as update_cloud:
+            update_cloud.return_value = {"result": True}
+            CloudHandler().update(bk_cloud_id, kwarg["bk_cloud_name"], kwarg["isp"], kwarg["ap_id"])
+            call_args = update_cloud.call_args
+            bk_cloud_vendor_scope = [str(bk_cloud_vendor) for bk_cloud_vendor in range(1, 19)]
+            self.assertIn(call_args[0][0]["bk_cloud_vendor"], bk_cloud_vendor_scope)

apps/node_man/tests/test_pericdic_tasks/test_sync_all_isp_to_cmdb.py

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available.
+Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+
+from unittest.mock import patch
+
+from django.test import TestCase
+
+from apps.node_man import models
+from apps.node_man.periodic_tasks.sync_all_isp_to_cmdb import (
+    sync_all_isp_to_cmdb_periodic_task,
+)
+from apps.node_man.tests.utils import MockClient, create_cloud_area
+
+
+class TestSyncAllIspToCmdb(TestCase):
+    @staticmethod
+    def init_db():
+        create_cloud_area(2)
+
+    @patch("apps.node_man.periodic_tasks.sync_all_isp_to_cmdb.client_v2", MockClient)
+    def test_sync_all_isp_to_cmdb(self):
+        self.init_db()
+        # 构造CMDB内置云区域ID
+        models.GlobalSettings.set_config(key=models.GlobalSettings.KeyEnum.CMDB_INTERNAL_CLOUD_IDS.value, value=[1])
+        models.Cloud.objects.filter(bk_cloud_id=2).update(isp="TencentCloud")
+        with patch("apps.node_man.periodic_tasks.sync_all_isp_to_cmdb.client_v2.cc.update_cloud_area") as update_cloud:
+            update_cloud.return_value = {"result": True}
+            sync_all_isp_to_cmdb_periodic_task()
+            call_args = update_cloud.call_args
+            bk_cloud_vendor_scope = [str(bk_cloud_vendor) for bk_cloud_vendor in range(1, 19)]
+            self.assertIn(call_args[0][0]["bk_cloud_vendor"], bk_cloud_vendor_scope)
+            self.assertNotIn(1, call_args[0][0])

env/__init__.py

@@ -70,6 +70,8 @@
     # 自动选择安装通道相关配置
     "BKAPP_DEFAULT_INSTALL_CHANNEL_ID",
     "BKAPP_AUTOMATIC_CHOICE_CLOUD_ID",
+    # 未分配管控区域ID
+    "BKAPP_UNASSIGNED_CLOUD_ID",
 ]
 
 # ===============================================================================
@@ -96,6 +98,7 @@
 )
 BKAPP_DEFAULT_INSTALL_CHANNEL_ID = get_type_env(key="BKAPP_DEFAULT_INSTALL_CHANNEL_ID", default=-1, _type=int)
 BKAPP_AUTOMATIC_CHOICE_CLOUD_ID = get_type_env(key="BKAPP_AUTOMATIC_CHOICE_CLOUD_ID", default=-1, _type=int)
+BKAPP_UNASSIGNED_CLOUD_ID = get_type_env(key="BKAPP_UNASSIGNED_CLOUD_ID", default=90000001, _type=int)
 
 # ===============================================================================
 # 日志