From 94cc39cb7e174841c80ea4de1c09cffe64620580 Mon Sep 17 00:00:00 2001 From: dcd <1151627903@qq.com> Date: Mon, 26 Aug 2024 16:23:31 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=90=91cmdb=E5=90=8C=E6=AD=A5?= =?UTF-8?q?=E4=BA=91=E5=8C=BA=E5=9F=9F=E6=9C=8D=E5=8A=A1=E5=95=86=20(close?= =?UTF-8?q?d=20#2386)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/node_man/constants.py | 21 +++++++ apps/node_man/handlers/cloud.py | 8 ++- apps/node_man/handlers/cmdb.py | 18 +++--- .../commands/sync_all_isp_to_cmdb.py | 19 ++++++ apps/node_man/models.py | 2 + apps/node_man/periodic_tasks/__init__.py | 1 + .../periodic_tasks/sync_all_isp_to_cmdb.py | 60 +++++++++++++++++++ .../tests/test_handlers/test_cloud.py | 39 ++++++++++++ .../test_sync_all_isp_to_cmdb.py | 40 +++++++++++++ 9 files changed, 198 insertions(+), 10 deletions(-) create mode 100644 apps/node_man/management/commands/sync_all_isp_to_cmdb.py create mode 100644 apps/node_man/periodic_tasks/sync_all_isp_to_cmdb.py create mode 100644 apps/node_man/tests/test_pericdic_tasks/test_sync_all_isp_to_cmdb.py diff --git a/apps/node_man/constants.py b/apps/node_man/constants.py index 6e965680f..4d3242d72 100644 --- a/apps/node_man/constants.py +++ b/apps/node_man/constants.py @@ -59,6 +59,7 @@ class TimeUnit: COLLECT_AUTO_TRIGGER_JOB_INTERVAL = 5 * TimeUnit.MINUTE SYNC_CMDB_CLOUD_AREA_INTERVAL = 10 * TimeUnit.SECOND SYNC_AGENT_STATUS_TASK_INTERVAL = 10 * TimeUnit.MINUTE +SYNC_ISP_TO_CMDB_INTERVAL = 1 * TimeUnit.DAY SYNC_PROC_STATUS_TASK_INTERVAL = settings.SYNC_PROC_STATUS_TASK_INTERVAL SYNC_BIZ_TO_GRAY_SCOPE_LIST_INTERVAL = 30 * TimeUnit.MINUTE @@ -559,6 +560,7 @@ def _get_member__alias_map(cls) -> Dict[Enum, str]: QUERY_CLOUD_LIMIT = 200 QUERY_HOST_SERVICE_TEMPLATE_LIMIT = 200 QUERY_MODULE_ID_THRESHOLD = 15 +UPDATE_CMDB_CLOUD_AREA_LIMIT = 50 VERSION_PATTERN = re.compile(r"[vV]?(\d+\.){1,5}\d+(-rc\d)?$") # 语义化版本正则，参考：https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string SEMANTIC_VERSION_PATTERN = re.compile( @@ -595,6 +597,25 @@ def _get_member__alias_map(cls) -> Dict[Enum, str]: MAX_HOST_IDS_LENGTH = 5000 # 操作系统对应账户名 OS_ACCOUNT = {"LINUX": LINUX_ACCOUNT, "WINDOWS": WINDOWS_ACCOUNT} +# NODEMAN云服务商对应CMDB接口云服务商映射 +CMDB_CLOUD_VENDOR_MAP = { + "Amazon": "1", + "Tencent": "2", + "Google": "3", + "MicroSoft": "4", + "PrivateCloud": "5", + "SalesForce": "6", + "Oracle": "7", + "IBM": "8", + "Aliyun": "9", + "ECloud": "10", + "UCloud": "11", + "MOS": "12", + "KSCLOUD": "13", + "baidu": "14", + "huawei": "15", + "capitalonline": "16", +} class ProxyFileFromType(Enum): diff --git a/apps/node_man/handlers/cloud.py b/apps/node_man/handlers/cloud.py index e015d38c2..3fcc82f7a 100644 --- a/apps/node_man/handlers/cloud.py +++ b/apps/node_man/handlers/cloud.py @@ -199,7 +199,8 @@ def create(self, params: dict, username: str): """ bk_cloud_name = params["bk_cloud_name"] - bk_cloud_id = CmdbHandler.get_or_create_cloud(bk_cloud_name) + bk_cloud_vendor = const.CMDB_CLOUD_VENDOR_MAP.get(params["isp"]) + bk_cloud_id = CmdbHandler.get_or_create_cloud(bk_cloud_name, bk_cloud_vendor=bk_cloud_vendor) if bk_cloud_name == str(DEFAULT_CLOUD_NAME): raise ValidationError(_("管控区域不可名为「直连区域」")) @@ -236,8 +237,9 @@ def update(bk_cloud_id: int, bk_cloud_name: str, isp: str, ap_id: int): if Cloud.objects.filter(bk_cloud_name=bk_cloud_name).exclude(bk_cloud_id=bk_cloud_id).exists(): raise ValidationError(_("管控区域名称不可重复")) - # 向CMDB修改管控区域名称 - CmdbHandler.rename_cloud(bk_cloud_id, bk_cloud_name) + # 向CMDB修改管控区域名称以及云服务商 + bk_cloud_vendor: str = const.CMDB_CLOUD_VENDOR_MAP.get(isp) + CmdbHandler.rename_cloud(bk_cloud_id, bk_cloud_name, bk_cloud_vendor=bk_cloud_vendor) cloud.bk_cloud_name = bk_cloud_name cloud.isp = isp diff --git a/apps/node_man/handlers/cmdb.py b/apps/node_man/handlers/cmdb.py index 06c7d7b0c..b371e4df8 100644 --- a/apps/node_man/handlers/cmdb.py +++ b/apps/node_man/handlers/cmdb.py @@ -322,12 +322,12 @@ def check_biz_permission(self, bk_biz_scope: list, action: str): raise PermissionDeniedError(action_name=action, apply_url=apply_url, permission=apply_data) @staticmethod - def add_cloud(bk_cloud_name): + def add_cloud(bk_cloud_name: str, bk_cloud_vendor: str = None): """ 新增管控区域 """ # 增删改查CMDB操作以admin用户进行 - data = client_v2.cc.create_cloud_area({"bk_cloud_name": bk_cloud_name}) + data = client_v2.cc.create_cloud_area({"bk_cloud_name": bk_cloud_name, "bk_cloud_vendor": bk_cloud_vendor}) return data.get("created", {}).get("id") @staticmethod @@ -363,20 +363,24 @@ def get_cloud(bk_cloud_name): raise CloudNotExistError @staticmethod - def rename_cloud(bk_cloud_id, bk_cloud_name): + def rename_cloud(bk_cloud_id: int, bk_cloud_name: str, bk_cloud_vendor: str = None): try: # 增删改查CMDB操作以admin用户进行 - client_v2.cc.update_cloud_area({"bk_cloud_id": bk_cloud_id, "bk_cloud_name": bk_cloud_name}) + client_v2.cc.update_cloud_area( + {"bk_cloud_id": bk_cloud_id, "bk_cloud_name": bk_cloud_name, "bk_cloud_vendor": bk_cloud_vendor} + ) except ComponentCallError as e: logger.error("esb->call update_cloud_area error %s" % e.message) - client_v2.cc.update_inst(bk_obj_id="plat", bk_inst_id=bk_cloud_id, bk_cloud_name=bk_cloud_name) + client_v2.cc.update_inst( + bk_obj_id="plat", bk_inst_id=bk_cloud_id, bk_cloud_name=bk_cloud_name, bk_cloud_vendor=bk_cloud_vendor + ) @classmethod - def get_or_create_cloud(cls, bk_cloud_name): + def get_or_create_cloud(cls, bk_cloud_name: str, bk_cloud_vendor: str = None): try: return cls.get_cloud(bk_cloud_name) except CloudNotExistError: - return cls.add_cloud(bk_cloud_name) + return cls.add_cloud(bk_cloud_name, bk_cloud_vendor=bk_cloud_vendor) def fetch_topo(self, bk_biz_id: int, with_biz_node: bool = False) -> List: """ diff --git a/apps/node_man/management/commands/sync_all_isp_to_cmdb.py b/apps/node_man/management/commands/sync_all_isp_to_cmdb.py new file mode 100644 index 000000000..568c260b2 --- /dev/null +++ b/apps/node_man/management/commands/sync_all_isp_to_cmdb.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from django.core.management.base import BaseCommand + +from apps.node_man.periodic_tasks import sync_all_isp_to_cmdb_periodic_task + + +class Command(BaseCommand): + def handle(self, **kwargs): + sync_all_isp_to_cmdb_periodic_task() diff --git a/apps/node_man/models.py b/apps/node_man/models.py index 91bf53f0e..9a19c7857 100644 --- a/apps/node_man/models.py +++ b/apps/node_man/models.py @@ -164,6 +164,8 @@ class KeyEnum(Enum): IP_CHOOSER_ENABLE_SHOW_REALTIME_AGENT_STATE = "IP_CHOOSER_ENABLE_SHOW_REALTIME_AGENT_STATE" # IP选择器详情接口实时展示agent状态业务白名单 IP_CHOOSER_BIZ_WHITELIST = "IP_CHOOSER_BIZ_WHITELIST" + # CMDB内置云区域IDS + CMDB_INTERNAL_CLOUD_IDS = "CMDB_INTERNAL_CLOUD_IDS" key = models.CharField(_("键"), max_length=255, db_index=True, primary_key=True) v_json = JSONField(_("值")) diff --git a/apps/node_man/periodic_tasks/__init__.py b/apps/node_man/periodic_tasks/__init__.py index 6a0bdc9c6..eaffce09a 100644 --- a/apps/node_man/periodic_tasks/__init__.py +++ b/apps/node_man/periodic_tasks/__init__.py @@ -15,6 +15,7 @@ clean_subscription_record_info_periodic_task, ) from .sync_agent_status_task import sync_agent_status_periodic_task # noqa +from .sync_all_isp_to_cmdb import sync_all_isp_to_cmdb_periodic_task # noqa from .sync_cmdb_cloud_area import sync_cmdb_cloud_area_periodic_task # noqa from .sync_cmdb_host import sync_cmdb_host_periodic_task # noqa from .sync_proc_status_task import sync_proc_status_periodic_task # noqa diff --git a/apps/node_man/periodic_tasks/sync_all_isp_to_cmdb.py b/apps/node_man/periodic_tasks/sync_all_isp_to_cmdb.py new file mode 100644 index 000000000..0c5078a32 --- /dev/null +++ b/apps/node_man/periodic_tasks/sync_all_isp_to_cmdb.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import time +from typing import Any, Dict, List + +from celery.task import periodic_task + +from apps.component.esbclient import client_v2 +from apps.exceptions import ComponentCallError +from apps.node_man import constants +from apps.node_man.models import Cloud, GlobalSettings +from apps.utils.basic import chunk_lists +from common.log import logger + + +def sync_all_isp_to_cmdb(task_id): + logger.info(f"{task_id} | Start syncing cloud isp info.") + # CMDB内置云区域不更新 + cmdb_internal_cloud_ids = GlobalSettings.get_config( + key=GlobalSettings.KeyEnum.CMDB_INTERNAL_CLOUD_IDS.value, default=[] + ) + cloud_info: List[Dict[str, Any]] = list(Cloud.objects.values("bk_cloud_id", "isp")) + # 分片请求：一次五十条 + for chunk_clouds in chunk_lists(cloud_info, constants.UPDATE_CMDB_CLOUD_AREA_LIMIT): + for cloud in chunk_clouds: + bk_cloud_id: int = cloud["bk_cloud_id"] + if bk_cloud_id in cmdb_internal_cloud_ids: + continue + bk_cloud_vendor: str = constants.CMDB_CLOUD_VENDOR_MAP.get(cloud["isp"]) + try: + client_v2.cc.update_cloud_area({"bk_cloud_id": bk_cloud_id, "bk_cloud_vendor": bk_cloud_vendor}) + except ComponentCallError as e: + logger.error("esb->call update_cloud_area error %s" % e.message) + client_v2.cc.update_inst(bk_obj_id="plat", bk_inst_id=bk_cloud_id, bk_cloud_vendor=bk_cloud_vendor) + # 休眠1秒避免一次性全量请求导致接口超频 + time.sleep(1) + + logger.info(f"{task_id} | Sync cloud isp info task complete.") + + +@periodic_task( + queue="default", + options={"queue": "default"}, + run_every=constants.SYNC_ISP_TO_CMDB_INTERVAL, +) +def sync_all_isp_to_cmdb_periodic_task(): + """ + 同步云服务商至CMDB + """ + task_id = sync_all_isp_to_cmdb_periodic_task.request.id + sync_all_isp_to_cmdb(task_id) diff --git a/apps/node_man/tests/test_handlers/test_cloud.py b/apps/node_man/tests/test_handlers/test_cloud.py index ea9e9873a..24d57d4f2 100644 --- a/apps/node_man/tests/test_handlers/test_cloud.py +++ b/apps/node_man/tests/test_handlers/test_cloud.py @@ -181,3 +181,42 @@ def test_list_cloud_name(self, *args, **kwargs): cloud_info = CloudHandler().list_cloud_name() self.assertEqual(len(cloud_info), 1) + + @patch("apps.node_man.handlers.cmdb.client_v2", MockClient) + def test_cloud_create_and_sync_isp(self): + with patch("apps.node_man.handlers.cmdb.client_v2.cc.search_cloud_area") as search_cloud: + search_cloud.return_value = {"info": []} + with patch("apps.node_man.handlers.cmdb.client_v2.cc.create_cloud_area") as create_cloud: + create_cloud.return_value = {"created": {"id": 10000}} + CloudHandler().create( + { + "isp": ["Tencent", "Aliyun", "AWS"][random.randint(0, 2)], + "ap_id": -1, + "bk_cloud_name": "".join(random.choice(DIGITS) for x in range(8)), + }, + "admin", + ) + call_args = create_cloud.call_args + bk_cloud_vendor_scope = [str(bk_cloud_vendor) for bk_cloud_vendor in range(1, 17)] + self.assertIn(call_args[0][0]["bk_cloud_vendor"], bk_cloud_vendor_scope) + + @patch("apps.node_man.handlers.cmdb.client_v2", MockClient) + def test_update_cloud_and_isp(self): + kwarg = { + "isp": ["Tencent", "Aliyun", "AWS"][random.randint(0, 2)], + "ap_id": -1, + "bk_cloud_name": "".join(random.choice(DIGITS) for x in range(8)), + } + cloud = CloudHandler().create(kwarg, "admin") + + # 测试更新isp + bk_cloud_id = cloud["bk_cloud_id"] + kwarg["ap_id"] = 1 + kwarg["bk_cloud_name"] = "cdtest" + + with patch("apps.node_man.handlers.cmdb.client_v2.cc.update_cloud_area") as update_cloud: + update_cloud.return_value = {"result": True} + CloudHandler().update(bk_cloud_id, kwarg["bk_cloud_name"], kwarg["isp"], kwarg["ap_id"]) + call_args = update_cloud.call_args + bk_cloud_vendor_scope = [str(bk_cloud_vendor) for bk_cloud_vendor in range(1, 17)] + self.assertIn(call_args[0][0]["bk_cloud_vendor"], bk_cloud_vendor_scope) diff --git a/apps/node_man/tests/test_pericdic_tasks/test_sync_all_isp_to_cmdb.py b/apps/node_man/tests/test_pericdic_tasks/test_sync_all_isp_to_cmdb.py new file mode 100644 index 000000000..5d73c8f42 --- /dev/null +++ b/apps/node_man/tests/test_pericdic_tasks/test_sync_all_isp_to_cmdb.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from unittest.mock import patch + +from django.test import TestCase + +from apps.node_man import models +from apps.node_man.periodic_tasks.sync_all_isp_to_cmdb import ( + sync_all_isp_to_cmdb_periodic_task, +) +from apps.node_man.tests.utils import MockClient, create_cloud_area + + +class TestSyncAllIspToCmdb(TestCase): + @staticmethod + def init_db(): + create_cloud_area(2) + + @patch("apps.node_man.periodic_tasks.sync_all_isp_to_cmdb.client_v2", MockClient) + def test_sync_all_isp_to_cmdb(self): + self.init_db() + # 构造CMDB内置云区域ID + models.GlobalSettings.set_config(key=models.GlobalSettings.KeyEnum.CMDB_INTERNAL_CLOUD_IDS.value, value=[1]) + models.Cloud.objects.filter(bk_cloud_id=2).update(isp="Tencent") + with patch("apps.node_man.periodic_tasks.sync_all_isp_to_cmdb.client_v2.cc.update_cloud_area") as update_cloud: + update_cloud.return_value = {"result": True} + sync_all_isp_to_cmdb_periodic_task() + call_args = update_cloud.call_args + bk_cloud_vendor_scope = [str(bk_cloud_vendor) for bk_cloud_vendor in range(1, 17)] + self.assertIn(call_args[0][0]["bk_cloud_vendor"], bk_cloud_vendor_scope) + self.assertNotIn(1, call_args[0][0])