From 08c9c9b0c42a772047645cca9315342d83287538 Mon Sep 17 00:00:00 2001 From: jpyoung3 <809608046@qq.com> Date: Fri, 13 Dec 2024 10:06:38 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=AE=89=E8=A3=85=E9=A2=84=E8=AE=BE?= =?UTF-8?q?=E6=8F=92=E4=BB=B6=E9=94=81=E5=AE=9A=E7=89=88=E6=9C=AC=20(close?= =?UTF-8?q?d=20#2482)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/backend/components/collections/plugin.py | 12 +- apps/backend/subscription/steps/adapter.py | 111 +++++++++++++++--- apps/backend/tests/plugin/utils.py | 74 +++--------- apps/node_man/models.py | 2 + 4 files changed, 124 insertions(+), 75 deletions(-) diff --git a/apps/backend/components/collections/plugin.py b/apps/backend/components/collections/plugin.py index db4aa1b52..6f22f4b5c 100644 --- a/apps/backend/components/collections/plugin.py +++ b/apps/backend/components/collections/plugin.py @@ -203,7 +203,7 @@ def get_package_by_process_status( """通过进程状态得到插件包对象""" host = self.get_host_by_process_status(process_status, common_data) policy_step_adapter = common_data.policy_step_adapter - package = policy_step_adapter.get_matching_package_obj(host.os_type, host.cpu_arch) + package = policy_step_adapter.get_matching_package_obj(host.os_type, host.cpu_arch, host.bk_biz_id) return package def get_plugin_root_by_process_status( @@ -280,11 +280,12 @@ def _execute(self, data, parent_data, common_data: PluginCommonData): # target_host_objs 的长度通常为1或2，此处也不必担心时间复杂度问题 # 指定 target_host 主要用于远程采集的场景，常见于第三方插件，如拨测 for host in target_host_objs: + bk_biz_id = host.bk_biz_id bk_host_id = host.bk_host_id os_type = host.os_type.lower() cpu_arch = host.cpu_arch group_id = create_group_id(subscription, subscription_instance.instance_info) - package = self.get_package(subscription_instance, policy_step_adapter, os_type, cpu_arch) + package = self.get_package(subscription_instance, policy_step_adapter, os_type, cpu_arch, bk_biz_id) ap_config = self.get_ap_config(ap_id_obj_map, host) setup_path, pid_path, log_path, data_path = self.get_plugins_paths( package, plugin_name, ap_config, group_id, subscription @@ -340,10 +341,11 @@ def get_package( policy_step_adapter: PolicyStepAdapter, os_type: str, cpu_arch: str, + bk_biz_id: int, ) -> models.Packages: """获取插件包对象""" try: - return policy_step_adapter.get_matching_package_obj(os_type, cpu_arch) + return policy_step_adapter.get_matching_package_obj(os_type, cpu_arch, bk_biz_id) except errors.PackageNotExists as error: # 插件包不支持或不存在时，记录异常信息，此实例不参与后续流程 self.move_insts_to_failed([subscription_instance.id], str(error)) @@ -979,7 +981,9 @@ def _execute(self, data, parent_data, common_data: PluginCommonData): # 根据配置模板和上下文变量渲染配置文件 rendered_configs = render_config_files_by_config_templates( - policy_step_adapter.get_matching_config_tmpl_objs(target_host.os_type, target_host.cpu_arch), + policy_step_adapter.get_matching_config_tmpl_objs( + target_host.os_type, target_host.cpu_arch, package, target_host.bk_biz_id, subscription_step.config + ), {"group_id": process_status.group_id}, context, package_obj=package, diff --git a/apps/backend/subscription/steps/adapter.py b/apps/backend/subscription/steps/adapter.py index 56e397578..1b324472b 100644 --- a/apps/backend/subscription/steps/adapter.py +++ b/apps/backend/subscription/steps/adapter.py @@ -16,6 +16,7 @@ from django.db.models import Max, Subquery, Value from django.utils.translation import ugettext as _ +from packaging import version from rest_framework import exceptions, serializers from apps.backend.subscription import errors @@ -284,19 +285,19 @@ def max_ids_by_key(self, contained_os_cpu_items: List[Dict[str, Any]]) -> List[i def format2policy_packages_new( self, plugin_id: int, plugin_name: str, plugin_version: str, config_templates: List[Dict[str, Any]] ) -> List[Dict[str, Any]]: - latest_flag: str = "latest" - is_tag: bool = Tag.objects.filter( - target_id=plugin_id, name=latest_flag, target_type=TargetType.PLUGIN.value - ).exists() + tags: List[str] = ["latest", "stable"] + is_tag: Dict[str, bool] = { + tag: Tag.objects.filter(target_id=plugin_id, name=tag, target_type=TargetType.PLUGIN.value).exists() + for tag in tags + } - if plugin_version != latest_flag or is_tag: - # 如果 latest 是 tag，走取指定版本的逻辑 - packages = models.Packages.objects.filter(project=plugin_name, version=plugin_version) - else: + if plugin_version in tags and not is_tag[plugin_version]: max_pkg_ids: List[int] = self.max_ids_by_key( list(models.Packages.objects.filter(project=plugin_name).values("id", "os", "cpu_arch")) ) packages = models.Packages.objects.filter(id__in=max_pkg_ids) + else: + packages = self.get_packages(plugin_name, plugin_version) if not packages: raise errors.PluginValidationError( @@ -306,11 +307,11 @@ def format2policy_packages_new( os_cpu__config_templates_map = defaultdict(list) for template in config_templates: is_main_template = template["is_main"] - if template["version"] != latest_flag or is_tag: - plugin_version_set = {plugin_version, "*"} + if plugin_version in tags and not is_tag[plugin_version]: + tag_packages_version_set = set(packages.values_list("version", flat=True)) + plugin_version_set = tag_packages_version_set | {"*"} else: - latest_packages_version_set = set(packages.values_list("version", flat=True)) - plugin_version_set = latest_packages_version_set | {"*"} + plugin_version_set = {plugin_version, "*"} max_config_tmpl_ids: typing.List[int] = self.max_ids_by_key( list( @@ -444,9 +445,18 @@ def get_matching_step_params(self, os_type: str = None, cpu_arch: str = None, os return self.os_key_params_map.get(os_key) return self.os_key_params_map.get(self.get_os_key(os_type, cpu_arch), {}) - def get_matching_package_obj(self, os_type: str, cpu_arch: str) -> models.Packages: + def get_matching_package_obj(self, os_type: str, cpu_arch: str, bk_biz_id: int) -> models.Packages: try: package = self.os_key_pkg_map[self.get_os_key(os_type, cpu_arch)] + version_str = getattr(package, "version", "") + tag_name__obj_map: Dict[str, Tag] = PluginTargetHelper.get_tag_name__obj_map( + target_id=self.plugin_desc.id, + ) + if version_str in tag_name__obj_map: + version_str = tag_name__obj_map[version_str].target_version + biz_version = self.get_biz_version(package, bk_biz_id) + if biz_version and version.Version(version_str) > version.Version(biz_version): + package = self.get_biz_package(package.project, os_type, cpu_arch, biz_version) except KeyError: msg = _("插件 [{name}] 不支持 系统:{os_type}-架构:{cpu_arch}-版本:{plugin_version}").format( name=self.plugin_name, @@ -466,5 +476,78 @@ def get_matching_package_obj(self, os_type: str, cpu_arch: str) -> models.Packag raise errors.PluginValidationError(msg) return package - def get_matching_config_tmpl_objs(self, os_type: str, cpu_arch: str) -> List[models.PluginConfigTemplate]: + def get_matching_config_tmpl_objs( + self, os_type: str, cpu_arch: str, package: models.Packages, bk_biz_id: int, config: Dict + ) -> List[models.PluginConfigTemplate]: + # 如果当前业务设定了业务最大版本，重新从数据库中获取 + if str(bk_biz_id) in self.plugin_version_config(): + config_tmpl = ( + models.PluginConfigTemplate.objects.filter( + name=config["config_templates"][0]["name"], + plugin_name=package.project, + plugin_version=package.version, + is_main=Value(1 if config["config_templates"][0]["is_main"] else 0), + ) + .order_by("-id") + .first() + ) + return config_tmpl return self.config_tmpl_obj_gby_os_key.get(self.get_os_key(os_type, cpu_arch), []) + + def get_biz_package(self, plugin_name: str, os_type: str, cpu_arch: str, biz_version: str): + """获取业务锁定版本的插件包""" + packages_all = self.get_packages(plugin_name, biz_version) + packages = packages_all.filter( + id__in=[pkg.id for pkg in packages_all if version.Version(pkg.version) <= version.Version(biz_version)] + ) + os_cpu__biz_pkg_map = {self.get_os_key(package.os, package.cpu_arch): package for package in packages} + if not os_cpu__biz_pkg_map: + raise errors.PluginValidationError( + msg="插件 [{name}-{versions}] 不存在".format(name=self.plugin_name, versions=biz_version) + ) + package = os_cpu__biz_pkg_map[self.get_os_key(os_type, cpu_arch)] + return package + + @staticmethod + def plugin_version_config(): + """业务锁定版本配置""" + plugin_version_config: Dict[str, Dict[str, str]] = models.GlobalSettings.get_config( + models.GlobalSettings.KeyEnum.PLUGIN_VERSION_CONFIG.value, default={} + ) + return plugin_version_config + + @staticmethod + def get_biz_version(package: models.Packages, bk_biz_id: int): + """获取业务锁定版本""" + plugin_version_config = PolicyStepAdapter.plugin_version_config() + biz_version = None + if str(bk_biz_id) in plugin_version_config: + biz_version_config = plugin_version_config[str(bk_biz_id)] + biz_version = next( + ( + biz_plugin_version + for biz_plugin_name, biz_plugin_version in biz_version_config.items() + if package.project == biz_plugin_name + ), + None, + ) + return biz_version + + def get_packages(self, plugin_name: str, plugin_version: str, biz_version: str = None): + """如果不存在某个版本，则取最大id的版本""" + all_packages = models.Packages.objects.filter(project=plugin_name).values("id", "os", "cpu_arch", "version") + version_packages = {pkg["id"]: pkg for pkg in all_packages if pkg["version"] == plugin_version} + package_ids = set(version_packages.keys()) + for os in constants.PLUGIN_OS_TUPLE: + for cpu_arch in constants.CPU_TUPLE: + if not any( + pkg["os"] == os and pkg["cpu_arch"] == cpu_arch + for pkg in all_packages + if pkg["version"] == plugin_version + ): + max_pkg_ids: List[int] = self.max_ids_by_key( + [pkg for pkg in all_packages if pkg["os"] == os and pkg["cpu_arch"] == cpu_arch] + ) + package_ids.update(max_pkg_ids) + packages = models.Packages.objects.filter(id__in=package_ids) + return packages diff --git a/apps/backend/tests/plugin/utils.py b/apps/backend/tests/plugin/utils.py index 1a873280e..e9c2addda 100644 --- a/apps/backend/tests/plugin/utils.py +++ b/apps/backend/tests/plugin/utils.py @@ -149,27 +149,10 @@ "type": "object", "required": True, "properties": { - "token": { - "title": "token", - "type": "string", - "required": True - }, - "logVerbosity": { - "title": "logVerbosity", - "type": "number", - "required": False, - "default": 5 - }, - "tempDir": { - "title": "tempDir", - "type": "string", - "required": False - }, - "uid": { - "title": "uid", - "type": "string", - "required": False - }, + "token": {"title": "token", "type": "string", "required": True}, + "logVerbosity": {"title": "logVerbosity", "type": "number", "required": False, "default": 5}, + "tempDir": {"title": "tempDir", "type": "string", "required": False}, + "uid": {"title": "uid", "type": "string", "required": False}, "labels": { "title": "labels", "type": "array", @@ -177,17 +160,8 @@ "title": "label", "type": "object", "required": False, - "properties": { - "key": { - "title": "key", - "type": "string" - }, - "value": { - "title": "value", - "type": "string" - } - } - } + "properties": {"key": {"title": "键", "type": "string"}, "value": {"title": "值", "type": "string"}}, + }, }, "apps": { "title": "apps", @@ -196,16 +170,8 @@ "title": "named_label", "type": "object", "properties": { - "name": { - "title": "name", - "type": "string", - "required": True - }, - "uid": { - "title": "uid", - "type": "string", - "required": False - }, + "name": {"title": "name", "type": "string", "required": True}, + "uid": {"title": "uid", "type": "string", "required": False}, "labels": { "title": "labels", "type": "array", @@ -215,21 +181,15 @@ "type": "object", "required": False, "properties": { - "key": { - "title": "key", - "type": "string" - }, - "value": { - "title": "value", - "type": "string" - } - } - } - } - } - } - } - } + "key": {"title": "键", "type": "string"}, + "value": {"title": "值", "type": "string"}, + }, + }, + }, + }, + }, + }, + }, } # 插件名称 diff --git a/apps/node_man/models.py b/apps/node_man/models.py index 8dc074de9..bd91264f0 100644 --- a/apps/node_man/models.py +++ b/apps/node_man/models.py @@ -170,6 +170,8 @@ class KeyEnum(Enum): INSTALL_CHANNEL_ID_NETWORK_SEGMENT = "INSTALL_CHANNEL_ID_NETWORK_SEGMENT" # 需要执行清理订阅的APP_CODE NEED_CLEAN_SUBSCRIPTION_APP_CODE = "NEED_CLEAN_SUBSCRIPTION_APP_CODE" + # 业务最大插件版本 + PLUGIN_VERSION_CONFIG = "PLUGIN_VERSION_CONFIG" key = models.CharField(_("键"), max_length=255, db_index=True, primary_key=True) v_json = JSONField(_("值"))