From 6ce463cf3579b97864a6eba1b49718b707273bcf Mon Sep 17 00:00:00 2001 From: jfuller Date: Tue, 24 Oct 2023 11:24:00 +0200 Subject: [PATCH] GRIF-72, GRIF-91, GRIF-173 provide searching on provides --- griffon/commands/queries.py | 27 ++++++- griffon/services/core_queries.py | 132 ++++++++++++++++++++++++++++--- 2 files changed, 142 insertions(+), 17 deletions(-) diff --git a/griffon/commands/queries.py b/griffon/commands/queries.py index 06dbe9a..78f2710 100644 --- a/griffon/commands/queries.py +++ b/griffon/commands/queries.py @@ -209,6 +209,20 @@ def retrieve_component_summary(ctx, component_name, strict_name_search): default=False, help="Search root Components (\033[1menabled by default\033[0m).", ) +@click.option( + "--search-provides", + "search_provides", + is_flag=True, + default=False, + help="Search root Components by provides children(\033[1menabled by default\033[0m).", +) +@click.option( + "--search-upstreams", + "search_upstreams", + is_flag=True, + default=False, + help="Search root Components by upstreams children (\033[1menabled by default\033[0m).", +) @click.option( "--search-related-url", "search_related_url", @@ -251,8 +265,8 @@ def retrieve_component_summary(ctx, component_name, strict_name_search): help="Search community Components.", ) @click.option( - "--search-upstreams", - "search_upstreams", + "--search-all-upstreams", + "search_all_upstreams", is_flag=True, default=False, help="Search for Components by upstream.", @@ -319,13 +333,15 @@ def get_product_contain_component( sfm2_flaw_id, flaw_mode, search_latest, + search_provides, + search_upstreams, search_related_url, filter_rh_naming, search_all, search_all_roots, search_redhat, search_community, - search_upstreams, + search_all_upstreams, no_community, no_middleware, no_upstream_affects, @@ -350,10 +366,13 @@ def get_product_contain_component( and not search_all_roots and not search_related_url and not search_community - and not search_upstreams + and not search_all_upstreams and not search_redhat + and not search_provides + and not search_upstreams ): ctx.params["search_latest"] = True + ctx.params["search_provides"] = True params = copy.deepcopy(ctx.params) params.pop("verbose") diff --git a/griffon/services/core_queries.py b/griffon/services/core_queries.py index 285ff85..eb306d6 100644 --- a/griffon/services/core_queries.py +++ b/griffon/services/core_queries.py @@ -167,8 +167,11 @@ class products_containing_specific_component_query: "strict_name_search", "affect_mode", "search_latest", + "search_provides", + "search_upstreams", "search_all", "search_all_roots", + "search_all_upstreams", "search_related_url", "search_community", "search_upstreams", @@ -257,8 +260,11 @@ class products_containing_component_query: "component_type", "strict_name_search", "search_latest", + "search_provides", + "search_upstreams", "search_all", "search_all_roots", + "search_all_upstreams", "search_related_url", "search_redhat", "search_community", @@ -286,11 +292,14 @@ def __init__(self, params: dict) -> None: self.search_related_url = self.params.get("search_related_url") self.search_redhat = self.params.get("search_redhat") self.search_community = self.params.get("search_community") - self.search_upstreams = self.params.get("search_upstreams") + self.search_all_upstreams = self.params.get("search_all_upstreams") self.filter_rh_naming = self.params.get("filter_rh_naming") self.no_community = self.params.get("no_community") + self.search_provides = self.params.get("search_provides") + self.search_upstreams = self.params.get("search_upstreams") if not self.no_community: self.community_session = CommunityComponentService.create_session() + self.include_inactive_product_streams = self.params.get("include_inactive_product_streams") def execute(self, status=None) -> List[Dict[str, Any]]: status.update("griffoning: searching component-registry.") @@ -300,37 +309,129 @@ def execute(self, status=None) -> List[Dict[str, Any]]: "include_fields": "purl,type,name,related_url,namespace,software_build,nvr,release,version,arch,product_streams.product_versions,product_streams.name,product_streams.ofuri,product_streams.active,product_streams.exclude_components", # noqa } if self.search_latest: - params["latest_components_by_streams"] = "True" if not self.strict_name_search: params["re_name"] = self.component_name else: params["name"] = self.component_name if self.ns: params["namespace"] = self.ns - status.update("griffoning: searching latest component(s).") + if not self.include_inactive_product_streams: + params["active_streams"] = "True" + params["root_components"] = "True" + params["latest_components_by_streams"] = "True" + status.update("griffoning: searching latest root component(s).") + latest_components_cnt = self.corgi_session.components.count(**params) + status.update(f"griffoning: found {latest_components_cnt} latest component(s).") + latest_components = self.corgi_session.components.retrieve_list_iterator_async(**params) + with multiprocessing.Pool() as pool: + status.update( + f"griffoning: found {latest_components_cnt} latest root component(s), retrieving sources & upstreams." # noqa + ) + for processed_component in pool.map( + partial(process_component, self.corgi_session), latest_components + ): + results.append(processed_component) + if not self.no_community: + status.update("griffoning: searching latest community root component(s).") + community_component_cnt = self.community_session.components.count(**params) + status.update( + f"griffoning: found {community_component_cnt} latest community root component(s)." # noqa + ) + latest_community_components = ( + self.community_session.components.retrieve_list_iterator_async(**params) + ) + with multiprocessing.Pool() as pool: + status.update( + f"griffoning: found {community_component_cnt} latest community root component(s), retrieving sources & upstreams." # noqa + ) + for processed_component in pool.map( + partial(process_component, self.community_session), + latest_community_components, + ): + results.append(processed_component) + + if self.search_provides: + if not self.strict_name_search: + params["re_provides_name"] = self.component_name + else: + params["provides_name"] = self.component_name + if self.ns: + params["namespace"] = self.ns + params["active_streams"] = "True" + if self.include_inactive_product_streams: + params["active_streams"] = "False" + params["latest_components_by_streams"] = "True" + status.update("griffoning: searching latest provided child component(s).") latest_components_cnt = self.corgi_session.components.count(**params) status.update(f"griffoning: found {latest_components_cnt} latest component(s).") latest_components = self.corgi_session.components.retrieve_list_iterator_async(**params) with multiprocessing.Pool() as pool: status.update( - f"griffoning: found {latest_components_cnt} latest component(s), retrieving sources & upstreams." # noqa + f"griffoning: found {latest_components_cnt} latest provides child component(s), retrieving sources & upstreams." # noqa ) for processed_component in pool.map( partial(process_component, self.corgi_session), latest_components ): results.append(processed_component) if not self.no_community: - status.update("griffoning: searching latest community component(s).") + status.update("griffoning: searching latest community provided child component(s).") community_component_cnt = self.community_session.components.count(**params) status.update( - f"griffoning: found {community_component_cnt} latest community component(s)." # noqa + f"griffoning: found {community_component_cnt} latest community provided child component(s)." # noqa ) latest_community_components = ( self.community_session.components.retrieve_list_iterator_async(**params) ) with multiprocessing.Pool() as pool: status.update( - f"griffoning: found {community_component_cnt} latest community component(s), retrieving sources & upstreams." # noqa + f"griffoning: found {community_component_cnt} latest community provided child component(s), retrieving sources & upstreams." # noqa + ) + for processed_component in pool.map( + partial(process_component, self.community_session), + latest_community_components, + ): + results.append(processed_component) + + if self.search_upstreams: + params["latest_components_by_streams"] = "True" + if not self.strict_name_search: + params["re_upstreams_name"] = self.component_name + else: + params["upstreams_name"] = self.component_name + if self.ns: + params["namespace"] = self.ns + if not self.include_inactive_product_streams: + params["active_streams"] = "True" + params["root_components"] = "True" + params["latest_components_by_streams"] = "True" + status.update("griffoning: searching latest upstreams child component(s).") + latest_components_cnt = self.corgi_session.components.count(**params) + status.update(f"griffoning: found {latest_components_cnt} latest component(s).") + latest_components = self.corgi_session.components.retrieve_list_iterator_async(**params) + with multiprocessing.Pool() as pool: + status.update( + f"griffoning: found {latest_components_cnt} latest upstreams child component(s), retrieving sources & upstreams." # noqa + # noqa + ) + for processed_component in pool.map( + partial(process_component, self.corgi_session), latest_components + ): + results.append(processed_component) + if not self.no_community: + status.update( + "griffoning: searching latest community upstreams child component(s)." + ) + community_component_cnt = self.community_session.components.count(**params) + status.update( + f"griffoning: found {community_component_cnt} latest community upstreams child component(s)." # noqa + ) + latest_community_components = ( + self.community_session.components.retrieve_list_iterator_async(**params) + ) + with multiprocessing.Pool() as pool: + status.update( + f"griffoning: found {community_component_cnt} latest community provided child component(s), retrieving sources & upstreams." # noqa + # noqa ) for processed_component in pool.map( partial(process_component, self.community_session), @@ -345,7 +446,8 @@ def execute(self, status=None) -> List[Dict[str, Any]]: params["namespace"] = self.ns if self.component_type: params["type"] = self.component_type - params["released_components"] = "True" + if not self.include_inactive_product_streams: + params["active_streams"] = "True" related_url_components_cnt = self.corgi_session.components.count(**params) status.update( f"griffoning: found {related_url_components_cnt} related url component(s)." @@ -390,7 +492,8 @@ def execute(self, status=None) -> List[Dict[str, Any]]: params["type"] = self.component_type if self.ns: params["namespace"] = self.ns - params["released_components"] = "True" + if not self.include_inactive_product_streams: + params["active_streams"] = "True" all_components_cnt = self.corgi_session.components.count(**params) status.update(f"griffoning: found {all_components_cnt} all component(s).") # TODO: remove max_results @@ -436,7 +539,8 @@ def execute(self, status=None) -> List[Dict[str, Any]]: params["name"] = self.component_name if self.ns: params["namespace"] = self.ns - params["released_components"] = "True" + if not self.include_inactive_product_streams: + params["active_streams"] = "True" all_src_components_cnt = self.corgi_session.components.count(**params) status.update(f"griffoning: found {all_src_components_cnt} all root component(s).") all_src_components = self.corgi_session.components.retrieve_list_iterator_async( @@ -455,9 +559,7 @@ def execute(self, status=None) -> List[Dict[str, Any]]: for c in all_src_community_components: results.append(c) - if self.search_upstreams: - # Note: upstreams only takes a purl ... so we must use re_upstreams for - # both strict and not strict search + if self.search_all_upstreams: params["namespace"] = "UPSTREAM" if not self.strict_name_search: params["re_name"] = self.component_name @@ -465,6 +567,8 @@ def execute(self, status=None) -> List[Dict[str, Any]]: params["name"] = self.component_name if self.component_type: params["type"] = self.component_type + if not self.include_inactive_product_streams: + params["active_streams"] = "True" upstream_components_cnt = self.corgi_session.components.count(**params) status.update(f"griffoning: found {upstream_components_cnt} upstream component(s).") upstream_components = self.corgi_session.components.retrieve_list_iterator_async( @@ -542,6 +646,8 @@ def execute(self, status=None) -> List[Dict[str, Any]]: params["name"] = self.component_name if self.ns: params["namespace"] = self.ns + if not self.include_inactive_product_streams: + params["active_streams"] = "True" all_community_components_cnt = self.community_session.components.count(**params) status.update( f"griffoning: found {all_community_components_cnt} community all component(s)." # noqa