From 264d645038fdf77c50a2c203da2def32f20bb6cb Mon Sep 17 00:00:00 2001 From: jfuller Date: Wed, 31 Jan 2024 12:03:20 +0100 Subject: [PATCH 1/3] minor refactor --- griffon/services/core_queries.py | 62 ++++++++------------------------ scripts/smoke-tests.sh | 5 +-- 2 files changed, 18 insertions(+), 49 deletions(-) diff --git a/griffon/services/core_queries.py b/griffon/services/core_queries.py index aaae004..98dc4fc 100644 --- a/griffon/services/core_queries.py +++ b/griffon/services/core_queries.py @@ -338,6 +338,13 @@ def execute(self, status=None) -> List[Dict[str, Any]]: "limit": 50, "include_fields": "purl,type,name,related_url,namespace,software_build,nvr,release,version,arch,product_streams.product_versions,product_streams.name,product_streams.ofuri,product_streams.active,product_streams.exclude_components,product_streams.relations", # noqa } + if not (self.include_inactive_product_streams): + params["active_streams"] = "True" + if self.exclude_unreleased: + params["released_components"] = "True" + if not self.include_container_roots: + params["type"] = "RPM" + params["arch"] = "src" component_name = self.component_name if not self.strict_name_search and not self.regex_name_search: @@ -351,12 +358,6 @@ def execute(self, status=None) -> List[Dict[str, Any]]: search_latest_params["name"] = component_name if self.ns: search_latest_params["namespace"] = self.ns - if not (self.include_inactive_product_streams): - search_latest_params["active_streams"] = "True" - if self.exclude_unreleased: - search_latest_params["released_components"] = "True" - if not self.include_container_roots: - search_latest_params["type"] = "RPM" search_latest_params["root_components"] = "True" search_latest_params["latest_components_by_streams"] = "True" status.update("searching latest root component(s).") @@ -404,12 +405,6 @@ def execute(self, status=None) -> List[Dict[str, Any]]: search_provides_params["provides_name"] = component_name if self.ns: search_provides_params["namespace"] = self.ns - if not (self.include_inactive_product_streams): - search_provides_params["active_streams"] = "True" - if self.exclude_unreleased: - search_provides_params["released_components"] = "True" - if not self.include_container_roots: - search_provides_params["type"] = "RPM" search_provides_params["latest_components_by_streams"] = "True" status.update("searching latest provided child component(s).") latest_components_cnt = self.corgi_session.components.count(**search_provides_params) @@ -460,12 +455,6 @@ def execute(self, status=None) -> List[Dict[str, Any]]: search_upstreams_params["upstreams_name"] = component_name if self.ns: search_upstreams_params["namespace"] = self.ns - if not (self.include_inactive_product_streams): - search_upstreams_params["active_streams"] = "True" - if self.exclude_unreleased: - search_upstreams_params["released_components"] = "True" - if not self.include_container_roots: - search_upstreams_params["type"] = "RPM" search_upstreams_params["latest_components_by_streams"] = "True" status.update("searching latest upstreams child component(s).") latest_components_cnt = self.corgi_session.components.count(**search_upstreams_params) @@ -513,13 +502,6 @@ def execute(self, status=None) -> List[Dict[str, Any]]: search_related_url_params["namespace"] = self.ns if self.component_type: search_related_url_params["type"] = self.component_type - if not (self.include_inactive_product_streams): - search_related_url_params["active_streams"] = "True" - if self.exclude_unreleased: - search_related_url_params["released_components"] = "True" - if not self.include_container_roots: - search_related_url_params["type"] = "RPM" - search_related_url_params["released_components"] = "True" related_url_components_cnt = self.corgi_session.components.count( **search_related_url_params, ) @@ -555,12 +537,6 @@ def execute(self, status=None) -> List[Dict[str, Any]]: search_all_params["type"] = self.component_type if self.ns: search_all_params["namespace"] = self.ns - if not (self.include_inactive_product_streams): - search_all_params["active_streams"] = "True" - if self.exclude_unreleased: - search_all_params["released_components"] = "True" - if not self.include_container_roots: - search_all_params["type"] = "RPM" all_components_cnt = self.corgi_session.components.count(**search_all_params) status.update(f"found {all_components_cnt} all component(s).") # TODO: remove max_results @@ -569,6 +545,8 @@ def execute(self, status=None) -> List[Dict[str, Any]]: ) status.update(f"found {all_components_cnt} all component(s).") for c in all_components: + c.upstreams = [] + c.sources = [] results.append(c) if not self.no_community: @@ -585,6 +563,8 @@ def execute(self, status=None) -> List[Dict[str, Any]]: ) ) for c in all_community_components: + c.upstreams = [] + c.sources = [] results.append(c) if self.search_all_roots: @@ -596,18 +576,14 @@ def execute(self, status=None) -> List[Dict[str, Any]]: search_all_roots_params["name"] = component_name if self.ns: search_all_roots_params["namespace"] = self.ns - if not (self.include_inactive_product_streams): - search_all_roots_params["active_streams"] = "True" - if self.exclude_unreleased: - search_all_roots_params["released_components"] = "True" - if not self.include_container_roots: - search_all_roots_params["type"] = "RPM" all_src_components_cnt = self.corgi_session.components.count(**search_all_roots_params) status.update(f"found {all_src_components_cnt} all root component(s).") all_src_components = self.corgi_session.components.retrieve_list_iterator_async( **search_all_roots_params, max_results=10000 ) for c in all_src_components: + c.upstreams = [] + c.sources = [] results.append(c) if not self.no_community: all_src_community_components_cnt = self.community_session.components.count( @@ -622,6 +598,8 @@ def execute(self, status=None) -> List[Dict[str, Any]]: f"found {all_src_community_components_cnt} community all root component(s)." # noqa ) for c in all_src_community_components: + c.upstreams = [] + c.sources = [] results.append(c) if self.search_all_upstreams: @@ -633,12 +611,6 @@ def execute(self, status=None) -> List[Dict[str, Any]]: search_all_upstreams_params["name"] = component_name if self.component_type: search_all_upstreams_params["type"] = self.component_type - if not (self.include_inactive_product_streams): - search_all_upstreams_params["active_streams"] = "True" - if self.exclude_unreleased: - search_all_upstreams_params["released_components"] = "True" - if not self.include_container_roots: - search_all_upstreams_params["type"] = "RPM" upstream_components_cnt = self.corgi_session.components.count( **search_all_upstreams_params ) @@ -726,10 +698,6 @@ def execute(self, status=None) -> List[Dict[str, Any]]: search_community_params["name"] = component_name if self.ns: search_community_params["namespace"] = self.ns - if not (self.include_inactive_product_streams): - search_community_params["active_streams"] = "True" - if self.exclude_unreleased: - search_community_params["released_components"] = "True" all_community_components_cnt = self.community_session.components.count( **search_community_params ) diff --git a/scripts/smoke-tests.sh b/scripts/smoke-tests.sh index c7d5363..d92fa28 100755 --- a/scripts/smoke-tests.sh +++ b/scripts/smoke-tests.sh @@ -88,5 +88,6 @@ griffon service products-contain-component webkitgtk4-jsc --include-product-stre griffon service products-contain-component -s bind-libs-lite --search-all griffon service products-contain-component v2v-conversion-host-ansible -vvv --include-container-roots --include-inactive-product-streams --include-product-streams-excluded-components --no-filter-rh-naming griffon service products-contain-component 'compat-sap-c++-12' -griffon service products-contain-component -r 'compat-sap-c++-12' -griffon service products-contain-component -s 'compat-sap-c++-12' +griffon service products-contain-component -r 'compat-sap-c\+\+-12' +#griffon service products-contain-component -s 'compat-sap-c++-12' +griffon service products-contain-component runc \ No newline at end of file From 6af0de749097cfcdc6ddf2e6bd89f3b70bd97918 Mon Sep 17 00:00:00 2001 From: jfuller Date: Fri, 2 Feb 2024 11:35:07 +0100 Subject: [PATCH 2/3] if --search-provides finds no children we should then fallback to --search-latest --- CHANGELOG.md | 3 ++ griffon/services/core_queries.py | 85 +++++++++++++++++--------------- scripts/smoke-tests.sh | 6 ++- 3 files changed, 52 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 842010f..8a12a45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Changed +* if default --search_provides finds no children then revert to --search-latest + ## [0.5.4] - 2024-01-23 ### Fixed * fixed searches with middleware CLI enabled (GRIF-221) diff --git a/griffon/services/core_queries.py b/griffon/services/core_queries.py index 98dc4fc..0381782 100644 --- a/griffon/services/core_queries.py +++ b/griffon/services/core_queries.py @@ -342,7 +342,7 @@ def execute(self, status=None) -> List[Dict[str, Any]]: params["active_streams"] = "True" if self.exclude_unreleased: params["released_components"] = "True" - if not self.include_container_roots: + if not (self.include_container_roots): params["type"] = "RPM" params["arch"] = "src" @@ -350,98 +350,101 @@ def execute(self, status=None) -> List[Dict[str, Any]]: if not self.strict_name_search and not self.regex_name_search: component_name = re.escape(component_name) - if self.search_latest: - search_latest_params = copy.deepcopy(params) + if self.search_provides: + search_provides_params = copy.deepcopy(params) if not (self.strict_name_search): - search_latest_params["re_name"] = component_name + search_provides_params["re_provides_name"] = component_name else: - search_latest_params["name"] = component_name + search_provides_params["provides_name"] = component_name if self.ns: - search_latest_params["namespace"] = self.ns - search_latest_params["root_components"] = "True" - search_latest_params["latest_components_by_streams"] = "True" - status.update("searching latest root component(s).") - latest_components_cnt = self.corgi_session.components.count(**search_latest_params) - status.update(f"found {latest_components_cnt} latest component(s).") + search_provides_params["namespace"] = self.ns + search_provides_params["latest_components_by_streams"] = "True" + status.update("searching latest provided child component(s).") + latest_components_cnt = self.corgi_session.components.count(**search_provides_params) + status.update(f"found {latest_components_cnt} latest provides component(s).") latest_components = self.corgi_session.components.retrieve_list_iterator_async( - **search_latest_params, max_results=10000 + **search_provides_params, max_results=10000 + ) + + status.update( + f"found {latest_components_cnt} latest provides child component(s)- retrieving children, sources & upstreams." # noqa ) - status.update(f"found {latest_components_cnt} latest root component(s).") # noqa with multiprocessing.Pool() as pool: for processed_component in pool.map( - partial(process_component, self.corgi_session, search_latest_params), + partial(process_component, self.corgi_session, search_provides_params), latest_components, ): results.append(processed_component) + # if we have found no children then search_latest for roots + if not (results): + self.search_latest = True if not self.no_community: - status.update("searching latest community root component(s).") + status.update("searching latest community provided child component(s).") community_component_cnt = self.community_session.components.count( - **search_latest_params + **search_provides_params ) status.update( - f"found {community_component_cnt} latest community root component(s)." # noqa + f"found {community_component_cnt} latest community provided child component(s)." # noqa ) latest_community_components = ( self.community_session.components.retrieve_list_iterator_async( - **search_latest_params, max_results=10000 + **search_provides_params, max_results=10000 ) ) status.update( - f"found {community_component_cnt} latest community root component(s)- retrieving children, sources & upstreams." # noqa + f"found {community_component_cnt} latest community provided child component(s)- retrieving children, sources & upstreams." # noqa ) with multiprocessing.Pool() as pool: for processed_component in pool.map( - partial(process_component, self.community_session, search_latest_params), + partial(process_component, self.community_session, search_provides_params), latest_community_components, ): results.append(processed_component) - if self.search_provides: - search_provides_params = copy.deepcopy(params) + if self.search_latest: + search_latest_params = copy.deepcopy(params) if not (self.strict_name_search): - search_provides_params["re_provides_name"] = component_name + search_latest_params["re_name"] = component_name else: - search_provides_params["provides_name"] = component_name + search_latest_params["name"] = component_name if self.ns: - search_provides_params["namespace"] = self.ns - search_provides_params["latest_components_by_streams"] = "True" - status.update("searching latest provided child component(s).") - latest_components_cnt = self.corgi_session.components.count(**search_provides_params) - status.update(f"found {latest_components_cnt} latest provides component(s).") + search_latest_params["namespace"] = self.ns + search_latest_params["root_components"] = "True" + search_latest_params["latest_components_by_streams"] = "True" + status.update("searching latest root component(s).") + latest_components_cnt = self.corgi_session.components.count(**search_latest_params) + status.update(f"found {latest_components_cnt} latest component(s).") latest_components = self.corgi_session.components.retrieve_list_iterator_async( - **search_provides_params, max_results=10000 - ) - - status.update( - f"found {latest_components_cnt} latest provides child component(s)- retrieving children, sources & upstreams." # noqa + **search_latest_params, max_results=10000 ) + status.update(f"found {latest_components_cnt} latest root component(s).") # noqa with multiprocessing.Pool() as pool: for processed_component in pool.map( - partial(process_component, self.corgi_session, search_provides_params), + partial(process_component, self.corgi_session, search_latest_params), latest_components, ): results.append(processed_component) if not self.no_community: - status.update("searching latest community provided child component(s).") + status.update("searching latest community root component(s).") community_component_cnt = self.community_session.components.count( - **search_provides_params + **search_latest_params ) status.update( - f"found {community_component_cnt} latest community provided child component(s)." # noqa + f"found {community_component_cnt} latest community root component(s)." # noqa ) latest_community_components = ( self.community_session.components.retrieve_list_iterator_async( - **search_provides_params, max_results=10000 + **search_latest_params, max_results=10000 ) ) status.update( - f"found {community_component_cnt} latest community provided child component(s)- retrieving children, sources & upstreams." # noqa + f"found {community_component_cnt} latest community root component(s)- retrieving children, sources & upstreams." # noqa ) with multiprocessing.Pool() as pool: for processed_component in pool.map( - partial(process_component, self.community_session, search_provides_params), + partial(process_component, self.community_session, search_latest_params), latest_community_components, ): results.append(processed_component) diff --git a/scripts/smoke-tests.sh b/scripts/smoke-tests.sh index d92fa28..ad85de6 100755 --- a/scripts/smoke-tests.sh +++ b/scripts/smoke-tests.sh @@ -90,4 +90,8 @@ griffon service products-contain-component v2v-conversion-host-ansible -vvv --i griffon service products-contain-component 'compat-sap-c++-12' griffon service products-contain-component -r 'compat-sap-c\+\+-12' #griffon service products-contain-component -s 'compat-sap-c++-12' -griffon service products-contain-component runc \ No newline at end of file +griffon service products-contain-component runc +griffon service products-contain-component github.com/go-redis/redis/v8/internal/hscan --include-container-roots --include-inactive-product-streams --no-filter-rh-naming --include-product-streams-excluded-components +griffon service products-contain-component pdf-generator -vvv --include-container-roots +griffon service products-contain-component -s wireshark -v +griffon service products-contain-component "hypershift-cloudwatch-loggging" --include-container-roots From e94ae032826a934150854d61f699e0284e4f1215 Mon Sep 17 00:00:00 2001 From: jfuller Date: Fri, 2 Feb 2024 11:43:46 +0100 Subject: [PATCH 3/3] pin osidb-bindings==3.6.0 --- CHANGELOG.md | 2 +- requirements/base.in | 2 +- requirements/base.txt | 8 ++++---- requirements/dev.txt | 15 ++++----------- requirements/lint.txt | 2 +- requirements/test.txt | 18 ++++++------------ 6 files changed, 17 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a12a45..b133c29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,12 +8,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed * if default --search_provides finds no children then revert to --search-latest +* pin osidb-bindings==3.6.0 ## [0.5.4] - 2024-01-23 ### Fixed * fixed searches with middleware CLI enabled (GRIF-221) - ## [0.5.3] - 2024-01-22 ### Fixed * fixed regex escaping logic for -r and -s diff --git a/requirements/base.in b/requirements/base.in index a7de8e8..2f24bbe 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -1,4 +1,4 @@ -osidb-bindings==3.5.1 +osidb-bindings==3.6.0 component-registry-bindings==1.4.0 click click-completion diff --git a/requirements/base.txt b/requirements/base.txt index 913d28d..8ee0958 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.9 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile --allow-unsafe --generate-hashes requirements/base.in @@ -458,9 +458,9 @@ multidict==6.0.4 \ # via # aiohttp # yarl -osidb-bindings==3.5.1 \ - --hash=sha256:70ac8e7e714611a126f0b73e48db93ff7769725d653fdcca47badcd9c1a4964c \ - --hash=sha256:a27b9211ad2bb9102a41fec6db375f51e480dbb42c8516f00e1be04ec135cb61 +osidb-bindings==3.6.0 \ + --hash=sha256:0be7fc0dc0a49a4fe77a997e85cfe1133d38e7cc6d51cf4eb076a418d818391e \ + --hash=sha256:448ca7f065289b5b4ecab54884f9c0fe8bc9d50840d84cf12ffe0c71dc3f411b # via -r requirements/base.in packageurl-python==0.10.4 \ --hash=sha256:5c91334f942cd55d45eb0c67dd339a535ef90e25f05b9ec016ad188ed0ef9048 \ diff --git a/requirements/dev.txt b/requirements/dev.txt index e544656..ae8a4ca 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.9 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile --allow-unsafe --generate-hashes requirements/dev.in @@ -432,7 +432,6 @@ exceptiongroup==1.1.0 \ # via # -r requirements/lint.txt # -r requirements/test.txt - # pytest executing==1.2.0 \ --hash=sha256:0314a69e37426e3608aada02473b4161d4caf5a4b244d1d0c48072b8fee7bacc \ --hash=sha256:19da64c18d2d851112f09c287f8d3dbbdf725ab0e569077efb6cdcbd3497c107 @@ -869,9 +868,9 @@ mypy-extensions==1.0.0 \ # via # -r requirements/test.txt # mypy -osidb-bindings==3.5.1 \ - --hash=sha256:70ac8e7e714611a126f0b73e48db93ff7769725d653fdcca47badcd9c1a4964c \ - --hash=sha256:a27b9211ad2bb9102a41fec6db375f51e480dbb42c8516f00e1be04ec135cb61 +osidb-bindings==3.6.0 \ + --hash=sha256:0be7fc0dc0a49a4fe77a997e85cfe1133d38e7cc6d51cf4eb076a418d818391e \ + --hash=sha256:448ca7f065289b5b4ecab54884f9c0fe8bc9d50840d84cf12ffe0c71dc3f411b # via # -r requirements/base.txt # -r requirements/test.txt @@ -1104,12 +1103,6 @@ tomli==2.0.1 \ # via # -r requirements/lint.txt # -r requirements/test.txt - # build - # coverage - # ipdb - # mypy - # pyproject-hooks - # pytest traitlets==5.9.0 \ --hash=sha256:9e6ec080259b9a5940c797d58b613b5e31441c2257b87c2e795c5228ae80d2d8 \ --hash=sha256:f6cde21a9c68cf756af02035f72d5a723bf607e862e7be33ece505abf4a3bad9 diff --git a/requirements/lint.txt b/requirements/lint.txt index 776b0cd..fa734ca 100644 --- a/requirements/lint.txt +++ b/requirements/lint.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.9 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile --allow-unsafe --generate-hashes requirements/lint.in diff --git a/requirements/test.txt b/requirements/test.txt index 968b1fe..fb61a58 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.9 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile --allow-unsafe --generate-hashes requirements/test.in @@ -295,9 +295,7 @@ decorator==5.1.1 \ exceptiongroup==1.1.0 \ --hash=sha256:327cbda3da756e2de031a3107b81ab7b3770a602c4d16ca618298c526f4bec1e \ --hash=sha256:bcb67d800a4497e1b404c2dd44fca47d3b7a5e5433dbab67f96c1a685cdfdf23 - # via - # -r requirements/test.in - # pytest + # via -r requirements/test.in frozenlist==1.4.0 \ --hash=sha256:007df07a6e3eb3e33e9a1fe6a9db7af152bbd8a185f9aaa6ece10a3529e3e1c6 \ --hash=sha256:008eb8b31b3ea6896da16c38c1b136cb9fec9e249e77f6211d479db79a4eaf01 \ @@ -661,9 +659,9 @@ mypy-extensions==1.0.0 \ --hash=sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d \ --hash=sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782 # via mypy -osidb-bindings==3.5.1 \ - --hash=sha256:70ac8e7e714611a126f0b73e48db93ff7769725d653fdcca47badcd9c1a4964c \ - --hash=sha256:a27b9211ad2bb9102a41fec6db375f51e480dbb42c8516f00e1be04ec135cb61 +osidb-bindings==3.6.0 \ + --hash=sha256:0be7fc0dc0a49a4fe77a997e85cfe1133d38e7cc6d51cf4eb076a418d818391e \ + --hash=sha256:448ca7f065289b5b4ecab54884f9c0fe8bc9d50840d84cf12ffe0c71dc3f411b # via -r requirements/base.txt packageurl-python==0.10.4 \ --hash=sha256:5c91334f942cd55d45eb0c67dd339a535ef90e25f05b9ec016ad188ed0ef9048 \ @@ -788,11 +786,7 @@ six==1.16.0 \ tomli==2.0.1 \ --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \ --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f - # via - # -r requirements/test.in - # coverage - # mypy - # pytest + # via -r requirements/test.in types-requests==2.28.11.14 \ --hash=sha256:232792870b60adb07d23175451ab4e6190021b0c584edf052d92d9b993118f06 \ --hash=sha256:f84613b0d4c5d0eeb7879dfa05e14a3702b9c1f7a4ee81dfe9b4321b13fe93a1