From 08e2c5d874a942810226ca689f0a8d0acaa9b037 Mon Sep 17 00:00:00 2001 From: Jakub Frejlach Date: Thu, 23 Nov 2023 15:49:10 +0100 Subject: [PATCH 1/8] Move envvar checks from __init__.py to CLI In __ini__.py zsh/bash autocompletion eval has problems with the direct print statements and tries to evaluate the text which is printed --- griffon/__init__.py | 57 +++++++++++++++++++++++++-------------------- griffon/cli.py | 3 +++ 2 files changed, 35 insertions(+), 25 deletions(-) diff --git a/griffon/__init__.py b/griffon/__init__.py index 043fb8b..eb08e9b 100644 --- a/griffon/__init__.py +++ b/griffon/__init__.py @@ -21,32 +21,8 @@ __version__ = "0.3.8" # TODO: Deprecate CORGI_API_URL completely in the next version or two -if "CORGI_API_URL" in os.environ: - print( - ( - f"{Style.BOLD}{Color.YELLOW}WARNING: CORGI_API_URL will be deprecated " - "in the next version of Griffon in favour of CORGI_SERVER_URL, please " - f"switch to the new environment variable.{Style.RESET}" - ) - ) - -if "CORGI_SERVER_URL" not in os.environ and "CORGI_API_URL" not in os.environ: - print("Must set CORGI_SERVER_URL environment variable.") - exit(1) CORGI_SERVER_URL = os.getenv("CORGI_SERVER_URL", os.getenv("CORGI_API_URL")) - -# TODO: Deprecate CORGI_API_URL completely in the next version or two -if "OSIDB_API_URL" in os.environ: - print( - ( - f"{Style.BOLD}{Color.YELLOW}WARNING: OSIDB_API_URL will be deprecated " - "in the next version of Griffon in favour of OSIDB_SERVER_URL, please " - f"switch to the new environment variable.{Style.RESET}" - ) - ) -if "OSIDB_SERVER_URL" not in os.environ and "OSIDB_API_URL" not in os.environ: - print("Must set OSIDB_SERVER_URL environment variable.") - exit(1) +# TODO: Deprecate OSIDB_API_URL completely in the next version or two OSIDB_SERVER_URL = os.getenv("OSIDB_SERVER_URL", os.getenv("OSIDB_API_URL")) OSIDB_USERNAME = os.getenv("OSIDB_USERNAME", "") @@ -70,6 +46,37 @@ RELATED_MODELS_MAPPING = {Flaw: {"affects": Affect}, Affect: {"trackers": Tracker}} +def check_envvars(): + """Check that all necessary envvars are set""" + + # TODO: Deprecate CORGI_API_URL completely in the next version or two + if "CORGI_API_URL" in os.environ: + print( + ( + f"{Style.BOLD}{Color.YELLOW}WARNING: CORGI_API_URL will be deprecated " + "in the next version of Griffon in favour of CORGI_SERVER_URL, please " + f"switch to the new environment variable.{Style.RESET}" + ) + ) + + if "CORGI_SERVER_URL" not in os.environ and "CORGI_API_URL" not in os.environ: + print("Must set CORGI_SERVER_URL environment variable.") + exit(1) + + # TODO: Deprecate OSIDB_API_URL completely in the next version or two + if "OSIDB_API_URL" in os.environ: + print( + ( + f"{Style.BOLD}{Color.YELLOW}WARNING: OSIDB_API_URL will be deprecated " + "in the next version of Griffon in favour of OSIDB_SERVER_URL, please " + f"switch to the new environment variable.{Style.RESET}" + ) + ) + if "OSIDB_SERVER_URL" not in os.environ and "OSIDB_API_URL" not in os.environ: + print("Must set OSIDB_SERVER_URL environment variable.") + exit(1) + + def config_logging(level="INFO"): # if set to 'DEBUG' then we want all the http conversation if level == "DEBUG": diff --git a/griffon/cli.py b/griffon/cli.py index e954fdd..7940c76 100644 --- a/griffon/cli.py +++ b/griffon/cli.py @@ -8,6 +8,7 @@ import click_completion from griffon import ( + check_envvars, config_logging, get_config_option, list_config_sections, @@ -134,6 +135,8 @@ def cli( ): """Red Hat product security CLI""" + check_envvars() + if ctx.invoked_subcommand is None: click.echo(ctx.parent.get_help()) From cdaa6c3ed8c51860843e4c9e8f0755e38cb0d6ed Mon Sep 17 00:00:00 2001 From: Jakub Frejlach Date: Thu, 23 Nov 2023 16:06:45 +0100 Subject: [PATCH 2/8] Change community corgi env var URL from API to server Leftover from OSIDB/CORGI API to SERVER change --- docs/user_guide.md | 2 +- griffon/__init__.py | 22 ++++++++++++++----- .../entities/community_component_registry.py | 4 ++-- griffon/services/core_queries.py | 6 ++--- 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/docs/user_guide.md b/docs/user_guide.md index 96c9f55..6d3d25e 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -67,7 +67,7 @@ export CORGI_SERVER_URL="https://" and the following is set to enable searching community components: ```commandline -export COMMUNITY_COMPONENTS_API_URL="https://component-registry.fedoraproject.org" +export COMMUNITY_COMPONENTS_SERVER_URL="https://component-registry.fedoraproject.org" ``` Your system must be properly authorised to access all these systems. diff --git a/griffon/__init__.py b/griffon/__init__.py index eb08e9b..c195f01 100644 --- a/griffon/__init__.py +++ b/griffon/__init__.py @@ -29,9 +29,11 @@ OSIDB_PASSWORD = os.getenv("OSIDB_PASSWORD", "") OSIDB_AUTH_METHOD = os.getenv("OSIDB_AUTH_METHOD", "kerberos") +# TODO: Deprecate COMMUNITY_COMPONENTS_API_URL completely in the next version or two # required to enable --search-community -COMMUNITY_COMPONENTS_API_URL = os.getenv( - "COMMUNITY_COMPONENTS_API_URL", "https://component-registry.fedoraproject.org" +COMMUNITY_COMPONENTS_SERVER_URL = os.getenv( + "COMMUNITY_COMPONENTS_SERVER_URL", + os.getenv("COMMUNITY_COMPONENTS_API_URL", "https://component-registry.fedoraproject.org"), ) # TODO: temporary hack required to enable searching of middleware @@ -58,11 +60,21 @@ def check_envvars(): f"switch to the new environment variable.{Style.RESET}" ) ) - if "CORGI_SERVER_URL" not in os.environ and "CORGI_API_URL" not in os.environ: print("Must set CORGI_SERVER_URL environment variable.") exit(1) + # TODO: Deprecate COMMUNITY_COMPONENTS_API_URL completely in the next version or two + if "COMMUNITY_COMPONENTS_API_URL" in os.environ: + print( + ( + f"{Style.BOLD}{Color.YELLOW}WARNING: COMMUNITY_COMPONENTS_API_URL " + "will be deprecated in the next version of Griffon in favour of " + "COMMUNITY_COMPONENTS_SERVER_URL, please switch to the new environment " + f"variable.{Style.RESET}" + ) + ) + # TODO: Deprecate OSIDB_API_URL completely in the next version or two if "OSIDB_API_URL" in os.environ: print( @@ -294,10 +306,10 @@ def create_session(): """init corgi session""" try: return component_registry_bindings.new_session( - component_registry_server_uri=COMMUNITY_COMPONENTS_API_URL, + component_registry_server_uri=COMMUNITY_COMPONENTS_SERVER_URL, ) except: # noqa - console.log(f"{COMMUNITY_COMPONENTS_API_URL} is not accessible.") + console.log(f"{COMMUNITY_COMPONENTS_SERVER_URL } is not accessible.") exit(1) @staticmethod diff --git a/griffon/commands/entities/community_component_registry.py b/griffon/commands/entities/community_component_registry.py index ae6da1a..e60178c 100644 --- a/griffon/commands/entities/community_component_registry.py +++ b/griffon/commands/entities/community_component_registry.py @@ -33,7 +33,7 @@ ) from griffon import ( - COMMUNITY_COMPONENTS_API_URL, + COMMUNITY_COMPONENTS_SERVER_URL, CORGI_SERVER_URL, CommunityComponentService, progress_bar, @@ -65,7 +65,7 @@ def commmunity_components_grp(ctx): # COMPONENTS -@commmunity_components_grp.group(help=f"{COMMUNITY_COMPONENTS_API_URL}/api/v1/components") +@commmunity_components_grp.group(help=f"{COMMUNITY_COMPONENTS_SERVER_URL}/api/v1/components") @click.pass_context def components(ctx): """Corgi Components.""" diff --git a/griffon/services/core_queries.py b/griffon/services/core_queries.py index d9762c7..bbe5f5d 100644 --- a/griffon/services/core_queries.py +++ b/griffon/services/core_queries.py @@ -12,7 +12,7 @@ from component_registry_bindings.bindings.python_client.models import Component from griffon import ( - COMMUNITY_COMPONENTS_API_URL, + COMMUNITY_COMPONENTS_SERVER_URL, CORGI_SERVER_URL, OSIDB_SERVER_URL, CommunityComponentService, @@ -931,7 +931,7 @@ def execute(self, status=None) -> List[Dict[str, Any]]: "link_affect": f"{OSIDB_SERVER_URL}/osidb/api/v1/affects/{affect.uuid}", # noqa "link_cve": f"{OSIDB_SERVER_URL}/osidb/api/v1/flaws/{flaw.cve_id}", # noqa "link_component": f"{CORGI_SERVER_URL}/api/v1/components?name={affect.ps_component}&latest_components_by_streams=True", # noqa - "link_community_component": f"{COMMUNITY_COMPONENTS_API_URL}/api/v1/components?name={affect.ps_component}&latest_components_by_streams=True", # noqa + "link_community_component": f"{COMMUNITY_COMPONENTS_SERVER_URL}/api/v1/components?name={affect.ps_component}&latest_components_by_streams=True", # noqa "flaw_cve_id": flaw.cve_id, "title": flaw.title, "flaw_resolution": flaw.resolution, @@ -1020,7 +1020,7 @@ def execute(self, status=None) -> List[Dict[str, Any]]: "link_affect": f"{OSIDB_SERVER_URL}/osidb/api/v1/affects/{affect.uuid}", # noqa "link_cve": f"{OSIDB_SERVER_URL}/osidb/api/v1/flaws/{flaw.cve_id}", # noqa "link_component": f"{CORGI_SERVER_URL}/api/v1/components?name={affect.ps_component}&latest_components_by_streams=True", # noqa - "link_community_component": f"{COMMUNITY_COMPONENTS_API_URL}/api/v1/components?name={affect.ps_component}&latest_components_by_streams=True", # noqa + "link_community_component": f"{COMMUNITY_COMPONENTS_SERVER_URL}/api/v1/components?name={affect.ps_component}&latest_components_by_streams=True", # noqa "flaw_cve_id": flaw.cve_id, "title": flaw.title, "flaw_state": flaw.state, From 5f9744465b3fdfc4e2c105eb3c77020e35fae7fd Mon Sep 17 00:00:00 2001 From: Jakub Frejlach Date: Mon, 27 Nov 2023 16:33:20 +0100 Subject: [PATCH 3/8] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d22d0d..2dedcc6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 service products-contain-component * OSIDB_API_URL environment variable changed to OSIDB_SERVER_URL * CORGI_API_URL environment variable changed to CORGI_SERVER_URL +* COMMUNITY_COMPONENTS_API_URL environment variable changed to + COMMUNITY_COMPONENTS_SERVER_URL * Migrated from setup.py to pyproject.toml ### Added From d4c4450c68a4805ccc594d15f150ea6d9bd1fd40 Mon Sep 17 00:00:00 2001 From: Jakub Frejlach Date: Thu, 23 Nov 2023 16:41:40 +0100 Subject: [PATCH 4/8] Split MIDDLEWARE_CLI command on spaces This is for users who doesn't use single command but run middleware cli through python. Python subprocess requires all parts of the command to be separated --- griffon/commands/queries.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/griffon/commands/queries.py b/griffon/commands/queries.py index adeb18d..f64b74a 100644 --- a/griffon/commands/queries.py +++ b/griffon/commands/queries.py @@ -404,7 +404,9 @@ def get_product_contain_component( # TODO: interim hack for middleware if component_name and MIDDLEWARE_CLI and not no_middleware: operation_status.update("searching deptopia middleware.") - mw_command = [MIDDLEWARE_CLI, component_name, "-e", "maven", "--json"] + + # Use split for users who runs middleware via python + mw_command = [*MIDDLEWARE_CLI.split(), component_name, "-e", "maven", "--json"] if strict_name_search: mw_command.append("-s") proc = subprocess.run( From 50794d3740ba363794c4ecba909e32bfdcef4a6c Mon Sep 17 00:00:00 2001 From: Jakub Frejlach Date: Mon, 27 Nov 2023 16:29:41 +0100 Subject: [PATCH 5/8] Fix custom command check on tab completion Check for mutually exclusive groups or one of groups should only run when griffon itself is running. During tab completion these checks should be disabled. --- CHANGELOG.md | 2 ++ griffon/commands/custom_commands.py | 49 +++++++++++++++-------------- 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2dedcc6..677b592 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed * standardized progress bar accross whole Griffon which fixes no_progress_bar functionality +* fixed tab autocompletion for mutually exclusive and one of + option/argument groups ## [0.3.8] - 2023-10-18 ### Added diff --git a/griffon/commands/custom_commands.py b/griffon/commands/custom_commands.py index d21b8bc..41a465c 100644 --- a/griffon/commands/custom_commands.py +++ b/griffon/commands/custom_commands.py @@ -27,30 +27,33 @@ class BaseGroupParameter: """ def handle_parse_result(self, ctx, opts, args): - if self.mutually_exclusive_group: - if opts.get(self.name) is None: - pass # skip check for not supplied click.Arguments - elif self.name in opts and any( - opt in opts for opt in self.mutually_exclusive_group if opts.get(opt) is not None - ): - raise GriffonUsageError( - ( - f"{Style.BOLD}{self.name} cannot be used with " - f"{', '.join(self.mutually_exclusive_group)}.{Style.RESET}" - ), - ctx=ctx, - ) - - if self.required_group: - group_set = set( - opt for opt in opts if opt in self.required_group and opts.get(opt) is not None - ) - if not any(group_set): - raise GriffonUsageError( - f"{Style.BOLD}At least one of {', '.join(self.required_group)} " - f"is required.{Style.RESET}", - ctx=ctx, + if not ctx.resilient_parsing: + if self.mutually_exclusive_group: + if opts.get(self.name) is None: + pass # skip check for not supplied click.Arguments + elif self.name in opts and any( + opt in opts + for opt in self.mutually_exclusive_group + if opts.get(opt) is not None + ): + raise GriffonUsageError( + ( + f"{Style.BOLD}{self.name} cannot be used with " + f"{', '.join(self.mutually_exclusive_group)}.{Style.RESET}" + ), + ctx=ctx, + ) + + if self.required_group: + group_set = set( + opt for opt in opts if opt in self.required_group and opts.get(opt) is not None ) + if not any(group_set): + raise GriffonUsageError( + f"{Style.BOLD}At least one of {', '.join(self.required_group)} " + f"is required.{Style.RESET}", + ctx=ctx, + ) return super().handle_parse_result(ctx, opts, args) From fa14c8453509f0782b802f9ef366acb55b6e90d1 Mon Sep 17 00:00:00 2001 From: Jakub Frejlach Date: Fri, 8 Dec 2023 13:24:17 +0100 Subject: [PATCH 6/8] Fix filtering middleware data * consider all middleware data retrieved by middleware CLI active * delegate maven build type filter to middleware CLI --- griffon/commands/queries.py | 76 ++++++++++++++++++++----------------- 1 file changed, 42 insertions(+), 34 deletions(-) diff --git a/griffon/commands/queries.py b/griffon/commands/queries.py index f64b74a..516237b 100644 --- a/griffon/commands/queries.py +++ b/griffon/commands/queries.py @@ -406,7 +406,15 @@ def get_product_contain_component( operation_status.update("searching deptopia middleware.") # Use split for users who runs middleware via python - mw_command = [*MIDDLEWARE_CLI.split(), component_name, "-e", "maven", "--json"] + mw_command = [ + *MIDDLEWARE_CLI.split(), + component_name, + "-e", + "maven", + "-b", + "maven", + "--json", + ] if strict_name_search: mw_command.append("-s") proc = subprocess.run( @@ -421,39 +429,39 @@ def get_product_contain_component( # if search_all: # mw_components.extend(mw_json["deps"]) for build in mw_components: - if build["build_type"] == "maven": - component = { - "product_versions": [{"name": build["ps_module"]}], - "product_streams": [ - { - "name": build["ps_update_stream"], - "product_versions": [{"name": build["ps_module"]}], - } - ], - "product_active": True, - "type": build["build_type"], - "name": build["build_name"], - "nvr": build["build_nvr"], - "upstreams": [], - "sources": [], - "software_build": { - "build_id": build["build_id"], - "source": build["build_repo"], - }, - } - if "sources" in build: - for deps in build["sources"]: - for dep in deps["dependencies"]: - components = [] - components.append( - { - "name": dep.get("name"), - "nvr": dep.get("nvr"), - "type": dep.get("type"), - } - ) - component["sources"] = components - q.append(component) + component = { + "product_versions": [{"name": build["ps_module"]}], + "product_streams": [ + { + "name": build["ps_update_stream"], + "product_versions": [{"name": build["ps_module"]}], + "active": True, # assume all product streams as active + } + ], + "product_active": True, + "type": build["build_type"], + "name": build["build_name"], + "nvr": build["build_nvr"], + "upstreams": [], + "sources": [], + "software_build": { + "build_id": build["build_id"], + "source": build["build_repo"], + }, + } + if "sources" in build: + for deps in build["sources"]: + for dep in deps["dependencies"]: + components = [] + components.append( + { + "name": dep.get("name"), + "nvr": dep.get("nvr"), + "type": dep.get("type"), + } + ) + component["sources"] = components + q.append(component) except Exception: logger.warning("problem accessing deptopia.") From 7a9f1b8dd2765579a4169091103eb3a0022491c9 Mon Sep 17 00:00:00 2001 From: Jakub Frejlach Date: Fri, 8 Dec 2023 13:25:43 +0100 Subject: [PATCH 7/8] Standardize work with related data sources and upstreams are transformed to raw JSON now so we avoid problems when mixing middleware CLI data and component registry data (eg. Component registry bindings models vs JSON) --- griffon/output.py | 52 +++++++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/griffon/output.py b/griffon/output.py index b3b65b1..524ea20 100644 --- a/griffon/output.py +++ b/griffon/output.py @@ -8,6 +8,9 @@ import re import click +from component_registry_bindings.bindings.python_client.types import ( + ComponentRegistryModel, +) from packageurl import PackageURL from rich.console import Console from rich.text import Text @@ -29,25 +32,38 @@ class DEST(enum.Enum): FILE = "file" +def raw_json_transform_related(data: dict, name): + """normalise ralated data which were piggy-backed to data""" + transformed = [] + related_data = data.get(name) + if related_data is not None: + if isinstance(related_data, list): + transformed = [ + item.to_dict() if isinstance(item, ComponentRegistryModel) else item + for item in related_data + ] + + return transformed + + def raw_json_transform(data, show_count: bool) -> dict: """normalise all data to dict""" if type(data) is list: results = [] - for d in data: - if type(d) is dict: - results.append(d) - else: - results.append(d.to_dict()) + for item in data: + transformed = item if type(item) is dict else item.to_dict() + for related_data in ("upstreams", "sources"): + transformed[related_data] = raw_json_transform_related(transformed, related_data) + results.append(transformed) output = { "results": results, } if show_count: output["count"] = len(results) # type: ignore else: - if type(data) is dict: - output = data - else: - output = data.to_dict() + transformed = data if type(data) is dict else data.to_dict() + for related_data in ("upstreams", "sources"): + transformed[related_data] = raw_json_transform_related(transformed, related_data) return output @@ -274,7 +290,7 @@ def generate_affects( for nvr in result_tree[pv][ps][component_name].keys(): if result_tree[pv][ps][component_name][nvr]["sources"]: source_names = [ - source.name + source["name"] for source in result_tree[pv][ps][component_name][nvr]["sources"] if source.namespace == "REDHAT" ] @@ -413,7 +429,7 @@ def text_output_products_contain_component( list( set( [ - source.name + source["name"] for source in result_tree[pv][ps][cn][nvr][ "upstreams" ] @@ -439,7 +455,7 @@ def text_output_products_contain_component( list( set( [ - source.name + source["name"] for source in result_tree[pv][ps][cn][nvr][ "sources" ] @@ -503,7 +519,7 @@ def text_output_products_contain_component( list( set( [ - source.name + source["name"] for source in result_tree[pv][ps][cn][nvr][ "upstreams" ] @@ -530,7 +546,7 @@ def text_output_products_contain_component( list( set( [ - source.name + source["name"] for source in result_tree[pv][ps][cn][nvr][ "sources" ] @@ -598,7 +614,7 @@ def text_output_products_contain_component( list( set( [ - source.name + source["name"] for source in result_tree[pv][ps][cn][nvr][ "upstreams" ] @@ -620,7 +636,7 @@ def text_output_products_contain_component( list( set( [ - source.name + source["name"] for source in result_tree[pv][ps][cn][nvr][ "sources" ] @@ -684,7 +700,7 @@ def text_output_products_contain_component( list( set( [ - source.nvr + source["nvr"] for source in result_tree[pv][ps][cn][nvr][ "upstreams" ] @@ -706,7 +722,7 @@ def text_output_products_contain_component( list( set( [ - source.nvr + source["nvr"] for source in result_tree[pv][ps][cn][nvr][ "sources" ] From a3868203a8e465dba1643ec3cd8b7b65115fda92 Mon Sep 17 00:00:00 2001 From: Jakub Frejlach Date: Fri, 8 Dec 2023 13:27:56 +0100 Subject: [PATCH 8/8] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 677b592..ffb9523 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 no_progress_bar functionality * fixed tab autocompletion for mutually exclusive and one of option/argument groups +* data obtained from middleware CLI are now correctly filtered ## [0.3.8] - 2023-10-18 ### Added