From b541cf2e70f64d7e3c43fbb3b8e7db44277cfa9b Mon Sep 17 00:00:00 2001 From: jfuller Date: Wed, 17 Jan 2024 11:46:05 +0100 Subject: [PATCH 1/8] refactor products-contains-component output and added --include-container-roots --- CHANGELOG.md | 7 + griffon/commands/queries.py | 11 +- griffon/output.py | 940 ++++++++++++++++++++----------- griffon/services/core_queries.py | 139 +++-- griffon/static/default_griffonrc | 1 + 5 files changed, 723 insertions(+), 375 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19e0e9e..57d4710 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Changed +* refactor output format of service products-contain-component (GRIF-209) +* container roots are excluded by default (need to use --include-container-roots) + +### Added +* --include-container_roots in service products-contain-component + ## [0.4.2] - 2024-01-14 ### Added * GRIFFON_VERIFY_SSL environment variable diff --git a/griffon/commands/queries.py b/griffon/commands/queries.py index c054ba6..dc81686 100644 --- a/griffon/commands/queries.py +++ b/griffon/commands/queries.py @@ -363,6 +363,13 @@ def retrieve_component_summary(ctx, component_name, strict_name_search, operatio help="Regex search.", mutually_exclusive_group=["strict_name_search"], ) +@click.option( + "--include-container-roots", + "include_container_roots", + is_flag=True, + default=get_config_option("default", "include_container_roots", False), + help="Include OCI root components in output.", +) @click.pass_context @progress_bar(is_updatable=True) def get_product_contain_component( @@ -395,12 +402,13 @@ def get_product_contain_component( verbose, operation_status, regex_name_search, + include_container_roots, ): # with console_status(ctx) as operation_status: """List products of a latest component.""" if verbose: ctx.obj["VERBOSE"] = verbose - + ctx.obj["INCLUDE_CONTAINER_ROOTS"] = include_container_roots if ( not search_latest and not search_all @@ -419,6 +427,7 @@ def get_product_contain_component( params.pop("sfm2_flaw_id") params.pop("flaw_mode") params.pop("affect_mode") + params.pop("include_container_roots") if component_name: q = query_service.invoke( core_queries.products_containing_component_query, params, status=operation_status diff --git a/griffon/output.py b/griffon/output.py index 51303ae..ddb9d8b 100644 --- a/griffon/output.py +++ b/griffon/output.py @@ -52,7 +52,7 @@ def raw_json_transform(data, show_count: bool) -> dict: results = [] for item in data: transformed = item if type(item) is dict else item.to_dict() - for related_data in ("upstreams", "sources"): + for related_data in ("upstreams", "sources", "provides"): transformed[related_data] = raw_json_transform_related(transformed, related_data) results.append(transformed) output = { @@ -175,6 +175,27 @@ def text_output_product_summary(ctx, output, format, exclude_products, no_wrap=F ctx.exit() +def process_excludes_condition( + exclude_products, + ps_exclude_components, + exclude_components, + ps_version_name, + component_name, + include_product_stream_excluded_components, +): + # .griffonrc defined exclude product streams + if not any([re.search(match, ps_version_name) for match in exclude_products]): + # product stream defined exclude components + if ( + not any([re.search(match, component_name) for match in ps_exclude_components]) + or include_product_stream_excluded_components + ): + # .griffonrc defined exclude components + if not any([re.search(match, component_name) for match in exclude_components]): + return True + return False + + def generate_normalised_results( output, exclude_products, @@ -185,53 +206,89 @@ def generate_normalised_results( ): normalised_results = list() if "results" in output: - for item in output["results"]: + # ensure unique result set + seen = set() + results = [] + for obj in output["results"]: + if obj["purl"] not in seen: + seen.add(obj["purl"]) + results.append(obj) + for item in results: for ps in item["product_streams"]: # only include component from active product stream if ps.get("active") or include_inactive_product_streams: - # .griffonrc defined exclude product streams - if not any( - [ - re.search(match, ps["product_versions"][0]["name"]) - for match in exclude_products - ] + if process_excludes_condition( + exclude_products, + ps.get("exclude_components", []), + exclude_components, + ps["product_versions"][0]["name"], + item["name"], + include_product_stream_excluded_components, ): - # product stream defined exclude components - if ( - not any( - [ - re.search(match, item["name"]) - for match in ps.get("exclude_components", []) - ] - ) - or include_product_stream_excluded_components - ): - # .griffonrc defined exclude components - if not any( - [re.search(match, item["name"]) for match in exclude_components] - ): - c = { - "product_version": ps["product_versions"][0]["name"], - "product_stream": ps.get("name"), - "product_stream_active": ps.get("active"), - "namespace": item.get("namespace"), - "name": item.get("name"), - "nvr": item.get("nvr"), - "type": item.get("type"), - "arch": item.get("arch"), - "version": item.get("version"), - "related_url": item.get("related_url"), - "purl": item.get("purl"), - "sources": item.get("sources"), - "upstreams": item.get("upstreams"), - } - if "software_build" in item: - c["build_source_url"] = item["software_build"].get("source") - # output type filter - if output_type_filter is None: - normalised_results.append(c) - if item.get("type") == output_type_filter: - normalised_results.append(c) + c = { + "product_version": ps["product_versions"][0]["name"], + "product_stream": ps.get("name"), + "product_stream_active": ps.get("active"), + "namespace": item.get("namespace"), + "name": item.get("name"), + "nvr": item.get("nvr"), + "type": item.get("type"), + "arch": item.get("arch"), + "version": item.get("version"), + "related_url": item.get("related_url"), + "purl": item.get("purl"), + } + if "software_build" in item: + c["build_source_url"] = item["software_build"].get("source") + c["build_type"] = item["software_build"].get("build_type") + + if "sources" in item: + sources = [] + for source in item.get("sources"): + if process_excludes_condition( + exclude_products, + ps.get("exclude_components", []), + exclude_components, + ps["product_versions"][0]["name"], + source["name"], + include_product_stream_excluded_components, + ): + sources.append(source) + c["sources"] = sources + + if "upstreams" in item: + upstreams = [] + for upstream in item.get("upstreams"): + if process_excludes_condition( + exclude_products, + ps.get("exclude_components", []), + exclude_components, + ps["product_versions"][0]["name"], + upstream["name"], + include_product_stream_excluded_components, + ): + upstreams.append(upstream) + c["upstreams"] = upstreams + + if "provides" in item: + provides = [] + for provide in item.get("provides"): + if process_excludes_condition( + exclude_products, + ps.get("exclude_components", []), + exclude_components, + ps["product_versions"][0]["name"], + provide["name"], + include_product_stream_excluded_components, + ): + provides.append(provide) + c["provides"] = provides + + # output type filter + if output_type_filter is None: + normalised_results.append(c) + if item.get("type") == output_type_filter: + normalised_results.append(c) return normalised_results @@ -326,6 +383,21 @@ def generate_affects( return affects +def process_product_color(build_type: str) -> str: + """return color used for product, currently only based on component build type""" + if build_type == "APP_INTERFACE": + return "blue" + if build_type == "CENTOS": + return "cyan" + if build_type == "PNC": + return "red" + if build_type == "KOJI": + return "yellow" + if build_type == "PYXIS": + return "green" + return "magenta" + + def text_output_products_contain_component( ctx, output, @@ -377,378 +449,602 @@ def text_output_products_contain_component( generate_affects(ctx, result_tree, exclude_components, flaw_operation, no_wrap=False) else: - if ctx.obj["VERBOSE"] == 0: # product_version X component_name - for pv in result_tree.keys(): - component_names = set() - for ps in result_tree[pv].keys(): - component_names.update(result_tree[pv][ps].keys()) - # we should only show component name if both {component name} and {component name-container} exists # noqa - if ( - search_component_name in component_names - and f"{search_component_name}-container" in component_names - ): - component_names.remove(f"{search_component_name}-container") - for cn in sorted(component_names): - # highlight search term - dep_name = re.sub(re.escape(cn), f"[b]{cn}[/b]", cn) - dep = f"[grey93]{dep_name}[/grey93]" - console.print( - Text(pv, style="magenta b u"), - dep, - no_wrap=no_wrap, - ) - if ctx.obj["VERBOSE"] == 1: # product_stream X nvr x related_url + if ctx.obj["VERBOSE"] == 0: # product_version X root component nvr for pv in result_tree.keys(): for ps in result_tree[pv].keys(): for cn in sorted(result_tree[pv][ps].keys()): # select the latest nvr (from sorted list) nvr = list(result_tree[pv][ps][cn].keys())[-1] - # highlight search term - dep_name = nvr - try: - dep_name = re.sub( - re.escape(search_component_name), - f"[b]{search_component_name}[/b]", - nvr, + if result_tree[pv][ps][cn][nvr]["type"] != "OCI" or ( + result_tree[pv][ps][cn][nvr]["type"] == "OCI" + and ctx.obj["INCLUDE_CONTAINER_ROOTS"] + ): + product_color = process_product_color( + result_tree[pv][ps][cn][nvr]["build_type"] ) - except re.error: - pass - dep = f"[grey93]{dep_name} ({result_tree[pv][ps][cn][nvr]['type']})[/grey93]" # noqa - related_url = result_tree[pv][ps][cn][nvr].get("related_url") - try: - if result_tree[pv][ps][cn][nvr]["related_url"]: - related_url = re.sub( + dep_name = nvr + # highlight search term + try: + dep_name = re.sub( re.escape(search_component_name), f"[b]{search_component_name}[/b]", - result_tree[pv][ps][cn][nvr]["related_url"], + dep_name, ) - except re.error: - pass - if result_tree[pv][ps][cn][nvr]["upstreams"]: - upstream_component_names = sorted( - list( - set( + except re.error: + pass + dep = f"[grey93]{dep_name}[/grey93]" # noqa + if result_tree[pv][ps][cn][nvr]["upstreams"]: + upstream_component_names = sorted( + list( [ - source["name"] - for source in result_tree[pv][ps][cn][nvr][ + f"{upstream['name']} {upstream['version']}" + for upstream in result_tree[pv][ps][cn][nvr][ "upstreams" ] ] ) ) - ) - if len(upstream_component_names) > 0: - upstream_component_name = ( - f"[cyan]{upstream_component_names[0]}[/cyan]" + for upstream_component_name in upstream_component_names: + console.print( + Text(pv, style="bright_red b"), + upstream_component_name, + no_wrap=no_wrap, + ) + if result_tree[pv][ps][cn][nvr]["sources"]: + source_component_names = sorted( + list( + set( + [ + f"{source['name']} {source['version']}" + for source in result_tree[pv][ps][cn][nvr][ + "sources" + ] + ] + ) + ) ) - if len(upstream_component_names) > 1: - upstream_component_name = f"[cyan]{upstream_component_names[0]} and {len(upstream_component_names) - 1} more[/cyan]" # noqa + for source_component_name in source_component_names: + console.print( + Text(pv, style="bright_red b"), + source_component_name, + no_wrap=no_wrap, + ) + if not (result_tree[pv][ps][cn][nvr]["upstreams"]) and not ( + result_tree[pv][ps][cn][nvr]["sources"] + ): console.print( - Text(ps, style="magenta b u"), - upstream_component_name, + Text(pv, style=f"{product_color} b"), dep, - f"([grey]{related_url}[/grey])", no_wrap=no_wrap, ) - if result_tree[pv][ps][cn][nvr]["sources"]: - source_component_names = sorted( - list( - set( + if ( + ctx.obj["VERBOSE"] == 1 + ): # product_stream X root component nvr (type) x child components [nvr (type)] + for pv in result_tree.keys(): + for ps in result_tree[pv].keys(): + for cn in sorted(result_tree[pv][ps].keys()): + # select the latest nvr (from sorted list) + nvr = list(result_tree[pv][ps][cn].keys())[-1] + if result_tree[pv][ps][cn][nvr]["type"] != "OCI" or ( + result_tree[pv][ps][cn][nvr]["type"] == "OCI" + and ctx.obj["INCLUDE_CONTAINER_ROOTS"] + ): + product_color = process_product_color( + result_tree[pv][ps][cn][nvr]["build_type"] + ) + # highlight search term + # dep_name = nvr + dep_name = nvr + try: + dep_name = re.sub( + re.escape(search_component_name), + f"[b]{search_component_name}[/b]", + dep_name, + ) + except re.error: + pass + dep = f"[grey93]{dep_name} ({result_tree[pv][ps][cn][nvr]['type']})[/grey93]" # noqa + related_url = result_tree[pv][ps][cn][nvr].get("related_url") + try: + if result_tree[pv][ps][cn][nvr]["related_url"]: + related_url = re.sub( + re.escape(search_component_name), + f"[b]{search_component_name}[/b]", + result_tree[pv][ps][cn][nvr]["related_url"], + ) + except re.error: + pass + provides_components = [] + if "provides" in result_tree[pv][ps][cn][nvr]: + provides_component_names = set( + [ + provide["name"] + for provide in result_tree[pv][ps][cn][nvr]["provides"] + ] + ) + for provide_name in provides_component_names: + versions = set( [ - source["name"] - for source in result_tree[pv][ps][cn][nvr][ - "sources" + provide["version"] + for provide in result_tree[pv][ps][cn][nvr][ + "provides" ] + if provide["name"] == provide_name ] ) + types = set( + [ + provide["type"] + for provide in result_tree[pv][ps][cn][nvr][ + "provides" + ] + if provide["name"] == provide_name + ] + ) + provides_components.append( + f"{provide_name} {(','.join(versions))} ({(',').join(types)})" # noqa + ) + if result_tree[pv][ps][cn][nvr]["upstreams"]: + upstream_component_names = sorted( + list( + set( + [ + f"{upstream['nvr']} ({upstream['type']})" + for upstream in result_tree[pv][ps][cn][nvr][ + "upstreams" + ] + ] + ) + ) ) - ) - if len(source_component_names) > 0: - source_component_name = ( - f"[red]{source_component_names[0]}[/red]" + if len(upstream_component_names) > 0: + upstream_component_name = ( + f"[cyan]{upstream_component_names[0]}[/cyan]" + ) + if len(upstream_component_names) > 1: + upstream_component_name = f"[cyan]{upstream_component_names[0]} and {len(upstream_component_names) - 1} more[/cyan]" # noqa + console.print( + Text(ps, style=f"{product_color} b"), + upstream_component_name, + dep, + no_wrap=no_wrap, + ) + if result_tree[pv][ps][cn][nvr]["sources"]: + source_component_names = sorted( + list( + set( + [ + f"{source['nvr']} ({source['type']})" + for source in result_tree[pv][ps][cn][nvr][ + "sources" + ] + ] + ) + ) ) - if len(source_component_names) > 1: - source_component_name = f"[red]{source_component_names[0]} and {len(source_component_names) - 1} more[/red]" # noqa + if len(source_component_names) > 0: + source_component_name = ( + f"[red]{source_component_names[0]}[/red]" + ) + if len(source_component_names) > 1: + source_component_name = f"[red]{source_component_names[0]} and {len(source_component_names) - 1} more[/red]" # noqa + console.print( + Text(ps, style=f"{product_color} b"), + source_component_name, + dep, + no_wrap=no_wrap, + ) + if not (result_tree[pv][ps][cn][nvr]["upstreams"]) and not ( + result_tree[pv][ps][cn][nvr]["sources"] + ): + try: + child_dep_names = re.sub( + re.escape(search_component_name), + f"[b]{search_component_name}[/b]", + ", ".join(provides_components), + ) + except re.error: + pass + console.print( - Text(ps, style="magenta b u"), - source_component_name, + Text(ps, style=f"{product_color} b"), dep, - f"([grey]{related_url}[/grey])", + f"[{child_dep_names}]", no_wrap=no_wrap, ) - if not (result_tree[pv][ps][cn][nvr]["upstreams"]) and not ( - result_tree[pv][ps][cn][nvr]["sources"] - ): - console.print( - Text(ps, style="magenta b u"), - dep, - f"([grey]{related_url}[/grey])", - no_wrap=no_wrap, - ) - if ctx.obj["VERBOSE"] == 2: # product_stream X nvr x related_url x build_source_url + if ( + ctx.obj["VERBOSE"] == 2 + ): # product_stream X root component nvr (type:arch) x child components [name {versions} (type:{arches})] x related_url x build_source_url # noqa for pv in result_tree.keys(): for ps in result_tree[pv].keys(): for cn in sorted(result_tree[pv][ps].keys()): + # select the latest nvr (from sorted list) nvr = list(result_tree[pv][ps][cn].keys())[-1] - # highlight search term - dep_name = nvr - try: - dep_name = re.sub( - re.escape(search_component_name), - f"[b]{search_component_name}[/b]", - nvr, + if result_tree[pv][ps][cn][nvr]["type"] != "OCI" or ( + result_tree[pv][ps][cn][nvr]["type"] == "OCI" + and ctx.obj["INCLUDE_CONTAINER_ROOTS"] + ): + product_color = process_product_color( + result_tree[pv][ps][cn][nvr]["build_type"] ) - except re.error: - pass - dep = f"[grey93]{dep_name} ({result_tree[pv][ps][cn][nvr]['type']})[/grey93]" # noqa - related_url = result_tree[pv][ps][cn][nvr].get("related_url") - try: - if result_tree[pv][ps][cn][nvr]["related_url"]: - related_url = re.sub( + # highlight search term + dep_name = nvr + try: + dep_name = re.sub( re.escape(search_component_name), f"[b]{search_component_name}[/b]", - result_tree[pv][ps][cn][nvr]["related_url"], + dep_name, + ) + except re.error: + pass + dep = f"[grey93]{dep_name} ({result_tree[pv][ps][cn][nvr]['type']}:{result_tree[pv][ps][cn][nvr]['arch']})[/grey93]" # noqa + related_url = result_tree[pv][ps][cn][nvr].get("related_url") + try: + if result_tree[pv][ps][cn][nvr]["related_url"]: + related_url = re.sub( + re.escape(search_component_name), + f"[b]{search_component_name}[/b]", + result_tree[pv][ps][cn][nvr]["related_url"], + ) + except re.error: + pass + build_source_url = "" + if result_tree[pv][ps][cn][nvr]["build_source_url"]: + build_source_url = result_tree[pv][ps][cn][nvr][ + "build_source_url" + ] + provides_components = [] + if "provides" in result_tree[pv][ps][cn][nvr]: + provides_component_names = set( + [ + provide["name"] + for provide in result_tree[pv][ps][cn][nvr]["provides"] + ] ) - except re.error: - pass - build_source_url = "" - if result_tree[pv][ps][cn][nvr]["build_source_url"]: - build_source_url = result_tree[pv][ps][cn][nvr]["build_source_url"] - if result_tree[pv][ps][cn][nvr]["upstreams"]: - upstream_component_names = sorted( - list( - set( + for provide_name in provides_component_names: + versions = set( [ - source["name"] - for source in result_tree[pv][ps][cn][nvr][ - "upstreams" + provide["version"] + for provide in result_tree[pv][ps][cn][nvr][ + "provides" ] + if provide["name"] == provide_name ] ) - ) - ) - if len(upstream_component_names) > 0: - upstream_component_name = ( - f"[cyan]{upstream_component_names[0]}[/cyan]" - ) - if len(upstream_component_names) > 1: - upstream_component_name = f"[cyan]{upstream_component_names[0]} and {len(upstream_component_names) - 1} more[/cyan]" # noqa - console.print( - Text(ps, style="magenta b u"), - upstream_component_name, - dep, - f"([grey]{related_url}[/grey])", - f"([grey]{build_source_url}[/grey])", - no_wrap=no_wrap, - ) - if result_tree[pv][ps][cn][nvr]["sources"]: - source_component_names = sorted( - list( - set( + types = set( + [ + provide["type"] + for provide in result_tree[pv][ps][cn][nvr][ + "provides" + ] + if provide["name"] == provide_name + ] + ) + arches = set( [ - source["name"] - for source in result_tree[pv][ps][cn][nvr][ - "sources" + provide["arch"] + for provide in result_tree[pv][ps][cn][nvr][ + "provides" ] + if provide["name"] == provide_name ] ) + provides_components.append( + f"{provide_name} {(','.join(versions))} ({(',').join(types)}:{','.join(arches)})" # noqa + ) + if result_tree[pv][ps][cn][nvr]["upstreams"]: + upstream_component_names = sorted( + list( + set( + [ + f"{upstream['nvr']} ({upstream['type']}:{upstream['arch']})" # noqa + for upstream in result_tree[pv][ps][cn][nvr][ + "upstreams" + ] + ] + ) + ) ) - ) - if len(source_component_names) > 0: - source_component_name = ( - f"[red]{source_component_names[0]}[/red]" + if len(upstream_component_names) > 0: + upstream_component_name = ( + f"[cyan]{upstream_component_names[0]}[/cyan]" + ) + if len(upstream_component_names) > 1: + upstream_component_name = f"[cyan]{upstream_component_names[0]} and {len(upstream_component_names) - 1} more[/cyan]" # noqa + console.print( + Text(ps, style=f"{product_color} b"), + upstream_component_name, + dep, + f"[grey]{related_url}[/grey]", + no_wrap=no_wrap, + ) + if result_tree[pv][ps][cn][nvr]["sources"]: + source_component_names = sorted( + list( + set( + [ + f"{source['nvr']} ({source['type']}:{source['arch']})" # noqa + for source in result_tree[pv][ps][cn][nvr][ + "sources" + ] + ] + ) + ) ) - if len(source_component_names) > 1: - source_component_name = f"[red]{source_component_names[0]} and {len(source_component_names) - 1} more[/red]" # noqa + if len(source_component_names) > 0: + source_component_name = ( + f"[red]{source_component_names[0]}[/red]" + ) + if len(source_component_names) > 1: + source_component_name = f"[red]{source_component_names[0]} and {len(source_component_names) - 1} more[/red]" # noqa + console.print( + Text(ps, style=f"{product_color} b"), + source_component_name, + dep, + f"[grey]{related_url}[/grey]", + no_wrap=no_wrap, + ) + if not (result_tree[pv][ps][cn][nvr]["upstreams"]) and not ( + result_tree[pv][ps][cn][nvr]["sources"] + ): + try: + child_dep_names = re.sub( + re.escape(search_component_name), + f"[b]{search_component_name}[/b]", + ", ".join(provides_components), + ) + except re.error: + pass + console.print( - Text(ps, style="magenta b u"), - source_component_name, + Text(ps, style=f"{product_color} b"), dep, - f"([grey]{related_url}[/grey])", - f"([grey]{build_source_url}[/grey])", + f"[{child_dep_names}]", + f"[i][grey]{related_url}[/grey][/i]", + f"[i][grey]{build_source_url}[/grey][/i]", + width=1000, no_wrap=no_wrap, ) - if not (result_tree[pv][ps][cn][nvr]["upstreams"]) and not ( - result_tree[pv][ps][cn][nvr]["sources"] - ): - console.print( - Text(ps, style="magenta b u"), - dep, - f"([grey]{related_url}[/grey])", - no_wrap=no_wrap, - ) if ( ctx.obj["VERBOSE"] == 3 - ): # product_stream X nvr (full source/upstreams) x related_url x build_source_url + ): # product_stream X root component nvr (type:arch) x child components [ nvr (type:arch)] x related_url x build_source_url # noqa for pv in result_tree.keys(): for ps in result_tree[pv].keys(): for cn in sorted(result_tree[pv][ps].keys()): # select the latest nvr (from sorted list) nvr = list(result_tree[pv][ps][cn].keys())[-1] - # highlight search term - dep_name = nvr - try: - dep_name = re.sub( - re.escape(search_component_name), - f"[b]{search_component_name}[/b]", - nvr, + if result_tree[pv][ps][cn][nvr]["type"] != "OCI" or ( + result_tree[pv][ps][cn][nvr]["type"] == "OCI" + and ctx.obj["INCLUDE_CONTAINER_ROOTS"] + ): + product_color = process_product_color( + result_tree[pv][ps][cn][nvr]["build_type"] ) - except re.error: - pass - dep = f"[grey93]{dep_name} ({result_tree[pv][ps][cn][nvr]['type']})[/grey93]" # noqa - related_url = result_tree[pv][ps][cn][nvr].get("related_url") - try: - if result_tree[pv][ps][cn][nvr]["related_url"]: - related_url = re.sub( + # highlight search term + dep_name = nvr + try: + dep_name = re.sub( re.escape(search_component_name), f"[b]{search_component_name}[/b]", - result_tree[pv][ps][cn][nvr]["related_url"], + dep_name, ) - except re.error: - pass - build_source_url = "" - if result_tree[pv][ps][cn][nvr]["build_source_url"]: - build_source_url = result_tree[pv][ps][cn][nvr]["build_source_url"] - if result_tree[pv][ps][cn][nvr]["upstreams"]: - upstream_component_names = sorted( - list( - set( - [ - source["name"] - for source in result_tree[pv][ps][cn][nvr][ - "upstreams" + except re.error: + pass + dep = f"[grey93]{dep_name} ({result_tree[pv][ps][cn][nvr]['type']}:{result_tree[pv][ps][cn][nvr]['arch']})[/grey93]" # noqa + related_url = result_tree[pv][ps][cn][nvr].get("related_url") + try: + if result_tree[pv][ps][cn][nvr]["related_url"]: + related_url = re.sub( + re.escape(search_component_name), + f"[b]{search_component_name}[/b]", + result_tree[pv][ps][cn][nvr]["related_url"], + ) + except re.error: + pass + build_source_url = "" + if result_tree[pv][ps][cn][nvr]["build_source_url"]: + build_source_url = result_tree[pv][ps][cn][nvr][ + "build_source_url" + ] + provides_components = [] + if "provides" in result_tree[pv][ps][cn][nvr]: + for provide in result_tree[pv][ps][cn][nvr]["provides"]: + provides_components.append( + f"{provide['nvr']} ({provide['type']}:{provide['arch']})" # noqa + ) + if result_tree[pv][ps][cn][nvr]["upstreams"]: + upstream_component_names = sorted( + list( + set( + [ + f"{upstream['nvr']} ({upstream['type']}:{upstream['arch']})" # noqa + for upstream in result_tree[pv][ps][cn][nvr][ + "upstreams" + ] ] - ] + ) ) ) - ) - for upstream_name in upstream_component_names: - console.print( - Text(ps, style="magenta b u"), - f"[cyan]{upstream_name}[/cyan]", - dep, - f"([grey]{related_url}[/grey])", - f"([grey]{build_source_url}[/grey])", - no_wrap=no_wrap, - ) - if result_tree[pv][ps][cn][nvr]["sources"]: - source_component_names = sorted( - list( - set( - [ - source["name"] - for source in result_tree[pv][ps][cn][nvr][ - "sources" + if len(upstream_component_names) > 0: + upstream_component_name = ( + f"[cyan]{upstream_component_names[0]}[/cyan]" + ) + if len(upstream_component_names) > 1: + upstream_component_name = f"[cyan]{upstream_component_names[0]} and {len(upstream_component_names) - 1} more[/cyan]" # noqa + console.print( + Text(ps, style=f"{product_color} b"), + upstream_component_name, + dep, + f"[grey]{related_url}[/grey]", + no_wrap=no_wrap, + ) + if result_tree[pv][ps][cn][nvr]["sources"]: + source_component_names = sorted( + list( + set( + [ + f"{source['nvr']} ({source['type']}:{source['arch']})" # noqa + for source in result_tree[pv][ps][cn][nvr][ + "sources" + ] ] - ] + ) ) ) - ) - for source_name in source_component_names: + if len(source_component_names) > 0: + source_component_name = ( + f"[red]{source_component_names[0]}[/red]" + ) + if len(source_component_names) > 1: + source_component_name = f"[red]{source_component_names[0]} and {len(source_component_names) - 1} more[/red]" # noqa + console.print( + Text(ps, style=f"{product_color} b"), + source_component_name, + dep, + f"[grey]{related_url}[/grey]", + no_wrap=no_wrap, + ) + if not (result_tree[pv][ps][cn][nvr]["upstreams"]) and not ( + result_tree[pv][ps][cn][nvr]["sources"] + ): + try: + child_dep_names = re.sub( + re.escape(search_component_name), + f"[b]{search_component_name}[/b]", + ", ".join(provides_components), + ) + except re.error: + pass + console.print( - Text(ps, style="magenta b u"), - f"[light_blue]{source_name}[/light_blue]", + Text(ps, style=f"{product_color} b"), dep, - f"([grey]{related_url}[/grey])", - f"([grey]{build_source_url}[/grey])", + f"[{child_dep_names}]", + f"[i][grey]{related_url}[/grey][/i]", + f"[i][grey]{build_source_url}[/grey][/i]", + width=1000, no_wrap=no_wrap, ) - if not (result_tree[pv][ps][cn][nvr]["upstreams"]) and not ( - result_tree[pv][ps][cn][nvr]["sources"] - ): - console.print( - Text(ps, style="magenta b u"), - dep, - f"([grey]{related_url}[/grey])", - f"([grey]{build_source_url}[/grey])", - no_wrap=no_wrap, - ) if ( ctx.obj["VERBOSE"] > 3 - ): # product_stream X nvr (full source/upstreams) x related_url x build_source_url + ): # product_stream X root component purl x child components [ purl ] x related_url x build_source_url # noqa for pv in result_tree.keys(): for ps in result_tree[pv].keys(): for cn in sorted(result_tree[pv][ps].keys()): # select the latest nvr (from sorted list) nvr = list(result_tree[pv][ps][cn].keys())[-1] - # highlight search term - dep_name = nvr - try: - dep_name = re.sub( - re.escape(search_component_name), - f"[b]{search_component_name}[/b]", - nvr, + if result_tree[pv][ps][cn][nvr]["type"] != "OCI" or ( + result_tree[pv][ps][cn][nvr]["type"] == "OCI" + and ctx.obj["INCLUDE_CONTAINER_ROOTS"] + ): + product_color = process_product_color( + result_tree[pv][ps][cn][nvr]["build_type"] ) - except re.error: - pass - dep = f"[grey93]{dep_name} ({result_tree[pv][ps][cn][nvr]['type']})[/grey93]" # noqa - related_url = result_tree[pv][ps][cn][nvr].get("related_url") - try: - if result_tree[pv][ps][cn][nvr]["related_url"]: - related_url = re.sub( + # highlight search term + dep_name = result_tree[pv][ps][cn][nvr]["purl"] + try: + dep_name = re.sub( re.escape(search_component_name), f"[b]{search_component_name}[/b]", - result_tree[pv][ps][cn][nvr]["related_url"], + dep_name, ) - except re.error: - pass - build_source_url = "" - if result_tree[pv][ps][cn][nvr]["build_source_url"]: - build_source_url = result_tree[pv][ps][cn][nvr]["build_source_url"] - if result_tree[pv][ps][cn][nvr]["upstreams"]: - upstream_component_names = sorted( - list( - set( - [ - source["nvr"] - for source in result_tree[pv][ps][cn][nvr][ - "upstreams" + except re.error: + pass + dep = f"[grey93]{dep_name}[/grey93]" # noqa + related_url = result_tree[pv][ps][cn][nvr].get("related_url") + try: + if result_tree[pv][ps][cn][nvr]["related_url"]: + related_url = re.sub( + re.escape(search_component_name), + f"[b]{search_component_name}[/b]", + result_tree[pv][ps][cn][nvr]["related_url"], + ) + except re.error: + pass + build_source_url = "" + if result_tree[pv][ps][cn][nvr]["build_source_url"]: + build_source_url = result_tree[pv][ps][cn][nvr][ + "build_source_url" + ] + provides_components = [] + if "provides" in result_tree[pv][ps][cn][nvr]: + for provide in result_tree[pv][ps][cn][nvr]["provides"]: + provides_components.append(f"{provide['purl']}") + if result_tree[pv][ps][cn][nvr]["upstreams"]: + upstream_component_names = sorted( + list( + set( + [ + upstream["purl"] + for upstream in result_tree[pv][ps][cn][nvr][ + "upstreams" + ] ] - ] + ) ) ) - ) - for upstream_name in upstream_component_names: - console.print( - Text(ps, style="magenta b u"), - f"[cyan]{upstream_name}[/cyan]", - dep, - f"([grey]{related_url}[/grey])", - f"([grey]{build_source_url}[/grey])", - no_wrap=no_wrap, - ) - if result_tree[pv][ps][cn][nvr]["sources"]: - source_component_names = sorted( - list( - set( - [ - source["nvr"] - for source in result_tree[pv][ps][cn][nvr][ - "sources" + if len(upstream_component_names) > 0: + upstream_component_name = ( + f"[cyan]{upstream_component_names[0]}[/cyan]" + ) + if len(upstream_component_names) > 1: + upstream_component_name = f"[cyan]{upstream_component_names[0]} and {len(upstream_component_names) - 1} more[/cyan]" # noqa + console.print( + Text(ps, style=f"{product_color} b u"), + upstream_component_name, + dep, + f"[grey]{related_url}[/grey]", + f"[grey]{build_source_url}[/grey]", + no_wrap=no_wrap, + ) + if result_tree[pv][ps][cn][nvr]["sources"]: + source_component_names = sorted( + list( + set( + [ + source["purl"] + for source in result_tree[pv][ps][cn][nvr][ + "sources" + ] ] - ] + ) ) ) - ) - for source_name in source_component_names: + if len(source_component_names) > 0: + source_component_name = ( + f"[red]{source_component_names[0]}[/red]" + ) + if len(source_component_names) > 1: + source_component_name = f"[red]{source_component_names[0]} and {len(source_component_names) - 1} more[/red]" # noqa + console.print( + Text(ps, style=f"{product_color} b u"), + source_component_name, + dep, + f"[grey]{related_url}[/grey]", + f"[grey]{build_source_url}[/grey]", + no_wrap=no_wrap, + ) + if not (result_tree[pv][ps][cn][nvr]["upstreams"]) and not ( + result_tree[pv][ps][cn][nvr]["sources"] + ): + try: + child_dep_names = re.sub( + re.escape(search_component_name), + f"[b]{search_component_name}[/b]", + ", ".join(provides_components), + ) + except re.error: + pass + console.print( - Text(ps, style="magenta b u"), - f"[light_blue]{source_name}[/light_blue]", + Text(ps, style=f"{product_color} b"), dep, - f"([grey]{related_url}[/grey])", - f"([grey]{build_source_url}[/grey])", + f"[{child_dep_names}]", + f"[i][grey]{related_url}[/grey][/i]", + f"[i][grey]{build_source_url}[/grey][/i]", + width=10000, no_wrap=no_wrap, ) - if not (result_tree[pv][ps][cn][nvr]["upstreams"]) and not ( - result_tree[pv][ps][cn][nvr]["sources"] - ): - console.print( - Text(ps, style="magenta b u"), - dep, - f"([grey]{related_url}[/grey])", - f"([grey]{build_source_url}[/grey])", - no_wrap=no_wrap, - ) ctx.exit() diff --git a/griffon/services/core_queries.py b/griffon/services/core_queries.py index 3d77460..06ae9ef 100644 --- a/griffon/services/core_queries.py +++ b/griffon/services/core_queries.py @@ -217,20 +217,13 @@ def async_retrieve_sources(self, purl): params = { "limit": 200, "root_components": "True", - "released_components": "True", "provides": purl, - "include_fields": "type,nvr,purl,name,namespace,download_url,related_url", + "include_fields": "type,nvr,purl,name,version,namespace,download_url,related_url", } - # TODO: remove cnt and max_result and cnt > 10000 on next stage-> prod deployment - cnt = self.components.count(**params) - if cnt == 0: - return [] try: - if cnt > 10000: - return list(self.components.retrieve_list_iterator_async(max_results=5000, **params)) - return list(self.components.retrieve_list_iterator_async(max_results=10000, **params)) + return list(self.components.retrieve_list_iterator_async(**params)) except Exception as e: - logger.warning(f"{type(e).__name__} - problem retrieving all of {purl} {cnt} sources.") + logger.warning(f"{type(e).__name__} - problem retrieving all of {purl} sources.") return [] @@ -239,27 +232,44 @@ def async_retrieve_upstreams(self, purl): "limit": 200, "root_components": "True", "upstreams": purl, - "include_fields": "type,nvr,purl,name,namespace,download_url,related_url", + "include_fields": "type,nvr,purl,name,version,namespace,download_url,related_url", } - # TODO: remove cnt and max_result and cnt > 10000 on next stage-> prod deployment - cnt = self.components.count(**params) - if cnt == 0: + try: + return list(self.components.retrieve_list_iterator_async(**params)) + except Exception as e: + logger.warning(f"{type(e).__name__} - problem retrieving all of {purl} upstreams.") return [] + + +def async_retrieve_provides(self, urlparams, purl): + params = { + "limit": 200, + "sources": purl, + "include_fields": "type,arch,nvr,purl,version,name,namespace,download_url,related_url", + } + if "name" in urlparams: + params["name"] = urlparams["name"] + if "provides_name" in urlparams: + params["name"] = urlparams["provides_name"] + if "re_name" in urlparams: + params["re_name"] = urlparams["re_name"] + if "re_provides_name" in urlparams: + params["re_name"] = urlparams["re_provides_name"] + if "namespace" in urlparams: + params["namespace"] = urlparams["namespace"] try: - if cnt > 10000: - return list(self.components.retrieve_list_iterator_async(max_results=5000, **params)) - return list(self.components.retrieve_list_iterator_async(max_results=10000, **params)) + return list(self.components.retrieve_list_iterator_async(**params)) except Exception as e: - logger.warning(f"{type(e).__name__} - problem retrieving all of {purl} {cnt} upstreams.") + logger.warning(f"{type(e).__name__} - problem retrieving all of {purl} provides.") return [] -def process_component(session, c): +def process_component(session, urlparams, c): """perform any neccessary sub retrievals.""" - # only process provided components - if c.software_build is None: + if c.sources: c.sources = async_retrieve_sources(session, c.purl) - c.upstreams = async_retrieve_upstreams(session, c.purl) + c.upstreams = async_retrieve_upstreams(session, c.purl) + c.provides = async_retrieve_provides(session, urlparams, c.purl) return c @@ -350,8 +360,12 @@ def execute(self, status=None) -> List[Dict[str, Any]]: **search_latest_params ) status.update(f"found {latest_components_cnt} latest root component(s).") # noqa - for c in latest_components: - results.append(c) + with multiprocessing.Pool() as pool: + for processed_component in pool.map( + partial(process_component, self.corgi_session, search_latest_params), + latest_components, + ): + results.append(processed_component) if not self.no_community: status.update("searching latest community root component(s).") @@ -367,10 +381,14 @@ def execute(self, status=None) -> List[Dict[str, Any]]: ) ) status.update( - f"found {community_component_cnt} latest community root component(s), retrieving sources & upstreams." # noqa + f"found {community_component_cnt} latest community root component(s)- retrieving children, sources & upstreams." # noqa ) - for c in latest_community_components: - results.append(c) + with multiprocessing.Pool() as pool: + for processed_component in pool.map( + partial(process_component, self.corgi_session, search_latest_params), + latest_community_components, + ): + results.append(processed_component) if self.search_provides: search_provides_params = copy.deepcopy(params) @@ -390,10 +408,14 @@ def execute(self, status=None) -> List[Dict[str, Any]]: **search_provides_params ) status.update( - f"found {latest_components_cnt} latest provides child component(s), retrieving sources & upstreams." # noqa + f"found {latest_components_cnt} latest provides child component(s)- retrieving children, sources & upstreams." # noqa ) - for c in latest_components: - results.append(c) + with multiprocessing.Pool() as pool: + for processed_component in pool.map( + partial(process_component, self.corgi_session, search_provides_params), + latest_components, + ): + results.append(processed_component) if not self.no_community: status.update("searching latest community provided child component(s).") @@ -409,10 +431,14 @@ def execute(self, status=None) -> List[Dict[str, Any]]: ) ) status.update( - f"found {community_component_cnt} latest community provided child component(s), retrieving sources & upstreams." # noqa + f"found {community_component_cnt} latest community provided child component(s)- retrieving children, sources & upstreams." # noqa ) - for c in latest_community_components: - results.append(c) + with multiprocessing.Pool() as pool: + for processed_component in pool.map( + partial(process_component, self.corgi_session, search_provides_params), + latest_community_components, + ): + results.append(processed_component) if self.search_upstreams: search_upstreams_params = copy.deepcopy(params) @@ -435,10 +461,11 @@ def execute(self, status=None) -> List[Dict[str, Any]]: ) with multiprocessing.Pool() as pool: status.update( - f"found {latest_components_cnt} latest upstreams child component(s), retrieving sources & upstreams." # noqa + f"found {latest_components_cnt} latest upstreams child component(s)- retrieving children, sources & upstreams." # noqa ) for processed_component in pool.map( - partial(process_component, self.corgi_session), latest_components + partial(process_component, self.corgi_session, search_upstreams_params), + latest_components, ): results.append(processed_component) if not self.no_community: @@ -456,10 +483,10 @@ def execute(self, status=None) -> List[Dict[str, Any]]: ) with multiprocessing.Pool() as pool: status.update( - f"found {community_component_cnt} latest community provided child component(s), retrieving sources & upstreams." # noqa + f"found {community_component_cnt} latest community provided child component(s)- retrieving children, sources & upstreams." # noqa ) for processed_component in pool.map( - partial(process_component, self.community_session), + partial(process_component, self.community_session, search_upstreams_params), latest_community_components, ): results.append(processed_component) @@ -484,10 +511,11 @@ def execute(self, status=None) -> List[Dict[str, Any]]: ) with multiprocessing.Pool() as pool: status.update( - f"found {related_url_components_cnt} related url component(s), retrieving sources & upstreams." # noqa + f"found {related_url_components_cnt} related url component(s)- retrieving children, sources & upstreams." # noqa ) for processed_component in pool.map( - partial(process_component, self.corgi_session), related_url_components + partial(process_component, self.corgi_session, search_related_url_params), + related_url_components, ): results.append(processed_component) if not self.no_community: @@ -504,10 +532,12 @@ def execute(self, status=None) -> List[Dict[str, Any]]: ) with multiprocessing.Pool() as pool: status.update( - f"found {latest_community_url_components_cnt} related url community component(s), retrieving sources & upstreams." # noqa + f"found {latest_community_url_components_cnt} related url community component(s)- retrieving children, sources & upstreams." # noqa ) for processed_component in pool.map( - partial(process_component, self.community_session), + partial( + process_component, self.community_session, search_related_url_params + ), latest_community_url_components, ): results.append(processed_component) @@ -524,7 +554,7 @@ def execute(self, status=None) -> List[Dict[str, Any]]: search_all_params["namespace"] = self.ns if not (self.include_inactive_product_streams): search_all_params["active_streams"] = "True" - search_all_params["released_components"] = "True" + # search_all_params["released_components"] = "True" all_components_cnt = self.corgi_session.components.count(**search_all_params) status.update(f"found {all_components_cnt} all component(s).") # TODO: remove max_results @@ -534,10 +564,11 @@ def execute(self, status=None) -> List[Dict[str, Any]]: status.update(f"found {all_components_cnt} all component(s).") with multiprocessing.Pool() as pool: status.update( - f"found {all_components_cnt} all component(s), retrieving sources & upstreams." # noqa + f"found {all_components_cnt} all component(s)- retrieving children, sources & upstreams." # noqa ) for processed_component in pool.map( - partial(process_component, self.corgi_session), all_components + partial(process_component, self.corgi_session, search_all_params), + all_components, ): results.append(processed_component) @@ -556,10 +587,10 @@ def execute(self, status=None) -> List[Dict[str, Any]]: ) with multiprocessing.Pool() as pool: status.update( - f"found {all_community_components_cnt} community all component(s), retrieving sources & upstreams." # noqa + f"found {all_community_components_cnt} community all component(s)- retrieving children, sources & upstreams." # noqa ) for processed_component in pool.map( - partial(process_component, self.community_session), + partial(process_component, self.community_session, search_all_params), all_community_components, ): results.append(processed_component) @@ -575,7 +606,7 @@ def execute(self, status=None) -> List[Dict[str, Any]]: search_all_roots_params["namespace"] = self.ns if not (self.include_inactive_product_streams): search_all_roots_params["active_streams"] = "True" - search_all_roots_params["released_components"] = "True" + # search_all_roots_params["released_components"] = "True" all_src_components_cnt = self.corgi_session.components.count(**search_all_roots_params) status.update(f"found {all_src_components_cnt} all root component(s).") all_src_components = self.corgi_session.components.retrieve_list_iterator_async( @@ -619,7 +650,8 @@ def execute(self, status=None) -> List[Dict[str, Any]]: with multiprocessing.Pool() as pool: status.update(f"found {upstream_components_cnt} upstream component(s).") for processed_component in pool.map( - partial(process_component, self.corgi_session), upstream_components + partial(process_component, self.corgi_session, search_all_upstreams_params), + upstream_components, ): results.append(processed_component) if not self.no_community: @@ -636,10 +668,12 @@ def execute(self, status=None) -> List[Dict[str, Any]]: ) with multiprocessing.Pool() as pool: status.update( - f"found {commmunity_upstream_components_cnt} community upstream component(s), retrieving sources & upstreams." # noqa + f"found {commmunity_upstream_components_cnt} community upstream component(s)- retrieving children, sources & upstreams." # noqa ) for processed_component in pool.map( - partial(process_component, self.community_session), + partial( + process_component, self.community_session, search_all_upstreams_params + ), commmunity_upstream_components, ): results.append(processed_component) @@ -706,10 +740,11 @@ def execute(self, status=None) -> List[Dict[str, Any]]: ) with multiprocessing.Pool() as pool: status.update( - f"found {all_community_components_cnt} community all component(s), retrieving sources & upstreams." # noqa + f"found {all_community_components_cnt} community all component(s)- retrieving children, sources & upstreams." # noqa ) for processed_component in pool.map( - partial(process_component, self.community_session), all_community_components + partial(process_component, self.community_session, search_community_params), + all_community_components, ): results.append(processed_component) diff --git a/griffon/static/default_griffonrc b/griffon/static/default_griffonrc index 8fa3893..4134267 100644 --- a/griffon/static/default_griffonrc +++ b/griffon/static/default_griffonrc @@ -20,6 +20,7 @@ exclude_components = -container-source -javadoc -testlib -repolib +include_container_roots = False # profile sections (use with --profile {profile} flag) [cloud] From 51f81fe2206e4771cab1b5655d0b8326733597a8 Mon Sep 17 00:00:00 2001 From: jfuller Date: Wed, 17 Jan 2024 19:04:52 +0100 Subject: [PATCH 2/8] no need to check for unique list --- griffon/output.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/griffon/output.py b/griffon/output.py index ddb9d8b..314dd5b 100644 --- a/griffon/output.py +++ b/griffon/output.py @@ -206,14 +206,7 @@ def generate_normalised_results( ): normalised_results = list() if "results" in output: - # ensure unique result set - seen = set() - results = [] - for obj in output["results"]: - if obj["purl"] not in seen: - seen.add(obj["purl"]) - results.append(obj) - for item in results: + for item in output["results"]: for ps in item["product_streams"]: # only include component from active product stream if ps.get("active") or include_inactive_product_streams: From 51f26f1062789b876cfd5db2575eaa645e72abfb Mon Sep 17 00:00:00 2001 From: jfuller Date: Wed, 17 Jan 2024 19:07:09 +0100 Subject: [PATCH 3/8] revert back max limits --- griffon/services/core_queries.py | 89 +++++++++++--------------------- 1 file changed, 30 insertions(+), 59 deletions(-) diff --git a/griffon/services/core_queries.py b/griffon/services/core_queries.py index 06ae9ef..163ce2f 100644 --- a/griffon/services/core_queries.py +++ b/griffon/services/core_queries.py @@ -221,9 +221,9 @@ def async_retrieve_sources(self, purl): "include_fields": "type,nvr,purl,name,version,namespace,download_url,related_url", } try: - return list(self.components.retrieve_list_iterator_async(**params)) + return list(self.components.retrieve_list_iterator_async(**params, max_results=5000)) except Exception as e: - logger.warning(f"{type(e).__name__} - problem retrieving all of {purl} sources.") + logger.warning(f"{type(e).__name__} - problem retrieving {purl} sources.") return [] @@ -235,9 +235,9 @@ def async_retrieve_upstreams(self, purl): "include_fields": "type,nvr,purl,name,version,namespace,download_url,related_url", } try: - return list(self.components.retrieve_list_iterator_async(**params)) + return list(self.components.retrieve_list_iterator_async(**params, max_results=5000)) except Exception as e: - logger.warning(f"{type(e).__name__} - problem retrieving all of {purl} upstreams.") + logger.warning(f"{type(e).__name__} - problem retrieving {purl} upstreams.") return [] @@ -258,9 +258,9 @@ def async_retrieve_provides(self, urlparams, purl): if "namespace" in urlparams: params["namespace"] = urlparams["namespace"] try: - return list(self.components.retrieve_list_iterator_async(**params)) + return list(self.components.retrieve_list_iterator_async(**params, max_results=5000)) except Exception as e: - logger.warning(f"{type(e).__name__} - problem retrieving all of {purl} provides.") + logger.warning(f"{type(e).__name__} - problem retrieving {purl} provides.") return [] @@ -333,7 +333,7 @@ def execute(self, status=None) -> List[Dict[str, Any]]: status.update("searching component-registry.") results = [] params = { - "limit": 50, + "limit": 120, "include_fields": "purl,type,name,related_url,namespace,software_build,nvr,release,version,arch,product_streams.product_versions,product_streams.name,product_streams.ofuri,product_streams.active,product_streams.exclude_components", # noqa } @@ -357,7 +357,7 @@ def execute(self, status=None) -> List[Dict[str, Any]]: latest_components_cnt = self.corgi_session.components.count(**search_latest_params) status.update(f"found {latest_components_cnt} latest component(s).") latest_components = self.corgi_session.components.retrieve_list_iterator_async( - **search_latest_params + **search_latest_params, max_results=10000 ) status.update(f"found {latest_components_cnt} latest root component(s).") # noqa with multiprocessing.Pool() as pool: @@ -377,7 +377,7 @@ def execute(self, status=None) -> List[Dict[str, Any]]: ) latest_community_components = ( self.community_session.components.retrieve_list_iterator_async( - **search_latest_params + **search_latest_params, max_results=10000 ) ) status.update( @@ -405,7 +405,7 @@ def execute(self, status=None) -> List[Dict[str, Any]]: latest_components_cnt = self.corgi_session.components.count(**search_provides_params) status.update(f"found {latest_components_cnt} latest provides component(s).") latest_components = self.corgi_session.components.retrieve_list_iterator_async( - **search_provides_params + **search_provides_params, max_results=10000 ) status.update( f"found {latest_components_cnt} latest provides child component(s)- retrieving children, sources & upstreams." # noqa @@ -427,7 +427,7 @@ def execute(self, status=None) -> List[Dict[str, Any]]: ) latest_community_components = ( self.community_session.components.retrieve_list_iterator_async( - **search_provides_params + **search_provides_params, max_results=10000 ) ) status.update( @@ -457,7 +457,7 @@ def execute(self, status=None) -> List[Dict[str, Any]]: latest_components_cnt = self.corgi_session.components.count(**search_upstreams_params) status.update(f"found {latest_components_cnt} latest component(s).") latest_components = self.corgi_session.components.retrieve_list_iterator_async( - **search_upstreams_params + **search_upstreams_params, max_results=10000 ) with multiprocessing.Pool() as pool: status.update( @@ -478,7 +478,7 @@ def execute(self, status=None) -> List[Dict[str, Any]]: ) latest_community_components = ( self.community_session.components.retrieve_list_iterator_async( - **search_upstreams_params + **search_upstreams_params, max_results=10000 ) ) with multiprocessing.Pool() as pool: @@ -503,21 +503,15 @@ def execute(self, status=None) -> List[Dict[str, Any]]: search_related_url_params["active_streams"] = "True" search_related_url_params["released_components"] = "True" related_url_components_cnt = self.corgi_session.components.count( - **search_related_url_params + **search_related_url_params, max_results=10000 ) status.update(f"found {related_url_components_cnt} related url component(s).") related_url_components = self.corgi_session.components.retrieve_list_iterator_async( **search_related_url_params ) - with multiprocessing.Pool() as pool: - status.update( - f"found {related_url_components_cnt} related url component(s)- retrieving children, sources & upstreams." # noqa - ) - for processed_component in pool.map( - partial(process_component, self.corgi_session, search_related_url_params), - related_url_components, - ): - results.append(processed_component) + for c in related_url_components: + results.append(c) + if not self.no_community: latest_community_url_components_cnt = self.community_session.components.count( **search_related_url_params @@ -527,20 +521,11 @@ def execute(self, status=None) -> List[Dict[str, Any]]: ) latest_community_url_components = ( self.community_session.components.retrieve_list_iterator_async( - **search_related_url_params + **search_related_url_params, max_results=10000 ) ) - with multiprocessing.Pool() as pool: - status.update( - f"found {latest_community_url_components_cnt} related url community component(s)- retrieving children, sources & upstreams." # noqa - ) - for processed_component in pool.map( - partial( - process_component, self.community_session, search_related_url_params - ), - latest_community_url_components, - ): - results.append(processed_component) + for c in latest_community_url_components: + results.append(c) if self.search_all: search_all_params = copy.deepcopy(params) @@ -554,7 +539,7 @@ def execute(self, status=None) -> List[Dict[str, Any]]: search_all_params["namespace"] = self.ns if not (self.include_inactive_product_streams): search_all_params["active_streams"] = "True" - # search_all_params["released_components"] = "True" + search_all_params["released_components"] = "True" all_components_cnt = self.corgi_session.components.count(**search_all_params) status.update(f"found {all_components_cnt} all component(s).") # TODO: remove max_results @@ -562,15 +547,8 @@ def execute(self, status=None) -> List[Dict[str, Any]]: **search_all_params, max_results=10000 ) status.update(f"found {all_components_cnt} all component(s).") - with multiprocessing.Pool() as pool: - status.update( - f"found {all_components_cnt} all component(s)- retrieving children, sources & upstreams." # noqa - ) - for processed_component in pool.map( - partial(process_component, self.corgi_session, search_all_params), - all_components, - ): - results.append(processed_component) + for c in all_components: + results.append(c) if not self.no_community: all_community_components_cnt = self.community_session.components.count( @@ -585,15 +563,8 @@ def execute(self, status=None) -> List[Dict[str, Any]]: **search_all_params, max_results=10000 ) ) - with multiprocessing.Pool() as pool: - status.update( - f"found {all_community_components_cnt} community all component(s)- retrieving children, sources & upstreams." # noqa - ) - for processed_component in pool.map( - partial(process_component, self.community_session, search_all_params), - all_community_components, - ): - results.append(processed_component) + for c in all_community_components: + results.append(c) if self.search_all_roots: search_all_roots_params = copy.deepcopy(params) @@ -606,11 +577,11 @@ def execute(self, status=None) -> List[Dict[str, Any]]: search_all_roots_params["namespace"] = self.ns if not (self.include_inactive_product_streams): search_all_roots_params["active_streams"] = "True" - # search_all_roots_params["released_components"] = "True" + search_all_roots_params["released_components"] = "True" all_src_components_cnt = self.corgi_session.components.count(**search_all_roots_params) status.update(f"found {all_src_components_cnt} all root component(s).") all_src_components = self.corgi_session.components.retrieve_list_iterator_async( - **search_all_roots_params + **search_all_roots_params, max_results=10000 ) for c in all_src_components: results.append(c) @@ -620,7 +591,7 @@ def execute(self, status=None) -> List[Dict[str, Any]]: ) all_src_community_components = ( self.community_session.components.retrieve_list_iterator_async( - **search_all_roots_params + **search_all_roots_params, max_results=10000 ) ) status.update( @@ -645,7 +616,7 @@ def execute(self, status=None) -> List[Dict[str, Any]]: ) status.update(f"found {upstream_components_cnt} upstream component(s).") upstream_components = self.corgi_session.components.retrieve_list_iterator_async( - **search_all_upstreams_params + **search_all_upstreams_params, max_results=10000 ) with multiprocessing.Pool() as pool: status.update(f"found {upstream_components_cnt} upstream component(s).") @@ -663,7 +634,7 @@ def execute(self, status=None) -> List[Dict[str, Any]]: ) commmunity_upstream_components = ( self.community_session.components.retrieve_list_iterator_async( - **search_all_upstreams_params + **search_all_upstreams_params, max_results=10000 ) ) with multiprocessing.Pool() as pool: From de8a702f76fb08de61de859f37a40ce91bdf0f79 Mon Sep 17 00:00:00 2001 From: jfuller Date: Wed, 17 Jan 2024 19:33:52 +0100 Subject: [PATCH 4/8] minor --- griffon/output.py | 85 ++++++++++++++++++++++++++--------------------- 1 file changed, 48 insertions(+), 37 deletions(-) diff --git a/griffon/output.py b/griffon/output.py index 314dd5b..76da99f 100644 --- a/griffon/output.py +++ b/griffon/output.py @@ -630,19 +630,21 @@ def text_output_products_contain_component( if not (result_tree[pv][ps][cn][nvr]["upstreams"]) and not ( result_tree[pv][ps][cn][nvr]["sources"] ): - try: - child_dep_names = re.sub( - re.escape(search_component_name), - f"[b]{search_component_name}[/b]", - ", ".join(provides_components), - ) - except re.error: - pass - + child_dep_names = "" + if provides_components: + try: + child_dep_names = re.sub( + re.escape(search_component_name), + f"[b]{search_component_name}[/b]", + ", ".join(provides_components), + ) + child_dep_names = f"[{child_dep_names}]" + except re.error: + pass console.print( Text(ps, style=f"{product_color} b"), dep, - f"[{child_dep_names}]", + child_dep_names, no_wrap=no_wrap, ) if ( @@ -780,19 +782,22 @@ def text_output_products_contain_component( if not (result_tree[pv][ps][cn][nvr]["upstreams"]) and not ( result_tree[pv][ps][cn][nvr]["sources"] ): - try: - child_dep_names = re.sub( - re.escape(search_component_name), - f"[b]{search_component_name}[/b]", - ", ".join(provides_components), - ) - except re.error: - pass + child_dep_names = "" + if provides_components: + try: + child_dep_names = re.sub( + re.escape(search_component_name), + f"[b]{search_component_name}[/b]", + ", ".join(provides_components), + ) + child_dep_names = f"[{child_dep_names}]" + except re.error: + pass console.print( Text(ps, style=f"{product_color} b"), dep, - f"[{child_dep_names}]", + child_dep_names, f"[i][grey]{related_url}[/grey][/i]", f"[i][grey]{build_source_url}[/grey][/i]", width=1000, @@ -900,19 +905,22 @@ def text_output_products_contain_component( if not (result_tree[pv][ps][cn][nvr]["upstreams"]) and not ( result_tree[pv][ps][cn][nvr]["sources"] ): - try: - child_dep_names = re.sub( - re.escape(search_component_name), - f"[b]{search_component_name}[/b]", - ", ".join(provides_components), - ) - except re.error: - pass + child_dep_names = "" + if provides_components: + try: + child_dep_names = re.sub( + re.escape(search_component_name), + f"[b]{search_component_name}[/b]", + ", ".join(provides_components), + ) + child_dep_names = f"[{child_dep_names}]" + except re.error: + pass console.print( Text(ps, style=f"{product_color} b"), dep, - f"[{child_dep_names}]", + child_dep_names, f"[i][grey]{related_url}[/grey][/i]", f"[i][grey]{build_source_url}[/grey][/i]", width=1000, @@ -1020,19 +1028,22 @@ def text_output_products_contain_component( if not (result_tree[pv][ps][cn][nvr]["upstreams"]) and not ( result_tree[pv][ps][cn][nvr]["sources"] ): - try: - child_dep_names = re.sub( - re.escape(search_component_name), - f"[b]{search_component_name}[/b]", - ", ".join(provides_components), - ) - except re.error: - pass + child_dep_names = "" + if provides_components: + try: + child_dep_names = re.sub( + re.escape(search_component_name), + f"[b]{search_component_name}[/b]", + ", ".join(provides_components), + ) + child_dep_names = f"[{child_dep_names}]" + except re.error: + pass console.print( Text(ps, style=f"{product_color} b"), dep, - f"[{child_dep_names}]", + child_dep_names, f"[i][grey]{related_url}[/grey][/i]", f"[i][grey]{build_source_url}[/grey][/i]", width=10000, From b5c93b8f6b971f184ce020dfbe2ba69758f06aa3 Mon Sep 17 00:00:00 2001 From: jfuller Date: Thu, 18 Jan 2024 09:22:16 +0100 Subject: [PATCH 5/8] add --exclude-unreleased on service products-contains-component --- CHANGELOG.md | 1 + griffon/commands/queries.py | 8 ++++++++ griffon/services/core_queries.py | 23 ++++++++++++++++++++--- griffon/static/default_griffonrc | 1 + 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57d4710..3bef181 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added * --include-container_roots in service products-contain-component +* --exclude-unreleased in service products-contain-component ## [0.4.2] - 2024-01-14 ### Added diff --git a/griffon/commands/queries.py b/griffon/commands/queries.py index dc81686..9bddf77 100644 --- a/griffon/commands/queries.py +++ b/griffon/commands/queries.py @@ -370,6 +370,13 @@ def retrieve_component_summary(ctx, component_name, strict_name_search, operatio default=get_config_option("default", "include_container_roots", False), help="Include OCI root components in output.", ) +@click.option( + "--exclude-unreleased", + "exclude_unreleased", + is_flag=True, + default=get_config_option("default", "exclude_unreleased", False), + help="Exclude unreleased components.", +) @click.pass_context @progress_bar(is_updatable=True) def get_product_contain_component( @@ -403,6 +410,7 @@ def get_product_contain_component( operation_status, regex_name_search, include_container_roots, + exclude_unreleased, ): # with console_status(ctx) as operation_status: """List products of a latest component.""" diff --git a/griffon/services/core_queries.py b/griffon/services/core_queries.py index 163ce2f..077059d 100644 --- a/griffon/services/core_queries.py +++ b/griffon/services/core_queries.py @@ -199,6 +199,7 @@ class products_containing_specific_component_query: "include_product_stream_excluded_components", "output_type_filter", "regex_name_search", + "exclude_unreleased", ] def __init__(self, params: dict) -> None: @@ -303,6 +304,7 @@ class products_containing_component_query: "include_product_stream_excluded_components", "output_type_filter", "regex_name_search", + "exclude_unreleased", ] def __init__(self, params: dict) -> None: @@ -328,6 +330,7 @@ def __init__(self, params: dict) -> None: if not self.no_community: self.community_session = CommunityComponentService.create_session() self.include_inactive_product_streams = self.params.get("include_inactive_product_streams") + self.exclude_unreleased = self.params.get("exclude_unreleased") def execute(self, status=None) -> List[Dict[str, Any]]: status.update("searching component-registry.") @@ -351,6 +354,8 @@ def execute(self, status=None) -> List[Dict[str, Any]]: search_latest_params["namespace"] = self.ns if not (self.include_inactive_product_streams): search_latest_params["active_streams"] = "True" + if self.exclude_unreleased: + search_latest_params["released_components"] = "True" search_latest_params["root_components"] = "True" search_latest_params["latest_components_by_streams"] = "True" status.update("searching latest root component(s).") @@ -400,6 +405,9 @@ def execute(self, status=None) -> List[Dict[str, Any]]: search_provides_params["namespace"] = self.ns if not (self.include_inactive_product_streams): search_provides_params["active_streams"] = "True" + if self.exclude_unreleased: + search_provides_params["released_components"] = "True" + search_provides_params["root_components"] = "True" search_provides_params["latest_components_by_streams"] = "True" status.update("searching latest provided child component(s).") latest_components_cnt = self.corgi_session.components.count(**search_provides_params) @@ -451,7 +459,8 @@ def execute(self, status=None) -> List[Dict[str, Any]]: search_upstreams_params["namespace"] = self.ns if not (self.include_inactive_product_streams): search_upstreams_params["active_streams"] = "True" - search_upstreams_params["released_components"] = "True" + if self.exclude_unreleased: + search_upstreams_params["released_components"] = "True" search_upstreams_params["latest_components_by_streams"] = "True" status.update("searching latest upstreams child component(s).") latest_components_cnt = self.corgi_session.components.count(**search_upstreams_params) @@ -501,6 +510,8 @@ def execute(self, status=None) -> List[Dict[str, Any]]: search_related_url_params["type"] = self.component_type if not (self.include_inactive_product_streams): search_related_url_params["active_streams"] = "True" + if self.exclude_unreleased: + search_related_url_params["released_components"] = "True" search_related_url_params["released_components"] = "True" related_url_components_cnt = self.corgi_session.components.count( **search_related_url_params, max_results=10000 @@ -539,7 +550,8 @@ def execute(self, status=None) -> List[Dict[str, Any]]: search_all_params["namespace"] = self.ns if not (self.include_inactive_product_streams): search_all_params["active_streams"] = "True" - search_all_params["released_components"] = "True" + if self.exclude_unreleased: + search_all_params["released_components"] = "True" all_components_cnt = self.corgi_session.components.count(**search_all_params) status.update(f"found {all_components_cnt} all component(s).") # TODO: remove max_results @@ -577,7 +589,8 @@ def execute(self, status=None) -> List[Dict[str, Any]]: search_all_roots_params["namespace"] = self.ns if not (self.include_inactive_product_streams): search_all_roots_params["active_streams"] = "True" - search_all_roots_params["released_components"] = "True" + if self.exclude_unreleased: + search_all_roots_params["released_components"] = "True" all_src_components_cnt = self.corgi_session.components.count(**search_all_roots_params) status.update(f"found {all_src_components_cnt} all root component(s).") all_src_components = self.corgi_session.components.retrieve_list_iterator_async( @@ -611,6 +624,8 @@ def execute(self, status=None) -> List[Dict[str, Any]]: search_all_upstreams_params["type"] = self.component_type if not (self.include_inactive_product_streams): search_all_upstreams_params["active_streams"] = "True" + if self.exclude_unreleased: + search_all_upstreams_params["released_components"] = "True" upstream_components_cnt = self.corgi_session.components.count( **search_all_upstreams_params ) @@ -698,6 +713,8 @@ def execute(self, status=None) -> List[Dict[str, Any]]: search_community_params["namespace"] = self.ns if not (self.include_inactive_product_streams): search_community_params["active_streams"] = "True" + if self.exclude_unreleased: + search_community_params["released_components"] = "True" all_community_components_cnt = self.community_session.components.count( **search_community_params ) diff --git a/griffon/static/default_griffonrc b/griffon/static/default_griffonrc index 4134267..45745e9 100644 --- a/griffon/static/default_griffonrc +++ b/griffon/static/default_griffonrc @@ -21,6 +21,7 @@ exclude_components = -container-source -testlib -repolib include_container_roots = False +exclude_unreleased = False # profile sections (use with --profile {profile} flag) [cloud] From b61ed8a2ef0f9d662fc71171abc601fa21c1f1d2 Mon Sep 17 00:00:00 2001 From: jfuller Date: Thu, 18 Jan 2024 09:48:23 +0100 Subject: [PATCH 6/8] ensure output set is unique --- griffon/output.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/griffon/output.py b/griffon/output.py index 76da99f..26fca8b 100644 --- a/griffon/output.py +++ b/griffon/output.py @@ -206,6 +206,18 @@ def generate_normalised_results( ): normalised_results = list() if "results" in output: + # ensure unique result set + seen = set() + results = [] + for obj in output["results"]: + if "purl" in obj: + if obj["purl"] not in seen: + seen.add(obj["purl"]) + results.append(obj) + else: + if obj["nvr"] not in seen: + seen.add(obj["nvr"]) + results.append(obj) for item in output["results"]: for ps in item["product_streams"]: # only include component from active product stream From 1dfbd03f82447b3e3930709a4666973bed85f77e Mon Sep 17 00:00:00 2001 From: jfuller Date: Thu, 18 Jan 2024 10:44:40 +0100 Subject: [PATCH 7/8] minor perf --- griffon/services/core_queries.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/griffon/services/core_queries.py b/griffon/services/core_queries.py index 077059d..693851e 100644 --- a/griffon/services/core_queries.py +++ b/griffon/services/core_queries.py @@ -216,7 +216,7 @@ def execute(self, status=None) -> dict: def async_retrieve_sources(self, purl): params = { - "limit": 200, + "limit": 120, "root_components": "True", "provides": purl, "include_fields": "type,nvr,purl,name,version,namespace,download_url,related_url", @@ -230,7 +230,7 @@ def async_retrieve_sources(self, purl): def async_retrieve_upstreams(self, purl): params = { - "limit": 200, + "limit": 120, "root_components": "True", "upstreams": purl, "include_fields": "type,nvr,purl,name,version,namespace,download_url,related_url", @@ -244,7 +244,7 @@ def async_retrieve_upstreams(self, purl): def async_retrieve_provides(self, urlparams, purl): params = { - "limit": 200, + "limit": 120, "sources": purl, "include_fields": "type,arch,nvr,purl,version,name,namespace,download_url,related_url", } @@ -336,7 +336,7 @@ def execute(self, status=None) -> List[Dict[str, Any]]: status.update("searching component-registry.") results = [] params = { - "limit": 120, + "limit": 50, "include_fields": "purl,type,name,related_url,namespace,software_build,nvr,release,version,arch,product_streams.product_versions,product_streams.name,product_streams.ofuri,product_streams.active,product_streams.exclude_components", # noqa } @@ -407,7 +407,6 @@ def execute(self, status=None) -> List[Dict[str, Any]]: search_provides_params["active_streams"] = "True" if self.exclude_unreleased: search_provides_params["released_components"] = "True" - search_provides_params["root_components"] = "True" search_provides_params["latest_components_by_streams"] = "True" status.update("searching latest provided child component(s).") latest_components_cnt = self.corgi_session.components.count(**search_provides_params) From 944713e39167d7ebe05687c97ad7e03359399912 Mon Sep 17 00:00:00 2001 From: jfuller Date: Thu, 18 Jan 2024 11:56:23 +0100 Subject: [PATCH 8/8] -r search should highlight matched term --- CHANGELOG.md | 1 + griffon/commands/queries.py | 1 + griffon/output.py | 80 ++++++++++++++++++------------------- scripts/smoke-tests.sh | 1 + 4 files changed, 41 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bef181..e2c06bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed * refactor output format of service products-contain-component (GRIF-209) * container roots are excluded by default (need to use --include-container-roots) +* when using -r highlight matched term ### Added * --include-container_roots in service products-contain-component diff --git a/griffon/commands/queries.py b/griffon/commands/queries.py index 9bddf77..b9decf8 100644 --- a/griffon/commands/queries.py +++ b/griffon/commands/queries.py @@ -417,6 +417,7 @@ def get_product_contain_component( if verbose: ctx.obj["VERBOSE"] = verbose ctx.obj["INCLUDE_CONTAINER_ROOTS"] = include_container_roots + ctx.obj["REGEX_NAME_SEARCH"] = regex_name_search if ( not search_latest and not search_all diff --git a/griffon/output.py b/griffon/output.py index 26fca8b..bbc631b 100644 --- a/griffon/output.py +++ b/griffon/output.py @@ -403,6 +403,10 @@ def process_product_color(build_type: str) -> str: return "magenta" +def highlight_search_term(search_pattern, text_value): + return re.sub(search_pattern, "[b]\\g<0>[/b]", text_value) + + def text_output_products_contain_component( ctx, output, @@ -410,7 +414,13 @@ def text_output_products_contain_component( exclude_components, no_wrap=False, ): - search_component_name = ctx.params["component_name"] + logger.info(ctx.obj["REGEX_NAME_SEARCH"]) + # if -r option used we need to escape it + search_component_name = ( + re.escape(ctx.params["component_name"]) + if not ctx.obj["REGEX_NAME_SEARCH"] + else ctx.params["component_name"] + ) # handle single component if ctx.params["purl"]: @@ -470,10 +480,8 @@ def text_output_products_contain_component( dep_name = nvr # highlight search term try: - dep_name = re.sub( - re.escape(search_component_name), - f"[b]{search_component_name}[/b]", - dep_name, + dep_name = highlight_search_term( + search_component_name, dep_name ) except re.error: pass @@ -541,10 +549,8 @@ def text_output_products_contain_component( # dep_name = nvr dep_name = nvr try: - dep_name = re.sub( - re.escape(search_component_name), - f"[b]{search_component_name}[/b]", - dep_name, + dep_name = highlight_search_term( + search_component_name, dep_name ) except re.error: pass @@ -552,9 +558,8 @@ def text_output_products_contain_component( related_url = result_tree[pv][ps][cn][nvr].get("related_url") try: if result_tree[pv][ps][cn][nvr]["related_url"]: - related_url = re.sub( - re.escape(search_component_name), - f"[b]{search_component_name}[/b]", + related_url = highlight_search_term( + search_component_name, result_tree[pv][ps][cn][nvr]["related_url"], ) except re.error: @@ -645,9 +650,8 @@ def text_output_products_contain_component( child_dep_names = "" if provides_components: try: - child_dep_names = re.sub( - re.escape(search_component_name), - f"[b]{search_component_name}[/b]", + child_dep_names = highlight_search_term( + search_component_name, ", ".join(provides_components), ) child_dep_names = f"[{child_dep_names}]" @@ -677,9 +681,8 @@ def text_output_products_contain_component( # highlight search term dep_name = nvr try: - dep_name = re.sub( - re.escape(search_component_name), - f"[b]{search_component_name}[/b]", + dep_name = highlight_search_term( + search_component_name, dep_name, ) except re.error: @@ -688,11 +691,11 @@ def text_output_products_contain_component( related_url = result_tree[pv][ps][cn][nvr].get("related_url") try: if result_tree[pv][ps][cn][nvr]["related_url"]: - related_url = re.sub( - re.escape(search_component_name), - f"[b]{search_component_name}[/b]", + related_url = highlight_search_term( + search_component_name, result_tree[pv][ps][cn][nvr]["related_url"], ) + except re.error: pass build_source_url = "" @@ -797,9 +800,8 @@ def text_output_products_contain_component( child_dep_names = "" if provides_components: try: - child_dep_names = re.sub( - re.escape(search_component_name), - f"[b]{search_component_name}[/b]", + child_dep_names = highlight_search_term( + search_component_name, ", ".join(provides_components), ) child_dep_names = f"[{child_dep_names}]" @@ -833,9 +835,8 @@ def text_output_products_contain_component( # highlight search term dep_name = nvr try: - dep_name = re.sub( - re.escape(search_component_name), - f"[b]{search_component_name}[/b]", + dep_name = highlight_search_term( + search_component_name, dep_name, ) except re.error: @@ -844,9 +845,8 @@ def text_output_products_contain_component( related_url = result_tree[pv][ps][cn][nvr].get("related_url") try: if result_tree[pv][ps][cn][nvr]["related_url"]: - related_url = re.sub( - re.escape(search_component_name), - f"[b]{search_component_name}[/b]", + related_url = highlight_search_term( + search_component_name, result_tree[pv][ps][cn][nvr]["related_url"], ) except re.error: @@ -920,9 +920,8 @@ def text_output_products_contain_component( child_dep_names = "" if provides_components: try: - child_dep_names = re.sub( - re.escape(search_component_name), - f"[b]{search_component_name}[/b]", + child_dep_names = highlight_search_term( + search_component_name, ", ".join(provides_components), ) child_dep_names = f"[{child_dep_names}]" @@ -956,9 +955,8 @@ def text_output_products_contain_component( # highlight search term dep_name = result_tree[pv][ps][cn][nvr]["purl"] try: - dep_name = re.sub( - re.escape(search_component_name), - f"[b]{search_component_name}[/b]", + dep_name = highlight_search_term( + search_component_name, dep_name, ) except re.error: @@ -967,9 +965,8 @@ def text_output_products_contain_component( related_url = result_tree[pv][ps][cn][nvr].get("related_url") try: if result_tree[pv][ps][cn][nvr]["related_url"]: - related_url = re.sub( - re.escape(search_component_name), - f"[b]{search_component_name}[/b]", + related_url = highlight_search_term( + search_component_name, result_tree[pv][ps][cn][nvr]["related_url"], ) except re.error: @@ -1043,9 +1040,8 @@ def text_output_products_contain_component( child_dep_names = "" if provides_components: try: - child_dep_names = re.sub( - re.escape(search_component_name), - f"[b]{search_component_name}[/b]", + child_dep_names = highlight_search_term( + search_component_name, ", ".join(provides_components), ) child_dep_names = f"[{child_dep_names}]" diff --git a/scripts/smoke-tests.sh b/scripts/smoke-tests.sh index 2351f53..63b10d1 100755 --- a/scripts/smoke-tests.sh +++ b/scripts/smoke-tests.sh @@ -80,3 +80,4 @@ griffon service component-flaws npm griffon service component-flaws python-marshmallow --affectedness AFFECTED griffon service products-contain-component --search-all --search-upstreams -s libxml2 -a griffon service products-contain-component -s grep -v --search-all +griffon service products-contain-component -r 'webkit.tk' -vv