diff --git a/griffon/__init__.py b/griffon/__init__.py index 476834e..5179df5 100644 --- a/griffon/__init__.py +++ b/griffon/__init__.py @@ -6,13 +6,17 @@ import logging import os from configparser import ConfigParser +from contextlib import contextmanager from functools import partial, wraps +from typing import Optional import component_registry_bindings import osidb_bindings from osidb_bindings.bindings.python_client.models import Affect, Flaw, Tracker -from pkg_resources import resource_filename # type: ignore +from pkg_resources import resource_filename +from rich.console import RenderableType # type: ignore from rich.logging import RichHandler +from rich.style import StyleType from griffon.output import console @@ -319,6 +323,44 @@ def get_fields(model, prefix=""): return fields +@contextmanager +def console_status(no_progress_bar): + """updatable console status progress bar""" + + class DisabledStatusObject: + """ + Dummy disabled status object for graceful handle of + no progress bar option + """ + + def __getattr__(self, attr): + def dummy_method(*args, **kwargs): + pass # Do nothing when any method is called + + return dummy_method + + class StatusObject: + """ + Status object for default Griffon status handling + """ + + def __init__(self, status): + self.status = status + + def update(self, status, *args, **kwargs): + self.status.update( + status=f"[magenta b]griffoning:[/magenta b] [bold]{status}[/bold]", *args, **kwargs + ) + + if no_progress_bar: + yield DisabledStatusObject() + else: + with console.status( + f"[magenta b]griffoning[/magenta b]", spinner="line" + ) as operation_status: + yield StatusObject(operation_status) + + def progress_bar( func=None, ): @@ -329,11 +371,8 @@ def progress_bar( @wraps(func) def wrapper(*args, **kwargs): obj: dict = args[0].obj - if obj.get("NO_PROGRESS_BAR"): - func(*args, **kwargs) - else: - with console.status("griffoning", spinner="line"): - func(*args, **kwargs) + with console_status(obj.get("NO_PROGRESS_BAR")) as operation_status: + func(*args, operation_status=operation_status, **kwargs) return wrapper diff --git a/griffon/commands/process.py b/griffon/commands/process.py index 9c2c03f..95e707e 100644 --- a/griffon/commands/process.py +++ b/griffon/commands/process.py @@ -32,7 +32,7 @@ @catch_exception(handle=(HTTPError)) @click.pass_context @progress_bar -def generate_affects_for_component_process(ctx, purl, cve_id): +def generate_affects_for_component_process(ctx, purl, cve_id, operation_status): """Generate affects for specific component.""" if not purl and not cve_id: click.echo(ctx.get_help()) diff --git a/griffon/commands/queries.py b/griffon/commands/queries.py index a880f5d..20d5de7 100644 --- a/griffon/commands/queries.py +++ b/griffon/commands/queries.py @@ -9,6 +9,7 @@ import logging import subprocess from json import loads +from time import sleep import click from component_registry_bindings.bindings.python_client.api.v1 import v1_components_list @@ -18,6 +19,7 @@ MIDDLEWARE_CLI, CorgiService, OSIDBService, + console_status, get_config_option, progress_bar, ) @@ -336,6 +338,7 @@ def retrieve_component_summary(ctx, component_name, strict_name_search): help="Verbose output, more detailed search results, can be used multiple times (e.g. -vvv).", ) # noqa @click.pass_context +@progress_bar def get_product_contain_component( ctx, component_name, @@ -364,231 +367,232 @@ def get_product_contain_component( include_product_stream_excluded_components, output_type_filter, verbose, + operation_status, ): - with console.status("griffoning", spinner="line") as operation_status: - """List products of a latest component.""" - if verbose: - ctx.obj["VERBOSE"] = verbose - - if ( - not search_latest - and not search_all - and not search_all_roots - and not search_related_url - and not search_community - and not search_all_upstreams - and not search_redhat - and not search_provides - and not search_upstreams - ): - ctx.params["search_latest"] = True - ctx.params["search_provides"] = True - - params = copy.deepcopy(ctx.params) - params.pop("verbose") - params.pop("sfm2_flaw_id") - params.pop("flaw_mode") - params.pop("affect_mode") - if component_name: - q = query_service.invoke( - core_queries.products_containing_component_query, params, status=operation_status - ) - if purl: - q = query_service.invoke( - core_queries.products_containing_specific_component_query, - params, - status=operation_status, - ) + # with console_status(ctx) as operation_status: + """List products of a latest component.""" + if verbose: + ctx.obj["VERBOSE"] = verbose - # TODO: interim hack for middleware - if component_name and MIDDLEWARE_CLI and not no_middleware: - operation_status.update("griffoning: searching deptopia middleware.", spinner="line") - mw_command = [MIDDLEWARE_CLI, component_name, "-e", "maven", "--json"] - if strict_name_search: - mw_command.append("-s") - proc = subprocess.run( - mw_command, - capture_output=True, - text=True, - ) - try: - mw_json = loads(proc.stdout) - mw_components = mw_json["deps"] - # TODO: need to determine if we use "build" or "deps" - # if search_all: - # mw_components.extend(mw_json["deps"]) - for build in mw_components: - if build["build_type"] == "maven": - component = { - "product_versions": [{"name": build["ps_module"]}], - "product_streams": [ - { - "name": build["ps_update_stream"], - "product_versions": [{"name": build["ps_module"]}], - } - ], - "product_active": True, - "type": build["build_type"], - "name": build["build_name"], - "nvr": build["build_nvr"], - "upstreams": [], - "sources": [], - "software_build": { - "build_id": build["build_id"], - "source": build["build_repo"], - }, - } - if "sources" in build: - for deps in build["sources"]: - for dep in deps["dependencies"]: - components = [] - components.append( - { - "name": dep.get("name"), - "nvr": dep.get("nvr"), - "type": dep.get("type"), - } - ) - component["sources"] = components - q.append(component) - except Exception: - logger.warning("problem accessing deptopia.") - - # TODO: in the short term affect handling will be mediated via sfm2 here in the operation itself # noqa - if ctx.params["sfm2_flaw_id"]: - operation_status.update("griffoning: invoking sfm2.", spinner="line") - - console.no_color = True - console.highlighter = None - operation_status.stop() - - # generate affects - output = raw_json_transform(q, True) - - exclude_products = [] - if get_config_option(ctx.obj["PROFILE"], "exclude"): - exclude_products = get_config_option(ctx.obj["PROFILE"], "exclude").split("\n") - exclude_components = [] - if get_config_option(ctx.obj["PROFILE"], "exclude_components"): - exclude_components = get_config_option( - ctx.obj["PROFILE"], "exclude_components" - ).split("\n") - normalised_results = generate_normalised_results( - output, - exclude_products, - exclude_components, - output_type_filter, - include_inactive_product_streams, - include_product_stream_excluded_components, + if ( + not search_latest + and not search_all + and not search_all_roots + and not search_related_url + and not search_community + and not search_all_upstreams + and not search_redhat + and not search_provides + and not search_upstreams + ): + ctx.params["search_latest"] = True + ctx.params["search_provides"] = True + + params = copy.deepcopy(ctx.params) + params.pop("verbose") + params.pop("sfm2_flaw_id") + params.pop("flaw_mode") + params.pop("affect_mode") + if component_name: + q = query_service.invoke( + core_queries.products_containing_component_query, params, status=operation_status + ) + if purl: + q = query_service.invoke( + core_queries.products_containing_specific_component_query, + params, + status=operation_status, + ) + + # TODO: interim hack for middleware + if component_name and MIDDLEWARE_CLI and not no_middleware: + operation_status.update("searching deptopia middleware.") + mw_command = [MIDDLEWARE_CLI, component_name, "-e", "maven", "--json"] + if strict_name_search: + mw_command.append("-s") + proc = subprocess.run( + mw_command, + capture_output=True, + text=True, + ) + try: + mw_json = loads(proc.stdout) + mw_components = mw_json["deps"] + # TODO: need to determine if we use "build" or "deps" + # if search_all: + # mw_components.extend(mw_json["deps"]) + for build in mw_components: + if build["build_type"] == "maven": + component = { + "product_versions": [{"name": build["ps_module"]}], + "product_streams": [ + { + "name": build["ps_update_stream"], + "product_versions": [{"name": build["ps_module"]}], + } + ], + "product_active": True, + "type": build["build_type"], + "name": build["build_name"], + "nvr": build["build_nvr"], + "upstreams": [], + "sources": [], + "software_build": { + "build_id": build["build_id"], + "source": build["build_repo"], + }, + } + if "sources" in build: + for deps in build["sources"]: + for dep in deps["dependencies"]: + components = [] + components.append( + { + "name": dep.get("name"), + "nvr": dep.get("nvr"), + "type": dep.get("type"), + } + ) + component["sources"] = components + q.append(component) + except Exception: + logger.warning("problem accessing deptopia.") + + # TODO: in the short term affect handling will be mediated via sfm2 here in the operation itself # noqa + if ctx.params["sfm2_flaw_id"]: + operation_status.update("invoking sfm2.") + + console.no_color = True + console.highlighter = None + operation_status.stop() + + # generate affects + output = raw_json_transform(q, True) + + exclude_products = [] + if get_config_option(ctx.obj["PROFILE"], "exclude"): + exclude_products = get_config_option(ctx.obj["PROFILE"], "exclude").split("\n") + exclude_components = [] + if get_config_option(ctx.obj["PROFILE"], "exclude_components"): + exclude_components = get_config_option(ctx.obj["PROFILE"], "exclude_components").split( + "\n" ) - result_tree = generate_result_tree(normalised_results) - affects = generate_affects(ctx, result_tree, exclude_components, "add", format="json") - - # attempt to import sfm2client module - try: - import sfm2client - except ImportError: - logger.warning("sfm2client library not found, cannot compare with flaw affects") - ctx.exit() - - # TODO: paramaterise into dotfile/env var - sfm2 = sfm2client.api.core.SFMApi({"url": "http://localhost:5600"}) - try: - flaw = sfm2.flaw.get(sfm2_flaw_id) - except Exception as e: - logger.warning(f"Could not retrieve flaw {sfm2_flaw_id}: {e}") - return - - if ctx.params["flaw_mode"] == "replace": - if affects: + normalised_results = generate_normalised_results( + output, + exclude_products, + exclude_components, + output_type_filter, + include_inactive_product_streams, + include_product_stream_excluded_components, + ) + result_tree = generate_result_tree(normalised_results) + affects = generate_affects(ctx, result_tree, exclude_components, "add", format="json") + + # attempt to import sfm2client module + try: + import sfm2client + except ImportError: + logger.warning("sfm2client library not found, cannot compare with flaw affects") + ctx.exit() + + # TODO: paramaterise into dotfile/env var + sfm2 = sfm2client.api.core.SFMApi({"url": "http://localhost:5600"}) + try: + flaw = sfm2.flaw.get(sfm2_flaw_id) + except Exception as e: + logger.warning(f"Could not retrieve flaw {sfm2_flaw_id}: {e}") + return + + if ctx.params["flaw_mode"] == "replace": + if affects: + console.print( + f"The following affects will REPLACE all flaw {flaw['id']}'s existing affects in \"new\" state:\n" # noqa + ) + for m in affects: console.print( - f"The following affects will REPLACE all flaw {flaw['id']}'s existing affects in \"new\" state:\n" # noqa + f"{m['product_version']}\t{m['component_name']}", + no_wrap=False, ) + + if click.confirm( + f"\nREPLACE flaw {flaw['id']}'s existing affects in \"new\" state with the above? THIS CANNOT BE UNDONE: ", # noqa + default=True, + ): + click.echo("Updating ...") + # only discard affects in 'new' state, we should preserve all others so not to throw work away # noqa + new_affects = [a for a in flaw["affects"] if a["affected"] != "new"] + # get map of existing affects first, so that we don't try to add duplicates + existing = set((a["ps_module"], a["ps_component"]) for a in new_affects) for m in affects: - console.print( - f"{m['product_version']}\t{m['component_name']}", - no_wrap=False, + if (m["product_version"], m["component_name"]) in existing: + continue + new_affects.append( + { + "affected": "new", + "ps_component": m["component_name"], + "ps_module": m["product_version"], + } ) - - if click.confirm( - f"\nREPLACE flaw {flaw['id']}'s existing affects in \"new\" state with the above? THIS CANNOT BE UNDONE: ", # noqa - default=True, - ): - click.echo("Updating ...") - # only discard affects in 'new' state, we should preserve all others so not to throw work away # noqa - new_affects = [a for a in flaw["affects"] if a["affected"] != "new"] - # get map of existing affects first, so that we don't try to add duplicates - existing = set((a["ps_module"], a["ps_component"]) for a in new_affects) - for m in affects: - if (m["product_version"], m["component_name"]) in existing: - continue - new_affects.append( - { - "affected": "new", - "ps_component": m["component_name"], - "ps_module": m["product_version"], - } - ) - try: - sfm2.flaw.update(flaw["id"], data={"affects": new_affects}) - except Exception as e: - msg = e.response.json() - logger.warning(f"Failed to update flaw: {e}: {msg}") - console.print("Operation done.") - ctx.exit() - - click.echo("No affects were added to flaw.") - else: - console.print("No affects to add to flaw.") + try: + sfm2.flaw.update(flaw["id"], data={"affects": new_affects}) + except Exception as e: + msg = e.response.json() + logger.warning(f"Failed to update flaw: {e}: {msg}") + console.print("Operation done.") + ctx.exit() + + click.echo("No affects were added to flaw.") else: - missing = [] - for affect in affects: - flaw_has_affect = False - for a in flaw.get("affects"): - if a.get("ps_module") == affect.get("product_version") and a.get( - "ps_component" - ) == affect.get("component_name"): - flaw_has_affect = True - if not flaw_has_affect: - missing.append(affect) - - if missing: - console.log("Flaw is missing the following affect entries:\n") + console.print("No affects to add to flaw.") + else: + missing = [] + for affect in affects: + flaw_has_affect = False + for a in flaw.get("affects"): + if a.get("ps_module") == affect.get("product_version") and a.get( + "ps_component" + ) == affect.get("component_name"): + flaw_has_affect = True + if not flaw_has_affect: + missing.append(affect) + + if missing: + console.log("Flaw is missing the following affect entries:\n") + for m in missing: + console.print( + f"{m['product_version']}\t{m['component_name']}", + no_wrap=False, + ) + if click.confirm( + "Would you like to add them? ", + default=True, + ): + click.echo("Updating ...") + + updated_affects = flaw.get("affects")[:] for m in missing: - console.print( - f"{m['product_version']}\t{m['component_name']}", - no_wrap=False, + updated_affects.append( + { + "affected": "new", + "ps_component": m["component_name"], + "ps_module": m["product_version"], + } ) - if click.confirm( - "Would you like to add them? ", - default=True, - ): - click.echo("Updating ...") - - updated_affects = flaw.get("affects")[:] - for m in missing: - updated_affects.append( - { - "affected": "new", - "ps_component": m["component_name"], - "ps_module": m["product_version"], - } - ) - try: - sfm2.flaw.update(flaw["id"], data={"affects": updated_affects}) - except Exception as e: - msg = e.response.json() - logger.warning(f"Failed to update flaw: {e}: {msg}") - console.print("Operation done.") - ctx.exit() - click.echo("No affects were added to flaw.") - - else: - console.print("Flaw is not missing any affect entries") + try: + sfm2.flaw.update(flaw["id"], data={"affects": updated_affects}) + except Exception as e: + msg = e.response.json() + logger.warning(f"Failed to update flaw: {e}: {msg}") + console.print("Operation done.") + ctx.exit() + click.echo("No affects were added to flaw.") - ctx.exit() + else: + console.print("Flaw is not missing any affect entries") - cprint(q, ctx=ctx) + ctx.exit() + + cprint(q, ctx=ctx) @queries_grp.command( @@ -643,6 +647,7 @@ def get_component_contain_component( namespace, strict_name_search, verbose, + operation_status, ): """List components that contain component.""" if verbose: @@ -686,7 +691,8 @@ def get_component_contain_component( help="Generate spdx manifest (json).", ) @click.pass_context -def get_product_manifest_query(ctx, product_stream_name, ofuri, spdx_json_format): +@progress_bar +def get_product_manifest_query(ctx, product_stream_name, ofuri, spdx_json_format, operation_status): """List components of a specific product version.""" if spdx_json_format: ctx.ensure_object(dict) @@ -736,7 +742,10 @@ def get_product_manifest_query(ctx, product_stream_name, ofuri, spdx_json_format help="Verbose output, more detailed search results, can be used multiple times (e.g. -vvv).", ) # noqa @click.pass_context -def get_product_latest_components_query(ctx, product_stream_name, ofuri, verbose, **params): +@progress_bar +def get_product_latest_components_query( + ctx, product_stream_name, ofuri, verbose, operation_status, **params +): """List components of a specific product version.""" if verbose: ctx.obj["VERBOSE"] = verbose @@ -780,7 +789,8 @@ def get_product_latest_components_query(ctx, product_stream_name, ofuri, verbose help="Generate spdx manifest (json).", ) @click.pass_context -def retrieve_component_manifest(ctx, component_uuid, purl, spdx_json_format): +@progress_bar +def retrieve_component_manifest(ctx, component_uuid, purl, spdx_json_format, operation_status): """Retrieve Component manifest.""" if spdx_json_format: ctx.ensure_object(dict) @@ -829,7 +839,14 @@ def retrieve_component_manifest(ctx, component_uuid, purl, spdx_json_format): @click.pass_context @progress_bar def components_affected_by_specific_cve_query( - ctx, cve_id, affectedness, affect_resolution, affect_impact, component_type, namespace + ctx, + cve_id, + affectedness, + affect_resolution, + affect_impact, + component_type, + namespace, + operation_status, ): """List components affected by specific CVE.""" q = query_service.invoke(core_queries.components_affected_by_specific_cve_query, ctx.params) @@ -843,7 +860,7 @@ def components_affected_by_specific_cve_query( @click.argument("cve_id", required=True, type=click.STRING, shell_complete=get_cve_ids) @click.pass_context @progress_bar -def product_versions_affected_by_cve_query(ctx, cve_id): +def product_versions_affected_by_cve_query(ctx, cve_id, operation_status): """List Products affected by a CVE.""" q = query_service.invoke( core_queries.products_versions_affected_by_specific_cve_query, ctx.params @@ -913,6 +930,7 @@ def cves_for_specific_component_query( affect_resolution, affect_impact, strict_name_search, + operation_status, ): """List flaws of a specific component.""" q = query_service.invoke(core_queries.cves_for_specific_component_query, ctx.params) @@ -986,6 +1004,7 @@ def cves_for_specific_product_query( affect_impact, affect_resolution, strict_name_search, + operation_status, ): """List flaws of a specific product.""" q = query_service.invoke(core_queries.cves_for_specific_product_query, ctx.params) diff --git a/griffon/commands/reports.py b/griffon/commands/reports.py index 58884ba..4832f8a 100644 --- a/griffon/commands/reports.py +++ b/griffon/commands/reports.py @@ -42,7 +42,16 @@ def reports_grp(ctx): @click.pass_context @progress_bar def generate_affects_report( - ctx, product_version_name, all, show_components, show_products, purl, name, product_name, ofuri + ctx, + product_version_name, + all, + show_components, + show_products, + purl, + name, + product_name, + ofuri, + operation_status, ): """A report operation""" if not all and not product_version_name: @@ -55,7 +64,7 @@ def generate_affects_report( @click.option("--all", is_flag=True, default=True, help="Show summary report on all entities.") @click.pass_context @progress_bar -def generate_entity_report(ctx, all): +def generate_entity_report(ctx, all, operation_status): """A report operation""" if not all: click.echo(ctx.get_help()) @@ -76,7 +85,7 @@ def generate_entity_report(ctx, all): ) @click.pass_context @progress_bar -def generate_license_report(ctx, product_stream_name, purl, exclude_children): +def generate_license_report(ctx, product_stream_name, purl, exclude_children, operation_status): """A report operation""" if not product_stream_name and not purl: click.echo(ctx.get_help()) diff --git a/griffon/services/core_queries.py b/griffon/services/core_queries.py index 9a56ab6..4b9e27a 100644 --- a/griffon/services/core_queries.py +++ b/griffon/services/core_queries.py @@ -303,7 +303,7 @@ def __init__(self, params: dict) -> None: self.include_inactive_product_streams = self.params.get("include_inactive_product_streams") def execute(self, status=None) -> List[Dict[str, Any]]: - status.update("griffoning: searching component-registry.") + status.update("searching component-registry.") results = [] params = { "limit": 50, @@ -321,27 +321,27 @@ def execute(self, status=None) -> List[Dict[str, Any]]: search_latest_params["active_streams"] = "True" search_latest_params["root_components"] = "True" search_latest_params["latest_components_by_streams"] = "True" - status.update("griffoning: searching latest root component(s).") + status.update("searching latest root component(s).") latest_components_cnt = self.corgi_session.components.count(**search_latest_params) - status.update(f"griffoning: found {latest_components_cnt} latest component(s).") + status.update(f"found {latest_components_cnt} latest component(s).") latest_components = self.corgi_session.components.retrieve_list_iterator_async( **search_latest_params ) with multiprocessing.Pool() as pool: status.update( - f"griffoning: found {latest_components_cnt} latest root component(s), retrieving sources & upstreams." # noqa + f"found {latest_components_cnt} latest root component(s), retrieving sources & upstreams." # noqa ) for processed_component in pool.map( partial(process_component, self.corgi_session), latest_components ): results.append(processed_component) if not self.no_community: - status.update("griffoning: searching latest community root component(s).") + status.update("searching latest community root component(s).") community_component_cnt = self.community_session.components.count( **search_latest_params ) status.update( - f"griffoning: found {community_component_cnt} latest community root component(s)." # noqa + f"found {community_component_cnt} latest community root component(s)." # noqa ) latest_community_components = ( self.community_session.components.retrieve_list_iterator_async( @@ -350,7 +350,7 @@ def execute(self, status=None) -> List[Dict[str, Any]]: ) with multiprocessing.Pool() as pool: status.update( - f"griffoning: found {community_component_cnt} latest community root component(s), retrieving sources & upstreams." # noqa + f"found {community_component_cnt} latest community root component(s), retrieving sources & upstreams." # noqa ) for processed_component in pool.map( partial(process_component, self.community_session), @@ -370,27 +370,27 @@ def execute(self, status=None) -> List[Dict[str, Any]]: search_provides_params["active_streams"] = "True" search_provides_params["root_components"] = "True" search_provides_params["latest_components_by_streams"] = "True" - status.update("griffoning: searching latest provided child component(s).") + status.update("searching latest provided child component(s).") latest_components_cnt = self.corgi_session.components.count(**search_provides_params) - status.update(f"griffoning: found {latest_components_cnt} latest component(s).") + status.update(f"found {latest_components_cnt} latest component(s).") latest_components = self.corgi_session.components.retrieve_list_iterator_async( **search_provides_params ) with multiprocessing.Pool() as pool: status.update( - f"griffoning: found {latest_components_cnt} latest provides child component(s), retrieving sources & upstreams." # noqa + f"found {latest_components_cnt} latest provides child component(s), retrieving sources & upstreams." # noqa ) for processed_component in pool.map( partial(process_component, self.corgi_session), latest_components ): results.append(processed_component) if not self.no_community: - status.update("griffoning: searching latest community provided child component(s).") + status.update("searching latest community provided child component(s).") community_component_cnt = self.community_session.components.count( **search_provides_params ) status.update( - f"griffoning: found {community_component_cnt} latest community provided child component(s)." # noqa + f"found {community_component_cnt} latest community provided child component(s)." # noqa ) latest_community_components = ( self.community_session.components.retrieve_list_iterator_async( @@ -399,7 +399,7 @@ def execute(self, status=None) -> List[Dict[str, Any]]: ) with multiprocessing.Pool() as pool: status.update( - f"griffoning: found {community_component_cnt} latest community provided child component(s), retrieving sources & upstreams." # noqa + f"found {community_component_cnt} latest community provided child component(s), retrieving sources & upstreams." # noqa ) for processed_component in pool.map( partial(process_component, self.community_session), @@ -420,29 +420,27 @@ def execute(self, status=None) -> List[Dict[str, Any]]: search_upstreams_params["active_streams"] = "True" search_upstreams_params["released_components"] = "True" search_upstreams_params["latest_components_by_streams"] = "True" - status.update("griffoning: searching latest upstreams child component(s).") + status.update("searching latest upstreams child component(s).") latest_components_cnt = self.corgi_session.components.count(**search_upstreams_params) - status.update(f"griffoning: found {latest_components_cnt} latest component(s).") + status.update(f"found {latest_components_cnt} latest component(s).") latest_components = self.corgi_session.components.retrieve_list_iterator_async( **search_upstreams_params ) with multiprocessing.Pool() as pool: status.update( - f"griffoning: found {latest_components_cnt} latest upstreams child component(s), retrieving sources & upstreams." # noqa + f"found {latest_components_cnt} latest upstreams child component(s), retrieving sources & upstreams." # noqa ) for processed_component in pool.map( partial(process_component, self.corgi_session), latest_components ): results.append(processed_component) if not self.no_community: - status.update( - "griffoning: searching latest community upstreams child component(s)." - ) + status.update("searching latest community upstreams child component(s).") community_component_cnt = self.community_session.components.count( **search_upstreams_params ) status.update( - f"griffoning: found {community_component_cnt} latest community upstreams child component(s)." # noqa + f"found {community_component_cnt} latest community upstreams child component(s)." # noqa ) latest_community_components = ( self.community_session.components.retrieve_list_iterator_async( @@ -451,7 +449,7 @@ def execute(self, status=None) -> List[Dict[str, Any]]: ) with multiprocessing.Pool() as pool: status.update( - f"griffoning: found {community_component_cnt} latest community provided child component(s), retrieving sources & upstreams." # noqa + f"found {community_component_cnt} latest community provided child component(s), retrieving sources & upstreams." # noqa ) for processed_component in pool.map( partial(process_component, self.community_session), @@ -473,15 +471,13 @@ def execute(self, status=None) -> List[Dict[str, Any]]: related_url_components_cnt = self.corgi_session.components.count( **search_related_url_params ) - status.update( - f"griffoning: found {related_url_components_cnt} related url component(s)." - ) + status.update(f"found {related_url_components_cnt} related url component(s).") related_url_components = self.corgi_session.components.retrieve_list_iterator_async( **search_related_url_params ) with multiprocessing.Pool() as pool: status.update( - f"griffoning: found {related_url_components_cnt} related url component(s), retrieving sources & upstreams." # noqa + f"found {related_url_components_cnt} related url component(s), retrieving sources & upstreams." # noqa ) for processed_component in pool.map( partial(process_component, self.corgi_session), related_url_components @@ -492,7 +488,7 @@ def execute(self, status=None) -> List[Dict[str, Any]]: **search_related_url_params ) status.update( - f"griffoning: found {latest_community_url_components_cnt} related url community component(s)." # noqa + f"found {latest_community_url_components_cnt} related url community component(s)." # noqa ) latest_community_url_components = ( self.community_session.components.retrieve_list_iterator_async( @@ -501,7 +497,7 @@ def execute(self, status=None) -> List[Dict[str, Any]]: ) with multiprocessing.Pool() as pool: status.update( - f"griffoning: found {latest_community_url_components_cnt} related url community component(s), retrieving sources & upstreams." # noqa + f"found {latest_community_url_components_cnt} related url community component(s), retrieving sources & upstreams." # noqa ) for processed_component in pool.map( partial(process_component, self.community_session), @@ -523,15 +519,15 @@ def execute(self, status=None) -> List[Dict[str, Any]]: search_all_params["active_streams"] = "True" search_all_params["released_components"] = "True" all_components_cnt = self.corgi_session.components.count(**search_all_params) - status.update(f"griffoning: found {all_components_cnt} all component(s).") + status.update(f"found {all_components_cnt} all component(s).") # TODO: remove max_results all_components = self.corgi_session.components.retrieve_list_iterator_async( **search_all_params, max_results=10000 ) - status.update(f"griffoning: found {all_components_cnt} all component(s).") + status.update(f"found {all_components_cnt} all component(s).") with multiprocessing.Pool() as pool: status.update( - f"griffoning: found {all_components_cnt} all component(s), retrieving sources & upstreams." # noqa + f"found {all_components_cnt} all component(s), retrieving sources & upstreams." # noqa ) for processed_component in pool.map( partial(process_component, self.corgi_session), all_components @@ -543,7 +539,7 @@ def execute(self, status=None) -> List[Dict[str, Any]]: **search_all_params ) status.update( - f"griffoning: found {all_community_components_cnt} community all component(s)." # noqa + f"found {all_community_components_cnt} community all component(s)." # noqa ) # TODO: remove max_results all_community_components = ( @@ -553,7 +549,7 @@ def execute(self, status=None) -> List[Dict[str, Any]]: ) with multiprocessing.Pool() as pool: status.update( - f"griffoning: found {all_community_components_cnt} community all component(s), retrieving sources & upstreams." # noqa + f"found {all_community_components_cnt} community all component(s), retrieving sources & upstreams." # noqa ) for processed_component in pool.map( partial(process_component, self.community_session), @@ -574,7 +570,7 @@ def execute(self, status=None) -> List[Dict[str, Any]]: search_all_roots_params["active_streams"] = "True" search_all_roots_params["released_components"] = "True" all_src_components_cnt = self.corgi_session.components.count(**search_all_roots_params) - status.update(f"griffoning: found {all_src_components_cnt} all root component(s).") + status.update(f"found {all_src_components_cnt} all root component(s).") all_src_components = self.corgi_session.components.retrieve_list_iterator_async( **search_all_roots_params ) @@ -590,7 +586,7 @@ def execute(self, status=None) -> List[Dict[str, Any]]: ) ) status.update( - f"griffoning: found {all_src_community_components_cnt} community all root component(s)." # noqa + f"found {all_src_community_components_cnt} community all root component(s)." # noqa ) for c in all_src_community_components: results.append(c) @@ -610,12 +606,12 @@ def execute(self, status=None) -> List[Dict[str, Any]]: upstream_components_cnt = self.corgi_session.components.count( **search_all_upstreams_params ) - status.update(f"griffoning: found {upstream_components_cnt} upstream component(s).") + status.update(f"found {upstream_components_cnt} upstream component(s).") upstream_components = self.corgi_session.components.retrieve_list_iterator_async( **search_all_upstreams_params ) with multiprocessing.Pool() as pool: - status.update(f"griffoning: found {upstream_components_cnt} upstream component(s).") + status.update(f"found {upstream_components_cnt} upstream component(s).") for processed_component in pool.map( partial(process_component, self.corgi_session), upstream_components ): @@ -625,7 +621,7 @@ def execute(self, status=None) -> List[Dict[str, Any]]: **search_all_upstreams_params ) status.update( - f"griffoning: found {commmunity_upstream_components_cnt} community upstream component(s)." # noqa + f"found {commmunity_upstream_components_cnt} community upstream component(s)." # noqa ) commmunity_upstream_components = ( self.community_session.components.retrieve_list_iterator_async( @@ -634,7 +630,7 @@ def execute(self, status=None) -> List[Dict[str, Any]]: ) with multiprocessing.Pool() as pool: status.update( - f"griffoning: found {commmunity_upstream_components_cnt} community upstream component(s), retrieving sources & upstreams." # noqa + f"found {commmunity_upstream_components_cnt} community upstream component(s), retrieving sources & upstreams." # noqa ) for processed_component in pool.map( partial(process_component, self.community_session), @@ -695,7 +691,7 @@ def execute(self, status=None) -> List[Dict[str, Any]]: **search_community_params ) status.update( - f"griffoning: found {all_community_components_cnt} community all component(s)." # noqa + f"found {all_community_components_cnt} community all component(s)." # noqa ) all_community_components = ( self.community_session.components.retrieve_list_iterator_async( @@ -704,7 +700,7 @@ def execute(self, status=None) -> List[Dict[str, Any]]: ) with multiprocessing.Pool() as pool: status.update( - f"griffoning: found {all_community_components_cnt} community all component(s), retrieving sources & upstreams." # noqa + f"found {all_community_components_cnt} community all component(s), retrieving sources & upstreams." # noqa ) for processed_component in pool.map( partial(process_component, self.community_session), all_community_components