From 6ab54e3e87cb10e305ab7a5f8081087bdb08b4ca Mon Sep 17 00:00:00 2001 From: Thomas Gurung <97883332+tommyg-cld@users.noreply.github.com> Date: Tue, 19 Sep 2023 23:00:57 +0100 Subject: [PATCH] Add support for `regen_derived` command * Add new command to help customers invalidate derived versions of transformations --- cloudinary_cli/modules/__init__.py | 2 + cloudinary_cli/modules/regen_derived.py | 99 +++++++++++++++++++++++++ cloudinary_cli/utils/api_utils.py | 26 ++++++- 3 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 cloudinary_cli/modules/regen_derived.py diff --git a/cloudinary_cli/modules/__init__.py b/cloudinary_cli/modules/__init__.py index f1a3086..996eab9 100644 --- a/cloudinary_cli/modules/__init__.py +++ b/cloudinary_cli/modules/__init__.py @@ -2,10 +2,12 @@ from .migrate import migrate from .sync import sync from .upload_dir import upload_dir +from .regen_derived import regen_derived commands = [ upload_dir, make, migrate, sync, + regen_derived ] diff --git a/cloudinary_cli/modules/regen_derived.py b/cloudinary_cli/modules/regen_derived.py new file mode 100644 index 0000000..a0b948b --- /dev/null +++ b/cloudinary_cli/modules/regen_derived.py @@ -0,0 +1,99 @@ +from click import command, argument, option +from cloudinary_cli.utils.utils import print_help_and_exit +from cloudinary_cli.utils.api_utils import handle_api_command, regen_derived_version +from cloudinary import api +from cloudinary_cli.utils.utils import confirm_action, run_tasks_concurrently +from cloudinary_cli.defaults import logger + +DEFAULT_MAX_RESULTS = 500 + + +@command("regen_derived", + short_help="""Regenerate all derived assets pertaining \ + to a named transformation, or transformation string.""", + help=""" +\b +Regenerate all derived assets pertaining to a specific named transformation, or transformation string. +Use this after updating a named transformation to invalidate and repopulate the cache with up-to-date versions of the assets. +Format: cld regen_derived +e.g. cld regen_derived t_named -A -ea -enu http://mywebhook.com +""") +@argument("trans_str") +@option("-enu", "--eager_notification_url", help="Webhook notification URL.") +@option("-ea", "--eager_async", is_flag=True, default=False, + help="Generate asynchronously.") +@option("-A", "--auto_paginate", is_flag=True, default=False, + help="Auto-paginate Admin API calls.") +@option("-F", "--force", is_flag=True, + help="Skip initial and auto-paginate confirmation.") +@option("-n", "--max_results", nargs=1, default=10, + help="""The maximum number of results to return. + Default: 10, maximum: 500.""") +@option("-w", "--concurrent_workers", type=int, default=30, + help="Specify the number of concurrent network threads.") +def regen_derived(trans_str, eager_notification_url, + eager_async, auto_paginate, force, + max_results, concurrent_workers): + + if not any(trans_str): + print_help_and_exit() + + if not force: + if not confirm_action( + f"Running this module will explicity " + f"re-generate all the related derived assets " + f"which will cause an increase in your transformation costs " + f"based on the number of derived assets re-generated.\n" + f"If running in auto-paginate (-A) mode, " + f"multiple Admin API (rate-limited) calls will be made.\n" + f"Continue? (y/N)"): + logger.info("Stopping.") + exit() + else: + logger.info("Continuing. You may use the -F " + "flag to skip confirmation.") + + if auto_paginate: + max_results = DEFAULT_MAX_RESULTS + force = True + + params = ('transformation', trans_str, f'max_results={max_results}') + trans_details = handle_api_command(params, (), (), None, None, None, + doc_url="", api_instance=api, + api_name="admin", + auto_paginate=auto_paginate, + force=force, return_data=True) + derived_resources = trans_details.get('derived') + if not derived_resources: + logger.info("No derived assets are using this transformation.") + exit() + + is_named = trans_details.get('named') + eager_trans = normalise_trans_name(trans_str) if is_named else trans_str + + progress_msg = f"Regenerating {len(derived_resources)} derived asset(s)" + if eager_async: + progress_msg += f" with eager_async={eager_async}" + logger.info(f"{progress_msg}...") + + regen_conc_list = [] + for derived in derived_resources: + public_id = derived.get('public_id') + delivery_type = derived.get('type') + res_type = derived.get('resource_type') + regen_conc_list.append((public_id, delivery_type, res_type, + eager_trans, eager_async, + eager_notification_url)) + + run_tasks_concurrently(regen_derived_version, regen_conc_list, + concurrent_workers) + complete_msg = ('Regeneration in progress' + if eager_async else 'Regeneration complete') + logger.info(f"{complete_msg}. It may take up to 10 mins " + "to see the changes. Please contact support " + "if you still see the old media.") + return True + + +def normalise_trans_name(trans_name): + return trans_name if trans_name.startswith('t_') else 't_' + trans_name diff --git a/cloudinary_cli/utils/api_utils.py b/cloudinary_cli/utils/api_utils.py index 5014711..7d25176 100644 --- a/cloudinary_cli/utils/api_utils.py +++ b/cloudinary_cli/utils/api_utils.py @@ -48,6 +48,25 @@ def query_cld_folder(folder): return files +def regen_derived_version(public_id, delivery_type, res_type, + eager_trans, eager_async, + eager_notification_url): + options = {"type": delivery_type, "resource_type": res_type, + "eager": eager_trans, "eager_async": eager_async, + "eager_notification_url": eager_notification_url, + "overwrite": True, "invalidate": True} + try: + exp_res = uploader.explicit(public_id, **options) + derived_url = f'{exp_res.get("eager")[0].get("secure_url")}' + msg = ('Processing' if options.get('eager_async') else 'Regenerated') + f' {derived_url}' + logger.info(style(msg, fg="green")) + except Exception as e: + error_msg = (f"Failed to regenerate {public_id} of type: " + f"{options.get('type')} and resource_type: " + f"{options.get('resource_type')}") + log_exception(e, error_msg) + + def upload_file(file_path, options, uploaded=None, failed=None): uploaded = uploaded if uploaded is not None else {} failed = failed if failed is not None else {} @@ -157,10 +176,12 @@ def handle_api_command( api_name, auto_paginate=False, force=False, - filter_fields=None): + filter_fields=None, + return_data=False): """ Used by Admin and Upload API commands """ + if doc: return launch(doc_url) @@ -185,6 +206,9 @@ def handle_api_command( if auto_paginate: res = handle_auto_pagination(res, func, args, kwargs, force, filter_fields) + if return_data: + return res + print_json(res) if save: