Adding changes per discussion

cloudinary_cli/modules/__init__.py

@@ -2,12 +2,12 @@
 from .migrate import migrate
 from .sync import sync
 from .upload_dir import upload_dir
-from .regenerate_all_derived_by_transformation import regenerate_all_derived_by_transformation
+from .regen_derived import regen_derived
 
 commands = [
     upload_dir,
     make,
     migrate,
     sync,
-    regenerate_all_derived_by_transformation
+    regen_derived
 ]

cloudinary_cli/modules/regen_derived.py

@@ -0,0 +1,102 @@
+from click import command, argument, option
+from cloudinary_cli.utils.utils import print_help_and_exit
+from cloudinary_cli.utils.api_utils import handle_api_command, regen_derived_version
+from cloudinary import api
+from cloudinary_cli.utils.utils import confirm_action, run_tasks_concurrently
+from cloudinary_cli.defaults import logger
+
+DEFAULT_MAX_RESULTS = 500
+
+@command("regen_derived",
+         short_help="""Regenerate all derived of a transformation.""",
+         help="""
+\b
+Regenerate all derived versions of a named transformations.
+Format: cld regen_dervied <transformation_name> <command options>
+e.g. cld regen_derived t_named -A -ea -enu http://mywebhook.com
+""")
+@argument("trans_str")
+@option("-enu", "--eager_notification_url", help="Webhook notification URL.")
+@option("-ea", "--eager_async", is_flag=True, default=False,
+        help="Generate asynchronously.")
+@option("-A", "--auto_paginate", is_flag=True, default=False,
+        help="Will auto paginate Admin API calls.")
+@option("-F", "--force", is_flag=True,
+        help="Skip confirmation when running --auto-paginate/-A.")
+@option("-n", "--max_results", nargs=1, default=10,
+        help="""The maximum number of derived results to return.
+              Default: 10, maximum: 500.""")
+@option("-w", "--concurrent_workers", type=int, default=30,
+        help="Specify the number of concurrent network threads.")
+def regen_derived(trans_str, eager_notification_url,
+                  eager_async, auto_paginate, force,
+                  max_results, concurrent_workers):
+
+    if not any(trans_str):
+        print_help_and_exit()
+
+    if not force:
+        if not confirm_action(
+            f"Running this module will explicity "
+            f"re-generate all the related derived versions "
+            f"which will cause an increase in your transformation costs "
+            f"based on the number of derived re-generated.\n"
+            f"If running in auto_paginate (-A) mode, "
+            f"multiple Admin API (rate-limited) calls will be used as well.\n"
+            f"Continue? (y/N)"):
+            logger.info("Stopping.")
+            exit()
+        else:
+            logger.info("Continuing. You may use the -F "
+                        "flag to skip confirmation.")
+
+    if auto_paginate:
+        max_results = DEFAULT_MAX_RESULTS
+        force = True
+
+    params = ('transformation', trans_str, f'max_results={max_results}')
+    res = handle_api_command(params, (), (), None, None, None, doc_url="",
+                             api_instance=api, api_name="admin",
+                             auto_paginate=auto_paginate, force=force,
+                             return_data=True)
+    derived_resources = res.get('derived')
+    if not derived_resources:
+        logger.info("No derived resources using this transformation.")
+        exit()
+
+    is_named = res.get('named')
+    if is_named:
+        eager_trans = normalise_trans_name(trans_str)
+
+    progress_msg = f"Regenerating {len(derived_resources)} derived version(s)"
+    if eager_async:
+        logger.info(f"{progress_msg} "
+                    f"with eager_async={eager_async}...")
+    else:
+        logger.info(f"{progress_msg}...")
+
+    regen_conc_list = []
+    for derived in derived_resources:
+        public_id = derived.get('public_id')
+        delivery_type = derived.get('type')
+        res_type = derived.get('resource_type')
+        options = {"type": delivery_type, "resource_type": res_type,
+                   "eager": eager_trans, "eager_async": eager_async,
+                   "eager_notification_url": eager_notification_url,
+                   "overwrite": True, "invalidate": True}
+        regen_conc_list.append((public_id, options))
+
+    run_tasks_concurrently(regen_derived_version, regen_conc_list,
+                           concurrent_workers)
+
+    logger.info("Regen complete. It may take up to 10 mins "
+                "to see the changes. Please contact support "
+                "if you still see the old media.")
+    return True
+
+
+def normalise_trans_name(trans_name):
+    normalised_trans_name = trans_name
+    if not normalised_trans_name.startswith('t_'):
+        normalised_trans_name = 't_' + normalised_trans_name
+    return normalised_trans_name

cloudinary_cli/utils/api_utils.py

@@ -48,6 +48,22 @@ def query_cld_folder(folder):
     return files
 
 
+def regen_derived_version(public_id, options):
+    try:
+        exp_res = uploader.explicit(public_id, **options)
+        derived_url = f'{exp_res.get("eager")[0].get("secure_url")}'
+        if (options.get('eager_async')):
+            msg = f'Processing {derived_url}'
+        else:
+            msg = f'Regenerated {derived_url}'
+        logger.info(style(msg, fg="green"))
+    except Exception as e:
+        error_msg = (f"Failed to regenerate {public_id} of type: "
+                     f"{options.get('type')} and resource_type: "
+                     f"{options.get('resource_type')}")
+        log_exception(e, error_msg)
+
+
 def upload_file(file_path, options, uploaded=None, failed=None):
     uploaded = uploaded if uploaded is not None else {}
     failed = failed if failed is not None else {}
@@ -145,29 +161,6 @@ def handle_command(
     return call_api(func, args, kwargs)
 
 
-def handle_module_command(params, optional_parameter,
-                          optional_parameter_parsed, api_instance,
-                          api_name, auto_paginate, force):
-    try:
-        func, args, kwargs = get_command_params(params, optional_parameter,
-                                                optional_parameter_parsed,
-                                                api_instance, api_name)
-    except Exception as e:
-        log_exception(e)
-        return False
-
-    try:
-        res = call_api(func, args, kwargs)
-    except Exception:
-        return False
-
-    if auto_paginate:
-        res = handle_auto_pagination(res, func, args, kwargs,
-                                     force=force, filter_fields=None)
-
-    return res
-
-
 def handle_api_command(
         params,
         optional_parameter,
@@ -180,10 +173,12 @@ def handle_api_command(
         api_name,
         auto_paginate=False,
         force=False,
-        filter_fields=None):
+        filter_fields=None,
+        return_data=False):
     """
     Used by Admin and Upload API commands
     """
+
     if doc:
         return launch(doc_url)
 
@@ -208,7 +203,10 @@ def handle_api_command(
     if auto_paginate:
         res = handle_auto_pagination(res, func, args, kwargs, force, filter_fields)
 
-    print_json(res)
+    if return_data:
+        return res
+    else:
+        print_json(res)
 
     if save:
         write_json_to_file(res, save)