diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 6380c71..697cac9 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.6.4 +current_version = 0.7.0 commit = True tag = False parse = ^ diff --git a/README.md b/README.md index d892593..9612a8f 100644 --- a/README.md +++ b/README.md @@ -35,21 +35,25 @@ Usage: fichub_cli [OPTIONS] COMMAND [ARGS]... https://github.com/FicHub/fichub-cli/issues Failed downloads will be saved in the `err.log` file in the current - directory. + directory Options: - -u, --url TEXT The url of the fanfiction enclosed within quotes - -i, --infile TEXT Path to a file to read URLs from - -l, --list-url TEXT Enter a comma separated list of urls to download, enclosed within quotes - -v, --verbose Verbose - -o, --out-dir TEXT Path to the Output directory for files (default: Current Directory) - --format TEXT Download Format: epub (default), mobi, pdf or html [default: epub] - --force / --no-force Force overwrite of an existing file [default: no-force] - -ss, --supported-sites List of supported sites - -d, --debug Show the log in the console for debugging - --log / --no-log Save the logfile for debugging [default: no-log] - --version / --no-version Display version & quit [default: no-version] - --help Show this message and exit. + -u, --url TEXT The url of the fanfiction enclosed within quotes + -i, --infile TEXT Path to a file to read URLs from + -l, --list-url TEXT Enter a comma separated list of urls to download, + enclosed within quotes + -v, --verbose Show fic stats + -o, --out-dir TEXT Path to the Output directory for files (default: + Current Directory) + --format TEXT Download Format: epub (default), mobi, pdf or html + [default: epub] + --force Force overwrite of an existing file + -ss, --supported-sites List of supported sites + -d, --debug Show the log in the console for debugging + --changelog Save the changelog file + --debug-log Save the logfile for debugging + --version Display version & quit + --help Show this message and exit. ``` # Default Configuration @@ -80,6 +84,12 @@ fichub_cli -i urls.txt fichub_cli -l "https://www.fanfiction.net/s/11191235/1/Harry-Potter-and-the-Prince-of-Slytherin,https://www.fanfiction.net/s/13720575/1/A-Cadmean-Victory-Remastered" ``` +- To generate a changelog of the download + +``` +fichub_cli -i urls.txt --changelog +``` + --- **NOTE** diff --git a/fichub_cli/__init__.py b/fichub_cli/__init__.py index 364e7ba..49e0fc1 100644 --- a/fichub_cli/__init__.py +++ b/fichub_cli/__init__.py @@ -1 +1 @@ -__version__ = "0.6.4" +__version__ = "0.7.0" diff --git a/fichub_cli/cli.py b/fichub_cli/cli.py index c0791f2..80a24ef 100644 --- a/fichub_cli/cli.py +++ b/fichub_cli/cli.py @@ -73,7 +73,7 @@ def default( "epub", help="Download Format: epub (default), mobi, pdf or html"), force: bool = typer.Option( - False, help="Force overwrite of an existing file", is_flag=True), + False, "--force", help="Force overwrite of an existing file", is_flag=True), supported_sites: bool = typer.Option( False, "-ss", "--supported-sites", help="List of supported sites", is_flag=True), @@ -81,14 +81,17 @@ def default( debug: bool = typer.Option( False, "-d", " --debug", help="Show the log in the console for debugging", is_flag=True), - log: bool = typer.Option( - False, help="Save the logfile for debugging", is_flag=True), + changelog: bool = typer.Option( + False, "--changelog", help="Save the changelog file", is_flag=True), + + debug_log: bool = typer.Option( + False, "--debug-log", help="Save the logfile for debugging", is_flag=True), automated: bool = typer.Option( False, "-a", "--automated", help="For internal testing only", is_flag=True, hidden=True), version: bool = typer.Option( - False, help="Display version & quit", is_flag=True) + False, "--version", help="Display version & quit", is_flag=True) ): """ A CLI for the fichub.net API @@ -110,7 +113,7 @@ def default( Fore.BLUE + "Skipping default command to run sub-command.") return - if log: + if debug_log: logger.remove() # remove all existing handlers logger.add(f"fichub_cli - {timestamp}.log") debug = True @@ -121,13 +124,13 @@ def default( format_type = get_format_type(format) if infile: - fic = FetchData(format_type, out_dir, force, - debug, automated, verbose) + fic = FetchData(format_type, out_dir, force, debug, changelog, + automated, verbose) fic.get_fic_with_infile(infile) elif list_url: - fic = FetchData(format_type, out_dir, force, - debug, automated, verbose) + fic = FetchData(format_type, out_dir, force, debug, changelog, + automated, verbose) fic.get_fic_with_list(list_url) elif url: @@ -170,7 +173,7 @@ def default( if fic.exit_status == 1: typer.echo( Fore.RED + - "\nDownload failed for one or more URLs! Check " + Style.RESET_ALL + + "\nThe CLI ran into some errors! Check " + Style.RESET_ALL + Fore.YELLOW + "err.log" + Style.RESET_ALL + Fore.RED + " in the current directory!" + Style.RESET_ALL) diff --git a/fichub_cli/utils/fetch_data.py b/fichub_cli/utils/fetch_data.py index c11e7a7..6beac6a 100644 --- a/fichub_cli/utils/fetch_data.py +++ b/fichub_cli/utils/fetch_data.py @@ -21,18 +21,19 @@ from .logging import init_log, download_processing_log, \ verbose_log from .processing import check_url, save_data, \ - urls_preprocessing, check_output_log + urls_preprocessing, check_output_log, build_changelog bar_format = "{l_bar}{bar}| {n_fmt}/{total_fmt}, {rate_fmt}{postfix}, ETA: {remaining}" class FetchData: def __init__(self, format_type=0, out_dir="", force=False, - debug=False, automated=False, verbose=False): + debug=False, changelog=False, automated=False, verbose=False): self.format_type = format_type self.out_dir = out_dir self.force = force self.debug = debug + self.changelog = changelog self.automated = automated self.exit_status = 0 self.verbose = verbose @@ -57,15 +58,15 @@ def get_fic_with_infile(self, infile: str): f"{infile} file could not be found. Please enter a valid file path.") exit(1) - urls = urls_preprocessing(urls_input, self.debug) - + urls, urls_input_dedup = urls_preprocessing(urls_input, self.debug) + downloaded_urls, no_updates_urls, err_urls = [], [], [] if urls: init_log(self.debug, self.force) with tqdm(total=len(urls), ascii=False, unit="file", bar_format=bar_format) as pbar: for url in urls: - + url_exit_status = 0 download_processing_log(self.debug, url) supported_url, self.exit_status = check_url( url, self.debug, self.exit_status) @@ -85,7 +86,7 @@ def get_fic_with_infile(self, infile: str): self.exit_status = 1 else: - self.exit_status = save_data( + self.exit_status, url_exit_status = save_data( self.out_dir, fic.file_name, fic.download_url, self.debug, self.force, fic.cache_hash, self.exit_status, @@ -93,6 +94,11 @@ def get_fic_with_infile(self, infile: str): with open("output.log", "a") as file: file.write(f"{url}\n") + + if url_exit_status == 0: + downloaded_urls.append(url) + else: + no_updates_urls.append(url) pbar.update(1) # Error: 'FicHub' object has no attribute 'file_name' @@ -100,6 +106,7 @@ def get_fic_with_infile(self, infile: str): except AttributeError: with open("err.log", "a") as file: file.write(url.strip()+"\n") + err_urls.append(url) pbar.update(1) self.exit_status = 1 pass # skip the unsupported url @@ -107,6 +114,7 @@ def get_fic_with_infile(self, infile: str): else: # skip the unsupported url with open("err.log", "a") as file: file.write(url.strip()+"\n") + err_urls.append(url) pbar.update(1) continue @@ -114,20 +122,25 @@ def get_fic_with_infile(self, infile: str): typer.echo(Fore.RED + "No new urls found! If output.log exists, please clear it.") + if self.changelog: + build_changelog(urls_input, urls_input_dedup, urls, + downloaded_urls, err_urls, no_updates_urls, self.out_dir) + def get_fic_with_list(self, list_url: str): if self.debug: logger.info("-l flag used!") urls_input = list_url.split(",") - urls = urls_preprocessing(urls_input, self.debug) - + urls, urls_input_dedup = urls_preprocessing(urls_input, self.debug) + downloaded_urls, no_updates_urls, err_urls = [], [], [] if urls: init_log(self.debug, self.force) with tqdm(total=len(urls), ascii=False, unit="file", bar_format=bar_format) as pbar: for url in urls: + url_exit_status = 0 download_processing_log(self.debug, url) supported_url, self.exit_status = check_url( url, self.debug, self.exit_status) @@ -148,13 +161,19 @@ def get_fic_with_list(self, list_url: str): self.exit_status = 1 else: - self.exit_status = save_data( + self.exit_status, url_exit_status = save_data( self.out_dir, fic.file_name, fic.download_url, self.debug, self.force, fic.cache_hash, self.exit_status, self.automated) with open("output.log", "a") as file: file.write(f"{url}\n") + + if url_exit_status == 0: + downloaded_urls.append(url) + else: + no_updates_urls.append(url) + pbar.update(1) # Error: 'FicHub' object has no attribute 'file_name' @@ -162,6 +181,7 @@ def get_fic_with_list(self, list_url: str): except AttributeError: with open("err.log", "a") as file: file.write(url.strip()+"\n") + err_urls.append(url) pbar.update(1) self.exit_status = 1 pass # skip the unsupported url @@ -169,12 +189,17 @@ def get_fic_with_list(self, list_url: str): else: # skip the unsupported url with open("err.log", "a") as file: file.write(url.strip()+"\n") + err_urls.append(url) pbar.update(1) continue else: typer.echo(Fore.RED + "No new urls found! If output.log exists, please clear it.") + if self.changelog: + build_changelog(urls_input, urls_input_dedup, urls, + downloaded_urls, err_urls, no_updates_urls, self.out_dir) + def get_fic_with_url(self, url_input: str): if self.debug: @@ -207,11 +232,10 @@ def get_fic_with_url(self, url_input: str): self.exit_status = 1 else: - self.exit_status = save_data( + self.exit_status, _ = save_data( self.out_dir, fic.file_name, fic.download_url, self.debug, self.force, fic.cache_hash, self.exit_status, self.automated) - with open("output.log", "a") as file: file.write(f"{url[0]}\n") pbar.update(1) diff --git a/fichub_cli/utils/fichub.py b/fichub_cli/utils/fichub.py index 6f78106..79b87d1 100644 --- a/fichub_cli/utils/fichub.py +++ b/fichub_cli/utils/fichub.py @@ -112,12 +112,11 @@ def get_fic_metadata(self, url: str, format_type: int): # Reason: Unsupported URL except (KeyError, UnboundLocalError) as e: if self.debug: - logger.error(str(e)) - - self.exit_status = 1 - if self.debug: + logger.error(f"Error: {str(e)} not found!") logger.error( f"Skipping unsupported URL: {url}") + + self.exit_status = 1 tqdm.write( Fore.RED + f"\nSkipping unsupported URL: {url}" + Style.RESET_ALL + Fore.CYAN + diff --git a/fichub_cli/utils/logging.py b/fichub_cli/utils/logging.py index 829bd80..c2bc229 100644 --- a/fichub_cli/utils/logging.py +++ b/fichub_cli/utils/logging.py @@ -62,4 +62,4 @@ def verbose_log(debug: bool, fic): # Error: KeyError: 'meta' # Reason: Unsupported url except KeyError: - pass + pass \ No newline at end of file diff --git a/fichub_cli/utils/processing.py b/fichub_cli/utils/processing.py index fe0c25b..c021e33 100644 --- a/fichub_cli/utils/processing.py +++ b/fichub_cli/utils/processing.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from datetime import datetime from typing import Tuple import re import os @@ -87,18 +88,20 @@ def save_data(out_dir: str, file_name: str, download_url: str, exit_status: int, automated: bool) -> int: ebook_file = os.path.join(out_dir, file_name) + try: hash_flag = check_hash(ebook_file, cache_hash) except FileNotFoundError: hash_flag = False + url_exit_status = 0 if os.path.exists(ebook_file) and force is False and hash_flag is True: - exit_status = 1 + exit_status = url_exit_status = 1 if debug: logger.warning( - "The hash of the local file & the remote file is the same.") + "The md5 hash of the local file & the remote file are the same.") logger.error( f"{ebook_file} is already the latest version. Skipping download. Use --force flag to overwrite.") @@ -127,7 +130,7 @@ def save_data(out_dir: str, file_name: str, download_url: str, tqdm.write(Fore.RED + "Output directory doesn't exist. Exiting!") sys.exit(1) - return exit_status + return exit_status, url_exit_status def check_hash(ebook_file: str, cache_hash: str) -> bool: @@ -137,12 +140,7 @@ def check_hash(ebook_file: str, cache_hash: str) -> bool: ebook_hash = hashlib.md5(data).hexdigest() - if ebook_hash.strip() == cache_hash.strip(): - hash_flag = True - else: - hash_flag = False - - return hash_flag + return ebook_hash.strip() == cache_hash.strip() def out_dir_exists_check(out_dir): @@ -189,7 +187,8 @@ def check_output_log(urls_input, debug): tqdm.write( Fore.BLUE + f"After comparing with output.log, total URLs: {len(urls)}") if debug: - logger.info(f"After comparing with output.log, total URLs: {len(urls)}") + logger.info( + f"After comparing with output.log, total URLs: {len(urls)}") return urls @@ -221,7 +220,7 @@ def urls_preprocessing(urls_input, debug): tqdm.write(Fore.BLUE + f"URLs found: {len(urls_input)}") urls_input_dedup = list(dict.fromkeys(urls_input)) tqdm.write( - Fore.BLUE + f"After Deduplication, total URLs: {len(urls_input_dedup)}") + Fore.BLUE + f"After removing duplicates, total URLs: {len(urls_input_dedup)}") if debug: logger.info(f"URLs found: {len(urls_input)}") @@ -235,4 +234,48 @@ def urls_preprocessing(urls_input, debug): except FileNotFoundError: urls = urls_input_dedup - return urls + return urls, urls_input_dedup + + +def build_changelog(urls_input, urls_input_dedup, urls, downloaded_urls, + err_urls, no_updates_urls, out_dir): + timestamp = datetime.now().strftime("%Y-%m-%d T%H%M%S") + with open(os.path.join(out_dir, f"CHANGELOG - {timestamp}.txt"), 'w') as file: + file.write(f"""# Changelog +Total URLs given as input: {len(urls_input)} +Total URLs after removing duplicates: {len(urls_input_dedup)} +Total URLs after comparing with the output.log: {len(urls)} +Total URLs/Files downloaded: {len(downloaded_urls)} +Total URLs causing Download Errors: {len(err_urls)} +Total URLs without any updates: {len(no_updates_urls)} +""") + + if urls_input: + file.write("\n\n## URLs given as Input") + for url in urls_input: + file.write(f"\n{url}") + + if urls_input_dedup: + file.write("\n\n## URLs after removing duplicates") + for url in urls_input_dedup: + file.write(f"\n{url}") + + if urls: + file.write("\n\n## URLs after comparing with the output.log") + for url in urls: + file.write(f"\n{url}") + + if downloaded_urls: + file.write("\n\n## URLs/Files Downloaded") + for url in downloaded_urls: + file.write(f"\n{url}") + + if err_urls: + file.write("\n\n## URLs causing Download Errors") + for url in err_urls: + file.write(f"\n{url}") + + if no_updates_urls: + file.write("\n\n## URLs without any updates") + for url in no_updates_urls: + file.write(f"\n{url}") diff --git a/setup.py b/setup.py index 6f187b3..13d1982 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ description="A CLI for the fichub.net API", long_description=long_description, long_description_content_type="text/markdown", - version="0.6.4", + version="0.7.0", license='Apache License', url="https://github.com/FicHub/fichub-cli", packages=find_packages(include=['fichub_cli', 'fichub_cli.*']), diff --git a/tests/test_cli.py b/tests/test_cli.py index 8e7b25d..d916317 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -61,4 +61,4 @@ def test_cli_version(): assert not result.exception assert result.exit_code == 0 - assert result.output.strip() == 'fichub-cli: v0.6.4' + assert result.output.strip() == 'fichub-cli: v0.7.0'