diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 460cc3a..5e1fd86 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.7.1 +current_version = 0.8.0 commit = True tag = False parse = ^ @@ -19,6 +19,8 @@ values = rc [bumpversion:file:setup.py] +search = version="{current_version}" +replace = version="{new_version}" [bumpversion:file:fichub_cli/__init__.py] search = __version__ = "{current_version}" diff --git a/README.md b/README.md index 9612a8f..19bf73c 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,8 @@ Options: -d, --debug Show the log in the console for debugging --changelog Save the changelog file --debug-log Save the logfile for debugging + --config-init Initialize the CLI config files + --config-info Show the CLI config info --version Display version & quit --help Show this message and exit. ``` @@ -93,7 +95,12 @@ fichub_cli -i urls.txt --changelog --- **NOTE** -`--out-dir` or `-o` can be used in all the above commands to select an output directory. + +- `--out-dir` or `-o` can be used in all the above commands to select an output directory. + +- Using the `--config-init` flag, users can re-initialize/overwrite the config files to default. + +- Using the `--config-info` flag, users can get all the info about the config file and its settings. --- diff --git a/fichub_cli/__init__.py b/fichub_cli/__init__.py index a5f830a..777f190 100644 --- a/fichub_cli/__init__.py +++ b/fichub_cli/__init__.py @@ -1 +1 @@ -__version__ = "0.7.1" +__version__ = "0.8.0" diff --git a/fichub_cli/cli.py b/fichub_cli/cli.py index bf7447b..231d0db 100644 --- a/fichub_cli/cli.py +++ b/fichub_cli/cli.py @@ -25,7 +25,7 @@ from .utils.fetch_data import FetchData from .utils.processing import get_format_type, out_dir_exists_check, \ - appdir_exists_check, check_cli_outdated + appdir_exists_check, appdir_builder, appdir_config_info, check_cli_outdated from fichub_cli import __version__ init(autoreset=True) # colorama init @@ -46,6 +46,7 @@ # check if app directory exists, if not, create it appdir_exists_check(app_dirs) + # check if the cli is outdated check_cli_outdated("fichub-cli", __version__) @@ -87,6 +88,12 @@ def default( debug_log: bool = typer.Option( False, "--debug-log", help="Save the logfile for debugging", is_flag=True), + config_init: bool = typer.Option( + False, "--config-init", help="Initialize the CLI config files", is_flag=True), + + config_info: bool = typer.Option( + False, "--config-info", help="Show the CLI config info", is_flag=True), + automated: bool = typer.Option( False, "-a", "--automated", help="For internal testing only", is_flag=True, hidden=True), @@ -103,6 +110,14 @@ def default( Failed downloads will be saved in the `err.log` file in the current directory """ + if config_init: + # initialize/overwrite the config files + appdir_builder(app_dirs) + + if config_info: + # overwrite the config files + appdir_config_info(app_dirs) + # Check if the output directory exists if input is given if not out_dir == "": out_dir_exists_check(out_dir) diff --git a/fichub_cli/utils/fetch_data.py b/fichub_cli/utils/fetch_data.py index 6beac6a..1d84a47 100644 --- a/fichub_cli/utils/fetch_data.py +++ b/fichub_cli/utils/fetch_data.py @@ -60,71 +60,77 @@ def get_fic_with_infile(self, infile: str): urls, urls_input_dedup = urls_preprocessing(urls_input, self.debug) downloaded_urls, no_updates_urls, err_urls = [], [], [] - if urls: - init_log(self.debug, self.force) - with tqdm(total=len(urls), ascii=False, - unit="file", bar_format=bar_format) as pbar: - - for url in urls: - url_exit_status = 0 - download_processing_log(self.debug, url) - supported_url, self.exit_status = check_url( - url, self.debug, self.exit_status) - if supported_url: - try: - fic = FicHub(self.debug, self.automated, - self.exit_status) - fic.get_fic_metadata(url, self.format_type) - - if self.verbose: - verbose_log(self.debug, fic) - - # update the exit status - self.exit_status = fic.exit_status - if fic.file_name is None: - self.exit_status = 1 - - else: - self.exit_status, url_exit_status = save_data( - self.out_dir, fic.file_name, - fic.download_url, self.debug, self.force, - fic.cache_hash, self.exit_status, - self.automated) - - with open("output.log", "a") as file: - file.write(f"{url}\n") + try: + if urls: + init_log(self.debug, self.force) + with tqdm(total=len(urls), ascii=False, + unit="file", bar_format=bar_format) as pbar: + + for url in urls: + url_exit_status = 0 + download_processing_log(self.debug, url) + supported_url, self.exit_status = check_url( + url, self.debug, self.exit_status) + if supported_url: + try: + fic = FicHub(self.debug, self.automated, + self.exit_status) + fic.get_fic_metadata(url, self.format_type) + + if self.verbose: + verbose_log(self.debug, fic) + + # update the exit status + self.exit_status = fic.exit_status + + if fic.file_name is None: + self.exit_status = 1 - if url_exit_status == 0: - downloaded_urls.append(url) else: - no_updates_urls.append(url) - pbar.update(1) + self.exit_status, url_exit_status = save_data( + self.out_dir, fic.file_name, + fic.download_url, self.debug, self.force, + fic.cache_hash, self.exit_status, + self.automated) + + with open("output.log", "a") as file: + file.write(f"{url}\n") + + if url_exit_status == 0: + downloaded_urls.append(url) + else: + no_updates_urls.append(url) + pbar.update(1) + + # Error: 'FicHub' object has no attribute 'file_name' + # Reason: Unsupported URL + except AttributeError: + with open("err.log", "a") as file: + file.write(url.strip()+"\n") + err_urls.append(url) + pbar.update(1) + self.exit_status = 1 + pass # skip the unsupported url - # Error: 'FicHub' object has no attribute 'file_name' - # Reason: Unsupported URL - except AttributeError: + else: # skip the unsupported url with open("err.log", "a") as file: file.write(url.strip()+"\n") err_urls.append(url) pbar.update(1) - self.exit_status = 1 - pass # skip the unsupported url + continue - else: # skip the unsupported url - with open("err.log", "a") as file: - file.write(url.strip()+"\n") - err_urls.append(url) - pbar.update(1) - continue + else: + typer.echo(Fore.RED + + "No new urls found! If output.log exists, please clear it.") - else: - typer.echo(Fore.RED + - "No new urls found! If output.log exists, please clear it.") + except KeyboardInterrupt: + pass - if self.changelog: - build_changelog(urls_input, urls_input_dedup, urls, - downloaded_urls, err_urls, no_updates_urls, self.out_dir) + finally: + if self.changelog: + build_changelog(urls_input, urls_input_dedup, urls, + downloaded_urls, err_urls, no_updates_urls, self.out_dir) def get_fic_with_list(self, list_url: str): @@ -134,71 +140,77 @@ def get_fic_with_list(self, list_url: str): urls_input = list_url.split(",") urls, urls_input_dedup = urls_preprocessing(urls_input, self.debug) downloaded_urls, no_updates_urls, err_urls = [], [], [] - if urls: - init_log(self.debug, self.force) - with tqdm(total=len(urls), ascii=False, - unit="file", bar_format=bar_format) as pbar: - for url in urls: - url_exit_status = 0 - download_processing_log(self.debug, url) - supported_url, self.exit_status = check_url( - url, self.debug, self.exit_status) - - if supported_url: - try: - fic = FicHub(self.debug, self.automated, - self.exit_status) - fic.get_fic_metadata(url, self.format_type) + try: + if urls: + init_log(self.debug, self.force) + with tqdm(total=len(urls), ascii=False, + unit="file", bar_format=bar_format) as pbar: - if self.verbose: - verbose_log(self.debug, fic) + for url in urls: + url_exit_status = 0 + download_processing_log(self.debug, url) + supported_url, self.exit_status = check_url( + url, self.debug, self.exit_status) - # update the exit status - self.exit_status = fic.exit_status + if supported_url: + try: + fic = FicHub(self.debug, self.automated, + self.exit_status) + fic.get_fic_metadata(url, self.format_type) - if fic.file_name is None: - self.exit_status = 1 + if self.verbose: + verbose_log(self.debug, fic) - else: - self.exit_status, url_exit_status = save_data( - self.out_dir, fic.file_name, - fic.download_url, self.debug, self.force, - fic.cache_hash, self.exit_status, self.automated) + # update the exit status + self.exit_status = fic.exit_status - with open("output.log", "a") as file: - file.write(f"{url}\n") + if fic.file_name is None: + self.exit_status = 1 - if url_exit_status == 0: - downloaded_urls.append(url) else: - no_updates_urls.append(url) - - pbar.update(1) + self.exit_status, url_exit_status = save_data( + self.out_dir, fic.file_name, + fic.download_url, self.debug, self.force, + fic.cache_hash, self.exit_status, self.automated) + + with open("output.log", "a") as file: + file.write(f"{url}\n") + + if url_exit_status == 0: + downloaded_urls.append(url) + else: + no_updates_urls.append(url) + + pbar.update(1) + + # Error: 'FicHub' object has no attribute 'file_name' + # Reason: Unsupported URL + except AttributeError: + with open("err.log", "a") as file: + file.write(url.strip()+"\n") + err_urls.append(url) + pbar.update(1) + self.exit_status = 1 + pass # skip the unsupported url - # Error: 'FicHub' object has no attribute 'file_name' - # Reason: Unsupported URL - except AttributeError: + else: # skip the unsupported url with open("err.log", "a") as file: file.write(url.strip()+"\n") err_urls.append(url) pbar.update(1) - self.exit_status = 1 - pass # skip the unsupported url - - else: # skip the unsupported url - with open("err.log", "a") as file: - file.write(url.strip()+"\n") - err_urls.append(url) - pbar.update(1) - continue - else: - typer.echo(Fore.RED + - "No new urls found! If output.log exists, please clear it.") - - if self.changelog: - build_changelog(urls_input, urls_input_dedup, urls, - downloaded_urls, err_urls, no_updates_urls, self.out_dir) + continue + else: + typer.echo(Fore.RED + + "No new urls found! If output.log exists, please clear it.") + + except KeyboardInterrupt: + pass + + finally: + if self.changelog: + build_changelog(urls_input, urls_input_dedup, urls, + downloaded_urls, err_urls, no_updates_urls, self.out_dir) def get_fic_with_url(self, url_input: str): diff --git a/fichub_cli/utils/processing.py b/fichub_cli/utils/processing.py index c021e33..b1ff8bb 100644 --- a/fichub_cli/utils/processing.py +++ b/fichub_cli/utils/processing.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json from datetime import datetime from typing import Tuple import re @@ -153,12 +154,36 @@ def out_dir_exists_check(out_dir): os.makedirs(out_dir) +def appdir_builder(app_dirs): + tqdm.write( + f"Creating App directory: {app_dirs.user_data_dir}") + os.makedirs(app_dirs.user_data_dir, exist_ok=True) + + tqdm.write( + f"Building the config file: {os.path.join(app_dirs.user_data_dir, 'config.json')}") + config = {'db_up_time_format': r'%Y-%m-%dT%H:%M:%S%z', + 'fic_up_time_format': r'%Y-%m-%dT%H:%M:%S'} + with open(os.path.join(app_dirs.user_data_dir, "config.json"), 'w') as f: + json.dump(config, f) + + def appdir_exists_check(app_dirs): """Check if the app directory exists, if not, create it.""" if not os.path.isdir(app_dirs.user_data_dir): - tqdm.write( - f"App directory: {app_dirs.user_data_dir} does not exist. Running makedirs.") - os.makedirs(app_dirs.user_data_dir) + appdir_builder(app_dirs) + + +def appdir_config_info(app_dirs): + tqdm.write(f"App directory: {app_dirs.user_data_dir}") + tqdm.write( + f"CLI Config file: {os.path.join(app_dirs.user_data_dir, 'config.json')}") + + tqdm.write("\nConfig settings:") + with open(os.path.join(app_dirs.user_data_dir, 'config.json'), "r") as f: + config = json.load(f) + + for key, value in config.items(): + tqdm.write(f"{key}: {value}") def list_diff(urls_input, urls_output): @@ -171,14 +196,18 @@ def list_diff(urls_input, urls_output): def check_output_log(urls_input, debug): if debug: - logger.info("Checking output.log") + logger.info("Checking output.log and err.log") try: - urls_output = [] + urls_list = [] if os.path.exists("output.log"): with open("output.log", "r") as f: - urls_output = f.read().splitlines() + urls_list = f.read().splitlines() + + if os.path.exists("err.log"): + with open("err.log", "r") as f: + urls_list.extend(f.read().splitlines()) - urls = list_diff(urls_input, urls_output) + urls = list_diff(urls_input, urls_list) # if output.log doesnt exist, when run 1st time except FileNotFoundError: diff --git a/setup.py b/setup.py index 92f58e0..0e7062b 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ description="A CLI for the fichub.net API", long_description=long_description, long_description_content_type="text/markdown", - version="0.7.1", + version="0.8.0", license='Apache License', url="https://github.com/FicHub/fichub-cli", packages=find_packages(include=['fichub_cli', 'fichub_cli.*']), diff --git a/tests/test_cli.py b/tests/test_cli.py index c6d293f..be315db 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -61,4 +61,4 @@ def test_cli_version(): assert not result.exception assert result.exit_code == 0 - assert result.output.strip() == 'fichub-cli: v0.7.1' + assert result.output.strip() == 'fichub-cli: v0.8.0'