Added datetime format config & the ability to stop the CLI mid-exec a…

…nd generate the changelog (#17)

* Bugfix: CLI is not counting the stories that errored out

The errored out URLs were being written to `err.log`, not the `output.log`
and when the CLI was restarted and it checked `output.log`, instead of
skipping error URLS, it was trying to download them again.

Fixed by adding `err.log` to the `check_output()` function so err.log
was also checked along with `output.log`

* Added config for db and fic last_updated format

The CLI will read the datetime format from the `config.json` which
gives the users the option to change the format from default to whatever
they want.

* Added the ability to stop the CLI mid-exec and generate the changelog

Added a `KeyboardInterrupt` exception which ignores the exception then
builds the changelog instead of quiting the CLI due to the exception.

* Bump version: 0.7.1 → 0.8.0

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.7.1
+current_version = 0.8.0
 commit = True
 tag = False
 parse = ^
@@ -19,6 +19,8 @@ values =
 	rc
 
 [bumpversion:file:setup.py]
+search = version="{current_version}"
+replace = version="{new_version}"
 
 [bumpversion:file:fichub_cli/__init__.py]
 search = __version__ = "{current_version}"

README.md

@@ -52,6 +52,8 @@ Options:
   -d,  --debug            Show the log in the console for debugging
   --changelog             Save the changelog file
   --debug-log             Save the logfile for debugging
+  --config-init           Initialize the CLI config files
+  --config-info           Show the CLI config info
   --version               Display version & quit
   --help                  Show this message and exit.
 ```
@@ -93,7 +95,12 @@ fichub_cli -i urls.txt --changelog
 ---
 
 **NOTE**
-`--out-dir` or `-o` can be used in all the above commands to select an output directory.
+
+- `--out-dir` or `-o` can be used in all the above commands to select an output directory.
+
+- Using the `--config-init` flag, users can re-initialize/overwrite the config files to default.
+
+- Using the `--config-info` flag, users can get all the info about the config file and its settings.
 
 ---
 

fichub_cli/__init__.py

@@ -1 +1 @@
-__version__ = "0.7.1"
+__version__ = "0.8.0"

fichub_cli/cli.py

@@ -25,7 +25,7 @@
 
 from .utils.fetch_data import FetchData
 from .utils.processing import get_format_type, out_dir_exists_check, \
-    appdir_exists_check, check_cli_outdated
+    appdir_exists_check, appdir_builder, appdir_config_info, check_cli_outdated
 from fichub_cli import __version__
 
 init(autoreset=True)  # colorama init
@@ -46,6 +46,7 @@
 
 # check if app directory exists, if not, create it
 appdir_exists_check(app_dirs)
+
 # check if the cli is outdated
 check_cli_outdated("fichub-cli", __version__)
 
@@ -87,6 +88,12 @@ def default(
     debug_log: bool = typer.Option(
         False, "--debug-log", help="Save the logfile for debugging", is_flag=True),
 
+    config_init: bool = typer.Option(
+        False, "--config-init", help="Initialize the CLI config files", is_flag=True),
+
+    config_info: bool = typer.Option(
+        False, "--config-info", help="Show the CLI config info", is_flag=True),
+
     automated: bool = typer.Option(
         False, "-a", "--automated", help="For internal testing only", is_flag=True, hidden=True),
 
@@ -103,6 +110,14 @@ def default(
     Failed downloads will be saved in the `err.log` file in the current directory
     """
 
+    if config_init:
+        # initialize/overwrite the config files
+        appdir_builder(app_dirs)
+
+    if config_info:
+        # overwrite the config files
+        appdir_config_info(app_dirs)
+
     # Check if the output directory exists if input is given
     if not out_dir == "":
         out_dir_exists_check(out_dir)

fichub_cli/utils/fetch_data.py

@@ -60,71 +60,77 @@ def get_fic_with_infile(self, infile: str):
 
         urls, urls_input_dedup = urls_preprocessing(urls_input, self.debug)
         downloaded_urls, no_updates_urls, err_urls = [], [], []
-        if urls:
-            init_log(self.debug, self.force)
-            with tqdm(total=len(urls), ascii=False,
-                      unit="file", bar_format=bar_format) as pbar:
-
-                for url in urls:
-                    url_exit_status = 0
-                    download_processing_log(self.debug, url)
-                    supported_url, self.exit_status = check_url(
-                        url, self.debug, self.exit_status)
-                    if supported_url:
-                        try:
-                            fic = FicHub(self.debug, self.automated,
-                                         self.exit_status)
-                            fic.get_fic_metadata(url, self.format_type)
-
-                            if self.verbose:
-                                verbose_log(self.debug, fic)
-
-                            # update the exit status
-                            self.exit_status = fic.exit_status
 
-                            if fic.file_name is None:
-                                self.exit_status = 1
-
-                            else:
-                                self.exit_status, url_exit_status = save_data(
-                                    self.out_dir, fic.file_name,
-                                    fic.download_url, self.debug, self.force,
-                                    fic.cache_hash, self.exit_status,
-                                    self.automated)
-
-                                with open("output.log", "a") as file:
-                                    file.write(f"{url}\n")
+        try:
+            if urls:
+                init_log(self.debug, self.force)
+                with tqdm(total=len(urls), ascii=False,
+                          unit="file", bar_format=bar_format) as pbar:
+
+                    for url in urls:
+                        url_exit_status = 0
+                        download_processing_log(self.debug, url)
+                        supported_url, self.exit_status = check_url(
+                            url, self.debug, self.exit_status)
+                        if supported_url:
+                            try:
+                                fic = FicHub(self.debug, self.automated,
+                                             self.exit_status)
+                                fic.get_fic_metadata(url, self.format_type)
+
+                                if self.verbose:
+                                    verbose_log(self.debug, fic)
+
+                                # update the exit status
+                                self.exit_status = fic.exit_status
+
+                                if fic.file_name is None:
+                                    self.exit_status = 1
 
-                                if url_exit_status == 0:
-                                    downloaded_urls.append(url)
                                 else:
-                                    no_updates_urls.append(url)
-                            pbar.update(1)
+                                    self.exit_status, url_exit_status = save_data(
+                                        self.out_dir, fic.file_name,
+                                        fic.download_url, self.debug, self.force,
+                                        fic.cache_hash, self.exit_status,
+                                        self.automated)
+
+                                    with open("output.log", "a") as file:
+                                        file.write(f"{url}\n")
+
+                                    if url_exit_status == 0:
+                                        downloaded_urls.append(url)
+                                    else:
+                                        no_updates_urls.append(url)
+                                pbar.update(1)
+
+                            # Error: 'FicHub' object has no attribute 'file_name'
+                            # Reason: Unsupported URL
+                            except AttributeError:
+                                with open("err.log", "a") as file:
+                                    file.write(url.strip()+"\n")
+                                err_urls.append(url)
+                                pbar.update(1)
+                                self.exit_status = 1
+                                pass  # skip the unsupported url
 
-                        # Error: 'FicHub' object has no attribute 'file_name'
-                        # Reason: Unsupported URL
-                        except AttributeError:
+                        else:  # skip the unsupported url
                             with open("err.log", "a") as file:
                                 file.write(url.strip()+"\n")
                             err_urls.append(url)
                             pbar.update(1)
-                            self.exit_status = 1
-                            pass  # skip the unsupported url
+                            continue
 
-                    else:  # skip the unsupported url
-                        with open("err.log", "a") as file:
-                            file.write(url.strip()+"\n")
-                        err_urls.append(url)
-                        pbar.update(1)
-                        continue
+            else:
+                typer.echo(Fore.RED +
+                           "No new urls found! If output.log exists, please clear it.")
 
-        else:
-            typer.echo(Fore.RED +
-                       "No new urls found! If output.log exists, please clear it.")
+        except KeyboardInterrupt:
+            pass
 
-        if self.changelog:
-            build_changelog(urls_input, urls_input_dedup, urls,
-                            downloaded_urls, err_urls, no_updates_urls, self.out_dir)
+        finally:
+            if self.changelog:
+                build_changelog(urls_input, urls_input_dedup, urls,
+                                downloaded_urls, err_urls, no_updates_urls, self.out_dir)
 
     def get_fic_with_list(self, list_url: str):
 
@@ -134,71 +140,77 @@ def get_fic_with_list(self, list_url: str):
         urls_input = list_url.split(",")
         urls, urls_input_dedup = urls_preprocessing(urls_input, self.debug)
         downloaded_urls, no_updates_urls, err_urls = [], [], []
-        if urls:
-            init_log(self.debug, self.force)
-            with tqdm(total=len(urls), ascii=False,
-                      unit="file", bar_format=bar_format) as pbar:
 
-                for url in urls:
-                    url_exit_status = 0
-                    download_processing_log(self.debug, url)
-                    supported_url,  self.exit_status = check_url(
-                        url, self.debug, self.exit_status)
-
-                    if supported_url:
-                        try:
-                            fic = FicHub(self.debug, self.automated,
-                                         self.exit_status)
-                            fic.get_fic_metadata(url, self.format_type)
+        try:
+            if urls:
+                init_log(self.debug, self.force)
+                with tqdm(total=len(urls), ascii=False,
+                          unit="file", bar_format=bar_format) as pbar:
 
-                            if self.verbose:
-                                verbose_log(self.debug, fic)
+                    for url in urls:
+                        url_exit_status = 0
+                        download_processing_log(self.debug, url)
+                        supported_url,  self.exit_status = check_url(
+                            url, self.debug, self.exit_status)
 
-                            # update the exit status
-                            self.exit_status = fic.exit_status
+                        if supported_url:
+                            try:
+                                fic = FicHub(self.debug, self.automated,
+                                             self.exit_status)
+                                fic.get_fic_metadata(url, self.format_type)
 
-                            if fic.file_name is None:
-                                self.exit_status = 1
+                                if self.verbose:
+                                    verbose_log(self.debug, fic)
 
-                            else:
-                                self.exit_status, url_exit_status = save_data(
-                                    self.out_dir, fic.file_name,
-                                    fic.download_url, self.debug, self.force,
-                                    fic.cache_hash, self.exit_status, self.automated)
+                                # update the exit status
+                                self.exit_status = fic.exit_status
 
-                                with open("output.log", "a") as file:
-                                    file.write(f"{url}\n")
+                                if fic.file_name is None:
+                                    self.exit_status = 1
 
-                                if url_exit_status == 0:
-                                    downloaded_urls.append(url)
                                 else:
-                                    no_updates_urls.append(url)
-
-                            pbar.update(1)
+                                    self.exit_status, url_exit_status = save_data(
+                                        self.out_dir, fic.file_name,
+                                        fic.download_url, self.debug, self.force,
+                                        fic.cache_hash, self.exit_status, self.automated)
+
+                                    with open("output.log", "a") as file:
+                                        file.write(f"{url}\n")
+
+                                    if url_exit_status == 0:
+                                        downloaded_urls.append(url)
+                                    else:
+                                        no_updates_urls.append(url)
+
+                                pbar.update(1)
+
+                            # Error: 'FicHub' object has no attribute 'file_name'
+                            # Reason: Unsupported URL
+                            except AttributeError:
+                                with open("err.log", "a") as file:
+                                    file.write(url.strip()+"\n")
+                                err_urls.append(url)
+                                pbar.update(1)
+                                self.exit_status = 1
+                                pass  # skip the unsupported url
 
-                        # Error: 'FicHub' object has no attribute 'file_name'
-                        # Reason: Unsupported URL
-                        except AttributeError:
+                        else:  # skip the unsupported url
                             with open("err.log", "a") as file:
                                 file.write(url.strip()+"\n")
                             err_urls.append(url)
                             pbar.update(1)
-                            self.exit_status = 1
-                            pass  # skip the unsupported url
-
-                    else:  # skip the unsupported url
-                        with open("err.log", "a") as file:
-                            file.write(url.strip()+"\n")
-                        err_urls.append(url)
-                        pbar.update(1)
-                        continue
-        else:
-            typer.echo(Fore.RED +
-                       "No new urls found! If output.log exists, please clear it.")
-
-        if self.changelog:
-            build_changelog(urls_input, urls_input_dedup, urls,
-                            downloaded_urls, err_urls, no_updates_urls, self.out_dir)
+                            continue
+            else:
+                typer.echo(Fore.RED +
+                           "No new urls found! If output.log exists, please clear it.")
+
+        except KeyboardInterrupt:
+            pass
+
+        finally:
+            if self.changelog:
+                build_changelog(urls_input, urls_input_dedup, urls,
+                                downloaded_urls, err_urls, no_updates_urls, self.out_dir)
 
     def get_fic_with_url(self, url_input: str):