Skip to content

Commit

Permalink
Added changelog feature (#16)
Browse files Browse the repository at this point in the history 
Bump version: 0.6.4 → 0.7.0

- Added `--changelog` command which can be used to get a detailed
  log about the download like dedup, errors, updates etc.

- Renamed `--log` to `--debug-log`

- Logging Cleanup
  • Loading branch information
@arzkar
arzkar authored May 2, 2022
1 parent a45f36b commit 317e85b
Show file tree
Hide file tree
Showing 10 changed files with 134 additions and 55 deletions.
2 changes: 1 addition & 1 deletion .bumpversion.cfg
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[bumpversion]
current_version = 0.6.4
current_version = 0.7.0
commit = True
tag = False
parse = ^
Expand Down
36 changes: 23 additions & 13 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,21 +35,25 @@ Usage: fichub_cli [OPTIONS] COMMAND [ARGS]...
https://github.com/FicHub/fichub-cli/issues
Failed downloads will be saved in the `err.log` file in the current
directory.
directory
Options:
-u, --url TEXT The url of the fanfiction enclosed within quotes
-i, --infile TEXT Path to a file to read URLs from
-l, --list-url TEXT Enter a comma separated list of urls to download, enclosed within quotes
-v, --verbose Verbose
-o, --out-dir TEXT Path to the Output directory for files (default: Current Directory)
--format TEXT Download Format: epub (default), mobi, pdf or html [default: epub]
--force / --no-force Force overwrite of an existing file [default: no-force]
-ss, --supported-sites List of supported sites
-d, --debug Show the log in the console for debugging
--log / --no-log Save the logfile for debugging [default: no-log]
--version / --no-version Display version & quit [default: no-version]
--help Show this message and exit.
-u, --url TEXT The url of the fanfiction enclosed within quotes
-i, --infile TEXT Path to a file to read URLs from
-l, --list-url TEXT Enter a comma separated list of urls to download,
enclosed within quotes
-v, --verbose Show fic stats
-o, --out-dir TEXT Path to the Output directory for files (default:
Current Directory)
--format TEXT Download Format: epub (default), mobi, pdf or html
[default: epub]
--force Force overwrite of an existing file
-ss, --supported-sites List of supported sites
-d, --debug Show the log in the console for debugging
--changelog Save the changelog file
--debug-log Save the logfile for debugging
--version Display version & quit
--help Show this message and exit.
```

# Default Configuration
Expand Down Expand Up @@ -80,6 +84,12 @@ fichub_cli -i urls.txt
fichub_cli -l "https://www.fanfiction.net/s/11191235/1/Harry-Potter-and-the-Prince-of-Slytherin,https://www.fanfiction.net/s/13720575/1/A-Cadmean-Victory-Remastered"
```

- To generate a changelog of the download

```
fichub_cli -i urls.txt --changelog
```

---

**NOTE**
Expand Down
2 changes: 1 addition & 1 deletion fichub_cli/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = "0.6.4"
__version__ = "0.7.0"
23 changes: 13 additions & 10 deletions fichub_cli/cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,22 +73,25 @@ def default(
"epub", help="Download Format: epub (default), mobi, pdf or html"),

force: bool = typer.Option(
False, help="Force overwrite of an existing file", is_flag=True),
False, "--force", help="Force overwrite of an existing file", is_flag=True),

supported_sites: bool = typer.Option(
False, "-ss", "--supported-sites", help="List of supported sites", is_flag=True),

debug: bool = typer.Option(
False, "-d", " --debug", help="Show the log in the console for debugging", is_flag=True),

log: bool = typer.Option(
False, help="Save the logfile for debugging", is_flag=True),
changelog: bool = typer.Option(
False, "--changelog", help="Save the changelog file", is_flag=True),

debug_log: bool = typer.Option(
False, "--debug-log", help="Save the logfile for debugging", is_flag=True),

automated: bool = typer.Option(
False, "-a", "--automated", help="For internal testing only", is_flag=True, hidden=True),

version: bool = typer.Option(
False, help="Display version & quit", is_flag=True)
False, "--version", help="Display version & quit", is_flag=True)
):
"""
A CLI for the fichub.net API
Expand All @@ -110,7 +113,7 @@ def default(
Fore.BLUE + "Skipping default command to run sub-command.")
return

if log:
if debug_log:
logger.remove() # remove all existing handlers
logger.add(f"fichub_cli - {timestamp}.log")
debug = True
Expand All @@ -121,13 +124,13 @@ def default(

format_type = get_format_type(format)
if infile:
fic = FetchData(format_type, out_dir, force,
debug, automated, verbose)
fic = FetchData(format_type, out_dir, force, debug, changelog,
automated, verbose)
fic.get_fic_with_infile(infile)

elif list_url:
fic = FetchData(format_type, out_dir, force,
debug, automated, verbose)
fic = FetchData(format_type, out_dir, force, debug, changelog,
automated, verbose)
fic.get_fic_with_list(list_url)

elif url:
Expand Down Expand Up @@ -170,7 +173,7 @@ def default(
if fic.exit_status == 1:
typer.echo(
Fore.RED +
"\nDownload failed for one or more URLs! Check " + Style.RESET_ALL +
"\nThe CLI ran into some errors! Check " + Style.RESET_ALL +
Fore.YELLOW + "err.log" + Style.RESET_ALL + Fore.RED +
" in the current directory!" + Style.RESET_ALL)

Expand Down
46 changes: 35 additions & 11 deletions fichub_cli/utils/fetch_data.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,18 +21,19 @@
from .logging import init_log, download_processing_log, \
verbose_log
from .processing import check_url, save_data, \
urls_preprocessing, check_output_log
urls_preprocessing, check_output_log, build_changelog

bar_format = "{l_bar}{bar}| {n_fmt}/{total_fmt}, {rate_fmt}{postfix}, ETA: {remaining}"


class FetchData:
def __init__(self, format_type=0, out_dir="", force=False,
debug=False, automated=False, verbose=False):
debug=False, changelog=False, automated=False, verbose=False):
self.format_type = format_type
self.out_dir = out_dir
self.force = force
self.debug = debug
self.changelog = changelog
self.automated = automated
self.exit_status = 0
self.verbose = verbose
Expand All @@ -57,15 +58,15 @@ def get_fic_with_infile(self, infile: str):
f"{infile} file could not be found. Please enter a valid file path.")
exit(1)

urls = urls_preprocessing(urls_input, self.debug)

urls, urls_input_dedup = urls_preprocessing(urls_input, self.debug)
downloaded_urls, no_updates_urls, err_urls = [], [], []
if urls:
init_log(self.debug, self.force)
with tqdm(total=len(urls), ascii=False,
unit="file", bar_format=bar_format) as pbar:

for url in urls:

url_exit_status = 0
download_processing_log(self.debug, url)
supported_url, self.exit_status = check_url(
url, self.debug, self.exit_status)
Expand All @@ -85,49 +86,61 @@ def get_fic_with_infile(self, infile: str):
self.exit_status = 1

else:
self.exit_status = save_data(
self.exit_status, url_exit_status = save_data(
self.out_dir, fic.file_name,
fic.download_url, self.debug, self.force,
fic.cache_hash, self.exit_status,
self.automated)

with open("output.log", "a") as file:
file.write(f"{url}\n")

if url_exit_status == 0:
downloaded_urls.append(url)
else:
no_updates_urls.append(url)
pbar.update(1)

# Error: 'FicHub' object has no attribute 'file_name'
# Reason: Unsupported URL
except AttributeError:
with open("err.log", "a") as file:
file.write(url.strip()+"\n")
err_urls.append(url)
pbar.update(1)
self.exit_status = 1
pass # skip the unsupported url

else: # skip the unsupported url
with open("err.log", "a") as file:
file.write(url.strip()+"\n")
err_urls.append(url)
pbar.update(1)
continue

else:
typer.echo(Fore.RED +
"No new urls found! If output.log exists, please clear it.")

if self.changelog:
build_changelog(urls_input, urls_input_dedup, urls,
downloaded_urls, err_urls, no_updates_urls, self.out_dir)

def get_fic_with_list(self, list_url: str):

if self.debug:
logger.info("-l flag used!")

urls_input = list_url.split(",")
urls = urls_preprocessing(urls_input, self.debug)

urls, urls_input_dedup = urls_preprocessing(urls_input, self.debug)
downloaded_urls, no_updates_urls, err_urls = [], [], []
if urls:
init_log(self.debug, self.force)
with tqdm(total=len(urls), ascii=False,
unit="file", bar_format=bar_format) as pbar:

for url in urls:
url_exit_status = 0
download_processing_log(self.debug, url)
supported_url, self.exit_status = check_url(
url, self.debug, self.exit_status)
Expand All @@ -148,33 +161,45 @@ def get_fic_with_list(self, list_url: str):
self.exit_status = 1

else:
self.exit_status = save_data(
self.exit_status, url_exit_status = save_data(
self.out_dir, fic.file_name,
fic.download_url, self.debug, self.force,
fic.cache_hash, self.exit_status, self.automated)

with open("output.log", "a") as file:
file.write(f"{url}\n")

if url_exit_status == 0:
downloaded_urls.append(url)
else:
no_updates_urls.append(url)

pbar.update(1)

# Error: 'FicHub' object has no attribute 'file_name'
# Reason: Unsupported URL
except AttributeError:
with open("err.log", "a") as file:
file.write(url.strip()+"\n")
err_urls.append(url)
pbar.update(1)
self.exit_status = 1
pass # skip the unsupported url

else: # skip the unsupported url
with open("err.log", "a") as file:
file.write(url.strip()+"\n")
err_urls.append(url)
pbar.update(1)
continue
else:
typer.echo(Fore.RED +
"No new urls found! If output.log exists, please clear it.")

if self.changelog:
build_changelog(urls_input, urls_input_dedup, urls,
downloaded_urls, err_urls, no_updates_urls, self.out_dir)

def get_fic_with_url(self, url_input: str):

if self.debug:
Expand Down Expand Up @@ -207,11 +232,10 @@ def get_fic_with_url(self, url_input: str):
self.exit_status = 1

else:
self.exit_status = save_data(
self.exit_status, _ = save_data(
self.out_dir, fic.file_name,
fic.download_url, self.debug, self.force,
fic.cache_hash, self.exit_status, self.automated)

with open("output.log", "a") as file:
file.write(f"{url[0]}\n")
pbar.update(1)
Expand Down
7 changes: 3 additions & 4 deletions fichub_cli/utils/fichub.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,12 +112,11 @@ def get_fic_metadata(self, url: str, format_type: int):
# Reason: Unsupported URL
except (KeyError, UnboundLocalError) as e:
if self.debug:
logger.error(str(e))

self.exit_status = 1
if self.debug:
logger.error(f"Error: {str(e)} not found!")
logger.error(
f"Skipping unsupported URL: {url}")

self.exit_status = 1
tqdm.write(
Fore.RED + f"\nSkipping unsupported URL: {url}" +
Style.RESET_ALL + Fore.CYAN +
Expand Down
2 changes: 1 addition & 1 deletion fichub_cli/utils/logging.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,4 +62,4 @@ def verbose_log(debug: bool, fic):
# Error: KeyError: 'meta'
# Reason: Unsupported url
except KeyError:
pass
pass
Loading

0 comments on commit 317e85b

Please sign in to comment.