diff --git a/README.md b/README.md index 54211c4..9a77263 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,10 @@ Currently `mloader` supports these commands ```bash Usage: mloader [OPTIONS] [URLS]... + Command-line tool to download manga from mangaplus + Options: + --version Show the version and exit. -o, --out Save directory (not a file) [default: mloader_downloads] -r, --raw Save raw images [default: False] @@ -48,5 +51,12 @@ Options: -s, --split Split combined images [default: False] -c, --chapter INTEGER Chapter id -t, --title INTEGER Title id + -b, --begin INTEGER RANGE Minimal chapter to try to download + [default: 0;x>=0] + -e, --end INTEGER RANGE Maximal chapter to try to download [x>=1] + -l, --last Download only the last chapter for title + [default: False] + --chapter-title Include chapter titles in filenames + [default: False] --help Show this message and exit. ``` \ No newline at end of file diff --git a/mloader/__main__.py b/mloader/__main__.py index 4bfd1da..82af7d6 100644 --- a/mloader/__main__.py +++ b/mloader/__main__.py @@ -1,6 +1,7 @@ import logging import re import sys +from functools import partial from typing import Optional, Set import click @@ -145,6 +146,35 @@ def validate_ids(ctx: click.Context, param, value): expose_value=False, callback=validate_ids, ) +@click.option( + "--begin", + "-b", + type=click.IntRange(min=0), + default=0, + show_default=True, + help="Minimal chapter to try to download", +) +@click.option( + "--end", + "-e", + type=click.IntRange(min=1), + help="Maximal chapter to try to download", +) +@click.option( + "--last", + "-l", + is_flag=True, + default=False, + show_default=True, + help="Download only the last chapter for title", +) +@click.option( + "--chapter-title", + is_flag=True, + default=False, + show_default=True, + help="Include chapter titles in filenames", +) @click.argument("urls", nargs=-1, callback=validate_urls, expose_value=False) @click.pass_context def main( @@ -153,6 +183,10 @@ def main( raw: bool, quality: str, split: bool, + begin: int, + end: int, + last: bool, + chapter_title: bool, chapters: Optional[Set[int]] = None, titles: Optional[Set[int]] = None, ): @@ -160,11 +194,23 @@ def main( if not any((chapters, titles)): click.echo(ctx.get_help()) return + end = end or float("inf") log.info("Started export") - loader = MangaLoader(RawExporter if raw else CBZExporter, quality, split) + exporter = RawExporter if raw else CBZExporter + exporter = partial( + exporter, destination=out_dir, add_chapter_title=chapter_title + ) + + loader = MangaLoader(exporter, quality, split) try: - loader.download(title_ids=titles, chapter_ids=chapters, dst=out_dir) + loader.download( + title_ids=titles, + chapter_ids=chapters, + min_chapter=begin, + max_chapter=end, + last_chapter=last, + ) except Exception: log.exception("Failed to download manga") log.info("SUCCESS") diff --git a/mloader/__version__.py b/mloader/__version__.py index 57c882f..383345f 100644 --- a/mloader/__version__.py +++ b/mloader/__version__.py @@ -9,5 +9,5 @@ __title__ = "mloader" __description__ = "Command-line tool to download manga from mangaplus" __url__ = "https://github.com/hurlenko/mloader" -__version__ = "1.1.5" +__version__ = "1.1.6" __license__ = "GPLv3" diff --git a/mloader/constants.py b/mloader/constants.py new file mode 100644 index 0000000..d47496b --- /dev/null +++ b/mloader/constants.py @@ -0,0 +1,23 @@ +from enum import Enum + + +class Language(Enum): + eng = 0 + spa = 1 + ind = 3 + por = 4 + rus = 5 + tha = 6 + + +class ChapterType(Enum): + latest = 0 + sequence = 1 + nosequence = 2 + + +class PageType(Enum): + single = 0 + left = 1 + right = 2 + double = 3 diff --git a/mloader/exporter.py b/mloader/exporter.py index 1b4b423..7a4c4db 100644 --- a/mloader/exporter.py +++ b/mloader/exporter.py @@ -1,18 +1,12 @@ -import re -import string import zipfile from abc import ABCMeta, abstractmethod -from enum import Enum from itertools import chain from pathlib import Path from typing import Union, Optional +from mloader.constants import Language from mloader.response_pb2 import Title, Chapter - - -class Language(Enum): - eng = 0 - spa = 1 +from mloader.utils import escape_path, is_oneshot, chapter_name_to_int class ExporterBase(metaclass=ABCMeta): @@ -22,10 +16,12 @@ def __init__( title: Title, chapter: Chapter, next_chapter: Optional[Chapter] = None, + add_chapter_title: bool = False, ): self.destination = destination - self.title_name = self.escape_path(title.name).title() - self.is_oneshot = self._is_oneshot(chapter.name, chapter.sub_title) + self.add_chapter_title = add_chapter_title + self.title_name = escape_path(title.name).title() + self.is_oneshot = is_oneshot(chapter.name, chapter.sub_title) self.is_extra = chapter.name == "ex" self._extra_info = [] @@ -33,8 +29,8 @@ def __init__( if self.is_oneshot: self._extra_info.append("[Oneshot]") - if self.is_extra: - self._extra_info.append(f"[{self.escape_path(chapter.sub_title)}]") + if self.is_extra or self.add_chapter_title: + self._extra_info.append(f"[{escape_path(chapter.sub_title)}]") self._chapter_prefix = self._format_chapter_prefix( self.title_name, @@ -47,19 +43,6 @@ def __init__( (self._chapter_prefix, self._chapter_suffix) ) - def _is_oneshot(self, chapter_name: str, chapter_subtitle: str) -> bool: - for name in (chapter_name, chapter_subtitle): - name = name.lower() - if "one" in name and "shot" in name: - return True - return False - - def _chapter_name_to_int(self, name: str) -> Optional[int]: - try: - return int(name.lstrip("#")) - except ValueError: - return None - def _format_chapter_prefix( self, title_name: str, @@ -78,17 +61,17 @@ def _format_chapter_prefix( chapter_num = 0 elif self.is_extra and next_chapter_name: suffix = "x1" - chapter_num = self._chapter_name_to_int(next_chapter_name) + chapter_num = chapter_name_to_int(next_chapter_name) if chapter_num is not None: chapter_num -= 1 prefix = "c" if chapter_num < 1000 else "d" else: - chapter_num = self._chapter_name_to_int(chapter_name) + chapter_num = chapter_name_to_int(chapter_name) if chapter_num is not None: prefix = "c" if chapter_num < 1000 else "d" if chapter_num is None: - chapter_num = self.escape_path(chapter_name) + chapter_num = escape_path(chapter_name) components.append(f"{prefix}{chapter_num:0>3}{suffix}") components.append("(web)") @@ -103,13 +86,10 @@ def format_page_name(self, page: Union[int, range], ext=".jpg") -> str: else: page = f"p{page:0>3}" - ext = ext.lstrip('.') + ext = ext.lstrip(".") return f"{self._chapter_prefix} - {page} {self._chapter_suffix}.{ext}" - def escape_path(self, path: str) -> str: - return re.sub(r"[^\w]+", " ", path).strip(string.punctuation + " ") - def close(self): pass @@ -119,14 +99,8 @@ def add_image(self, image_data: bytes, index: Union[int, range]): class RawExporter(ExporterBase): - def __init__( - self, - destination: str, - title: Title, - chapter: Chapter, - next_chapter: Optional[Chapter] = None, - ): - super().__init__(destination, title, chapter, next_chapter) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self.path = Path(self.destination, self.title_name) self.path.mkdir(parents=True, exist_ok=True) @@ -136,15 +110,8 @@ def add_image(self, image_data: bytes, index: Union[int, range]): class CBZExporter(ExporterBase): - def __init__( - self, - destination: str, - title: Title, - chapter: Chapter, - next_chapter: Optional[Chapter] = None, - compression=zipfile.ZIP_DEFLATED, - ): - super().__init__(destination, title, chapter, next_chapter) + def __init__(self, compression=zipfile.ZIP_DEFLATED, *args, **kwargs): + super().__init__(*args, **kwargs) self.path = Path(self.destination, self.title_name) self.path.mkdir(parents=True, exist_ok=True) self.path = self.path.joinpath(self.chapter_name).with_suffix(".cbz") diff --git a/mloader/loader.py b/mloader/loader.py index 11f3f34..7f42940 100644 --- a/mloader/loader.py +++ b/mloader/loader.py @@ -1,41 +1,35 @@ import logging -from enum import Enum from functools import lru_cache from itertools import chain, count -from typing import Type, Union, Dict, Set, Collection, Optional +from typing import Union, Dict, Set, Collection, Optional, Callable import click from requests import Session -from mloader.exporter import ExporterBase, CBZExporter -from mloader.response_pb2 import Response, MangaViewer, TitleDetailView +from mloader.constants import PageType +from mloader.exporter import ExporterBase +from mloader.response_pb2 import ( + Response, + MangaViewer, + TitleDetailView, + Chapter, + Title, +) +from mloader.utils import chapter_name_to_int log = logging.getLogger() -MangaList = Dict[int, Set[int]] - - -class ChapterType(Enum): - latest = 0 - sequence = 1 - nosequence = 2 - - -class PageType(Enum): - single = 0 - left = 1 - right = 2 - double = 3 +MangaList = Dict[int, Set[int]] # Title ID: Set[Chapter ID] class MangaLoader: def __init__( self, - exporter_cls: Type[ExporterBase] = CBZExporter, + exporter: Callable[[Title, Chapter, Optional[Chapter]], ExporterBase], quality: str = "super_high", split: bool = False, ): - self.exporter_cls = exporter_cls + self.exporter = exporter self.quality = quality self.split = split self._api_url = "https://jumpg-webapi.tokyo-cdn.com" @@ -71,13 +65,21 @@ def _load_pages(self, chapter_id: Union[str, int]) -> MangaViewer: @lru_cache(None) def _get_title_details(self, title_id: Union[str, int]) -> TitleDetailView: resp = self.session.get( - f"{self._api_url}/api/title_detail", params={"title_id": title_id}, + f"{self._api_url}/api/title_detail", params={"title_id": title_id} ) return Response.FromString(resp.content).success.title_detail_view def _normalize_ids( - self, title_ids: Collection[int], chapter_ids: Collection[int], + self, + title_ids: Collection[int], + chapter_ids: Collection[int], + min_chapter: int, + max_chapter: int, + last_chapter: bool = False, ) -> MangaList: + # mloader allows you to mix chapters and titles(collections of chapters) + # This method tries to merge them while trying to avoid unnecessary + # http requests if not any((title_ids, chapter_ids)): raise ValueError("Expected at least one title or chapter id") title_ids = set(title_ids or []) @@ -86,27 +88,40 @@ def _normalize_ids( for cid in chapter_ids: viewer = self._load_pages(cid) title_id = viewer.title_id + # Fetching details for this chapter also downloads all other + # visible chapters for the same title. if title_id in title_ids: title_ids.remove(title_id) - mangas.setdefault(title_id, set()).update( - x.chapter_id for x in viewer.chapters - ) + mangas.setdefault(title_id, []).extend(viewer.chapters) else: - mangas.setdefault(title_id, set()).add(cid) + # Should result in one chapter + mangas.setdefault(title_id, []).extend( + c for c in viewer.chapters if c.chapter_id == cid + ) for tid in title_ids: - title_details = self._get_title_details(tid) - mangas[tid] = { - x.chapter_id - for x in chain( - title_details.first_chapter_list, - title_details.last_chapter_list, - ) - } + details = self._get_title_details(tid) + mangas[tid] = list( + chain(details.first_chapter_list, details.last_chapter_list) + ) + + for tid in mangas: + if last_chapter: + chapters = mangas[tid][-1:] + else: + chapters = [ + c + for c in mangas[tid] + if min_chapter + <= (chapter_name_to_int(c.name) or 0) + <= max_chapter + ] + + mangas[tid] = set(c.chapter_id for c in chapters) return mangas - def _download(self, manga_list: MangaList, dst: str): + def _download(self, manga_list: MangaList): manga_num = len(manga_list) for title_index, (title_id, chapters) in enumerate( manga_list.items(), 1 @@ -130,7 +145,9 @@ def _download(self, manga_list: MangaList, dst: str): f" {chapter_index}/{chapter_num}) " f"Chapter {chapter_name}: {chapter.sub_title}" ) - exporter = self.exporter_cls(dst, title, chapter, next_chapter) + exporter = self.exporter( + title=title, chapter=chapter, next_chapter=next_chapter + ) pages = [ p.manga_page for p in viewer.pages if p.manga_page.image_url ] @@ -155,6 +172,11 @@ def download( *, title_ids: Optional[Collection[int]] = None, chapter_ids: Optional[Collection[int]] = None, - dst: str = ".", + min_chapter: int, + max_chapter: int, + last_chapter: bool = False, ): - self._download(self._normalize_ids(title_ids, chapter_ids), dst) + manga_list = self._normalize_ids( + title_ids, chapter_ids, min_chapter, max_chapter, last_chapter + ) + self._download(manga_list) diff --git a/mloader/utils.py b/mloader/utils.py new file mode 100644 index 0000000..9c473d6 --- /dev/null +++ b/mloader/utils.py @@ -0,0 +1,22 @@ +import re +import string +from typing import Optional + + +def is_oneshot(chapter_name: str, chapter_subtitle: str) -> bool: + for name in (chapter_name, chapter_subtitle): + name = name.lower() + if "one" in name and "shot" in name: + return True + return False + + +def chapter_name_to_int(name: str) -> Optional[int]: + try: + return int(name.lstrip("#")) + except ValueError: + return None + + +def escape_path(path: str) -> str: + return re.sub(r"[^\w]+", " ", path).strip(string.punctuation + " ")