From dcba44fda595d497b615c15541bcc47827626c4a Mon Sep 17 00:00:00 2001 From: Indrawan I Date: Mon, 20 Feb 2023 07:07:02 +0700 Subject: [PATCH] feat: split client and remake boilerplate (#3) * old prototype * redesign * update * split client and utils to complier * pre release --- README.md | 5 +---- jigoku/__init__.py | 6 +++--- jigoku/app.py | 17 ++++++----------- jigoku/client/__init__.py | 0 jigoku/{utils => client}/scrape_pages.py | 4 +--- jigoku/{utils => client}/scrape_posts.py | 8 ++------ jigoku/constant/__init__.py | 0 jigoku/{utils/constant.py => constant/base.py} | 0 setup.py | 2 +- 9 files changed, 14 insertions(+), 28 deletions(-) create mode 100644 jigoku/client/__init__.py rename jigoku/{utils => client}/scrape_pages.py (99%) rename jigoku/{utils => client}/scrape_posts.py (98%) create mode 100644 jigoku/constant/__init__.py rename jigoku/{utils/constant.py => constant/base.py} (100%) diff --git a/README.md b/README.md index 66aa485..fab4a88 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ Jigoku apply connection retry on failed, every your network changes or something - Pure scraping, does not hit the API - Download with ease - Interactive prompt +- ~~Multithreaded~~ too nasty, low coverage ## Site support @@ -85,10 +86,6 @@ Jigoku supports the following imageboards: NOTE: Python 3.7 or above -Jigoku depends on -- [requests](https://pypi.org/project/requests/) Python HTTP Client -- [beautifulsoup4](https://pypi.org/project/beautifulsoup4/) HTML Parser - ## Installation It's fairly simple to use jigoku diff --git a/jigoku/__init__.py b/jigoku/__init__.py index 2471304..c9b02c6 100644 --- a/jigoku/__init__.py +++ b/jigoku/__init__.py @@ -1,6 +1,6 @@ -__version__ = "2.2.2" +__version__ = "2.2.3" from .app import main from .utils.log import log_data, log_time, get_hostname from .utils.disk import get_size, clean_html -from .utils.scrape_posts import download_from_multiple_posts -from .utils.scrape_pages import download_from_multiple_pages \ No newline at end of file +from .client.scrape_posts import download_from_multiple_posts +from .client.scrape_pages import download_from_multiple_pages \ No newline at end of file diff --git a/jigoku/app.py b/jigoku/app.py index 5b624c8..d68bb96 100644 --- a/jigoku/app.py +++ b/jigoku/app.py @@ -3,28 +3,23 @@ import time from inputimeout import inputimeout, TimeoutOccurred from jigoku.utils.log import log_time -from jigoku.utils.scrape_posts import download_from_multiple_posts -from jigoku.utils.scrape_pages import download_from_multiple_pages +from jigoku.client.scrape_posts import download_from_multiple_posts +from jigoku.client.scrape_pages import download_from_multiple_pages -# from jigoku.utils.log import log_time -## which is takes all galleries - -## from utils.log import log_time - try: file = inputimeout( - prompt="[+] Please enter file you want to bulk download (e.g. file.txt): ", + prompt="[+] Please enter file you want to bulk download (e.g. file.txt): ", timeout=30 ) - + download_by = inputimeout( prompt="[+] Please choose (1) Multiple posts or (2) Multiple pages: ", timeout=30, ) - + select_type = inputimeout( - prompt="[+] Select type image results (1) Original size or (2) Smaller size: ", + prompt="[+] Select type image results (1) Original size or (2) Smaller size: ", timeout=30 ) diff --git a/jigoku/client/__init__.py b/jigoku/client/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/jigoku/utils/scrape_pages.py b/jigoku/client/scrape_pages.py similarity index 99% rename from jigoku/utils/scrape_pages.py rename to jigoku/client/scrape_pages.py index 217f6f2..8b050a6 100644 --- a/jigoku/utils/scrape_pages.py +++ b/jigoku/client/scrape_pages.py @@ -2,12 +2,10 @@ import os import time import re -import pkg_resources from bs4 import BeautifulSoup from jigoku.utils.log import log_data, get_hostname from jigoku.utils.disk import get_size, clean_html -from jigoku.utils.constant import Jigoku -version = pkg_resources.require("jigoku")[0].version +from jigoku.constant.base import Jigoku jgx = Jigoku() diff --git a/jigoku/utils/scrape_posts.py b/jigoku/client/scrape_posts.py similarity index 98% rename from jigoku/utils/scrape_posts.py rename to jigoku/client/scrape_posts.py index a958f8a..eaf40c2 100644 --- a/jigoku/utils/scrape_posts.py +++ b/jigoku/client/scrape_posts.py @@ -2,14 +2,10 @@ import os import time import re -import pkg_resources from bs4 import BeautifulSoup from jigoku.utils.log import log_data from jigoku.utils.disk import get_size -from jigoku.utils.constant import Jigoku -version = pkg_resources.require("jigoku")[0].version - -# from jigoku.utils.log import log_data, get_hostname +from jigoku.constant.base import Jigoku jgx = Jigoku() @@ -155,7 +151,7 @@ def download_from_multiple_posts(file: str, select_type: str) -> None: if not os.path.exists(final_name): img_count += 1 ##current_progress = f"{img_count} / {len(f.readlines()) + 1}" - image = requests.get(final_img, stream=True) + image = requests.get(final_img, stream=True, headers=jgx.with_headers) with open(final_name, "wb") as f: f.write(image.content) diff --git a/jigoku/constant/__init__.py b/jigoku/constant/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/jigoku/utils/constant.py b/jigoku/constant/base.py similarity index 100% rename from jigoku/utils/constant.py rename to jigoku/constant/base.py diff --git a/setup.py b/setup.py index eae2778..3adecfe 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ "Issue tracker": "https://github.com/sinkaroid/jigoku/issues/new/choose", "Documentation": "https://github.com/sinkaroid/jigoku/wiki", }, - packages=['jigoku', 'jigoku.utils'], + packages=['jigoku', 'jigoku.constant', 'jigoku.utils', 'jigoku.client'], license='MIT', classifiers=[ 'Development Status :: 3 - Alpha',