From 71a8b69f2877cd93adb04ac6c33be95bd3a9d564 Mon Sep 17 00:00:00 2001 From: thezak48 Date: Tue, 14 Nov 2023 01:27:42 +0000 Subject: [PATCH] build(Build Docker Images): :construction_worker: --- .dockerignore | 7 + .github/workflows/release.yml | 76 ++++++++++ Dockerfile | 34 +++++ data/config.example.ini | 3 +- manga_dl.py | 259 +++++++++++++++++++++++----------- manga_dl/utilities/config.py | 4 +- manga_dl/utilities/logging.py | 4 +- 7 files changed, 298 insertions(+), 89 deletions(-) create mode 100644 .dockerignore create mode 100644 .github/workflows/release.yml create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..46562a8 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +.git +.vscode +renovate.json +.gitignore +data/* +systemd/* +README.md \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..215acca --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,76 @@ +name: build-and-push + +on: + push: + branches: + - "main" + tags: + - '*' + paths-ignore: + - 'data/**' + - 'systemd/**' + - 'README.md' + - 'renovate.json' + pull_request: + branches: + - "develop" + paths-ignore: + - 'data/**' + - 'systemd/**' + - 'README.md' + - 'renovate.json' + +jobs: + build-and-push-docker: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: buildx + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ghcr.io/thezak48/manga_dl + flavor: | + latest=auto + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor} + type=semver,pattern={{major}} + type=edge,branch=develop + type=ref,event=pr + type=sha + + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.CR_ZAK }} + + - name: Docker build and push + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: ${{ github.event.pull_request.head.repo.full_name == 'thezak48/manga_dl' || github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + build-args: | + BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') + VCS_REF=${{ github.sha }} + VERSION=${{ steps.meta.outputs.tags }} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..600bacf --- /dev/null +++ b/Dockerfile @@ -0,0 +1,34 @@ +FROM python:3.9-alpine + +RUN echo "http://dl-4.alpinelinux.org/alpine/v3.14/main" >> /etc/apk/repositories && \ + echo "http://dl-4.alpinelinux.org/alpine/v3.14/community" >> /etc/apk/repositories + + +RUN apk --no-cache add chromium chromium-chromedriver +RUN pip install --upgrade pip + +LABEL maintainer="thezak48" \ + org.opencontainers.image.created=$BUILD_DATE \ + org.opencontainers.image.url="https://github.com/thezak48/manga_dl" \ + org.opencontainers.image.source="https://github.com/thezak48/manga_dl" \ + org.opencontainers.image.version=$VERSION \ + org.opencontainers.image.revision=$VCS_REF \ + org.opencontainers.image.vendor="thezak48" \ + org.opencontainers.image.title="manga_dl" \ + org.opencontainers.image.description="manga_dl is an application to to download manga's, manhua's or manhwa's." + +ENV PUID=1000 +ENV PGID=1000 +ENV PYTHONUNBUFFERED=1 + +COPY . / + +RUN pip install --no-cache-dir -r requirements.txt + +RUN addgroup -g $PGID appgroup && \ + adduser -D -u $PUID -G appgroup appuser && \ + mkdir -p /config && \ + chown -R appuser:appgroup /config +USER appuser + +CMD [ "python", "/manga_dl.py" ] \ No newline at end of file diff --git a/data/config.example.ini b/data/config.example.ini index 17dd7a5..3192019 100644 --- a/data/config.example.ini +++ b/data/config.example.ini @@ -2,5 +2,4 @@ mangas = ./data/manga.txt multi_threaded = True num_threads = 10 -save_location = ./data/manga -driver_path = /usr/bin/chromedriver \ No newline at end of file +save_location = ./data/manga \ No newline at end of file diff --git a/manga_dl.py b/manga_dl.py index 4728271..ac6581d 100644 --- a/manga_dl.py +++ b/manga_dl.py @@ -5,12 +5,14 @@ python manga_dl.py manga [options] save_location """ import argparse -import configparser import concurrent.futures import os import signal import sys +import time from urllib.parse import unquote, urlparse +from datetime import datetime +from datetime import timedelta from manga_dl.utilities.logging import setup_logging @@ -43,13 +45,21 @@ def submit(self, fn, *args, **kwargs): super().submit(fn, *args, **kwargs) -log = setup_logging() +if os.path.exists("/.dockerenv"): + dirname = "config" +else: + dirname = "data" + +log = setup_logging(dirname) config = ConfigHandler( - log, os.path.join(os.path.dirname(__file__), "data", "config.ini") + log, os.path.join(os.path.dirname(__file__), dirname, "config.ini") ) -driver_path = config.get("General", "driver_path") +if config.has_option("General", "driver_path"): + driver_path = config.get("General", "driver_path") +else: + driver_path = "/usr/bin/chromedriver" parser = argparse.ArgumentParser( description="Download download manga's, manhua's or manhwa's", @@ -77,6 +87,19 @@ def submit(self, fn, *args, **kwargs): default=config.getint("General", "num_threads"), help="Number of threads to use in case of multi-threading", ) +parser.add_argument( + "-r", + "--run", + action="store_true", + help="Run the script once and exit", +) +parser.add_argument( + "-sch", + "--schedule", + default="1440", + type=str, + help="Schedule to run every x minutes. (Default set to 1440 (1 day))", +) parser.add_argument( "-s", "--save_location", @@ -107,45 +130,79 @@ def get_website_class(url: str): save_location = args.save_location multi_threaded = args.multi_threaded +schedule = args.schedule -if os.path.isfile(args.manga): - with open(args.manga, "r", encoding="utf-8") as f: - manga_urls = [line.strip().rstrip("/") for line in f] -else: - manga_urls = [args.manga.rstrip("/")] - -progress = Progress() -try: - with progress.progress: - manga_task = progress.add_task("Downloading manga...", total=len(manga_urls)) - - for manga_url in manga_urls: - manga = get_website_class(manga_url) - if isinstance(manga, (Webtoons, Kaiscans)): - manga_name = manga_url - else: - manga_name = unquote(urlparse(manga_url).path.split("/")[-1]) - manga_id, title_id = manga.get_manga_id(manga_name) - - if manga_id: - chapters = manga.get_manga_chapters(manga_id=manga_id) - chapter_task = progress.add_task( - f"Downloading chapters for {title_id}", total=len(chapters) - ) - genres, summary = manga.get_manga_metadata(manga_name) - complete_dir = os.path.join(save_location, title_id) - existing_chapters = ( - set(os.listdir(complete_dir)) - if os.path.exists(complete_dir) - else set() - ) +def download_manga(): + if os.path.isfile(args.manga): + with open(args.manga, "r", encoding="utf-8") as f: + manga_urls = [line.strip().rstrip("/") for line in f] + else: + manga_urls = [args.manga.rstrip("/")] + + progress = Progress() + try: + with progress.progress: + manga_task = progress.add_task( + "Downloading manga...", total=len(manga_urls) + ) + + for manga_url in manga_urls: + manga = get_website_class(manga_url) + if isinstance(manga, (Webtoons, Kaiscans)): + manga_name = manga_url + else: + manga_name = unquote(urlparse(manga_url).path.split("/")[-1]) + manga_id, title_id = manga.get_manga_id(manga_name) + + if manga_id: + chapters = manga.get_manga_chapters(manga_id=manga_id) + chapter_task = progress.add_task( + f"Downloading chapters for {title_id}", total=len(chapters) + ) + genres, summary = manga.get_manga_metadata(manga_name) + + complete_dir = os.path.join(save_location, title_id) + existing_chapters = ( + set(os.listdir(complete_dir)) + if os.path.exists(complete_dir) + else set() + ) - if multi_threaded: - with concurrent.futures.ThreadPoolExecutor( - max_workers=args.num_threads - ) as executor: - futures = [] + if multi_threaded: + with concurrent.futures.ThreadPoolExecutor( + max_workers=args.num_threads + ) as executor: + futures = [] + for chapter_number, chapter_url in chapters: + if f"Ch. {chapter_number}.cbz" in existing_chapters: + log.info( + "%s Ch. %s already exists, skipping", + title_id, + chapter_number, + ) + progress.update(chapter_task, advance=1) + continue + + images = manga.get_chapter_images(url=chapter_url) + futures.append( + executor.submit( + ImageDownloader( + log, manga.headers_image + ).download_chapter, + chapter_number, + images, + title_id, + save_location, + progress, + genres, + summary, + complete_dir, + chapter_task, + ) + ) + + else: for chapter_number, chapter_url in chapters: if f"Ch. {chapter_number}.cbz" in existing_chapters: log.info( @@ -157,49 +214,83 @@ def get_website_class(url: str): continue images = manga.get_chapter_images(url=chapter_url) - futures.append( - executor.submit( - ImageDownloader( - log, manga.headers_image - ).download_chapter, - chapter_number, - images, - title_id, - save_location, - progress, - genres, - summary, - complete_dir, - chapter_task, - ) - ) - - else: - for chapter_number, chapter_url in chapters: - if f"Ch. {chapter_number}.cbz" in existing_chapters: - log.info( - "%s Ch. %s already exists, skipping", - title_id, + ImageDownloader(log, manga.headers_image).download_chapter( chapter_number, + images, + title_id, + save_location, + progress, + genres, + summary, + complete_dir, + chapter_task, ) - progress.update(chapter_task, advance=1) - continue - - images = manga.get_chapter_images(url=chapter_url) - ImageDownloader(log, manga.headers_image).download_chapter( - chapter_number, - images, - title_id, - save_location, - progress, - genres, - summary, - complete_dir, - chapter_task, - ) - - progress.update(manga_task, advance=1) - -except KeyboardInterrupt: - progress.exit() - sys.exit(0) + + progress.update(manga_task, advance=1) + + except KeyboardInterrupt: + progress.exit() + sys.exit(0) + + +def calc_next_run(schd, write_out=False): + """Calculates the next run time based on the schedule""" + current = datetime.now().strftime("%H:%M") + seconds = schd * 60 + time_to_run = datetime.now() + timedelta(minutes=schd) + time_to_run_str = time_to_run.strftime("%H:%M") + new_seconds = ( + datetime.strptime(time_to_run_str, "%H:%M") + - datetime.strptime(current, "%H:%M") + ).total_seconds() + time_until = "" + next_run = {} + if args.run is False: + next_run["next_run"] = time_to_run + if new_seconds < 0: + new_seconds += 86400 + if (seconds is None or new_seconds < seconds) and new_seconds > 0: + seconds = new_seconds + if seconds is not None: + hours = int(seconds // 3600) + minutes = int((seconds % 3600) // 60) + time_until = ( + f"{hours} Hour{'s' if hours > 1 else ''}{' and ' if minutes > 1 else ''}" + if hours > 0 + else "" + ) + time_until += ( + f"{minutes} Minute{'s' if minutes > 1 else ''}" if minutes > 0 else "" + ) + if write_out: + next_run[ + "next_run_str" + ] = f"Current Time: {current} | {time_until} until the next run at {time_to_run_str}" + else: + next_run["next_run"] = None + next_run["next_run_str"] = "" + return time_until, next_run + + +def main(): + """Main function of the script""" + try: + if args.run: + log.warning("Run Mode: Script will exit after completion.") + download_manga() + else: + while True: + log.warning( + "Schedule Mode: Script will run every %s minutes.", schedule + ) + download_manga() + _, next_run = calc_next_run(int(args.schedule), write_out=True) + log.info(next_run["next_run_str"]) + time.sleep(int(args.schedule) * 60) + except KeyboardInterrupt: + Progress().exit() + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/manga_dl/utilities/config.py b/manga_dl/utilities/config.py index 26902c6..236798c 100644 --- a/manga_dl/utilities/config.py +++ b/manga_dl/utilities/config.py @@ -25,13 +25,15 @@ def getint(self, section, option): def getboolean(self, section, option): return self.config.getboolean(section, option) + def has_option(self, section, option): + return self.config.has_option(section, option) + def _generate_default_config(self): self.config["General"] = { "mangas": "./data/manga.txt", "multi_threaded": "True", "num_threads": "10", "save_location": "./data/manga", - "driver_path": "/usr/bin/chromedriver", } with open(self.path, "w") as configfile: diff --git a/manga_dl/utilities/logging.py b/manga_dl/utilities/logging.py index 4dc9037..8d94692 100644 --- a/manga_dl/utilities/logging.py +++ b/manga_dl/utilities/logging.py @@ -10,7 +10,7 @@ from rich.logging import RichHandler -def setup_logging(): +def setup_logging(dirname): """Setup logging.""" sys.stdout.reconfigure(encoding="utf-8") console = Console() @@ -24,7 +24,7 @@ def setup_logging(): current_time = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") log_filename = f"manga_dl_{current_time}.log" - log_dir = "./data/logs" + log_dir = f"./{dirname}/logs" os.makedirs(log_dir, exist_ok=True) log_filepath = os.path.join(log_dir, log_filename) file_handler = RotatingFileHandler(