From b93381081f9720011762412769f5c8942397a86b Mon Sep 17 00:00:00 2001 From: prof79 Date: Wed, 23 Aug 2023 14:40:12 +0200 Subject: [PATCH 01/19] Added rewrite log. --- RewriteNotes.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 RewriteNotes.md diff --git a/RewriteNotes.md b/RewriteNotes.md new file mode 100644 index 0000000..bd6b782 --- /dev/null +++ b/RewriteNotes.md @@ -0,0 +1,5 @@ +# Rewrite Notes + +Notes about the challenges and important changes during the major rewrite. + +-- prof79 From 6c169c74c4feadc85e54cc3de63660858d61bd22 Mon Sep 17 00:00:00 2001 From: prof79 Date: Wed, 23 Aug 2023 14:48:27 +0200 Subject: [PATCH 02/19] Added ignore file for Python. --- .gitignore | 160 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..68bc17f --- /dev/null +++ b/.gitignore @@ -0,0 +1,160 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ From 3e322a747642a9d84f25531cb55f5142528ccd13 Mon Sep 17 00:00:00 2001 From: prof79 Date: Wed, 23 Aug 2023 15:22:31 +0200 Subject: [PATCH 03/19] Sorted requirements file. --- requirements.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index 1d9759c..314f414 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ -requests>=2.26.0 -loguru>=0.5.3 +av>=9.0.0 imagehash>=4.2.1 +loguru>=0.5.3 +m3u8>=3.0.0 pillow>=8.4.0 -python-dateutil>=2.8.2 plyvel-ci>=1.5.0 psutil>=5.9.0 -av>=9.0.0 -m3u8>=3.0.0 +python-dateutil>=2.8.2 +requests>=2.26.0 rich>=13.0.0 From 43c4eb52e2b1418759a663c1f35eed778422a9ce Mon Sep 17 00:00:00 2001 From: prof79 Date: Wed, 23 Aug 2023 15:23:16 +0200 Subject: [PATCH 04/19] Added requirements file for dev/IDE dependencies. --- requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 requirements-dev.txt diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..f0aa93a --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1 @@ +mypy From f4afc899b082ee315ad366505504f6541ced812d Mon Sep 17 00:00:00 2001 From: prof79 Date: Wed, 30 Aug 2023 22:12:55 +0200 Subject: [PATCH 05/19] The initial of the full rewrite/refactoring - here goes nothing :D --- .gitignore | 4 + RewriteNotes.md | 26 + config/__init__.py | 14 + config/args.py | 406 ++++++++++ config/browser.py | 310 +++++++ config/config.py | 274 +++++++ config/fanslyconfig.py | 217 +++++ config/modes.py | 13 + config/validation.py | 347 ++++++++ download/__init__.py | 5 + download/account.py | 87 ++ download/collections.py | 47 ++ download/common.py | 105 +++ download/core.py | 25 + download/downloadstate.py | 51 ++ download/m3u8.py | 133 +++ download/media.py | 170 ++++ download/messages.py | 79 ++ download/single.py | 100 +++ download/timeline.py | 137 ++++ download/types.py | 12 + errors/__init__.py | 123 +++ fansly_downloader.py | 1608 ++++--------------------------------- fileio/dedupe.py | 115 +++ fileio/fnmanip.py | 197 +++++ media/__init__.py | 14 + media/media.py | 207 +++++ media/mediaitem.py | 55 ++ pathio/__init__.py | 10 + pathio/pathio.py | 108 +++ requirements-dev.txt | 3 + textio/__init__.py | 25 + textio/textio.py | 124 +++ updater/__init__.py | 65 ++ updater/utils.py | 274 +++++++ utils/common.py | 109 +++ utils/config_util.py | 209 ----- utils/datetime.py | 44 + utils/update_util.py | 207 ----- utils/web.py | 200 +++++ 40 files changed, 4374 insertions(+), 1885 deletions(-) create mode 100644 config/__init__.py create mode 100644 config/args.py create mode 100644 config/browser.py create mode 100644 config/config.py create mode 100644 config/fanslyconfig.py create mode 100644 config/modes.py create mode 100644 config/validation.py create mode 100644 download/__init__.py create mode 100644 download/account.py create mode 100644 download/collections.py create mode 100644 download/common.py create mode 100644 download/core.py create mode 100644 download/downloadstate.py create mode 100644 download/m3u8.py create mode 100644 download/media.py create mode 100644 download/messages.py create mode 100644 download/single.py create mode 100644 download/timeline.py create mode 100644 download/types.py create mode 100644 errors/__init__.py create mode 100644 fileio/dedupe.py create mode 100644 fileio/fnmanip.py create mode 100644 media/__init__.py create mode 100644 media/media.py create mode 100644 media/mediaitem.py create mode 100644 pathio/__init__.py create mode 100644 pathio/pathio.py create mode 100644 textio/__init__.py create mode 100644 textio/textio.py create mode 100644 updater/__init__.py create mode 100644 updater/utils.py create mode 100644 utils/common.py delete mode 100644 utils/config_util.py create mode 100644 utils/datetime.py delete mode 100644 utils/update_util.py create mode 100644 utils/web.py diff --git a/.gitignore b/.gitignore index 68bc17f..21776ad 100644 --- a/.gitignore +++ b/.gitignore @@ -158,3 +158,7 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +# User-specific +*_fansly/ +config_args.ini diff --git a/RewriteNotes.md b/RewriteNotes.md index bd6b782..b034d5f 100644 --- a/RewriteNotes.md +++ b/RewriteNotes.md @@ -2,4 +2,30 @@ Notes about the challenges and important changes during the major rewrite. +* Used an IMHO good IDE - Visual Studio Code with the official Python module and Pylance - which has great code quality (potentially unininitalized variables, unused variables, ...) and refactoring features (renaming, find usages, ...). There is also an excellent TODO Tree extension. +* Code developed and tested on Python 3.11.4 x64 for Windows. +* Introduced static typing where I thought of it especially functions and classes. This makes functions interfaces and their intentions clear, where code breaks them and what is expected to be passed around where. +* Documented functions as much as I could to also improve my understanding of the code. This also led to function renames to express their purpose more clearly. +* Partitioned code into functions to modularize it. +* Spread out functions in modules and separate files especially long functions. +* Switched to Context Managers (`with`) where appropriate to avoid potentially leaking resources on unexpected exceptions. +* Switched to the often more elegant `pathlib` library and `Path` where fesasible. Only `os.walk()` has no alternative yet. +* Used `fallback=` in `configparser` getters to be more resilient and allow minimal configuration files. Default values match the contents of the original `config.ini` from the repository. +* Renamed `utilise_duplicate_threshold` to `use_duplicate_threshold` but code retains compatibility with older `config.ini` files. +* Introduced `use_suffix` option in `config.ini`. If I have a folder for all my Fansly downloads, I do not see the point of suffixing every creator's subfolder with `_fansly`. This especially makes no sense any more since the rewritten code does not need to parse out the creator's folder from an arbitrary path in odd places. I did, however, retain the previous behavior - it defaults to `True`. +* Having the program version in a config file (`config.ini`) makes no sense and is potentially dangerous. The program version should (and is now) in a proper file heading block and can be read from there. Versioning a config file might make sense in some cases but all the changes to config structure so far can easily be handled by the config read/validation functions without reliance on versioning. +* While reworking `delete_deprecated_files()` I found a bug - `os.path.splitext()` includes the full path to the file name thus the `in` must have always failed. (See https://docs.python.org/3/library/os.path.html - "`root + ext == path`") +* I corrected the ratios for `rich`'s `TextColumn`/`BarColumn`: They are `int`s now, I assumed 2/5 thus 1; 5 (were: 0.355; 2). +* Switched to `Enum`s (`StrEnum`) for `download_mode` and `download_type` to be explicit and prevent magical string values floating around +* I made all hard errors `raise` and introduced an `interactive` flag (also as `config.ini` option) to bypass any `Press to continue` prompts. Thus you can now automate/schedule it using `Task Scheduler` on Windows or `cron` on Linux/UNIX or just have it run through while you are away. What is more, this also helps multiple user processing - an account info error due to invalid user name should not prevent all other users from being downloaded and can be caught in the user loop. +* There also now distinct program exit codes to facilitate automation. +* There is now a `prompt_on_exit` setting. This might seem redundant to `interactive` but helps for semi-automated runs (`interactive=False`) letting the computer run and when coming back wanting to see what happened meanwhile in the console window. For a truly automated/scheduled experience you need to set both `interactive` and `prompt_on_exit` to `False`. +* I made - hopefully - arg parsing foolproof by using a separate temporary `config_args.ini` for that scenario. Thus the validation functions, which may alter and save the config, are prevented from overwriting the users' original configuration as are other cases where `save_config_or_raise()` might be called. +* What's to difficult to test for me is the self-updating functionality. I rewrote it to the best of my knowledge but since it references the original repo and such, only accepted PR and time will tell. +* Renamed `use_suffix` to `use_folder_suffix` to be more clear and better convey meaning. +* All `config.ini` options have finally been implemented as command-line arguments. +* Since everything is now properly defaulted and handled (I hope) you are now able to run `Fansly Downloader` with empty config files or no `config.ini` at all :D You just need the executable and are good to go. +* Als worked around an issue in Firefox token retrieval where differently encoded DBs woult throw `sqlite3.OperationalError`. +* `Fansly Downloader` now also logs output to file, at least all stuff going through `loguru`. Log-rollover is at 1 MiB size and keeping the last 5 files. + -- prof79 diff --git a/config/__init__.py b/config/__init__.py new file mode 100644 index 0000000..c21c86f --- /dev/null +++ b/config/__init__.py @@ -0,0 +1,14 @@ +"""Configuration File Manipulation""" + + +from .config import copy_old_config_values, load_config +from .fanslyconfig import FanslyConfig +from .validation import validate_adjust_config + + +__all__ = [ + 'copy_old_config_values', + 'load_config', + 'validate_adjust_config', + 'FanslyConfig', +] diff --git a/config/args.py b/config/args.py new file mode 100644 index 0000000..3f2f640 --- /dev/null +++ b/config/args.py @@ -0,0 +1,406 @@ +"""Argument Parsing and Configuration Mapping""" + + +import argparse + +from functools import partial +from pathlib import Path + +from .config import parse_items_from_line, sanitize_creator_names +from .fanslyconfig import FanslyConfig +from .modes import DownloadMode + +from errors import ConfigError +from textio import print_debug, print_warning +from utils.common import is_valid_post_id, save_config_or_raise + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Fansly Downloader scrapes media content from one or more Fansly creators. " + "Settings will be taken from config.ini or internal defaults and " + "can be overriden with the following parameters.\n" + "Using the command-line will not overwrite config.ini.", + ) + + #region Essential Options + + parser.add_argument( + '-u', '--user', + required=False, + default=None, + metavar='USER', + dest='users', + help="A list of one or more Fansly creators you want to download " + "content from.\n" + "This overrides TargetedCreator > username in config.ini.", + nargs='+', + ) + parser.add_argument( + '-dir', '--directory', + required=False, + default=None, + dest='download_directory', + help="The base directory to store all creators' content in. " + "A subdirectory for each creator will be created automatically. " + "If you do not specify --no-folder-suffix, " + "each creator's folder will be suffixed with ""_fansly"". " + "Please remember to quote paths including spaces.", + ) + parser.add_argument( + '-t', '--token', + required=False, + default=None, + metavar='AUTHORIZATION_TOKEN', + dest='token', + help="The Fansly authorization token obtained from a browser session.", + ) + parser.add_argument( + '-ua', '--user-agent', + required=False, + default=None, + dest='user_agent', + help="The browser user agent string to use when communicating with " + "Fansly servers. This should ideally be set to the user agent " + "of the browser you use to view Fansly pages and where the " + "authorization token was obtained from.", + ) + + #endregion + + #region Download modes + + download_modes = parser.add_mutually_exclusive_group(required=False) + + download_modes.add_argument( + '--normal', + required=False, + default=False, + action='store_true', + dest='download_mode_normal', + help='Use "Normal" download mode. This will download messages and timeline media.', + ) + download_modes.add_argument( + '--messages', + required=False, + default=False, + action='store_true', + dest='download_mode_messages', + help='Use "Messages" download mode. This will download messages only.', + ) + download_modes.add_argument( + '--timeline', + required=False, + default=False, + action='store_true', + dest='download_mode_timeline', + help='Use "Timeline" download mode. This will download timeline content only.', + ) + download_modes.add_argument( + '--collection', + required=False, + default=False, + action='store_true', + dest='download_mode_collection', + help='Use "Collection" download mode. This will ony download a collection.', + ) + download_modes.add_argument( + '--single', + required=False, + default=None, + metavar='POST_ID', + dest='download_mode_single', + help='Use "Single" download mode. This will download a single post ' + "by ID from an arbitrary creator. " + "A post ID must be at least 10 characters and consist of digits only." + "Example - https://fansly.com/post/1283998432982 -> ID is: 1283998432982", + ) + + #endregion + + #region Other Options + + parser.add_argument( + '-ni', '--non-interactive', + required=False, + default=False, + action='store_true', + dest='non_interactive', + help="Do not ask for input during warnings and errors that need " + "your attention but can be automatically continued. " + "Setting this will download all media of all users without any " + "intervention.", + ) + parser.add_argument( + '-npox', '--no-prompt-on-exit', + required=False, + default=False, + action='store_true', + dest='no_prompt_on_exit', + help="Do not ask to press at the very end of the program. " + "Set this for a fully automated/headless experience.", + ) + parser.add_argument( + '-nfs', '--no-folder-suffix', + required=False, + default=False, + action='store_true', + dest='no_folder_suffix', + help='Do not add "_fansly" to the download folder of a creator.', + ) + parser.add_argument( + '-np', '--no-previews', + required=False, + default=False, + action='store_true', + dest='no_media_previews', + help="Do not download media previews (which may contain spam).", + ) + parser.add_argument( + '-hd', '--hide-downloads', + required=False, + default=False, + action='store_true', + dest='hide_downloads', + help="Do not show download information.", + ) + parser.add_argument( + '-nof', '--no-open-folder', + required=False, + default=False, + action='store_true', + dest='no_open_folder', + help="Do not open the download folder on creator completion.", + ) + parser.add_argument( + '-nsm', '--no-separate-messages', + required=False, + default=False, + action='store_true', + dest='no_separate_messages', + help="Do not separate messages into their own folder.", + ) + parser.add_argument( + '-nst', '--no-separate-timeline', + required=False, + default=False, + action='store_true', + dest='no_separate_timeline', + help="Do not separate timeline content into it's own folder.", + ) + parser.add_argument( + '-sp', '--separate-previews', + required=False, + default=False, + action='store_true', + dest='separate_previews', + help="Separate preview media (which may contain spam) into their own folder.", + ) + parser.add_argument( + '-udt', '--use-duplicate-threshold', + required=False, + default=False, + action='store_true', + dest='use_duplicate_threshold', + help="Use an internal de-deduplication threshold to not download " + "already downloaded media again.", + ) + + #endregion + + #region Developer/troubleshooting arguments + + parser.add_argument( + '--debug', + required=False, + default=False, + action='store_true', + help="Print debugging output. Only for developers or troubleshooting.", + ) + parser.add_argument( + '--updated-to', + required=False, + default=None, + help="This is for internal use of the self-updating functionality only.", + ) + + #endregion + + return parser.parse_args() + + +def check_attributes( + args: argparse.Namespace, + config: FanslyConfig, + arg_attribute: str, + config_attribute: str + ) -> None: + """A helper method to validate the presence of attributes (properties) + in `argparse.Namespace` and `FanslyConfig` objects for mapping + arguments. This is to locate code changes and typos. + + :param args: The arguments parsed. + :type args: argparse.Namespace + :param config: The Fansly Downloader configuration. + :type config: FanslyConfig + :param arg_attribute: The argument destination variable name. + :type arg_attribute: str + :param config_attribute: The configuration attribute/property name. + :type config_attribute: str + + :raise RuntimeError: Raised when an attribute does not exist. + + """ + if hasattr(args, arg_attribute) and hasattr(config, config_attribute): + return + + raise RuntimeError( + 'Internal argument configuration error - please contact the developer.' + f'(args.{arg_attribute} == {hasattr(args, arg_attribute)}, ' + f'config.{config_attribute} == {hasattr(config, config_attribute)})' + ) + + +def map_args_to_config(args: argparse.Namespace, config: FanslyConfig) -> None: + """Maps command-line arguments to the configuration object of + the current session. + + :param argparse.Namespace args: The command-line arguments + retrieved via argparse. + :param FanslyConfig config: The program configuration to map the + arguments to. + """ + if config.config_path is None: + raise RuntimeError('Internal error mapping arguments - configuration path not set. Load the config first.') + + config_overridden = False + + config.debug = args.debug + + if config.debug: + print_debug(f'Args: {args}') + print() + + if args.users is not None: + # If someone "abused" argparse like this: + # -u creater1, creator7 , lovedcreator + # ... then it's best to re-construct a line and fully parse. + users_line = ' '.join(args.users) + config.user_names = \ + sanitize_creator_names(parse_items_from_line(users_line)) + config_overridden = True + + if config.debug: + print_debug(f'Value of `args.users` is: {args.users}') + print_debug(f'`args.users` is None == {args.users is None}') + print_debug(f'`config.username` is: {config.user_names}') + print() + + if args.download_mode_normal: + config.download_mode = DownloadMode.NORMAL + config_overridden = True + + if args.download_mode_messages: + config.download_mode = DownloadMode.MESSAGES + config_overridden = True + + if args.download_mode_timeline: + config.download_mode = DownloadMode.TIMELINE + config_overridden = True + + if args.download_mode_collection: + config.download_mode = DownloadMode.COLLECTION + config_overridden = True + + if args.download_mode_single is not None: + post_id = args.download_mode_single + config.download_mode = DownloadMode.SINGLE + + if not is_valid_post_id(post_id): + raise ConfigError( + f"Argument error - '{post_id}' is not a valid post ID. " + "At least 10 characters/only digits required." + ) + + config.post_id = post_id + config_overridden = True + + # The code following avoids code duplication of checking an + # argument and setting the override flag for each argument. + # On the other hand, this certainly not refactoring/renaming friendly. + # But arguments following similar patterns can be changed or + # added more easily. + + # Simplify since args and config arguments will always be the same + check_attr = partial(check_attributes, args, config) + + # Not-None-settings to map + not_none_settings = [ + 'download_directory', + 'token', + 'user_agent', + 'updated_to', + ] + + # Sets config when arguments are not None + for attr_name in not_none_settings: + check_attr(attr_name, attr_name) + arg_attribute = getattr(args, attr_name) + + if arg_attribute is not None: + + if attr_name == 'download_directory': + setattr(config, attr_name, Path(arg_attribute)) + + else: + setattr(config, attr_name, arg_attribute) + + config_overridden = True + + # Do-settings to map to config + positive_bools = [ + 'separate_previews', + 'use_duplicate_threshold', + ] + + # Sets config to arguments when arguments are True + for attr_name in positive_bools: + check_attr(attr_name, attr_name) + arg_attribute = getattr(args, attr_name) + + if arg_attribute == True: + setattr(config, attr_name, arg_attribute) + config_overridden = True + + # Do-not-settings to map to config + negative_bool_map = [ + ('non_interactive', 'interactive'), + ('no_prompt_on_exit', 'prompt_on_exit'), + ('no_folder_suffix', 'use_folder_suffix'), + ('no_media_previews', 'download_media_previews'), + ('hide_downloads', 'show_downloads'), + ('no_open_folder', 'open_folder_when_finished'), + ('no_separate_messages', 'separate_messages'), + ('no_separate_timeline', 'separate_timeline'), + ('no_separate_messages', 'separate_messages'), + ] + + # Set config to the inverse (negation) of arguments that are True + for attributes in negative_bool_map: + arg_name = attributes[0] + config_name = attributes[1] + check_attr(arg_name, config_name) + + arg_attribute = getattr(args, arg_name) + + if arg_attribute == True: + setattr(config, config_name, not arg_attribute) + + if config_overridden: + print_warning( + "You have specified some command-line arguments that override config.ini settings.\n" + f"{20*' '}A separate, temporary config file will be generated for this session\n" + f"{20*' '}to prevent accidental changes to your original configuration.\n" + ) + config.config_path = config.config_path.parent / 'config_args.ini' + save_config_or_raise(config) diff --git a/config/browser.py b/config/browser.py new file mode 100644 index 0000000..230548f --- /dev/null +++ b/config/browser.py @@ -0,0 +1,310 @@ +"""Configuration Utilities""" + + +import json +import os +import os.path +import platform +import plyvel +import psutil +import sqlite3 +import traceback + +from time import sleep + +from textio import print_config + + +# Function to recursively search for "storage" folders and process SQLite files +def get_token_from_firefox_profile(directory: str) -> str | None: + """Gets a Fansly authorization token from a Firefox + configuration directory. + + :param str directory: The user's Firefox profile directory to analyze. + + :return: A Fansly authorization token if one could be found or None. + :rtype: str | None + """ + for root, _, files in os.walk(directory): + # Search the profile's "storage" folder + # TODO: It would probably be better to limit the search to the "default" subdirectory of "storage" + if "storage" in root: + for file in files: + if file.endswith(".sqlite"): + sqlite_file = os.path.join(root, file) + session_active_session = get_token_from_firefox_db(sqlite_file) + if session_active_session is not None: + return session_active_session + + # No match was found + return None + + +def get_token_from_firefox_db(sqlite_file_name: str, interactive: bool=True) -> str | None: + """Fetches the Fansly token from the Firefox SQLite configuration + database. + + :param str sqlite_file_name: The full path to the Firefox configuration + database. + + :return: The Fansly token if found or None otherwise. + :rtype: str | None + """ + session_active_session = None + + try: + with sqlite3.connect(sqlite_file_name) as conn: + cursor = conn.cursor() + + # Get all table names in the SQLite database + cursor.execute("SELECT name FROM sqlite_master WHERE type='table';") + tables = cursor.fetchall() + + for table in tables: + table_name = table[0] + cursor.execute(f"SELECT * FROM {table_name};") + rows = cursor.fetchall() + + # Read key-value pairs + for row in rows: + if row[0] == 'session_active_session': + session_active_session = json.loads(row[5].decode('utf-8'))['token'] + break + + return session_active_session + + except sqlite3.OperationalError as e: + # Got + # "sqlite3.OperationalError: Could not decode to UTF-8 column 'value' with text" + # all over the place. + # I guess this is from other databases with different encodings, + # maybe UTF-16. So just ignore. + pass + + except sqlite3.Error as e: + sqlite_error = str(e) + + if 'locked' in sqlite_error and 'irefox' in sqlite_file_name: + + if not interactive: + # Do not forcefully close user's browser in non-interactive/scheduled mode. + return None + + print_config( + f"Firefox browser is open but needs to be closed for automatic configurator" + f"\n{19*' '}to search your fansly account in the browsers storage." + f"\n{19*' '}Please save any important work within the browser & close the browser yourself" + f"\n{19*' '}or it will be closed automatically after continuing." + ) + + input(f"\n{19*' '}► Press to continue! ") + + close_browser_by_name('firefox') + + return get_token_from_firefox_db(sqlite_file_name, interactive) # recursively restart function + + else: + print(f"Unexpected Error processing SQLite file:\n{traceback.format_exc()}") + + except Exception: + print(f'Unexpected Error parsing Firefox SQLite databases:\n{traceback.format_exc()}') + + return None + + +def get_browser_config_paths() -> list[str]: + """Returns a list of file system paths where web browsers + would store configuration data in the current user profile and + depending on the operating system. + + This function returns paths of all supported browsers regardless + whether such a browser is installed or not. + + :return: A list of file system paths with potential web browser + configuration data. + :rtype: list[str] + """ + browser_paths = [] + + if platform.system() == 'Windows': + appdata = os.getenv('appdata') + local_appdata = os.getenv('localappdata') + + if appdata is None or local_appdata is None: + raise RuntimeError("Windows OS AppData environment variables are empty but shouldn't.") + + browser_paths = [ + os.path.join(local_appdata, 'Google', 'Chrome', 'User Data'), + os.path.join(local_appdata, 'Microsoft', 'Edge', 'User Data'), + os.path.join(appdata, 'Mozilla', 'Firefox', 'Profiles'), + os.path.join(appdata, 'Opera Software', 'Opera Stable'), + os.path.join(appdata, 'Opera Software', 'Opera GX Stable'), + os.path.join(local_appdata, 'BraveSoftware', 'Brave-Browser', 'User Data'), + ] + + elif platform.system() == 'Darwin': # macOS + home = os.path.expanduser("~") + # regarding safari comp: + # https://stackoverflow.com/questions/58479686/permissionerror-errno-1-operation-not-permitted-after-macos-catalina-update + + browser_paths = [ + os.path.join(home, 'Library', 'Application Support', 'Google', 'Chrome'), + os.path.join(home, 'Library', 'Application Support', 'Microsoft Edge'), + os.path.join(home, 'Library', 'Application Support', 'Firefox', 'Profiles'), + os.path.join(home, 'Library', 'Application Support', 'com.operasoftware.Opera'), + os.path.join(home, 'Library', 'Application Support', 'com.operasoftware.OperaGX'), + os.path.join(home, 'Library', 'Application Support', 'BraveSoftware'), + ] + + elif platform.system() == 'Linux': + home = os.path.expanduser("~") + + browser_paths = [ + os.path.join(home, '.config', 'google-chrome', 'Default'), + os.path.join(home, '.mozilla', 'firefox'), # firefox non-snap (couldn't verify with ubuntu) + os.path.join(home, 'snap', 'firefox', 'common', '.mozilla', 'firefox'), # firefox snap + os.path.join(home, '.config', 'opera'), # btw opera gx, does not exist for linux + os.path.join(home, '.config', 'BraveSoftware', 'Brave-Browser', 'Default'), + ] + + return browser_paths + + +def find_leveldb_folders(root_path: str) -> set[str]: + """Gets folders where leveldb (.ldb) files are located. + + :param str root_path: The directory path to start the search from. + + :return: A set of folder paths where leveldb files are located, + potentially empty. + :rtype: set[str] + """ + leveldb_folders = set() + + for root, dirs, files in os.walk(root_path): + for dir_name in dirs: + if 'leveldb' in dir_name.lower(): + leveldb_folders.add(os.path.join(root, dir_name)) + break + + for file in files: + if file.endswith('.ldb'): + leveldb_folders.add(root) + break + + return leveldb_folders + + +def close_browser_by_name(browser_name: str) -> None: + """Closes an active web browser application by name + eg. "Microsoft Edge" or "Opera Gx". + + :param str browser_name: The browser name. + """ + # microsoft edge names its process msedge + if browser_name == 'Microsoft Edge': + browser_name = 'msedge' + + # opera gx just names its process opera + elif browser_name == 'Opera Gx': + browser_name = 'opera' + + browser_processes = [ + proc for proc in psutil.process_iter(attrs=['name']) + if browser_name.lower() in proc.info['name'].lower() + ] + + closed = False # Flag to track if any process was closed + + if platform.system() == 'Windows': + for proc in browser_processes: + proc.terminate() + closed = True + + elif platform.system() == 'Darwin' or platform.system() == 'Linux': + for proc in browser_processes: + proc.kill() + closed = True + + if closed: + print_config(f"Succesfully closed {browser_name} browser.") + sleep(3) # give browser time to close its children processes + + +def parse_browser_from_string(browser_name: str) -> str: + """Returns a normalized browser name according to the input + or "Unknown". + + :param str browser_name: The web browser name to analyze. + + :return: A normalized (simplified/standardised) browser name + or "Unknown". + :rtype: str + """ + compatible_browsers = [ + 'Firefox', + 'Brave', + 'Opera GX', + 'Opera', + 'Chrome', + 'Edge' + ] + + for compatible_browser in compatible_browsers: + if compatible_browser.lower() in browser_name.lower(): + if compatible_browser.lower() == 'edge' and 'microsoft' in browser_name.lower(): + return 'Microsoft Edge' + else: + return compatible_browser + + return "Unknown" + + +def get_auth_token_from_leveldb_folder(leveldb_folder: str, interactive: bool=True) -> str | None: + """Gets a Fansly authorization token from a leveldb folder. + + :param str leveldb_folder: The leveldb folder. + + :return: A Fansly authorization token or None. + :rtype: str | None + """ + try: + db = plyvel.DB(leveldb_folder, compression='snappy') + + key = b'_https://fansly.com\x00\x01session_active_session' + value = db.get(key) + + if value: + session_active_session = value.decode('utf-8').replace('\x00', '').replace('\x01', '') + auth_token = json.loads(session_active_session).get('token') + db.close() + return auth_token + + else: + db.close() + return None + + except plyvel._plyvel.IOError as e: + error_message = str(e) + used_browser = parse_browser_from_string(error_message) + + if not interactive: + # Do not forcefully close user's browser in non-interactive/scheduled mode. + return None + + print_config( + f"{used_browser} browser is open but it needs to be closed for automatic configurator" + f"\n{19*' '}to search your Fansly account in the browser's storage." + f"\n{19*' '}Please save any important work within the browser & close the browser yourself" + f"\n{19*' '}or it will be closed automatically after continuing." + ) + + input(f"\n{19*' '}► Press to continue! ") + + close_browser_by_name(used_browser) + + # recursively restart function + return get_auth_token_from_leveldb_folder(leveldb_folder, interactive) + + except Exception: + return None diff --git a/config/config.py b/config/config.py new file mode 100644 index 0000000..9698df1 --- /dev/null +++ b/config/config.py @@ -0,0 +1,274 @@ +"""Configuration File Manipulation""" + + +import configparser +import os + +from configparser import ConfigParser +from os import getcwd +from os.path import join +from pathlib import Path + +from .fanslyconfig import FanslyConfig +from .modes import DownloadMode + +from errors import ConfigError +from textio import print_info, print_config, print_warning +from utils.common import save_config_or_raise +from utils.web import open_url + + +def parse_items_from_line(line: str) -> list[str]: + """Parses a list of items (eg. creator names) from a single line + as eg. read from a configuration file. + + :param str line: A single line containing eg. user names + separated by either spaces or commas (,). + + :return: A list of items (eg. names) parsed from the line. + :rtype: list[str] + """ + names: list[str] = [] + + if ',' in line: + names = line.split(',') + + else: + names = line.split() + + return names + + +def sanitize_creator_names(names: list[str]) -> set[str]: + """Sanitizes a list of creator names after they have been + parsed from a configuration file. + + This will: + + * remove empty names + * remove leading/trailing whitespace from a name + * remove a leading @ from a name + * remove duplicates + * lower-case each name (for de-duplication to work) + + :param list[str] names: A list of names to process. + + :return: A set of unique, sanitized creator names. + :rtype: set[str] + """ + return set( + name.strip().removeprefix('@').lower() + for name in names + if name.strip() + ) + + +def username_has_valid_length(name: str) -> bool: + if name is None: + return False + + return len(name) >= 4 and len(name) <= 30 + + +def username_has_valid_chars(name: str) -> bool: + if name is None: + return False + + invalid_chars = set(name) \ + - set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_") + + return not invalid_chars + + +def copy_old_config_values(): + """Copies configuration values from an old configuration file to + a new one. + + Only sections/values existing in the new configuration will be adjusted. + + The hardcoded file names are from `old_config.ini` to `config.ini`. + """ + current_directory = getcwd() + old_config_path = join(current_directory, 'old_config.ini') + new_config_path = join(current_directory, 'config.ini') + + if os.path.isfile(old_config_path) and os.path.isfile(new_config_path): + old_config = ConfigParser(interpolation=None) + old_config.read(old_config_path) + + new_config = ConfigParser(interpolation=None) + new_config.read(new_config_path) + + # iterate over each section in the old config + for section in old_config.sections(): + # check if the section exists in the new config + if new_config.has_section(section): + # iterate over each option in the section + for option in old_config.options(section): + # check if the option exists in the new config + if new_config.has_option(section, option): + # get the value from the old config and set it in the new config + value = old_config.get(section, option) + + # skip overwriting the version value + if section == 'Other' and option == 'version': + continue + + new_config.set(section, option, value) + + # save the updated new config + with open(new_config_path, 'w') as config_file: + new_config.write(config_file) + + +def load_config(config: FanslyConfig) -> None: + """Loads the program configuration from file. + + :param FanslyConfig config: The configuration object to fill. + """ + + print_info('Reading config.ini file ...') + print() + + config.config_path = Path.cwd() / 'config.ini' + + if not config.config_path.exists(): + print_warning("Configuration file config.ini not found.") + print_config("A default configuration file will be generated for you ...") + + with open(config.config_path, mode='w', encoding='utf-8'): + pass + + config._load_raw_config() + + try: + # WARNING: Do not use the save config helper until the very end! + # Since the settings from the config object are synced to the parser + # on save, all still uninitialized values from the partially loaded + # config would overwrite the existing configuration! + replace_me_str = 'ReplaceMe' + + #region TargetedCreator + + creator_section = 'TargetedCreator' + + if not config._parser.has_section(creator_section): + config._parser.add_section(creator_section) + + # Check for command-line override - already set? + if config.user_names is None: + user_names = config._parser.get(creator_section, 'Username', fallback=replace_me_str) # string + + config.user_names = \ + sanitize_creator_names(parse_items_from_line(user_names)) + + #endregion + + #region MyAccount + + account_section = 'MyAccount' + + if not config._parser.has_section(account_section): + config._parser.add_section(account_section) + + config.token = config._parser.get(account_section, 'Authorization_Token', fallback=replace_me_str) # string + config.user_agent = config._parser.get(account_section, 'User_Agent', fallback=replace_me_str) # string + + #endregion + + #region Other + + other_section = 'Other' + + # I obsoleted this ... + #config.current_version = config.parser.get('Other', 'version') # str + if config._parser.has_option(other_section, 'version'): + config._parser.remove_option(other_section, 'version') + + # Remove empty section + if config._parser.has_section(other_section) \ + and len(config._parser[other_section]) == 0: + config._parser.remove_section(other_section) + + #endregion + + #region Options + + options_section = 'Options' + + if not config._parser.has_section(options_section): + config._parser.add_section(options_section) + + # Local_directory, C:\MyCustomFolderFilePath -> str + config.download_directory = Path( + config._parser.get(options_section, 'download_directory', fallback='Local_directory') + ) + + # Normal (Timeline & Messages), Timeline, Messages, Single (Single by post id) or Collections -> str + download_mode = config._parser.get(options_section, 'download_mode', fallback='Normal') + config.download_mode = DownloadMode(download_mode.lower()) + + config.download_media_previews = config._parser.getboolean(options_section, 'download_media_previews', fallback=True) + config.open_folder_when_finished = config._parser.getboolean(options_section, 'open_folder_when_finished', fallback=True) + config.separate_messages = config._parser.getboolean(options_section, 'separate_messages', fallback=True) + config.separate_previews = config._parser.getboolean(options_section, 'separate_previews', fallback=False) + config.separate_timeline = config._parser.getboolean(options_section, 'separate_timeline', fallback=True) + config.show_downloads = config._parser.getboolean(options_section, 'show_downloads', fallback=True) + config.interactive = config._parser.getboolean(options_section, 'interactive', fallback=True) + config.prompt_on_exit = config._parser.getboolean(options_section, 'prompt_on_exit', fallback=True) + + # I renamed this to "use_duplicate_threshold" but retain older config.ini compatibility + # True, False -> boolean + if config._parser.has_option(options_section, 'utilise_duplicate_threshold'): + config.use_duplicate_threshold = config._parser.getboolean(options_section, 'utilise_duplicate_threshold', fallback=False) + config._parser.remove_option(options_section, 'utilise_duplicate_threshold') + + else: + config.use_duplicate_threshold = config._parser.getboolean(options_section, 'use_duplicate_threshold', fallback=False) + + # True, False -> boolean + if config._parser.has_option(options_section, 'use_suffix'): + config.use_folder_suffix = config._parser.getboolean(options_section, 'use_suffix', fallback=True) + config._parser.remove_option(options_section, 'use_suffix') + + else: + config.use_folder_suffix = config._parser.getboolean(options_section, 'use_folder_suffix', fallback=True) + + #endregion + + # Safe to save! :-) + save_config_or_raise(config) + + except configparser.NoOptionError as e: + error_string = str(e) + raise ConfigError(f"Your config.ini file is invalid, please download a fresh version of it from GitHub.\n{error_string}") + + except ValueError as e: + error_string = str(e) + + if 'a boolean' in error_string: + if config.interactive: + open_url('https://github.com/Avnsx/fansly-downloader/wiki/Explanation-of-provided-programs-&-their-functionality#4-configini') + + raise ConfigError( + f"'{error_string.rsplit('boolean: ')[1]}' is malformed in the configuration file! This value can only be True or False" + f"\n{17*' '}Read the Wiki > Explanation of provided programs & their functionality > config.ini [1]" + ) + + else: + if config.interactive: + open_url('https://github.com/Avnsx/fansly-downloader/wiki/Explanation-of-provided-programs-&-their-functionality#4-configini') + + raise ConfigError( + f"You have entered a wrong value in the config.ini file -> '{error_string}'" + f"\n{17*' '}Read the Wiki > Explanation of provided programs & their functionality > config.ini [2]" + ) + + except (KeyError, NameError) as key: + if config.interactive: + open_url('https://github.com/Avnsx/fansly-downloader/wiki/Explanation-of-provided-programs-&-their-functionality#4-configini') + + raise ConfigError( + f"'{key}' is missing or malformed in the configuration file!" + f"\n{17*' '}Read the Wiki > Explanation of provided programs & their functionality > config.ini [3]" + ) diff --git a/config/fanslyconfig.py b/config/fanslyconfig.py new file mode 100644 index 0000000..1f8c67c --- /dev/null +++ b/config/fanslyconfig.py @@ -0,0 +1,217 @@ +"""Configuration Class for Shared State""" + + +import requests + +from configparser import ConfigParser +from dataclasses import dataclass +from pathlib import Path + +from .modes import DownloadMode + + +@dataclass +class FanslyConfig(object): + #region Fields + + #region File-Independent Fields + + # Mandatory property + # This should be set to __version__ in the main script. + program_version: str + + # Define base threshold (used for when modules don't provide vars) + DUPLICATE_THRESHOLD: int = 50 + + # Configuration file + config_path: Path | None = None + + # Misc + token_from_browser_name: str | None = None + debug: bool = False + # If specified on the command-line + post_id: str | None = None + # Set on start after self-update + updated_to: str | None = None + + # Objects + _parser = ConfigParser(interpolation=None) + # Define requests session + http_session = requests.Session() + + #endregion + + #region config.ini Fields + + # TargetedCreator > username + user_names: set[str] | None = None + + # MyAccount + token: str | None = None + user_agent: str | None = None + + # Options + # "Normal" | "Timeline" | "Messages" | "Single" | "Collection" + download_mode: DownloadMode = DownloadMode.NORMAL + download_directory: (None | Path) = None + download_media_previews: bool = True + open_folder_when_finished: bool = True + separate_messages: bool = True + separate_previews: bool = False + separate_timeline: bool = True + show_downloads: bool = True + use_duplicate_threshold: bool = False + use_folder_suffix: bool = True + # Show input prompts or sleep - for automation/scheduling purposes + interactive: bool = True + # Should there be a "Press " prompt at the very end of the program? + # This helps for semi-automated runs (interactive=False) when coming back + # to the computer and wanting to see what happened in the console window. + prompt_on_exit: bool = True + + #endregion + + #endregion + + #region Methods + + def user_names_str(self) -> str | None: + """Returns a nicely formatted and alphabetically sorted list of + creator names - for console or config file output. + + :return: A single line of all creator names, alphabetically sorted + and separated by commas eg. "alice, bob, chris, dora". + Returns None if user_names is None. + :rtype: str | None + """ + if self.user_names is None: + return None + + return ', '.join(sorted(self.user_names)) + + + def download_mode_str(self) -> str: + """Gets `download_mod` as a string representation.""" + return str(self.download_mode).capitalize() + + + def _sync_settings(self) -> None: + """Syncs the settings of the config object + to the config parser/config file. + + This helper is required before saving. + """ + self._parser.set('TargetedCreator', 'username', self.user_names_str()) + + self._parser.set('MyAccount', 'authorization_token', self.token) + self._parser.set('MyAccount', 'user_agent', self.user_agent) + + if self.download_directory is None: + self._parser.set('Options', 'download_directory', 'Local_directory') + else: + self._parser.set('Options', 'download_directory', str(self.download_directory)) + + self._parser.set('Options', 'download_mode', self.download_mode_str()) + + # Booleans + self._parser.set('Options', 'show_downloads', str(self.show_downloads)) + self._parser.set('Options', 'download_media_previews', str(self.download_media_previews)) + self._parser.set('Options', 'open_folder_when_finished', str(self.open_folder_when_finished)) + self._parser.set('Options', 'separate_messages', str(self.separate_messages)) + self._parser.set('Options', 'separate_previews', str(self.separate_previews)) + self._parser.set('Options', 'separate_timeline', str(self.separate_timeline)) + self._parser.set('Options', 'use_duplicate_threshold', str(self.use_duplicate_threshold)) + self._parser.set('Options', 'use_folder_suffix', str(self.use_folder_suffix)) + self._parser.set('Options', 'interactive', str(self.interactive)) + self._parser.set('Options', 'prompt_on_exit', str(self.prompt_on_exit)) + + + def _load_raw_config(self) -> list[str]: + if self.config_path is None: + return [] + + else: + return self._parser.read(self.config_path) + + + def _save_config(self) -> bool: + if self.config_path is None: + return False + + else: + self._sync_settings() + + with self.config_path.open('w', encoding='utf-8') as f: + self._parser.write(f) + return True + + + def token_is_valid(self) -> bool: + if self.token is None: + return False + + return not any( + [ + len(self.token) < 50, + 'ReplaceMe' in self.token, + ] + ) + + + def useragent_is_valid(self) -> bool: + if self.user_agent is None: + return False + + return not any( + [ + len(self.user_agent) < 40, + 'ReplaceMe' in self.user_agent, + ] + ) + + + def get_unscrambled_token(self) -> str | None: + """Gets the unscrambled Fansly authorization token. + + Unscrambles the token if necessary. + + :return: The unscrambled Fansly authorization token. + :rtype: str | None + """ + + if self.token is not None: + F, c ='fNs', self.token + + if c[-3:] == F: + c = c.rstrip(F) + + A, B, C = [''] * len(c), 7, 0 + + for D in range(B): + for E in range(D, len(A), B) : A[E] = c[C]; C += 1 + + return ''.join(A) + + else: + return self.token + + return self.token + + + def http_headers(self) -> dict[str, str]: + token = self.get_unscrambled_token() + + if token is None or self.user_agent is None: + raise RuntimeError('Internal error generating HTTP headers - token and user agent not set.') + + headers = { + 'Accept': 'application/json, text/plain, */*', + 'Referer': 'https://fansly.com/', + 'accept-language': 'en-US,en;q=0.9', + 'authorization': token, + 'User-Agent': self.user_agent, + } + + return headers + + #endregion diff --git a/config/modes.py b/config/modes.py new file mode 100644 index 0000000..b70856d --- /dev/null +++ b/config/modes.py @@ -0,0 +1,13 @@ +"""Download Modes""" + + +from enum import StrEnum, auto + + +class DownloadMode(StrEnum): + NOTSET = auto() + COLLECTION = auto() + MESSAGES = auto() + NORMAL = auto() + SINGLE = auto() + TIMELINE = auto() diff --git a/config/validation.py b/config/validation.py new file mode 100644 index 0000000..53c8669 --- /dev/null +++ b/config/validation.py @@ -0,0 +1,347 @@ +"""Configuration Validation""" + + +from pathlib import Path +from time import sleep +from requests.exceptions import RequestException + +from .config import username_has_valid_chars, username_has_valid_length +from .fanslyconfig import FanslyConfig + +from errors import ConfigError +from pathio.pathio import ask_correct_dir +from textio import print_config, print_error, print_info, print_warning +from utils.common import save_config_or_raise +from utils.web import guess_user_agent, open_get_started_url + + +def validate_creator_names(config: FanslyConfig) -> bool: + """Validates the input value for `config_username` in `config.ini`. + + :param FanslyConfig config: The configuration object to validate. + + :return: True if all user names passed the test/could be remedied, + False otherwise. + :rtype: bool + """ + + if config.user_names is None: + return False + + # This is not only nice but also since this is a new list object, + # we won't be iterating over the list (set) being changed. + names = sorted(config.user_names) + list_changed = False + + for user in names: + validated_name = validate_adjust_creator_name(user, config.interactive) + + # Remove invalid names from set + if validated_name is None: + print_warning(f"Invalid creator name '{user}' will be removed from processing.") + config.user_names.remove(user) + list_changed = True + + # Has the user name been adjusted? (Interactive input) + elif user != validated_name: + config.user_names.remove(user) + config.user_names.add(validated_name) + list_changed = True + + print() + + # Save any potential changes + if list_changed: + save_config_or_raise(config) + + # No users left after validation -> error + if len(config.user_names) == 0: + return False + + else: + return True + + +def validate_adjust_creator_name(name: str, interactive: bool=False) -> str | None: + """Validates the name of a Fansly creator. + + :param name: The creator name to validate and potentially correct. + :type name: str + :param interactive: Prompts the user for a replacement name if an + invalid creator name is encountered. + :type interactive: bool + + :return: The potentially (interactively) adjusted creator name. + :rtype: str + """ + # validate input value for config_username in config.ini + while True: + usern_base_text = f"Invalid targeted creator name '@{name}':" + usern_error = False + + replaceme_str = 'ReplaceMe' + + if replaceme_str.lower() in name.lower(): + print_warning(f"Config.ini value '{name}' for TargetedCreator > Username is a placeholder value.") + usern_error = True + + if not usern_error and ' ' in name: + print_warning(f'{usern_base_text} name must not contain spaces.') + usern_error = True + + if not usern_error and not username_has_valid_length(name): + print_warning(f"{usern_base_text} must be between 4 and 30 characters long!\n") + usern_error = True + + if not usern_error and not username_has_valid_chars(name): + print_warning(f"{usern_base_text} should only contain\n{20*' '}alphanumeric characters, hyphens, or underscores!\n") + usern_error = True + + if not usern_error: + print_info(f"Name validation for @{name} successful!") + return name + + if interactive: + print_config( + f"Enter the username handle (eg. @MyCreatorsName or MyCreatorsName)" + f"\n{19*' '}of the Fansly creator you want to download content from." + ) + + name = input(f"\n{19*' '}► Enter a valid username: ") + name = name.strip().removeprefix('@') + + else: + return None + + +def validate_adjust_token(config: FanslyConfig) -> None: + """Validates the Fansly authorization token in the config + and analyzes installed browsers to automatically find tokens. + + :param FanslyConfig config: The configuration to validate and correct. + """ + # only if config_token is not set up already; verify if plyvel is installed + plyvel_installed, browser_name = False, None + + if not config.token_is_valid(): + try: + import plyvel + plyvel_installed = True + + except ImportError: + print_warning( + f"Fansly Downloader's automatic configuration for the authorization_token in the config.ini file will be skipped." + f"\n{20*' '}Your system is missing required plyvel (python module) builds by Siyao Chen (@liviaerxin)." + f"\n{20*' '}Installable with 'pip3 install plyvel-ci' or from github.com/liviaerxin/plyvel/releases/latest" + ) + + # semi-automatically set up value for config_token (authorization_token) based on the users input + if plyvel_installed and not config.token_is_valid(): + + # fansly-downloader plyvel dependant package imports + from config.browser import ( + find_leveldb_folders, + get_auth_token_from_leveldb_folder, + get_browser_config_paths, + get_token_from_firefox_profile, + parse_browser_from_string, + ) + + from utils.web import get_fansly_account_for_token + + print_warning( + f"Authorization token '{config.token}' is unmodified, missing or malformed" + f"\n{20*' '}in the configuration file." + ) + print_config( + f"Trying to automatically configure Fansly authorization token" + f"\n{19*' '}from any browser storage available on the local system ..." + ) + + browser_paths = get_browser_config_paths() + fansly_account = None + + for browser_path in browser_paths: + browser_fansly_token = None + + # if not firefox, process leveldb folders + if 'firefox' not in browser_path.lower(): + leveldb_folders = find_leveldb_folders(browser_path) + + for folder in leveldb_folders: + browser_fansly_token = get_auth_token_from_leveldb_folder(folder, interactive=config.interactive) + + if browser_fansly_token: + fansly_account = get_fansly_account_for_token(browser_fansly_token) + break # exit the inner loop if a valid processed_token is found + + # if firefox, process sqlite db instead + else: + browser_fansly_token = get_token_from_firefox_profile(browser_path) + + if browser_fansly_token: + fansly_account = get_fansly_account_for_token(browser_fansly_token) + + if all([fansly_account, browser_fansly_token]): + # we might also utilise this for guessing the useragent + browser_name = parse_browser_from_string(browser_path) + + if config.interactive: + # let user pick a account, to connect to fansly downloader + print_config(f"Do you want to link the account '{fansly_account}' to Fansly Downloader? (found in: {browser_name})") + + while True: + user_input_acc_verify = input(f"{19*' '}► Type either 'Yes' or 'No': ").strip().lower() + + if user_input_acc_verify.startswith('y') or user_input_acc_verify.startswith('n'): + break # break user input verification + + else: + print_error(f"Please enter either 'Yes' or 'No', to decide if you want to link to '{fansly_account}'.") + + else: + # Forcefully link account in interactive mode. + print_warning(f"Interactive mode is automtatically linking the account '{fansly_account}' to Fansly Downloader. (found in: {browser_name})") + user_input_acc_verify = 'y' + + # based on user input; write account username & auth token to config.ini + if user_input_acc_verify.startswith('y') and browser_fansly_token is not None: + config.token = browser_fansly_token + config.token_from_browser_name = browser_name + + save_config_or_raise(config) + + print_info(f"Success! Authorization token applied to config.ini file.\n") + + break # break whole loop + + # if no account auth was found in any of the users browsers + if fansly_account is None: + if config.interactive: + open_get_started_url() + + raise ConfigError( + f"Your Fansly account was not found in any of your browser's local storage." + f"\n{18*' '}Did you recently browse Fansly with an authenticated session?" + f"\n{18*' '}Please read & apply the 'Get-Started' tutorial." + ) + + # if users decisions have led to auth token still being invalid + if not config.token_is_valid(): + if config.interactive: + open_get_started_url() + + raise ConfigError( + f"Reached the end and the authorization token in config.ini file is still invalid!" + f"\n{18*' '}Please read & apply the 'Get-Started' tutorial." + ) + + +def validate_adjust_user_agent(config: FanslyConfig) -> None: + # validate input value for "user_agent" in config.ini + """Validates the input value for `user_agent` in `config.ini`. + + :param FanslyConfig config: The configuration to validate and correct. + """ + + # if no matches / error just set random UA + ua_if_failed = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36' + + based_on_browser = config.token_from_browser_name or 'Chrome' + + if not config.useragent_is_valid(): + print_warning(f"Browser user-agent '{config.user_agent}' in config.ini is most likely incorrect.") + + if config.token_from_browser_name is not None: + print_config( + f"Adjusting it with an educated guess based on the combination of your \n" + f"{19*' '}operating system & specific browser." + ) + + else: + print_config( + f"Adjusting it with an educated guess, hardcoded for Chrome browser." + f"\n{19*' '}If you're not using Chrome you might want to replace it in the config.ini file later on." + f"\n{19*' '}More information regarding this topic is on the Fansly Downloader Wiki." + ) + + try: + # thanks Jonathan Robson (@jnrbsn) - for continuously providing these up-to-date user-agents + user_agent_response = config.http_session.get( + 'https://jnrbsn.github.io/user-agents/user-agents.json', + headers = { + 'User-Agent': f"Avnsx/Fansly Downloader {config.program_version}", + 'accept-language': 'en-US,en;q=0.9' + } + ) + + if user_agent_response.status_code == 200: + config_user_agent = guess_user_agent( + user_agent_response.json(), + based_on_browser, + default_ua=ua_if_failed + ) + else: + config_user_agent = ua_if_failed + + except RequestException: + config_user_agent = ua_if_failed + + # save useragent modification to config file + config.user_agent = config_user_agent + + save_config_or_raise(config) + + print_info(f"Success! Applied a browser user-agent to config.ini file.\n") + + +def validate_adjust_download_directory(config: FanslyConfig) -> None: + """Validates the `download_directory` value from `config.ini` + and corrects it if possible. + + :param FanslyConfig config: The configuration to validate and correct. + """ + # if user didn't specify custom downloads path + if 'local_dir' in str(config.download_directory).lower(): + + config.download_directory = Path.cwd() + + print_info(f"Acknowledging local download directory: '{config.download_directory}'") + + # if user specified a correct custom downloads path + elif config.download_directory is not None \ + and config.download_directory.is_dir(): + + print_info(f"Acknowledging custom basis download directory: '{config.download_directory}'") + + else: # if their set directory, can't be found by the OS + print_warning( + f"The custom base download directory file path '{config.download_directory}' seems to be invalid!" + f"\n{20*' '}Please change it to a correct file path, for example: 'C:\\MyFanslyDownloads'" + f"\n{20*' '}An Explorer window to help you set the correct path will open soon!" + f"\n{20*' '}You may right-click inside the Explorer to create a new folder." + f"\n{20*' '}Select a folder and it will be used as the default download directory." + ) + + sleep(10) # give user time to realise instructions were given + + config.download_directory = ask_correct_dir() # ask user to select correct path using tkinters explorer dialog + + # save the config permanently into config.ini + save_config_or_raise(config) + + +def validate_adjust_config(config: FanslyConfig) -> None: + """Validates all input values from `config.ini` + and corrects them if possible. + + :param FanslyConfig config: The configuration to validate and correct. + """ + if not validate_creator_names(config): + raise ConfigError('Configuration error - no valid creator name specified.') + + validate_adjust_token(config) + + validate_adjust_user_agent(config) + + validate_adjust_download_directory(config) diff --git a/download/__init__.py b/download/__init__.py new file mode 100644 index 0000000..9b02ead --- /dev/null +++ b/download/__init__.py @@ -0,0 +1,5 @@ +"""Download Module""" + + +__all__: list[str] = [ +] diff --git a/download/account.py b/download/account.py new file mode 100644 index 0000000..6c45536 --- /dev/null +++ b/download/account.py @@ -0,0 +1,87 @@ +"""Fansly Account Information""" + + +import requests + +from typing import Any + +from .downloadstate import DownloadState + +from config import FanslyConfig +from config.modes import DownloadMode +from errors import ApiAccountInfoError, ApiAuthenticationError, ApiError +from textio import print_info + + +def get_creator_account_info(config: FanslyConfig, state: DownloadState) -> None: + print_info('Getting account information ...') + + if config.download_mode == DownloadMode.NOTSET: + message = 'Internal error getting account info - config download mode is not set.' + raise RuntimeError(message) + + # Collections are independent of creators and + # single posts may diverge from configured creators + if any([config.download_mode == DownloadMode.MESSAGES, + config.download_mode == DownloadMode.NORMAL, + config.download_mode == DownloadMode.TIMELINE]): + + account: dict[str, Any] = {} + + raw_response = requests.Response() + + try: + raw_response = config.http_session.get( + f"https://apiv3.fansly.com/api/v1/account?usernames={state.creator_name}", + headers=config.http_headers() + ) + + account = raw_response.json()['response'][0] + + state.creator_id = account['id'] + + except KeyError as e: + + if raw_response.status_code == 401: + message = \ + f"API returned unauthorized (24). This is most likely because of a wrong authorization token, in the configuration file.\n{21*' '}Used authorization token: '{config.token}'" \ + f'\n {str(e)}\n {raw_response.text}' + + raise ApiAuthenticationError(message) + + else: + message = \ + 'Bad response from fansly API (25). Please make sure your configuration file is not malformed.' \ + f'\n {str(e)}\n {raw_response.text}' + + raise ApiError(message) + + except IndexError as e: + message = \ + 'Bad response from fansly API (26). Please make sure your configuration file is not malformed; most likely misspelled the creator name.' \ + f'\n {str(e)}\n {raw_response.text}' + + raise ApiAccountInfoError(message) + + # below only needed by timeline; but wouldn't work without acc_req so it's here + # determine if followed + state.following = account.get('following', False) + + # determine if subscribed + state.subscribed = account.get('subscribed', False) + + # intentionally only put pictures into try / except block - its enough + try: + state.total_timeline_pictures = account['timelineStats']['imageCount'] + + except KeyError: + raise ApiAccountInfoError(f"Can not get timelineStats for creator username '{state.creator_name}'; you most likely misspelled it! (27)") + + state.total_timeline_videos = account['timelineStats']['videoCount'] + + # overwrite base dup threshold with custom 20% of total timeline content + config.DUPLICATE_THRESHOLD = int(0.2 * int(state.total_timeline_pictures + state.total_timeline_videos)) + + # timeline & messages will always use the creator name from config.ini, so we'll leave this here + print_info(f"Targeted creator: '{state.creator_name}'") + print() diff --git a/download/collections.py b/download/collections.py new file mode 100644 index 0000000..f62cc02 --- /dev/null +++ b/download/collections.py @@ -0,0 +1,47 @@ +"""Download Fansly Collections""" + + +from .common import process_download_accessible_media +from .downloadstate import DownloadState +from .types import DownloadType + +from config import FanslyConfig +from textio import input_enter_continue, print_error, print_info + + +def download_collections(config: FanslyConfig, state: DownloadState): + """Downloads Fansly purchased item collections.""" + + print_info(f"Starting Collections sequence. Buckle up and enjoy the ride!") + + # This is important for directory creation later on. + state.download_type = DownloadType.COLLECTIONS + + # send a first request to get all available "accountMediaId" ids, which are basically media ids of every graphic listed on /collections + collections_response = config.http_session.get( + 'https://apiv3.fansly.com/api/v1/account/media/orders/', + params={'limit': '9999','offset': '0','ngsw-bypass': 'true'}, + headers=config.http_headers() + ) + + if collections_response.status_code == 200: + collections_response = collections_response.json() + + # format all ids from /account/media/orders (collections) + accountMediaIds = ','.join( + [order['accountMediaId'] + for order in collections_response['response']['accountMediaOrders']] + ) + + # input them into /media?ids= to get all relevant information about each purchased media in a 2nd request + post_object = config.http_session.get( + f"https://apiv3.fansly.com/api/v1/account/media?ids={accountMediaIds}", + headers=config.http_headers()) + + post_object = post_object.json() + + process_download_accessible_media(config, state, post_object['response']) + + else: + print_error(f"Failed Collections download. Response code: {collections_response.status_code}\n{collections_response.text}", 23) + input_enter_continue(config.interactive) diff --git a/download/common.py b/download/common.py new file mode 100644 index 0000000..7ffaa1c --- /dev/null +++ b/download/common.py @@ -0,0 +1,105 @@ +"""Common Download Functions""" + + +import traceback + +from .downloadstate import DownloadState +from .media import download_media +from .types import DownloadType + +from config import FanslyConfig +from errors import DuplicateCountError +from media import MediaItem, parse_media_info +from pathio import set_create_directory_for_download +from textio import print_error, print_info, print_warning, input_enter_continue + + +def print_download_info(config: FanslyConfig) -> None: + ## starting here: stuff that literally every download mode uses, which should be executed at the very first everytime + if config.user_agent: + print_info(f"Using user-agent: '{config.user_agent[:28]} [...] {config.user_agent[-35:]}'") + + print_info(f"Open download folder when finished, is set to: '{config.open_folder_when_finished}'") + print_info(f"Downloading files marked as preview, is set to: '{config.download_media_previews}'") + print() + + if config.download_media_previews: + print_warning('Previews downloading is enabled; repetitive and/or emoji spammed media might be downloaded!') + print() + + +def process_download_accessible_media( + config: FanslyConfig, + state: DownloadState, + media_infos: list[dict], + post_id: str | None=None, + ) -> bool: + """Filters all media found in posts, messages, ... and downloads them. + + :param FanslyConfig config: The downloader configuration. + :param DownloadState state: The state and statistics of what is + currently being downloaded. + :param list[dict] media_infos: A list of media informations from posts, + timelines, messages, collections and so on. + :param str|None post_id: The post ID required for "Single" download mode. + + :return: "False" as a break indicator for "Timeline" downloads, + "True" otherwise. + :rtype: bool + """ + contained_posts: list[MediaItem] = [] + + # Timeline + + # loop through the list of dictionaries and find the highest quality media URL for each one + for media_info in media_infos: + try: + # add details into a list + contained_posts += [parse_media_info(state, media_info, post_id)] + + except Exception: + print_error(f"Unexpected error during parsing {state.download_type_str()} content;\n{traceback.format_exc()}", 42) + input_enter_continue(config.interactive) + + # summarise all scrapable & wanted media + accessible_media = [ + item for item in contained_posts + if item.download_url \ + and (item.is_preview == config.download_media_previews \ + or not item.is_preview) + ] + + # Special messages handling + original_duplicate_threshold = config.DUPLICATE_THRESHOLD + + if state.download_type == DownloadType.MESSAGES: + total_accessible_message_content = len(accessible_media) + + # Overwrite base dup threshold with 20% of total accessible content in messages. + # Don't forget to save/reset afterwards. + config.DUPLICATE_THRESHOLD = int(0.2 * total_accessible_message_content) + + # at this point we have already parsed the whole post object and determined what is scrapable with the code above + print_info(f"{state.creator_name} - amount of media in {state.download_type_str()}: {len(media_infos)} (scrapable: {len(accessible_media)})") + + set_create_directory_for_download(config, state) + + try: + # download it + download_media(config, state, accessible_media) + + except DuplicateCountError: + print_warning(f"Already downloaded all possible {state.download_type_str()} content! [Duplicate threshold exceeded {config.DUPLICATE_THRESHOLD}]") + # "Timeline" needs a way to break the loop. + if state.download_type == DownloadType.TIMELINE: + return False + + except Exception: + print_error(f"Unexpected error during {state.download_type_str()} download: \n{traceback.format_exc()}", 43) + input_enter_continue(config.interactive) + + finally: + # Reset DUPLICATE_THRESHOLD to the value it was before. + config.DUPLICATE_THRESHOLD = original_duplicate_threshold + + return True diff --git a/download/core.py b/download/core.py new file mode 100644 index 0000000..6a4ce37 --- /dev/null +++ b/download/core.py @@ -0,0 +1,25 @@ +"""Core Download Functions + +This sub-module exists to deal with circular module references +and still be convenient to use and not clutter the module namespace. +""" + + +from .account import get_creator_account_info +from .collections import download_collections +from .common import print_download_info +from .downloadstate import DownloadState +from .messages import download_messages +from .single import download_single_post +from .timeline import download_timeline + + +__all__ = [ + 'download_collections', + 'print_download_info', + 'download_messages', + 'download_single_post', + 'download_timeline', + 'DownloadState', + 'get_creator_account_info', +] diff --git a/download/downloadstate.py b/download/downloadstate.py new file mode 100644 index 0000000..cdae2aa --- /dev/null +++ b/download/downloadstate.py @@ -0,0 +1,51 @@ +"""Program Downloading State Management""" + + +from dataclasses import dataclass, field +from pathlib import Path + +from .types import DownloadType + + +@dataclass +class DownloadState(object): + #region Fields + + # Mandatory Field + creator_name: str + + download_type: DownloadType = DownloadType.NOTSET + + # Creator state + creator_id: str | None = None + following: bool = False + subscribed: bool = False + + base_path: Path | None = None + download_path: Path | None = None + + # Counters + pic_count: int = 0 + vid_count: int = 0 + duplicate_count: int = 0 + + total_timeline_pictures: int = 0 + total_timeline_videos: int = 0 + + # History + recent_audio_media_ids: set = field(default_factory=set) + recent_photo_media_ids: set = field(default_factory=set) + recent_video_media_ids: set = field(default_factory=set) + recent_audio_hashes: set = field(default_factory=set) + recent_photo_hashes: set = field(default_factory=set) + recent_video_hashes: set = field(default_factory=set) + + #endregion + + #region Methods + + def download_type_str(self) -> str: + """Gets `download_type` as a string representation.""" + return str(self.download_type).capitalize() + + #endregion diff --git a/download/m3u8.py b/download/m3u8.py new file mode 100644 index 0000000..1798de3 --- /dev/null +++ b/download/m3u8.py @@ -0,0 +1,133 @@ +"""Handles M3U8 Media""" + + +import av +import concurrent.futures +import io +import m3u8 + +from m3u8 import M3U8 +from pathlib import Path +from rich.table import Column +from rich.progress import BarColumn, TextColumn, Progress + +from config.fanslyconfig import FanslyConfig +from textio import print_error + + +def download_m3u8(config: FanslyConfig, m3u8_url: str, save_path: Path) -> bool: + """Download M3U8 content as MP4. + + :param FanslyConfig config: The downloader configuration. + :param str m3u8_url: The URL string of the M3U8 to download. + :param Path save_path: The suggested file to save the video to. + This will be changed to MP4 (.mp4). + + :return: True if successful or False otherwise. + :rtype: bool + """ + # parse m3u8_url for required strings + parsed_url = {k: v for k, v in [s.split('=') for s in m3u8_url.split('?')[-1].split('&')]} + + policy = parsed_url.get('Policy') + key_pair_id = parsed_url.get('Key-Pair-Id') + signature = parsed_url.get('Signature') + # re-construct original .m3u8 base link + m3u8_url = m3u8_url.split('.m3u8')[0] + '.m3u8' + # used for constructing .ts chunk links + split_m3u8_url = m3u8_url.rsplit('/', 1)[0] + # remove file extension (.m3u8) from save_path + save_path = save_path.parent / save_path.stem + + cookies = { + 'CloudFront-Key-Pair-Id': key_pair_id, + 'CloudFront-Policy': policy, + 'CloudFront-Signature': signature, + } + + # download the m3u8 playlist + playlist_content_req = config.http_session.get(m3u8_url, headers=config.http_headers(), cookies=cookies) + + if playlist_content_req.status_code != 200: + print_error(f'Failed downloading m3u8; at playlist_content request. Response code: {playlist_content_req.status_code}\n{playlist_content_req.text}', 12) + return False + + playlist_content = playlist_content_req.text + + # parse the m3u8 playlist content using the m3u8 library + playlist_obj: M3U8 = m3u8.loads(playlist_content) + + # get a list of all the .ts files in the playlist + ts_files = [segment.uri for segment in playlist_obj.segments if segment.uri.endswith('.ts')] + + # define a nested function to download a single .ts file and return the content + def download_ts(ts_file: str): + ts_url = f"{split_m3u8_url}/{ts_file}" + ts_response = config.http_session.get(ts_url, headers=config.http_headers(), cookies=cookies, stream=True) + buffer = io.BytesIO() + + for chunk in ts_response.iter_content(chunk_size=1024): + buffer.write(chunk) + + ts_content = buffer.getvalue() + + return ts_content + + # if m3u8 seems like it might be bigger in total file size; display loading bar + text_column = TextColumn(f"", table_column=Column(ratio=1)) + bar_column = BarColumn(bar_width=60, table_column=Column(ratio=5)) + + disable_loading_bar = False if len(ts_files) > 15 else True + + progress = Progress(text_column, bar_column, expand=True, transient=True, disable = disable_loading_bar) + + with progress: + with concurrent.futures.ThreadPoolExecutor() as executor: + ts_contents = [ + file for file in progress.track( + executor.map(download_ts, ts_files), + total=len(ts_files) + ) + ] + + segment = bytearray() + + for ts_content in ts_contents: + segment += ts_content + + input_container = av.open(io.BytesIO(segment), format='mpegts') + + video_stream = input_container.streams.video[0] + audio_stream = input_container.streams.audio[0] + + # define output container and streams + output_container = av.open(f"{save_path}.mp4", 'w') # add .mp4 file extension + + video_stream = output_container.add_stream(template=video_stream) + audio_stream = output_container.add_stream(template=audio_stream) + + start_pts = None + + for packet in input_container.demux(): + if packet.dts is None: + continue + + if start_pts is None: + start_pts = packet.pts + + packet.pts -= start_pts + packet.dts -= start_pts + + if packet.stream == input_container.streams.video[0]: + packet.stream = video_stream + + elif packet.stream == input_container.streams.audio[0]: + packet.stream = audio_stream + + output_container.mux(packet) + + # close containers + input_container.close() + output_container.close() + + return True diff --git a/download/media.py b/download/media.py new file mode 100644 index 0000000..4613e3b --- /dev/null +++ b/download/media.py @@ -0,0 +1,170 @@ +"""Fansly Download Functionality""" + +from pathlib import Path + +from rich.progress import Progress, BarColumn, TextColumn +from rich.table import Column +from PIL import Image, ImageFile + +from .downloadstate import DownloadState +from .m3u8 import download_m3u8 +from .types import DownloadType + +from config import FanslyConfig +from errors import DownloadError +from fileio.dedupe import dedupe_media_content +from media import MediaItem +from pathio import set_create_directory_for_download +from textio import input_enter_close, print_info, print_error, print_warning +from utils.common import exit +from errors import DuplicateCountError, MediaError + + +# tell PIL to be tolerant of files that are truncated +ImageFile.LOAD_TRUNCATED_IMAGES = True + +# turn off for our purpose unnecessary PIL safety features +Image.MAX_IMAGE_PIXELS = None + + +def download_media(config: FanslyConfig, state: DownloadState, accessible_media: list[MediaItem]): + """Downloads all media items to their respective target folders.""" + if state.download_type == DownloadType.NOTSET: + raise RuntimeError('Internal error during media download - download type not set on state.') + + # loop through the accessible_media and download the media files + for media_item in accessible_media: + # Verify that the duplicate count has not drastically spiked and + # and if it did verify that the spiked amount is significantly + # high to cancel scraping + if config.use_duplicate_threshold \ + and state.duplicate_count > config.DUPLICATE_THRESHOLD \ + and config.DUPLICATE_THRESHOLD >= 50: + raise DuplicateCountError(state.duplicate_count) + + # general filename construction & if content is a preview; add that into its filename + filename = media_item.get_file_name() + + # "None" safeguards + if media_item.mimetype is None: + raise MediaError('MIME type for media item not defined. Aborting.') + + if media_item.download_url is None: + raise MediaError('Download URL for media item not defined. Aborting.') + + # deduplication - part 1: decide if this media is even worth further processing; by media id + if any([media_item.media_id in state.recent_photo_media_ids, media_item.media_id in state.recent_video_media_ids]): + print_info(f"Deduplication [Media ID]: {media_item.mimetype.split('/')[-2]} '{filename}' → declined") + state.duplicate_count += 1 + continue + + else: + if 'image' in media_item.mimetype: + state.recent_photo_media_ids.add(media_item.media_id) + + elif 'video' in media_item.mimetype: + state.recent_video_media_ids.add(media_item.media_id) + + elif 'audio' in media_item.mimetype: + state.recent_audio_media_ids.add(media_item.media_id) + + base_directory = set_create_directory_for_download(config, state) + + # for collections downloads we just put everything into the same folder + if state.download_type == DownloadType.COLLECTIONS: + file_save_path = base_directory / filename + # compatibility for final "Download finished...!" print + file_save_dir = file_save_path + + # for every other type of download; we do want to determine the sub-directory to save the media file based on the mimetype + else: + if 'image' in media_item.mimetype: + file_save_dir = base_directory / "Pictures" + + elif 'video' in media_item.mimetype: + file_save_dir = base_directory / "Videos" + + elif 'audio' in media_item.mimetype: + file_save_dir = base_directory / "Audio" + + else: + # if the mimetype is neither image nor video, skip the download + print_warning(f"Unknown mimetype; skipping download for mimetype: '{media_item.mimetype}' | media_id: {media_item.media_id}") + continue + + # decides to separate previews or not + if media_item.is_preview and config.separate_previews: + file_save_path = file_save_dir / 'Previews' / filename + file_save_dir = file_save_dir / 'Previews' + + else: + file_save_path = file_save_dir / filename + + if not file_save_dir.exists(): + file_save_dir.mkdir(parents=True) + + # if show_downloads is True / downloads should be shown + if config.show_downloads: + print_info(f"Downloading {media_item.mimetype.split('/')[-2]} '{filename}'") + + if media_item.file_extension == 'm3u8': + # handle the download of a m3u8 file + file_downloaded = download_m3u8(config, m3u8_url=media_item.download_url, save_path=file_save_path) + + if file_downloaded: + state.pic_count += 1 if 'image' in media_item.mimetype else 0 + state.vid_count += 1 if 'video' in media_item.mimetype else 0 + + else: + # handle the download of a normal media file + response = config.http_session.get(media_item.download_url, stream=True, headers=config.http_headers()) + + if response.status_code == 200: + text_column = TextColumn(f"", table_column=Column(ratio=1)) + bar_column = BarColumn(bar_width=60, table_column=Column(ratio=5)) + + file_size = int(response.headers.get('content-length', 0)) + + # if file size is above 20 MB display loading bar + disable_loading_bar = False if file_size and file_size >= 20_000_000 else True + + progress = Progress(text_column, bar_column, expand=True, transient=True, disable = disable_loading_bar) + + task_id = progress.add_task('', total=file_size) + + progress.start() + + # iterate over the response data in chunks + content = bytearray() + + for chunk in response.iter_content(chunk_size=1024): + if chunk: + content += chunk + progress.advance(task_id, len(chunk)) + + progress.refresh() + progress.stop() + + file_hash = dedupe_media_content(state, content, media_item.mimetype, filename) + + # Is it a duplicate? + if file_hash is None: + continue + + # hacky overwrite for save_path to introduce file hash to filename + base_path, extension = file_save_path.parent / file_save_path.stem, file_save_path.suffix + file_save_path = Path(f"{base_path}_hash_{file_hash}{extension}") + + with open(file_save_path, 'wb') as f: + f.write(content) + + # we only count them if the file was actually written + state.pic_count += 1 if 'image' in media_item.mimetype else 0 + state.vid_count += 1 if 'video' in media_item.mimetype else 0 + + else: + raise DownloadError( + f"Download failed on filename: {filename} - due to a " + f"network error --> status_code: {response.status_code} " + f"| content: \n{response.content} [13]" + ) diff --git a/download/messages.py b/download/messages.py new file mode 100644 index 0000000..eecf006 --- /dev/null +++ b/download/messages.py @@ -0,0 +1,79 @@ +"""Message Downloading""" + + +from .common import process_download_accessible_media +from .downloadstate import DownloadState +from .types import DownloadType + +from config import FanslyConfig +from textio import input_enter_continue, print_error, print_info, print_warning + + +def download_messages(config: FanslyConfig, state: DownloadState): + # This is important for directory creation later on. + state.download_type = DownloadType.MESSAGES + + print_info(f"Initiating Messages procedure. Standby for results.") + print() + + groups_response = config.http_session.get( + 'https://apiv3.fansly.com/api/v1/group', + headers=config.http_headers() + ) + + if groups_response.status_code == 200: + groups_response = groups_response.json()['response']['groups'] + + # go through messages and check if we even have a chat history with the creator + group_id = None + + for group in groups_response: + for user in group['users']: + if user['userId'] == state.creator_id: + group_id = group['id'] + break + + if group_id: + break + + # only if we do have a message ("group") with the creator + if group_id: + + msg_cursor: str = '0' + + while True: + + params = {'groupId': group_id, 'limit': '25', 'ngsw-bypass': 'true'} + + if msg_cursor != '0': + params['before'] = msg_cursor + + messages_response = config.http_session.get( + 'https://apiv3.fansly.com/api/v1/message', + headers=config.http_headers(), + params=params, + ) + + if messages_response.status_code == 200: + + # post object contains: messages, accountMedia, accountMediaBundles, tips, tipGoals, stories + post_object = messages_response.json()['response'] + + process_download_accessible_media(config, state, post_object['accountMedia']) + + # get next cursor + try: + msg_cursor = post_object['messages'][-1]['id'] + + except IndexError: + break # break if end is reached + + else: + print_error(f"Failed messages download. messages_req failed with response code: {messages_response.status_code}\n{messages_response.text}", 30) + + elif group_id is None: + print_warning(f"Could not find a chat history with {state.creator_name}; skipping messages download ...") + + else: + print_error(f"Failed Messages download. Response code: {groups_response.status_code}\n{groups_response.text}", 31) + input_enter_continue(config.interactive) diff --git a/download/single.py b/download/single.py new file mode 100644 index 0000000..6c2dbbc --- /dev/null +++ b/download/single.py @@ -0,0 +1,100 @@ +"""Single Post Downloading""" + + +from fileio.dedupe import dedupe_init +from .common import process_download_accessible_media +from .core import DownloadState +from .types import DownloadType + +from config import FanslyConfig +from textio import input_enter_continue, print_error, print_info, print_warning +from utils.common import is_valid_post_id + + +def download_single_post(config: FanslyConfig, state: DownloadState): + """Downloads a single post.""" + + # This is important for directory creation later on. + state.download_type = DownloadType.SINGLE + + print_info(f"You have launched in Single Post download mode.") + + if config.post_id is not None: + print_info(f"Trying to download post {config.post_id} as specified on the command-line ...") + post_id = config.post_id + + elif not config.interactive: + raise RuntimeError( + 'Single Post downloading is not supported in non-interactive mode ' + 'unless a post ID is specified via command-line.' + ) + + else: + print_info(f"Please enter the ID of the post you would like to download." + f"\n{17*' '}After you click on a post, it will show in your browsers URL bar." + ) + print() + + while True: + post_id = input(f"\n{17*' '}► Post ID: ") + + if is_valid_post_id(post_id): + break + + else: + print_error(f"The input string '{post_id}' can not be a valid post ID." + f"\n{22*' '}The last few numbers in the url is the post ID" + f"\n{22*' '}Example: 'https://fansly.com/post/1283998432982'" + f"\n{22*' '}In the example, '1283998432982' would be the post ID.", + 17 + ) + + post_response = config.http_session.get( + 'https://apiv3.fansly.com/api/v1/post', + params={'ids': post_id, 'ngsw-bypass': 'true',}, + headers=config.http_headers() + ) + + if post_response.status_code == 200: + # From: "accounts" + creator_username, creator_display_name = None, None + + # post object contains: posts, aggregatedPosts, accountMediaBundles, accountMedia, accounts, tips, tipGoals, stories, polls + post_object = post_response.json()['response'] + + # if access to post content / post contains content + if post_object['accountMedia']: + + # parse post creator name + if creator_username is None: + # the post creators reliable accountId + state.creator_id = post_object['accountMedia'][0]['accountId'] + + creator_display_name, creator_username = next( + (account.get('displayName'), account.get('username')) + for account in post_object.get('accounts', []) + if account.get('id') == state.creator_id + ) + + # Override the creator's name with the one from the posting. + # Post ID could be from a different creator than specified + # in the config file. + state.creator_name = creator_username + + if creator_display_name and creator_username: + print_info(f"Inspecting a post by {creator_display_name} (@{creator_username})") + else: + print_info(f"Inspecting a post by {creator_username.capitalize()}") + + # Deferred deduplication init because directory may have changed + # depending on post creator (!= configured creator) + dedupe_init(config, state) + + process_download_accessible_media(config, state, post_object['accountMedia'], post_id) + + else: + print_warning(f"Could not find any accessible content in the single post.") + + else: + print_error(f"Failed single post download. Response code: {post_response.status_code}\n{post_response.text}", 20) + input_enter_continue(config.interactive) diff --git a/download/timeline.py b/download/timeline.py new file mode 100644 index 0000000..8747502 --- /dev/null +++ b/download/timeline.py @@ -0,0 +1,137 @@ +"""Timeline Downloads""" + + +import traceback + +from requests import Response +from time import sleep + +from .common import process_download_accessible_media +from .core import DownloadState +from .types import DownloadType + +from config import FanslyConfig +from errors import ApiError +from textio import input_enter_continue, print_debug, print_error, print_info, print_warning + + +def download_timeline(config: FanslyConfig, state: DownloadState) -> None: + + print_info(f"Executing Timeline functionality. Anticipate remarkable outcomes!") + print() + + # This is important for directory creation later on. + state.download_type = DownloadType.TIMELINE + + # this has to be up here so it doesn't get looped + timeline_cursor = 0 + + while True: + if timeline_cursor == 0: + print_info("Inspecting most recent Timeline cursor ...") + + else: + print_info(f"Inspecting Timeline cursor: {timeline_cursor}") + + timeline_response = Response() + + # Simple attempt to deal with rate limiting + for attempt in range(9999): + try: + # People with a high enough internet download speed & hardware specification will manage to hit a rate limit here + endpoint = "timelinenew" if attempt == 0 else "timeline" + + if config.debug: + print_debug(f'HTTP headers: {config.http_headers()}') + + timeline_response = config.http_session.get( + f"https://apiv3.fansly.com/api/v1/{endpoint}/{state.creator_id}?before={timeline_cursor}&after=0&wallId=&contentSearch=&ngsw-bypass=true", + headers=config.http_headers() + ) + + break # break if no errors happened; which means we will try parsing & downloading contents of that timeline_cursor + + except Exception: + if attempt == 0: + continue + + elif attempt == 1: + print_warning( + f"Uhm, looks like we've hit a rate limit ..." + f"\n{20 * ' '}Using a VPN might fix this issue entirely." + f"\n{20 * ' '}Regardless, will now try to continue the download infinitely, every 15 seconds." + f"\n{20 * ' '}Let me know if this logic worked out at any point in time" + f"\n{20 * ' '}by opening an issue ticket, please!" + ) + print('\n' + traceback.format_exc()) + + else: + print(f"Attempt {attempt} ...") + + sleep(15) + + try: + timeline_response.raise_for_status() + + if timeline_response.status_code == 200: + + post_object = timeline_response.json()['response'] + + if config.debug: + print_debug(f'Post object: {post_object}') + + if not process_download_accessible_media(config, state, post_object['accountMedia']): + # Break on deduplication error - already downloaded + break + + print() + + # get next timeline_cursor + try: + timeline_cursor = post_object['posts'][-1]['id'] + + except IndexError: + # break the whole while loop, if end is reached + break + + except Exception: + message = \ + 'Please copy & paste this on GitHub > Issues & provide a short explanation (34):'\ + f'\n{traceback.format_exc()}\n' + + raise ApiError(message) + + except KeyError: + print_error("Couldn't find any scrapable media at all!\ + \n This most likely happend because you're not following the creator, your authorisation token is wrong\ + \n or the creator is not providing unlocked content.", + 35 + ) + input_enter_continue(config.interactive) + + except Exception: + print_error(f"Unexpected error during Timeline download: \n{traceback.format_exc()}", 36) + input_enter_continue(config.interactive) + + # Check if atleast 20% of timeline was scraped; exluding the case when all the media was declined as duplicates + low_yield = False + + if state.pic_count <= state.total_timeline_pictures * 0.2 and state.duplicate_count <= state.total_timeline_pictures * 0.2: + print_warning(f"Low amount of Pictures scraped. Creators total Pictures: {state.total_timeline_pictures} | Downloaded: {state.pic_count}") + low_yield = True + + if state.vid_count <= state.total_timeline_videos * 0.2 and state.duplicate_count <= state.total_timeline_videos * 0.2: + print_warning(f"Low amount of Videos scraped. Creators total Videos: {state.total_timeline_videos} | Downloaded: {state.vid_count}") + low_yield = True + + if low_yield: + if not state.following: + print(f"{20*' '}Follow the creator to be able to scrape media!") + + if not state.subscribed: + print(f"{20*' '}Subscribe to the creator if you would like to get the entire content.") + + if not config.download_media_previews: + print(f"{20*' '}Try setting download_media_previews to True in the config.ini file. Doing so, will help if the creator has marked all his content as previews.") + + print() diff --git a/download/types.py b/download/types.py new file mode 100644 index 0000000..245f1e8 --- /dev/null +++ b/download/types.py @@ -0,0 +1,12 @@ +"""Download Types""" + + +from enum import StrEnum, auto + + +class DownloadType(StrEnum): + NOTSET = auto() + COLLECTIONS = auto() + MESSAGES = auto() + SINGLE = auto() + TIMELINE = auto() diff --git a/errors/__init__.py b/errors/__init__.py new file mode 100644 index 0000000..13872b1 --- /dev/null +++ b/errors/__init__.py @@ -0,0 +1,123 @@ +"""Errors/Exceptions""" + + +#region Constants + +EXIT_SUCCESS: int = 0 +EXIT_ERROR: int = -1 +EXIT_ABORT: int = -2 +UNEXPECTED_ERROR: int = -3 +API_ERROR: int = -4 +CONFIG_ERROR: int = -5 +DOWNLOAD_ERROR: int = -6 +SOME_USERS_FAILED: int = -7 +UPDATE_FAILED: int = -10 +UPDATE_MANUALLY: int = -11 +UPDATE_SUCCESS: int = 1 + +#endregion + +#region Exceptions + +class DuplicateCountError(RuntimeError): + """The purpose of this error is to prevent unnecessary computation or requests to fansly. + Will stop downloading, after reaching either the base DUPLICATE_THRESHOLD or 20% of total content. + + To maintain logical consistency, users have the option to disable this feature; + e.g. a user downloads only 20% of a creator's media and then cancels the download, afterwards tries + to update that folder -> the first 20% will report completed -> cancels the download -> other 80% missing + """ + + def __init__(self, duplicate_count): + self.duplicate_count = duplicate_count + self.message = f"Irrationally high rise in duplicates: {duplicate_count}" + super().__init__(self.message) + + +class ConfigError(RuntimeError): + """This error is raised when configuration data is invalid. + + Invalid data may have been provided by config.ini or command-line. + """ + + def __init__(self, *args): + super().__init__(*args) + + +class ApiError(RuntimeError): + """This error is raised when the Fansly API yields no or invalid results. + + This may be caused by authentication issues (invalid token), + invalid user names or - in rare cases - changes to the Fansly API itself. + """ + + def __init__(self, *args): + super().__init__(*args) + + +class ApiAuthenticationError(ApiError): + """This specific error is raised when the Fansly API + yields an authentication error. + + This may primarily be caused by an invalid token. + """ + + def __init__(self, *args): + super().__init__(*args) + + +class ApiAccountInfoError(ApiError): + """This specific error is raised when the Fansly API + for account information yields invalid results. + + This may primarily be caused by an invalid user name. + """ + + def __init__(self, *args): + super().__init__(*args) + + +class DownloadError(RuntimeError): + """This error is raised when a media item could not be downloaded. + + This may be caused by network errors, proxy errors, server outages + and so on. + """ + + def __init__(self, *args): + super().__init__(*args) + + +class MediaError(RuntimeError): + """This error is raised when data of a media item is invalid. + + This may be by programming errors or trace back to problems in + Fansly API calls. + """ + + def __init__(self, *args): + super().__init__(*args) + +#endregion + + +__all__ = [ + 'EXIT_ABORT', + 'EXIT_ERROR', + 'EXIT_SUCCESS', + 'API_ERROR', + 'CONFIG_ERROR', + 'DOWNLOAD_ERROR', + 'SOME_USERS_FAILED', + 'UNEXPECTED_ERROR', + 'UPDATE_FAILED', + 'UPDATE_MANUALLY', + 'UPDATE_SUCCESS', + 'ApiError', + 'ApiAccountInfoError', + 'ApiAuthenticationError', + 'ConfigError', + 'DownloadError', + 'DuplicateCountError', + 'MediaError', +] diff --git a/fansly_downloader.py b/fansly_downloader.py index 8b473a5..fc0b37a 100644 --- a/fansly_downloader.py +++ b/fansly_downloader.py @@ -1,1514 +1,184 @@ -# fix in future: audio needs to be properly transcoded from mp4 to mp3, instead of just saved as -import requests, os, re, base64, hashlib, imagehash, io, traceback, sys, platform, subprocess, concurrent.futures, json, m3u8, av, time, mimetypes, configparser -from random import randint -from tkinter import Tk, filedialog -from loguru import logger as log -from functools import partialmethod -from PIL import Image, ImageFile -from time import sleep as s -from rich.table import Column -from rich.progress import Progress, BarColumn, TextColumn -from configparser import RawConfigParser -from os.path import join, exists -from os import makedirs, getcwd -from utils.update_util import delete_deprecated_files, check_latest_release, apply_old_config_values - -# tell PIL to be tolerant of files that are truncated -ImageFile.LOAD_TRUNCATED_IMAGES = True - -# turn off for our purpose unnecessary PIL safety features -Image.MAX_IMAGE_PIXELS = None - -# define requests session -sess = requests.Session() - - -# cross-platform compatible, re-name downloaders terminal output window title -def set_window_title(title): - current_platform = platform.system() - if current_platform == 'Windows': - subprocess.call('title {}'.format(title), shell=True) - elif current_platform == 'Linux' or current_platform == 'Darwin': - subprocess.call(['printf', r'\33]0;{}\a'.format(title)]) -set_window_title('Fansly Downloader') - -# for pyinstaller compatibility -def exit(): - os._exit(0) - -# base64 code to display logo in console -print(base64.b64decode('CiAg4paI4paI4paI4paI4paI4paI4paI4pWXIOKWiOKWiOKWiOKWiOKWiOKVlyDilojilojilojilZcgICDilojilojilZfilojilojilojilojilojilojilojilZfilojilojilZcgIOKWiOKWiOKVlyAgIOKWiOKWiOKVlyAgICDilojilojilojilojilojilojilZcg4paI4paI4pWXICAgICAgICAgIOKWiOKWiOKWiOKWiOKWiOKVlyDilojilojilojilojilojilojilZcg4paI4paI4paI4paI4paI4paI4pWXIAogIOKWiOKWiOKVlOKVkOKVkOKVkOKVkOKVneKWiOKWiOKVlOKVkOKVkOKWiOKWiOKVl+KWiOKWiOKWiOKWiOKVlyAg4paI4paI4pWR4paI4paI4pWU4pWQ4pWQ4pWQ4pWQ4pWd4paI4paI4pWRICDilZrilojilojilZcg4paI4paI4pWU4pWdICAgIOKWiOKWiOKVlOKVkOKVkOKWiOKWiOKVl+KWiOKWiOKVkSAgICAgICAgIOKWiOKWiOKVlOKVkOKVkOKWiOKWiOKVl+KWiOKWiOKVlOKVkOKVkOKWiOKWiOKVl+KWiOKWiOKVlOKVkOKVkOKWiOKWiOKVlwogIOKWiOKWiOKWiOKWiOKWiOKVlyAg4paI4paI4paI4paI4paI4paI4paI4pWR4paI4paI4pWU4paI4paI4pWXIOKWiOKWiOKVkeKWiOKWiOKWiOKWiOKWiOKWiOKWiOKVl+KWiOKWiOKVkSAgIOKVmuKWiOKWiOKWiOKWiOKVlOKVnSAgICAg4paI4paI4pWRICDilojilojilZHilojilojilZEgICAgICAgICDilojilojilojilojilojilojilojilZHilojilojilojilojilojilojilZTilZ3ilojilojilojilojilojilojilZTilZ0KICDilojilojilZTilZDilZDilZ0gIOKWiOKWiOKVlOKVkOKVkOKWiOKWiOKVkeKWiOKWiOKVkeKVmuKWiOKWiOKVl+KWiOKWiOKVkeKVmuKVkOKVkOKVkOKVkOKWiOKWiOKVkeKWiOKWiOKVkSAgICDilZrilojilojilZTilZ0gICAgICDilojilojilZEgIOKWiOKWiOKVkeKWiOKWiOKVkSAgICAgICAgIOKWiOKWiOKVlOKVkOKVkOKWiOKWiOKVkeKWiOKWiOKVlOKVkOKVkOKVkOKVnSDilojilojilZTilZDilZDilZDilZ0gCiAg4paI4paI4pWRICAgICDilojilojilZEgIOKWiOKWiOKVkeKWiOKWiOKVkSDilZrilojilojilojilojilZHilojilojilojilojilojilojilojilZHilojilojilojilojilojilojilojilZfilojilojilZEgICAgICAg4paI4paI4paI4paI4paI4paI4pWU4pWd4paI4paI4paI4paI4paI4paI4paI4pWXICAgIOKWiOKWiOKVkSAg4paI4paI4pWR4paI4paI4pWRICAgICDilojilojilZEgICAgIAogIOKVmuKVkOKVnSAgICAg4pWa4pWQ4pWdICDilZrilZDilZ3ilZrilZDilZ0gIOKVmuKVkOKVkOKVkOKVneKVmuKVkOKVkOKVkOKVkOKVkOKVkOKVneKVmuKVkOKVkOKVkOKVkOKVkOKVkOKVneKVmuKVkOKVnSAgICAgICDilZrilZDilZDilZDilZDilZDilZ0g4pWa4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWdICAgIOKVmuKVkOKVnSAg4pWa4pWQ4pWd4pWa4pWQ4pWdICAgICDilZrilZDilZ0gICAgIAogICAgICAgICAgICAgICAgICAgICAgICBkZXZlbG9wZWQgb24gZ2l0aHViLmNvbS9Bdm5zeC9mYW5zbHktZG93bmxvYWRlcgo=').decode('utf-8')) - -# most of the time, we utilize this to display colored output rather than logging or prints -def output(level: int, log_type: str, color: str, mytext: str): - try: - log.level(log_type, no = level, color = color) - except TypeError: - pass # level failsafe - log.__class__.type = partialmethod(log.__class__.log, log_type) - log.remove() - log.add(sys.stdout, format = "{level} | {time:HH:mm} || {message}", level=log_type) - log.type(mytext) - -# mostly used to attempt to open fansly downloaders documentation -def open_url(url_to_open: str): - s(10) - try: - import webbrowser - webbrowser.open(url_to_open, new=0, autoraise=True) - except Exception: - pass - -output(1,'\n Info','','Reading config.ini file ...') -config = RawConfigParser() -config_path = join(getcwd(), 'config.ini') -if len(config.read(config_path)) != 1: - output(2,'\n [1]ERROR','', f"config.ini file not found or can not be read.\n{21*' '}Please download it & make sure it is in the same directory as fansly downloader") - input('\nPress Enter to close ...') - exit() - - -## starting here: self updating functionality -# if started with --update start argument -if len(sys.argv) > 1 and sys.argv[1] == '--update': - # config.ini backwards compatibility fix (≤ v0.4) -> fix spelling mistake "seperate" to "separate" - if 'seperate_messages' in config['Options']: - config['Options']['separate_messages'] = config['Options'].pop('seperate_messages') - if 'seperate_previews' in config['Options']: - config['Options']['separate_previews'] = config['Options'].pop('seperate_previews') - with open(config_path, 'w', encoding='utf-8') as f: - config.write(f) - - # config.ini backwards compatibility fix (≤ v0.4) -> config option "naming_convention" & "update_recent_download" removed entirely - options_to_remove = ['naming_convention', 'update_recent_download'] - for option in options_to_remove: - if option in config['Options']: - config['Options'].pop(option) - with open(config_path, 'w', encoding='utf-8') as f: - config.write(f) - output(3, '\n WARNING', '', f"Just removed \'{option}\' from the config.ini file,\n\ - {6*' '}as the whole option is no longer supported after version 0.3.5") - - # get the version string of what we've just been updated to - version_string = sys.argv[2] - - # check if old config.ini exists, compare each pre-existing value of it and apply it to new config.ini - apply_old_config_values() - - # temporary: delete deprecated files - delete_deprecated_files() - - # get release description and if existent; display it in terminal - check_latest_release(update_version = version_string, intend = 'update') - - # read the config.ini file for a last time - config.read(config_path) -else: - # check if a new version is available - check_latest_release(current_version = config.get('Other', 'version'), intend = 'check') - - -## read & verify config values -try: - # TargetedCreator - config_username = config.get('TargetedCreator', 'Username') # string - - # MyAccount - config_token = config.get('MyAccount', 'Authorization_Token') # string - config_useragent = config.get('MyAccount', 'User_Agent') # string - - # Options - download_mode = config.get('Options', 'download_mode').capitalize() # Normal (Timeline & Messages), Timeline, Messages, Single (Single by post id) or Collections -> str - show_downloads = config.getboolean('Options', 'show_downloads') # True, False -> boolean - download_media_previews = config.getboolean('Options', 'download_media_previews') # True, False -> boolean - open_folder_when_finished = config.getboolean('Options', 'open_folder_when_finished') # True, False -> boolean - separate_messages = config.getboolean('Options', 'separate_messages') # True, False -> boolean - separate_previews = config.getboolean('Options', 'separate_previews') # True, False -> boolean - separate_timeline = config.getboolean('Options', 'separate_timeline') # True, False -> boolean - utilise_duplicate_threshold = config.getboolean('Options', 'utilise_duplicate_threshold') # True, False -> boolean - download_directory = config.get('Options', 'download_directory') # Local_directory, C:\MyCustomFolderFilePath -> str - - # Other - current_version = config.get('Other', 'version') # str -except configparser.NoOptionError as e: - error_string = str(e) - output(2,'\n ERROR','', f"Your config.ini file is very malformed, please download a fresh version of it from GitHub.\n{error_string}") - input('\nPress Enter to close ...') - exit() -except ValueError as e: - error_string = str(e) - if 'a boolean' in error_string: - output(2,'\n [1]ERROR','', f"\'{error_string.rsplit('boolean: ')[1]}\' is malformed in the configuration file! This value can only be True or False\n\ - {6*' '}Read the Wiki > Explanation of provided programs & their functionality > config.ini") - open_url('https://github.com/Avnsx/fansly-downloader/wiki/Explanation-of-provided-programs-&-their-functionality#4-configini') - input('\nPress Enter to close ...') - exit() - else: - output(2,'\n [2]ERROR','', f"You have entered a wrong value in the config.ini file -> \'{error_string}\'\n\ - {6*' '}Read the Wiki > Explanation of provided programs & their functionality > config.ini") - open_url('https://github.com/Avnsx/fansly-downloader/wiki/Explanation-of-provided-programs-&-their-functionality#4-configini') - input('\nPress Enter to close ...') - exit() -except (KeyError, NameError) as key: - output(2,'\n [3]ERROR','', f"\'{key}\' is missing or malformed in the configuration file!\n\ - {6*' '}Read the Wiki > Explanation of provided programs & their functionality > config.ini") - open_url('https://github.com/Avnsx/fansly-downloader/wiki/Explanation-of-provided-programs-&-their-functionality#4-configini') - input('\nPress Enter to close ...') - exit() - - -# update window title with specific downloader version -set_window_title(f"Fansly Downloader v{current_version}") - - -# occasionally notfiy user to star repository -def remind_stargazing(): - stargazers_count, total_downloads = 0, 0 - - # depends on global variable current_version - stats_headers = {'user-agent': f"Avnsx/Fansly Downloader {current_version}", - 'referer': f"Avnsx/Fansly Downloader {current_version}", - 'accept-language': 'en-US,en;q=0.9'} - - # get total_downloads count - stargazers_check_request = requests.get('https://api.github.com/repos/avnsx/fansly-downloader/releases', allow_redirects = True, headers = stats_headers) - if not stargazers_check_request.ok: - return False - stargazers_check_request = stargazers_check_request.json() - for x in stargazers_check_request: - total_downloads += x['assets'][0]['download_count'] or 0 - - # get stargazers_count - downloads_check_request = requests.get('https://api.github.com/repos/avnsx/fansly-downloader', allow_redirects = True, headers = stats_headers) - if not downloads_check_request.ok: - return False - downloads_check_request = downloads_check_request.json() - stargazers_count = downloads_check_request['stargazers_count'] or 0 - - percentual_stars = round(stargazers_count / total_downloads * 100, 2) - - # display message (intentionally "lnfo" with lvl 4) - output(4,'\n lnfo','', f"Fansly Downloader was downloaded {total_downloads} times, but only {percentual_stars} % of You(!) have starred it.\n\ - {6*' '}Stars directly influence my willingness to continue maintaining the project.\n\ - {5*' '}Help the repository grow today, by leaving a star on it and sharing it to others online!") - s(15) - -if randint(1,100) <= 19: - try: - remind_stargazing() - except Exception: # irrelevant enough, to pass regardless what errors may happen - pass - - - -## starting here: general validation of all input values in config.ini - -# validate input value for config_username in config.ini -while True: - usern_base_text = f'Invalid targeted creators username value; ' - usern_error = False - - if 'ReplaceMe' in config_username: - output(3, '\n WARNING', '', f"Config.ini value for TargetedCreator > Username > \'{config_username}\'; is unmodified.") - usern_error = True - - # remove @ from username in config file & save changes - if '@' in config_username and not usern_error: - config_username = config_username.replace('@', '') - config.set('TargetedCreator', 'username', config_username) - with open(config_path, 'w', encoding='utf-8') as config_file: - config.write(config_file) - - # intentionally dont want to just .strip() spaces, because like this, it might give the user a food for thought, that he's supposed to enter the username tag after @ and not creators display name - if ' ' in config_username and not usern_error: - output(3, ' WARNING', '', f"{usern_base_text}must be a concatenated string. No spaces!\n") - usern_error = True - - if not usern_error: - if len(config_username) < 4 or len(config_username) > 30: - output(3, ' WARNING', '', f"{usern_base_text}must be between 4 and 30 characters long!\n") - usern_error = True - else: - invalid_chars = set(config_username) - set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_") - if invalid_chars: - output(3, ' WARNING', '', f"{usern_base_text}should only contain\n{20*' '}alphanumeric characters, hyphens, or underscores!\n") - usern_error = True - - if not usern_error: - output(1, '\n info', '', 'Username validation successful!') - if config_username != config['TargetedCreator']['username']: - config.set('TargetedCreator', 'username', config_username) - with open(config_path, 'w', encoding='utf-8') as config_file: - config.write(config_file) - break - else: - output(5,'\n Config','', f"Populate the value, with the username handle (e.g.: @MyCreatorsName)\n\ - {7*' '}of the fansly creator, whom you would like to download content from.") - config_username = input(f"\n{19*' '} ► Enter a valid username: ") - - - -# only if config_token is not set up already; verify if plyvel is installed -plyvel_installed, processed_from_path = False, None -if any([not config_token, 'ReplaceMe' in config_token]) or config_token and len(config_token) < 50: - try: - import plyvel - plyvel_installed = True - except ImportError: - output(3,'\n WARNING','', f"Fansly Downloaders automatic configuration for the authorization_token in the config.ini file will be skipped.\ - \n{20*' '}Your system is missing required plyvel (python module) builds by Siyao Chen (@liviaerxin).\ - \n{20*' '}Installable with \'pip3 install plyvel-ci\' or from github.com/liviaerxin/plyvel/releases/latest") - -# semi-automatically set up value for config_token (authorization_token) based on the users input -if plyvel_installed and any([not config_token, 'ReplaceMe' in config_token, config_token and len(config_token) < 50]): - - # fansly-downloader plyvel dependant package imports - from utils.config_util import ( - get_browser_paths, - parse_browser_from_string, - find_leveldb_folders, - get_auth_token_from_leveldb_folder, - process_storage_folders, - link_fansly_downloader_to_account - ) - - output(3,'\n WARNING','', f"Authorization token \'{config_token}\' is unmodified,\n\ - {12*' '}missing or malformed in the configuration file.\n\ - {12*' '}Will automatically configure by fetching fansly authorization token,\n\ - {12*' '}from all browser storages available on the local system.") - - browser_paths = get_browser_paths() - processed_account = None - - for path in browser_paths: - processed_token = None - - # if not firefox, process leveldb folders - if 'firefox' not in path.lower(): - leveldb_folders = find_leveldb_folders(path) - for folder in leveldb_folders: - processed_token = get_auth_token_from_leveldb_folder(folder) - if processed_token: - processed_account = link_fansly_downloader_to_account(processed_token) - break # exit the inner loop if a valid processed_token is found - - # if firefox, process sqlite db instead - else: - processed_token = process_storage_folders(path) - if processed_token: - processed_account = link_fansly_downloader_to_account(processed_token) - - if all([processed_account, processed_token]): - processed_from_path = parse_browser_from_string(path) # we might also utilise this for guessing the useragent - - # let user pick a account, to connect to fansly downloader - output(5,'\n Config','', f"Do you want to link the account \'{processed_account}\' to Fansly Downloader? (found in: {processed_from_path})") - while True: - user_input_acc_verify = input(f"{20*' '}► Type either \'Yes\' or \'No\': ").strip().lower() - if user_input_acc_verify == "yes" or user_input_acc_verify == "no": - break # break user input verification - else: - output(2,'\n ERROR','', f"Please enter either \'Yes\' or \'No\', to decide if you want to link to \'{processed_account}\'") - - # based on user input; write account username & auth token to config.ini - if user_input_acc_verify == "yes" and all([processed_account, processed_token]): - config_token = processed_token - config.set('MyAccount', 'authorization_token', config_token) - with open(config_path, 'w', encoding='utf-8') as f: - config.write(f) - output(1,'\n Info','', f"Success! Authorization token applied to config.ini file\n") - break # break whole loop - - # if no account auth, was found in any of the users browsers - if not processed_account: - output(2,'\n ERROR','', f"Your Fansly account was not found in any of your browser\'s local storage.\n\ - {10*' '}Did you not recently browse Fansly with an authenticated session?\ - {10*' '}Please read & apply the \'Get-Started\' tutorial instead.") - open_url('https://github.com/Avnsx/fansly-downloader/wiki/Get-Started') - input('\n Press Enter to close ..') - exit() - - # if users decisions have led to auth token still being invalid - elif any([not config_token, 'ReplaceMe' in config_token]) or config_token and len(config_token) < 50: - output(2,'\n ERROR','', f"Reached the end and the authentication token in config.ini file is still invalid!\n\ - {10*' '}Please read & apply the \'Get-Started\' tutorial instead.") - open_url('https://github.com/Avnsx/fansly-downloader/wiki/Get-Started') - input('\n Press Enter to close ..') - exit() - - -# validate input value for "user_agent" in config.ini -ua_if_failed = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36' # if no matches / error just set random UA -def guess_user_agent(user_agents: dict, based_on_browser: str = processed_from_path or 'Chrome'): - - if processed_from_path == 'Microsoft Edge': - based_on_browser = 'Edg' # msedge only reports "Edg" as its identifier - - # could do the same for opera, opera gx, brave. but those are not supported by @jnrbsn's repo. so we just return chrome ua - # in general his repo, does not provide the most accurate latest user-agents, if I am borred some time in the future, - # I might just write my own similar repo and use that instead - - try: - os_name = platform.system() - if os_name == "Windows": - for user_agent in user_agents: - if based_on_browser in user_agent and "Windows" in user_agent: - match = re.search(r'Windows NT ([\d.]+)', user_agent) - if match: - os_version = match.group(1) - if os_version in user_agent: - return user_agent - elif os_name == "Darwin": # macOS - for user_agent in user_agents: - if based_on_browser in user_agent and "Macintosh" in user_agent: - match = re.search(r'Mac OS X ([\d_.]+)', user_agent) - if match: - os_version = match.group(1).replace('_', '.') - if os_version in user_agent: - return user_agent - elif os_name == "Linux": - for user_agent in user_agents: - if based_on_browser in user_agent and "Linux" in user_agent: - match = re.search(r'Linux ([\d.]+)', user_agent) - if match: - os_version = match.group(1) - if os_version in user_agent: - return user_agent - except Exception: - output(2,'\n [4]ERROR','', f'Regexing user-agent from online source failed: {traceback.format_exc()}') - - output(3, '\n WARNING', '', f"Missing user-agent for {based_on_browser} & os: {os_name}. Set chrome & windows ua instead") - return ua_if_failed - -if not config_useragent or config_useragent and len(config_useragent) < 40 or 'ReplaceMe' in config_useragent: - output(3, '\n WARNING', '', f"Browser user-agent in config.ini \'{config_useragent}\', is most likely incorrect.") - if processed_from_path: - output(5,'\n Config','', f"Will adjust it with a educated guess;\n\ - {7*' '}based on the combination of your operating system & specific browser") - else: - output(5,'\n Config','', f"Will adjust it with a educated guess, hard-set for chrome browser.\n\ - {7*' '}If you're not using chrome, you might want to replace it in the config.ini file later on.\n\ - {7*' '}more information regarding this topic is on the fansly downloader Wiki.") - - try: - # thanks Jonathan Robson (@jnrbsn) - for continously providing these up-to-date user-agents - user_agent_req = requests.get('https://jnrbsn.github.io/user-agents/user-agents.json', headers = {'User-Agent': f"Avnsx/Fansly Downloader {current_version}", 'accept-language': 'en-US,en;q=0.9'}) - if user_agent_req.ok: - user_agent_req = user_agent_req.json() - config_useragent = guess_user_agent(user_agent_req) - else: - config_useragent = ua_if_failed - except requests.exceptions.RequestException: - config_useragent = ua_if_failed - - # save useragent modification to config file - config.set('MyAccount', 'user_agent', config_useragent) - with open(config_path, 'w', encoding='utf-8') as config_file: - config.write(config_file) - - output(1,'\n Info','', f"Success! Applied a browser user-agent to config.ini file\n") - - - -## starting here: general epoch timestamp to local timezone manipulation -# calculates offset from global utc time, to local systems time -def compute_timezone_offset(): - offset = time.timezone if (time.localtime().tm_isdst == 0) else time.altzone - diff_from_utc = int(offset / 60 / 60 * -1) - hours_in_seconds = diff_from_utc * 3600 * -1 - return diff_from_utc, hours_in_seconds - -# compute timezone offset and hours in seconds once -diff_from_utc, hours_in_seconds = compute_timezone_offset() - -# detect 12 vs 24 hour time format usage -time_format = 12 if ('AM' in time.strftime('%X') or 'PM' in time.strftime('%X')) else 24 - -# convert every epoch timestamp passed, to the time it was for the local computers timezone -def get_adjusted_datetime(epoch_timestamp: int, diff_from_utc: int = diff_from_utc, hours_in_seconds: int = hours_in_seconds): - adjusted_timestamp = epoch_timestamp + diff_from_utc * 3600 - adjusted_timestamp += hours_in_seconds - # start of strings are ISO 8601; so that they're sortable by Name after download - if time_format == 24: - return time.strftime("%Y-%m-%d_at_%H-%M", time.localtime(adjusted_timestamp)) - else: - return time.strftime("%Y-%m-%d_at_%I-%M-%p", time.localtime(adjusted_timestamp)) - - - -## starting here: current working directory generation & validation -# if the users custom provided filepath is invalid; a tkinter dialog will open during runtime, asking to adjust download path -def ask_correct_dir(): - global BASE_DIR_NAME - root = Tk() - root.withdraw() - BASE_DIR_NAME = filedialog.askdirectory() - if BASE_DIR_NAME: - output(1,'\n Info','', f"Chose folder file path {BASE_DIR_NAME}") - return BASE_DIR_NAME - else: - output(2,'\n [5]ERROR','', f"Could not register your chosen folder file path. Please close and start all over again!") - s(15) - exit() # this has to force exit - -# generate a base directory; every module (Timeline, Messages etc.) calls this to figure out the right directory path -BASE_DIR_NAME = None # required in global space -def generate_base_dir(creator_name_to_create_for: str, module_requested_by: str): - global BASE_DIR_NAME, download_directory, separate_messages, separate_timeline - if 'Local_dir' in download_directory: # if user didn't specify custom downloads path - if "Collection" in module_requested_by: - BASE_DIR_NAME = join(getcwd(), 'Collections') - elif "Message" in module_requested_by and separate_messages: - BASE_DIR_NAME = join(getcwd(), creator_name_to_create_for+'_fansly', 'Messages') - elif "Timeline" in module_requested_by and separate_timeline: - BASE_DIR_NAME = join(getcwd(), creator_name_to_create_for+'_fansly', 'Timeline') - else: - BASE_DIR_NAME = join(getcwd(), creator_name_to_create_for+'_fansly') # use local directory - elif os.path.isdir(download_directory): # if user specified a correct custom downloads path - if "Collection" in module_requested_by: - BASE_DIR_NAME = join(download_directory, 'Collections') - elif "Message" in module_requested_by and separate_messages: - BASE_DIR_NAME = join(download_directory, creator_name_to_create_for+'_fansly', 'Messages') - elif "Timeline" in module_requested_by and separate_timeline: - BASE_DIR_NAME = join(download_directory, creator_name_to_create_for+'_fansly', 'Timeline') - else: - BASE_DIR_NAME = join(download_directory, creator_name_to_create_for+'_fansly') # use their custom path & specify new folder for the current creator in it - output(1,' Info','', f"Acknowledging custom basis download directory: \'{download_directory}\'") - else: # if their set directory, can't be found by the OS - output(3,'\n WARNING','', f"The custom basis download directory file path: \'{download_directory}\'; seems to be invalid!\ - \n{20*' '}Please change it, to a correct file path for example: \'C:/MyFanslyDownloads\'\ - \n{20*' '}You could also just change it back to the default argument: \'Local_directory\'\n\ - \n{20*' '}A explorer window to help you set the correct path, will open soon!\n\ - \n{20*' '}Preferably right click inside the explorer, to create a new folder\ - \n{20*' '}Select it and the folder will be used as the default download directory") - s(10) # give user time to realise instructions were given - download_directory = ask_correct_dir() # ask user to select correct path using tkinters explorer dialog - config.set('Options', 'download_directory', download_directory) # set corrected path inside the config - # save the config permanently into config.ini - with open(config_path, 'w', encoding='utf-8') as f: - config.write(f) - if "Collection" in module_requested_by: - BASE_DIR_NAME = join(download_directory, 'Collections') - elif "Message" in module_requested_by and separate_messages: - BASE_DIR_NAME = join(download_directory, creator_name_to_create_for+'_fansly', 'Messages') - elif "Timeline" in module_requested_by and separate_timeline: - BASE_DIR_NAME = join(download_directory, creator_name_to_create_for+'_fansly', 'Timeline') - else: - BASE_DIR_NAME = join(download_directory, creator_name_to_create_for+'_fansly') # use their custom path & specify new folder for the current creator in it - - # validate BASE_DIR_NAME; if current download folder wasn't created with content separation, disable it for this download session too - correct_File_Hierarchy, tmp_BDR = True, BASE_DIR_NAME.partition('_fansly')[0] + '_fansly' - if os.path.isdir(tmp_BDR): - for directory in os.listdir(tmp_BDR): - if os.path.isdir(join(tmp_BDR, directory)): - if 'Pictures' in directory and any([separate_messages, separate_timeline]): - correct_File_Hierarchy = False - if 'Videos' in directory and any([separate_messages, separate_timeline]): - correct_File_Hierarchy = False - if not correct_File_Hierarchy: - output(3, '\n WARNING', '', f"Due to the presence of \'Pictures\' and \'Videos\' sub-directories in the current download folder;\ - \n{20*' '}content separation will remain disabled throughout this current downloading session.") - separate_messages, separate_timeline = False, False - - # utilize recursion to fix BASE_DIR_NAME generation - generate_base_dir(creator_name_to_create_for, module_requested_by) - - return BASE_DIR_NAME - - - -# utilized to open the download directory in file explorer; once the download process has finished -def open_location(filepath: str): - plat = platform.system() - - if not open_folder_when_finished: - return False - - if not os.path.isfile(filepath) and not os.path.isdir(filepath): - return False - - # tested below and they work to open folder locations - if plat == 'Windows': - os.startfile(filepath) # verified works - elif plat == 'Linux': - subprocess.run(['xdg-open', filepath], shell=False) # verified works - elif plat == 'Darwin': - subprocess.run(['open', filepath], shell=False) # verified works - - return True - - - -# un/scramble auth token -F, c ='fNs', config_token -if c[-3:]==F: - c=c.rstrip(F) - A,B,C=['']*len(c),7,0 - for D in range(B): - for E in range(D,len(A),B):A[E]=c[C];C+=1 - config_token = ''.join(A) - - -# general headers; which the whole code uses -headers = { - 'Accept': 'application/json, text/plain, */*', - 'Referer': 'https://fansly.com/', - 'accept-language': 'en-US,en;q=0.9', - 'authorization': config_token, - 'User-Agent': config_useragent, -} - - - -# m3u8 compability -def download_m3u8(m3u8_url: str, save_path: str): - # parse m3u8_url for required strings - parsed_url = {k: v for k, v in [s.split('=') for s in m3u8_url.split('?')[-1].split('&')]} - policy = parsed_url.get('Policy') - key_pair_id = parsed_url.get('Key-Pair-Id') - signature = parsed_url.get('Signature') - m3u8_url = m3u8_url.split('.m3u8')[0] + '.m3u8' # re-construct original .m3u8 base link - split_m3u8_url = m3u8_url.rsplit('/', 1)[0] # used for constructing .ts chunk links - save_path = save_path.rsplit('.m3u8')[0] # remove file_extension from save_path - - cookies = { - 'CloudFront-Key-Pair-Id': key_pair_id, - 'CloudFront-Policy': policy, - 'CloudFront-Signature': signature, - } - - # download the m3u8 playlist - playlist_content_req = sess.get(m3u8_url, headers=headers, cookies=cookies) - if not playlist_content_req.ok: - output(2,'\n [12]ERROR','', f'Failed downloading m3u8; at playlist_content request. Response code: {playlist_content_req.status_code}\n{playlist_content_req.text}') - return False - playlist_content = playlist_content_req.text - - # parse the m3u8 playlist content using the m3u8 library - playlist_obj = m3u8.loads(playlist_content) - - # get a list of all the .ts files in the playlist - ts_files = [segment.uri for segment in playlist_obj.segments if segment.uri.endswith('.ts')] - - # define a nested function to download a single .ts file and return the content - def download_ts(ts_file: str): - ts_url = f"{split_m3u8_url}/{ts_file}" - ts_response = sess.get(ts_url, headers=headers, cookies=cookies, stream=True) - buffer = io.BytesIO() - for chunk in ts_response.iter_content(chunk_size=1024): - buffer.write(chunk) - ts_content = buffer.getvalue() - return ts_content - - # if m3u8 seems like it might be bigger in total file size; display loading bar - text_column = TextColumn(f"", table_column=Column(ratio=0.355)) - bar_column = BarColumn(bar_width=60, table_column=Column(ratio=2)) - disable_loading_bar = False if len(ts_files) > 15 else True - progress = Progress(text_column, bar_column, expand=True, transient=True, disable = disable_loading_bar) - with progress: - with concurrent.futures.ThreadPoolExecutor() as executor: - ts_contents = [file for file in progress.track(executor.map(download_ts, ts_files), total=len(ts_files))] - - segment = bytearray() - for ts_content in ts_contents: - segment += ts_content - - input_container = av.open(io.BytesIO(segment), format='mpegts') - video_stream = input_container.streams.video[0] - audio_stream = input_container.streams.audio[0] - - # define output container and streams - output_container = av.open(f"{save_path}.mp4", 'w') # add .mp4 file extension - video_stream = output_container.add_stream(template=video_stream) - audio_stream = output_container.add_stream(template=audio_stream) - - start_pts = None - for packet in input_container.demux(): - if packet.dts is None: - continue - - if start_pts is None: - start_pts = packet.pts - - packet.pts -= start_pts - packet.dts -= start_pts - - if packet.stream == input_container.streams.video[0]: - packet.stream = video_stream - elif packet.stream == input_container.streams.audio[0]: - packet.stream = audio_stream - output_container.mux(packet) - - # close containers - input_container.close() - output_container.close() - - return True - +#!/usr/bin/env python3 +"""Fansly Downloader""" -# define base threshold (used for when modules don't provide vars) -DUPLICATE_THRESHOLD = 50 +__version__ = '0.5.0' +__date__ = '2023-08-30T21:24:00+02' +__maintainer__ = 'Avnsx (Mika C.)' +__copyright__ = f'Copyright (C) 2021-2023 by {__maintainer__}' +__authors__: list[str] = [] +__credits__: list[str] = [] -""" -The purpose of this error is to prevent unnecessary computation or requests to fansly. -Will stop downloading, after reaching either the base DUPLICATE_THRESHOLD or 20% of total content. +# TODO: Fix in future: audio needs to be properly transcoded from mp4 to mp3, instead of just saved as +# TODO: Maybe write a log file? -To maintain logical consistency, users have the option to disable this feature; -e.g. a user downloads only 20% of a creator's media and then cancels the download, afterwards tries -to update that folder -> the first 20% will report completed -> cancels the download -> other 80% missing -""" -class DuplicateCountError(Exception): - def __init__(self, duplicate_count): - self.duplicate_count = duplicate_count - self.message = f"Irrationally high rise in duplicates: {duplicate_count}" - super().__init__(self.message) -pic_count, vid_count, duplicate_count = 0, 0, 0 # count downloaded content & duplicates, from all modules globally +import base64 +import traceback -# deduplication functionality variables -recent_photo_media_ids, recent_video_media_ids, recent_audio_media_ids = set(), set(), set() -recent_photo_hashes, recent_video_hashes, recent_audio_hashes = set(), set(), set() - -def sort_download(accessible_media: dict): - # global required so we can use them at the end of the whole code in global space - global pic_count, vid_count, save_dir, recent_photo_media_ids, recent_video_media_ids, recent_audio_media_ids, recent_photo_hashes, recent_video_hashes, recent_audio_hashes, duplicate_count - - # loop through the accessible_media and download the media files - for post in accessible_media: - # extract the necessary information from the post - media_id = post['media_id'] - created_at = get_adjusted_datetime(post['created_at']) - mimetype = post['mimetype'] - download_url = post['download_url'] - file_extension = post['file_extension'] - is_preview = post['is_preview'] - - # verify that the duplicate count has not drastically spiked and in-case it did; verify that the spiked amount is significant enough to cancel scraping - if utilise_duplicate_threshold and duplicate_count > DUPLICATE_THRESHOLD and DUPLICATE_THRESHOLD > 50: - raise DuplicateCountError(duplicate_count) - - # general filename construction & if content is a preview; add that into its filename - filename = f"{created_at}_preview_id_{media_id}.{file_extension}" if is_preview else f"{created_at}_id_{media_id}.{file_extension}" - - # deduplication - part 1: decide if this media is even worth further processing; by media id - if any([media_id in recent_photo_media_ids, media_id in recent_video_media_ids]): - output(1,' Info','', f"Deduplication [Media ID]: {mimetype.split('/')[-2]} \'{filename}\' → declined") - duplicate_count += 1 - continue - else: - if 'image' in mimetype: - recent_photo_media_ids.add(media_id) - elif 'video' in mimetype: - recent_video_media_ids.add(media_id) - elif 'audio' in mimetype: - recent_audio_media_ids.add(media_id) - - # for collections downloads we just put everything into the same folder - if "Collection" in download_mode: - save_path = join(BASE_DIR_NAME, filename) - save_dir = join(BASE_DIR_NAME, filename) # compatibility for final "Download finished...!" print - - if not exists(BASE_DIR_NAME): - makedirs(BASE_DIR_NAME, exist_ok = True) - - # for every other type of download; we do want to determine the sub-directory to save the media file based on the mimetype - else: - if 'image' in mimetype: - save_dir = join(BASE_DIR_NAME, "Pictures") - elif 'video' in mimetype: - save_dir = join(BASE_DIR_NAME, "Videos") - elif 'audio' in mimetype: - save_dir = join(BASE_DIR_NAME, "Audio") - else: - # if the mimetype is neither image nor video, skip the download - output(3,'\n WARNING','', f"Unknown mimetype; skipping download for mimetype: \'{mimetype}\' | media_id: {media_id}") - continue - - # decides to separate previews or not - if is_preview and separate_previews: - save_path = join(save_dir, 'Previews', filename) - save_dir = join(save_dir, 'Previews') - else: - save_path = join(save_dir, filename) - - if not exists(save_dir): - makedirs(save_dir, exist_ok = True) - - # if show_downloads is True / downloads should be shown - if show_downloads: - output(1,' Info','', f"Downloading {mimetype.split('/')[-2]} \'{filename}\'") - - if file_extension == 'm3u8': - # handle the download of a m3u8 file - file_downloaded = download_m3u8(m3u8_url = download_url, save_path = save_path) - if file_downloaded: - pic_count += 1 if 'image' in mimetype else 0; vid_count += 1 if 'video' in mimetype else 0 - else: - # handle the download of a normal media file - response = sess.get(download_url, stream=True, headers=headers) - - if response.ok: - text_column = TextColumn(f"", table_column=Column(ratio=0.355)) - bar_column = BarColumn(bar_width=60, table_column=Column(ratio=2)) - file_size = int(response.headers.get('content-length', 0)) - disable_loading_bar = False if file_size and file_size >= 20000000 else True # if file size is above 20MB; display loading bar - progress = Progress(text_column, bar_column, expand=True, transient=True, disable = disable_loading_bar) - task_id = progress.add_task('', total=file_size) - progress.start() - # iterate over the response data in chunks - content = bytearray() - for chunk in response.iter_content(chunk_size=1024): - if chunk: - content += chunk - progress.advance(task_id, len(chunk)) - progress.refresh() - progress.stop() - - file_hash = None - # utilise hashing for images - if 'image' in mimetype: - # open the image - img = Image.open(io.BytesIO(content)) - - # calculate the hash of the resized image - photohash = str(imagehash.phash(img, hash_size = 16)) - - # deduplication - part 2.1: decide if this photo is even worth further processing; by hashing - if photohash in recent_photo_hashes: - output(1,' Info','', f"Deduplication [Hashing]: {mimetype.split('/')[-2]} \'{filename}\' → declined") - duplicate_count += 1 - continue - else: - recent_photo_hashes.add(photohash) - - # close the image - img.close() - - file_hash = photohash - - # utilise hashing for videos - elif 'video' in mimetype: - videohash = hashlib.md5(content).hexdigest() - - # deduplication - part 2.2: decide if this video is even worth further processing; by hashing - if videohash in recent_video_hashes: - output(1,' Info','', f"Deduplication [Hashing]: {mimetype.split('/')[-2]} \'{filename}\' → declined") - duplicate_count += 1 - continue - else: - recent_video_hashes.add(videohash) +from random import randint +from time import sleep + +from config import FanslyConfig, load_config, validate_adjust_config +from config.args import parse_args, map_args_to_config +from config.modes import DownloadMode +from download.core import * +from errors import * +from fileio.dedupe import dedupe_init +from textio import ( + input_enter_close, + input_enter_continue, + print_error, + print_info, + print_warning, + set_window_title, +) +from updater import self_update +from utils.common import exit, open_location +from utils.web import remind_stargazing - file_hash = videohash - - # utilise hashing for audio - elif 'audio' in mimetype: - audiohash = hashlib.md5(content).hexdigest() - # deduplication - part 2.2: decide if this audio is even worth further processing; by hashing - if audiohash in recent_audio_hashes: - output(1,' Info', '', f"Deduplication [Hashing]: {mimetype.split('/')[-2]} \'{filename}\' → declined") - duplicate_count += 1 - continue - else: - recent_audio_hashes.add(audiohash) +# tell PIL to be tolerant of files that are truncated +#ImageFile.LOAD_TRUNCATED_IMAGES = True - file_hash = audiohash - - # hacky overwrite for save_path to introduce file hash to filename - base_path, extension = os.path.splitext(save_path) - save_path = f"{base_path}_hash_{file_hash}{extension}" - - with open(save_path, 'wb') as f: - f.write(content) +# turn off for our purpose unnecessary PIL safety features +#Image.MAX_IMAGE_PIXELS = None - # we only count them if the file was actually written - pic_count += 1 if 'image' in mimetype else 0; vid_count += 1 if 'video' in mimetype else 0 - else: - output(2,'\n [13]ERROR','', f"Download failed on filename: {filename} - due to an network error --> status_code: {response.status_code} | content: \n{response.content}") - input() - exit() - # all functions call sort_download at the end; which means we leave this function open ended, so that the python executor can get back into executing in global space @ the end of the global space code / loop this function repetetively as seen in timeline code +def print_statistics(config: FanslyConfig, state: DownloadState) -> None: + print( + f"\n╔═\n Finished {config.download_mode_str()} type download of {state.pic_count} pictures & {state.vid_count} videos " \ + f"from @{state.creator_name}!\n Declined duplicates: {state.duplicate_count}" \ + f"\n Saved content in directory: '{state.base_path}'"\ + f"\n\n ✶ Please leave a Star on the GitHub Repository, if you are satisfied! ✶\n{74*' '}═╝") + sleep(10) -# whole code uses this; whenever any json response needs to get parsed from fansly api -def parse_media_info(media_info: dict, post_id = None): - # initialize variables - highest_variants_resolution_url, download_url, file_extension, metadata, default_normal_locations, default_normal_mimetype, mimetype = None, None, None, None, None, None, None - created_at, media_id, highest_variants_resolution, highest_variants_resolution_height, default_normal_height = 0, 0, 0, 0, 0 - # check if media is a preview - is_preview = media_info['previewId'] is not None +def main(config: FanslyConfig) -> int: + """The main logic of the downloader program. - # fix rare bug, of free / paid content being counted as preview - if is_preview: - if media_info['access']: - is_preview = False - - def simplify_mimetype(mimetype: str): - if mimetype == 'application/vnd.apple.mpegurl': - mimetype = 'video/mp4' - elif mimetype == 'audio/mp4': # another bug in fansly api, where audio is served as mp4 filetype .. - mimetype = 'audio/mp3' # i am aware that the correct mimetype would be "audio/mpeg", but we just simplify it - return mimetype - - # variables in api "media" = "default_" & "preview" = "preview" in our code - # parse normal basic (paid/free) media from the default location, before parsing its variants (later on we compare heights, to determine which one we want) - if not is_preview: - default_normal_locations = media_info['media']['locations'] - - default_details = media_info['media'] - default_normal_id = int(default_details['id']) - default_normal_created_at = int(default_details['createdAt']) - default_normal_mimetype = simplify_mimetype(default_details['mimetype']) - default_normal_height = default_details['height'] or 0 - - # if its a preview, we take the default preview media instead - elif is_preview: - default_normal_locations = media_info['preview']['locations'] - - default_details = media_info['preview'] - default_normal_id = int(media_info['preview']['id']) - default_normal_created_at = int(default_details['createdAt']) - default_normal_mimetype = simplify_mimetype(default_details['mimetype']) - default_normal_height = default_details['height'] or 0 - - if default_details['locations']: - default_normal_locations = default_details['locations'][0]['location'] - - # locally fixes fansly api highest current_variant_resolution height bug - def parse_variant_metadata(variant_metadata: str): - variant_metadata = json.loads(variant_metadata) - max_variant = max(variant_metadata['variants'], key=lambda variant: variant['h'], default=None) - # if a heighest height is not found, we just hope 1080p is available - if not max_variant: - return 1080 - # else parse through variants and find highest height - if max_variant['w'] < max_variant['h']: - max_variant['w'], max_variant['h'] = max_variant['h'], max_variant['w'] - return max_variant['h'] - - def parse_variants(content: dict, content_type: str): # content_type: media / preview - nonlocal metadata, highest_variants_resolution, highest_variants_resolution_url, download_url, media_id, created_at, highest_variants_resolution_height, default_normal_mimetype, mimetype - if content.get('locations'): - location_url = content['locations'][0]['location'] + :param config: The program configuration. + :type config: FanslyConfig - current_variant_resolution = (content['width'] or 0) * (content['height'] or 0) - if current_variant_resolution > highest_variants_resolution and default_normal_mimetype == simplify_mimetype(content['mimetype']): - highest_variants_resolution = current_variant_resolution - highest_variants_resolution_height = content['height'] or 0 - highest_variants_resolution_url = location_url - media_id = int(content['id']) - mimetype = simplify_mimetype(content['mimetype']) - - # if key-pair-id is not in there we'll know it's the new .m3u8 format, so we construct a generalised url, which we can pass relevant auth strings with - # note: this url won't actually work, its purpose is to just pass the strings through the download_url variable - if not 'Key-Pair-Id' in highest_variants_resolution_url: - try: - # use very specific metadata, bound to the specific media to get auth info - metadata = content['locations'][0]['metadata'] - highest_variants_resolution_url = f"{highest_variants_resolution_url.split('.m3u8')[0]}_{parse_variant_metadata(content['metadata'])}.m3u8?ngsw-bypass=true&Policy={metadata['Policy']}&Key-Pair-Id={metadata['Key-Pair-Id']}&Signature={metadata['Signature']}" - except KeyError:pass # we pass here and catch below - - """ - it seems like the date parsed here is actually the correct date, - which is directly attached to the content. but posts that could be uploaded - 8 hours ago, can contain images from 3 months ago. so the date we are parsing here, - might be the date, that the fansly CDN has first seen that specific content and the - content creator, just attaches that old content to a public post after e.g. 3 months. - - or createdAt & updatedAt are also just bugged out idk.. - note: images would be overwriting each other by filename, if hashing didnt provide uniqueness - else we would be forced to add randint(-1800, 1800) to epoch timestamps - """ - try: - created_at = int(content['updatedAt']) - except Exception: - created_at = int(media_info[content_type]['createdAt']) - download_url = highest_variants_resolution_url - - - # somehow unlocked / paid media: get download url from media location - if 'location' in media_info['media']: - variants = media_info['media']['variants'] - for content in variants: - parse_variants(content = content, content_type = 'media') - - # previews: if media location is not found, we work with the preview media info instead - if not download_url and 'preview' in media_info: - variants = media_info['preview']['variants'] - for content in variants: - parse_variants(content = content, content_type = 'preview') - - """ - so the way this works is; we have these 4 base variables defined all over this function. - parse_variants() will initially overwrite them with values from each contents variants above. - then right below, we will compare the values and decide which media has the higher resolution. (default populated content vs content from variants) - or if variants didn't provide a higher resolution at all, we just fall back to the default content + :return: The exit code of the program. + :rtype: int """ - if all([default_normal_locations, highest_variants_resolution_url, default_normal_height, highest_variants_resolution_height]) and all([default_normal_height > highest_variants_resolution_height, default_normal_mimetype == mimetype]) or not download_url: - # overwrite default variable values, which we will finally return; with the ones from the default media - media_id = default_normal_id - created_at = default_normal_created_at - mimetype = default_normal_mimetype - download_url = default_normal_locations - - # due to fansly may 2023 update - if download_url: - # parse file extension separately - file_extension = download_url.split('/')[-1].split('.')[-1].split('?')[0] - - if file_extension == 'mp4' and mimetype == 'audio/mp3': - file_extension = 'mp3' - - # if metadata didn't exist we need the user to notify us through github, because that would be detrimental - if not 'Key-Pair-Id' in download_url and not metadata: - output(2,'\n [14]ERROR','', f"Failed downloading a video! Please open a GitHub issue ticket called \'Metadata missing\' and copy paste this:\n\ - \n\tMetadata Missing\n\tpost_id: {post_id} & media_id: {media_id} & config_username: {config_username}\n") - input('Press Enter to attempt continuing download ...') - - return {'media_id': media_id, 'created_at': created_at, 'mimetype': mimetype, 'file_extension': file_extension, 'is_preview': is_preview, 'download_url': download_url} - - - -## starting here: deduplication functionality -# variables used: recent_photo_media_ids, recent_video_media_ids recent_audio_media_ids,, recent_photo_hashes, recent_video_hashes, recent_audio_hashes -# these are defined globally above sort_download() though + exit_code = EXIT_SUCCESS -# exclusively used for extracting media_id from pre-existing filenames -def extract_media_id(filename: str): - match = re.search(r'_id_(\d+)', filename) - if match: - return int(match.group(1)) - return None + # Update window title with specific downloader version + set_window_title(f"Fansly Downloader v{config.program_version}") -# exclusively used for extracting hash from pre-existing filenames -def extract_hash_from_filename(filename: str): - match = re.search(r'_hash_([a-fA-F0-9]+)', filename) - if match: - return match.group(1) - return None + # base64 code to display logo in console + print(base64.b64decode('CiAg4paI4paI4paI4paI4paI4paI4paI4pWXIOKWiOKWiOKWiOKWiOKWiOKVlyDilojilojilojilZcgICDilojilojilZfilojilojilojilojilojilojilojilZfilojilojilZcgIOKWiOKWiOKVlyAgIOKWiOKWiOKVlyAgICDilojilojilojilojilojilojilZcg4paI4paI4pWXICAgICAgICAgIOKWiOKWiOKWiOKWiOKWiOKVlyDilojilojilojilojilojilojilZcg4paI4paI4paI4paI4paI4paI4pWXIAogIOKWiOKWiOKVlOKVkOKVkOKVkOKVkOKVneKWiOKWiOKVlOKVkOKVkOKWiOKWiOKVl+KWiOKWiOKWiOKWiOKVlyAg4paI4paI4pWR4paI4paI4pWU4pWQ4pWQ4pWQ4pWQ4pWd4paI4paI4pWRICDilZrilojilojilZcg4paI4paI4pWU4pWdICAgIOKWiOKWiOKVlOKVkOKVkOKWiOKWiOKVl+KWiOKWiOKVkSAgICAgICAgIOKWiOKWiOKVlOKVkOKVkOKWiOKWiOKVl+KWiOKWiOKVlOKVkOKVkOKWiOKWiOKVl+KWiOKWiOKVlOKVkOKVkOKWiOKWiOKVlwogIOKWiOKWiOKWiOKWiOKWiOKVlyAg4paI4paI4paI4paI4paI4paI4paI4pWR4paI4paI4pWU4paI4paI4pWXIOKWiOKWiOKVkeKWiOKWiOKWiOKWiOKWiOKWiOKWiOKVl+KWiOKWiOKVkSAgIOKVmuKWiOKWiOKWiOKWiOKVlOKVnSAgICAg4paI4paI4pWRICDilojilojilZHilojilojilZEgICAgICAgICDilojilojilojilojilojilojilojilZHilojilojilojilojilojilojilZTilZ3ilojilojilojilojilojilojilZTilZ0KICDilojilojilZTilZDilZDilZ0gIOKWiOKWiOKVlOKVkOKVkOKWiOKWiOKVkeKWiOKWiOKVkeKVmuKWiOKWiOKVl+KWiOKWiOKVkeKVmuKVkOKVkOKVkOKVkOKWiOKWiOKVkeKWiOKWiOKVkSAgICDilZrilojilojilZTilZ0gICAgICDilojilojilZEgIOKWiOKWiOKVkeKWiOKWiOKVkSAgICAgICAgIOKWiOKWiOKVlOKVkOKVkOKWiOKWiOKVkeKWiOKWiOKVlOKVkOKVkOKVkOKVnSDilojilojilZTilZDilZDilZDilZ0gCiAg4paI4paI4pWRICAgICDilojilojilZEgIOKWiOKWiOKVkeKWiOKWiOKVkSDilZrilojilojilojilojilZHilojilojilojilojilojilojilojilZHilojilojilojilojilojilojilojilZfilojilojilZEgICAgICAg4paI4paI4paI4paI4paI4paI4pWU4pWd4paI4paI4paI4paI4paI4paI4paI4pWXICAgIOKWiOKWiOKVkSAg4paI4paI4pWR4paI4paI4pWRICAgICDilojilojilZEgICAgIAogIOKVmuKVkOKVnSAgICAg4pWa4pWQ4pWdICDilZrilZDilZ3ilZrilZDilZ0gIOKVmuKVkOKVkOKVkOKVneKVmuKVkOKVkOKVkOKVkOKVkOKVkOKVneKVmuKVkOKVkOKVkOKVkOKVkOKVkOKVneKVmuKVkOKVnSAgICAgICDilZrilZDilZDilZDilZDilZDilZ0g4pWa4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWdICAgIOKVmuKVkOKVnSAg4pWa4pWQ4pWd4pWa4pWQ4pWdICAgICDilZrilZDilZ0gICAgIAogICAgICAgICAgICAgICAgICAgICAgICBkZXZlbG9wZWQgb24gZ2l0aHViLmNvbS9Bdm5zeC9mYW5zbHktZG93bmxvYWRlcgo=').decode('utf-8')) -# exclusively used for adding hash to pre-existing filenames -def add_hash_to_filename(filename: str, file_hash: str): - base_name, extension = os.path.splitext(filename) - hash_suffix = f"_hash_{file_hash}{extension}" + load_config(config) - # adjust filename for 255 bytes filename limit, on all common operating systems - max_length = 250 - if len(base_name) + len(hash_suffix) > max_length: - base_name = base_name[:max_length - len(hash_suffix)] - - return f"{base_name}{hash_suffix}" - -# exclusively used for hashing images from pre-existing download directories -def hash_img(filepath: str): - try: - filename = os.path.basename(filepath) - - media_id = extract_media_id(filename) - if media_id: - recent_photo_media_ids.add(media_id) - - existing_hash = extract_hash_from_filename(filename) - if existing_hash: - recent_photo_hashes.add(existing_hash) - else: - img = Image.open(filepath) - file_hash = str(imagehash.phash(img, hash_size = 16)) - recent_photo_hashes.add(file_hash) - img.close() # close image - - new_filename = add_hash_to_filename(filename, file_hash) - new_filepath = join(os.path.dirname(filepath), new_filename) - os.rename(filepath, new_filepath) - filepath = new_filepath - except FileExistsError: - os.remove(filepath) - except Exception: - output(2,'\n [15]ERROR','', f"\nError processing image \'{filepath}\': {traceback.format_exc()}") - -# exclusively used for hashing videos & audio from pre-existing download directories -def hash_content(filepath: str, content_format: str): # former known as hash_video - global recent_video_hashes, recent_audio_hashes, recent_video_media_ids, recent_audio_media_ids - try: - filename = os.path.basename(filepath) - - media_id = extract_media_id(filename) - if media_id: - if content_format == 'video': - recent_video_media_ids.add(media_id) - elif content_format == 'audio': - recent_audio_media_ids.add(media_id) - - existing_hash = extract_hash_from_filename(filename) - if existing_hash: - if content_format == 'video': - recent_video_hashes.add(existing_hash) - elif content_format == 'audio': - recent_audio_hashes.add(existing_hash) - else: - h = hashlib.md5() - with open(filepath, 'rb') as f: - while (part := f.read(1_048_576)): - h.update(part) - file_hash = h.hexdigest() - if content_format == 'video': - recent_video_hashes.add(file_hash) - elif content_format == 'audio': - recent_audio_hashes.add(file_hash) - - new_filename = add_hash_to_filename(filename, file_hash) - new_filepath = join(os.path.dirname(filepath), new_filename) - os.rename(filepath, new_filepath) - filepath = new_filepath - except FileExistsError: - os.remove(filepath) - except Exception: - output(2,'\n [16]ERROR','', f"\nError processing {content_format} \'{filepath}\': {traceback.format_exc()}") - -# exclusively used for processing pre-existing files from previous downloads -def process_file(file_path: str): - mimetype, _ = mimetypes.guess_type(file_path) - if mimetype is not None: - if mimetype.startswith('image'): - hash_img(file_path) - elif mimetype.startswith('video'): - hash_content(file_path, content_format = 'video') - elif mimetype.startswith('audio'): - hash_content(file_path, content_format = 'audio') - -# exclusively used for processing pre-existing folders from previous downloads -def process_folder(folder_path: str): - with concurrent.futures.ThreadPoolExecutor() as executor: - for root, dirs, files in os.walk(folder_path): - file_paths = [join(root, file) for file in files] - executor.map(process_file, file_paths) - return True - - -if os.path.isdir(generate_base_dir(config_username, download_mode)): - output(1,' Info','', f"Deduplication is automatically enabled for;\n{17*' '}{BASE_DIR_NAME}") - - if process_folder(BASE_DIR_NAME): - output(1,' Info','', f"Deduplication process is complete! Each new download will now be compared\ - \n{17*' '}against a total of {len(recent_photo_hashes)} photo & {len(recent_video_hashes)} video hashes and corresponding media IDs.") + args = parse_args() + # Note that due to config._sync_settings(), command-line arguments + # may overwrite config.ini settings later on during validation + # when the config may be saved again. + # Thus a separate config_args.ini will be used for the session. + map_args_to_config(args, config) - # print("Recent Photo Hashes:", recent_photo_hashes) - # print("Recent Photo Media IDs:", recent_photo_media_ids) - # print("Recent Video Hashes:", recent_video_hashes) - # print("Recent Video Media IDs:", recent_video_media_ids) + self_update(config) + # occasionally notfiy user to star repository if randint(1,100) <= 19: - output(3, '\n WARNING', '', f"Reminder; If you remove id_NUMBERS or hash_STRING from filenames of previously downloaded files,\ - \n{20*' '}they will no longer be compatible with fansly downloaders deduplication algorithm") - # because adding information as metadata; requires specific configuration for each file type through PIL and that's too complex due to file types. maybe in the future I might decide to just save every image as .png and every video as .mp4 and add/read it as metadata - # or if someone contributes a function actually perfectly adding metadata to all common file types, that would be nice - - - -## starting here: stuff that literally every download mode uses, which should be executed at the very first everytime -if download_mode: - output(1,' Info','', f"Using user-agent: \'{config_useragent[:28]} [...] {config_useragent[-35:]}\'") - output(1,' Info','', f"Open download folder when finished, is set to: \'{open_folder_when_finished}\'") - output(1,' Info','', f"Downloading files marked as preview, is set to: \'{download_media_previews}\'") - - if download_media_previews:output(3,'\n WARNING','', 'Previews downloading is enabled; repetitive and/or emoji spammed media might be downloaded!') - - - -## starting here: download_mode = Single -if download_mode == 'Single': - output(1,' Info','', f"You have launched in Single Post download mode\ - \n{17*' '}Please enter the ID of the post you would like to download\ - \n{17*' '}After you click on a post, it will show in your browsers url bar") - - while True: - post_id = input(f"\n{17*' '}► Post ID: ") # str - if post_id.isdigit() and len(post_id) >= 10 and not any(char.isspace() for char in post_id): - break - else: - output(2,'\n [17]ERROR','', f"The input string \'{post_id}\' can not be a valid post ID.\ - \n{22*' '}The last few numbers in the url is the post ID\ - \n{22*' '}Example: \'https://fansly.com/post/1283998432982\'\ - \n{22*' '}In the example \'1283998432982\' would be the post ID") - - post_req = sess.get('https://apiv3.fansly.com/api/v1/post', params={'ids': post_id, 'ngsw-bypass': 'true',}, headers=headers) - - if post_req.status_code == 200: - creator_username, creator_display_name = None, None # from: "accounts" - accessible_media = None - contained_posts = [] - - # post object contains: posts, aggregatedPosts, accountMediaBundles, accountMedia, accounts, tips, tipGoals, stories, polls - post_object = post_req.json()['response'] - - # if access to post content / post contains content - if post_object['accountMedia']: - - # parse post creator name - if not creator_username: - creator_id = post_object['accountMedia'][0]['accountId'] # the post creators reliable accountId - creator_display_name, creator_username = next((account.get('displayName'), account.get('username')) for account in post_object.get('accounts', []) if account.get('id') == creator_id) - - if creator_display_name and creator_username: - output(1,' Info','', f"Inspecting a post by {creator_display_name} (@{creator_username})") - else: - output(1,' Info','', f"Inspecting a post by {creator_username.capitalize()}") - - # parse relevant details about the post - if not accessible_media: - # loop through the list of dictionaries and find the highest quality media URL for each one - for obj in post_object['accountMedia']: - try: - # add details into a list - contained_posts += [parse_media_info(obj, post_id)] - except Exception: - output(2,'\n [18]ERROR','', f"Unexpected error during parsing Single Post content; \n{traceback.format_exc()}") - input('\n Press Enter to attempt to continue ..') - - # summarise all scrapable & wanted media - accessible_media = [item for item in contained_posts if item.get('download_url') and (item.get('is_preview') == download_media_previews or not item.get('is_preview'))] - - # at this point we have already parsed the whole post object and determined what is scrapable with the code above - output(1,' Info','', f"Amount of Media linked to Single post: {len(post_object['accountMedia'])} (scrapable: {len(accessible_media)})") - - """ - generate a base dir based on various factors, except this time we ovewrite the username from config.ini - with the custom username we analysed through single post download mode's post_object. this is because - the user could've decide to just download some random creators post instead of the one that he currently - set as creator for > TargetCreator > username in config.ini - """ - generate_base_dir(creator_username, module_requested_by = 'Single') - - try: - # download it - sort_download(accessible_media) - except DuplicateCountError: - output(1,' Info','', f"Already downloaded all possible Single Post content! [Duplicate threshold exceeded {DUPLICATE_THRESHOLD}]") - except Exception: - output(2,'\n [19]ERROR','', f"Unexpected error during sorting Single Post download; \n{traceback.format_exc()}") - input('\n Press Enter to attempt to continue ..') - - else: - output(2, '\n WARNING', '', f"Could not find any accessible content in the single post.") - - else: - output(2,'\n [20]ERROR','', f"Failed single post download. Fetch post information request, response code: {post_req.status_code}\n{post_req.text}") - input('\n Press Enter to attempt to continue ..') - - - - -## starting here: download_mode = Collection(s) -if 'Collection' in download_mode: - output(1,'\n Info','', f"Starting Collections sequence. Buckle up and enjoy the ride!") - - # send a first request to get all available "accountMediaId" ids, which are basically media ids of every graphic listed on /collections - collections_req = sess.get('https://apiv3.fansly.com/api/v1/account/media/orders/', params={'limit': '9999','offset': '0','ngsw-bypass': 'true'}, headers=headers) - if collections_req.ok: - collections_req = collections_req.json() - - # format all ids from /account/media/orders (collections) - accountMediaIds = ','.join([order['accountMediaId'] for order in collections_req['response']['accountMediaOrders']]) - - # input them into /media?ids= to get all relevant information about each purchased media in a 2nd request - post_object = sess.get(f"https://apiv3.fansly.com/api/v1/account/media?ids={accountMediaIds}", headers=headers) - post_object = post_object.json() - - contained_posts = [] - - for obj in post_object['response']: - try: - # add details into a list - contained_posts += [parse_media_info(obj)] - except Exception: - output(2,'\n [21]ERROR','', f"Unexpected error during parsing Collections content; \n{traceback.format_exc()}") - input('\n Press Enter to attempt to continue ..') - - # count only amount of scrapable media (is_preview check not really necessary since everything in collections is always paid, but w/e) - accessible_media = [item for item in contained_posts if item.get('download_url') and (item.get('is_preview') == download_media_previews or not item.get('is_preview'))] - - output(1,' Info','', f"Amount of Media in Media Collection: {len(post_object['response'])} (scrapable: {len(accessible_media)})") - - generate_base_dir(config_username, module_requested_by = 'Collection') - try: - # download it - sort_download(accessible_media) - except DuplicateCountError: - output(1,' Info','', f"Already downloaded all possible Collections content! [Duplicate threshold exceeded {DUPLICATE_THRESHOLD}]") - except Exception: - output(2,'\n [22]ERROR','', f"Unexpected error during sorting Collections download; \n{traceback.format_exc()}") - input('\n Press Enter to attempt to continue ..') + remind_stargazing(config) + except Exception: # irrelevant enough, to pass regardless what errors may happen + pass - else: - output(2,'\n [23]ERROR','', f"Failed Collections download. Fetch collections request, response code: {collections_req.status_code}\n{collections_req.text}") - input('\n Press Enter to attempt to continue ..') + validate_adjust_config(config) + if config.user_names is None \ + or config.download_mode == DownloadMode.NOTSET: + raise RuntimeError('Internal error - user name and download mode should not be empty after validation.') + for creator_name in sorted(config.user_names): + try: + state = DownloadState(creator_name) + # Special treatment for deviating folder names later + if not config.download_mode == DownloadMode.SINGLE: + dedupe_init(config, state) -# here comes stuff that is required by Messages AND Timeline - so this is like a 'shared section' -if any(['Message' in download_mode, 'Timeline' in download_mode, 'Normal' in download_mode]): - try: - raw_req = sess.get(f"https://apiv3.fansly.com/api/v1/account?usernames={config_username}", headers=headers) - acc_req = raw_req.json()['response'][0] - creator_id = acc_req['id'] - except KeyError as e: - if raw_req.status_code == 401: - output(2,'\n [24]ERROR','', f"API returned unauthorized. This is most likely because of a wrong authorization token, in the configuration file.\n{21*' '}Used authorization token: \'{config_token}\'") - else: - output(2,'\n [25]ERROR','', 'Bad response from fansly API. Please make sure your configuration file is not malformed.') - print('\n'+str(e)) - print(raw_req.text) - input('\nPress Enter to close ...') - exit() - except IndexError as e: - output(2,'\n [26]ERROR','', 'Bad response from fansly API. Please make sure your configuration file is not malformed; most likely misspelled the creator name.') - print('\n'+str(e)) - print(raw_req.text) - input('\nPress Enter to close ...') - exit() - - # below only needed by timeline; but wouldn't work without acc_req so it's here - # determine if followed - try: - following = acc_req['following'] - except KeyError: - following = False - - # determine if subscribed - try: - subscribed = acc_req['subscribed'] - except KeyError: - subscribed = False - - # intentionally only put pictures into try / except block - its enough - try: - total_timeline_pictures = acc_req['timelineStats']['imageCount'] - except KeyError: - output(2,'\n [27]ERROR','', f"Can not get timelineStats for creator username \'{config_username}\'; most likely misspelled it!") - input('\nPress Enter to close ...') - exit() - total_timeline_videos = acc_req['timelineStats']['videoCount'] - - # overwrite base dup threshold with custom 20% of total timeline content - DUPLICATE_THRESHOLD = int(0.2 * int(total_timeline_pictures + total_timeline_videos)) - - # timeline & messages will always use the creator name from config.ini, so we'll leave this here - output(1,' Info','', f"Targeted creator: \'{config_username}\'") - - - -## starting here: download_mode = Message(s) -if any(['Message' in download_mode, 'Normal' in download_mode]): - output(1,' \n Info','', f"Initiating Messages procedure. Standby for results.") - - groups_req = sess.get('https://apiv3.fansly.com/api/v1/group', headers=headers) - - if groups_req.ok: - groups_req = groups_req.json()['response']['groups'] - - # go through messages and check if we even have a chat history with the creator - group_id = None - for group in groups_req: - for user in group['users']: - if user['userId'] == creator_id: - group_id = group['id'] - break - if group_id: - break - - # only if we do have a message ("group") with the creator - if group_id: - msg_cursor = 0 - while True: - messages_req = sess.get('https://apiv3.fansly.com/api/v1/message', headers = headers, params = {'groupId': group_id, 'before': msg_cursor, 'limit': '25', 'ngsw-bypass': 'true'} if msg_cursor else {'groupId': group_id, 'limit': '25', 'ngsw-bypass': 'true'}) - - if messages_req.status_code == 200: - accessible_media = None - contained_posts = [] - - # post object contains: messages, accountMedia, accountMediaBundles, tips, tipGoals, stories - post_object = messages_req.json()['response'] - - # parse relevant details about the post - if not accessible_media: - # loop through the list of dictionaries and find the highest quality media URL for each one - for obj in post_object['accountMedia']: - try: - # add details into a list - contained_posts += [parse_media_info(obj)] - except Exception: - output(2,'\n [28]ERROR','', f"Unexpected error during parsing Messages content; \n{traceback.format_exc()}") - input('\n Press Enter to attempt to continue ..') - - # summarise all scrapable & wanted media - accessible_media = [item for item in contained_posts if item.get('download_url') and (item.get('is_preview') == download_media_previews or not item.get('is_preview'))] - - total_accessible_messages_content = len(accessible_media) - - # overwrite base dup threshold with 20% of total accessible content in messages - DUPLICATE_THRESHOLD = int(0.2 * total_accessible_messages_content) - - # at this point we have already parsed the whole post object and determined what is scrapable with the code above - output(1,' Info','', f"Amount of Media in Messages with {config_username}: {len(post_object['accountMedia'])} (scrapable: {total_accessible_messages_content})") - - generate_base_dir(config_username, module_requested_by = 'Messages') - - try: - # download it - sort_download(accessible_media) - except DuplicateCountError: - output(1,' Info','', f"Already downloaded all possible Messages content! [Duplicate threshold exceeded {DUPLICATE_THRESHOLD}]") - except Exception: - output(2,'\n [29]ERROR','', f"Unexpected error during sorting Messages download; \n{traceback.format_exc()}") - input('\n Press Enter to attempt to continue ..') - - # get next cursor - try: - msg_cursor = post_object['messages'][-1]['id'] - except IndexError: - break # break if end is reached - else: - output(2,'\n [30]ERROR','', f"Failed messages download. messages_req failed with response code: {messages_req.status_code}\n{messages_req.text}") - - elif group_id is None: - output(2, ' WARNING', '', f"Could not find a chat history with {config_username}; skipping messages download ..") - else: - output(2,'\n [31]ERROR','', f"Failed Messages download. Fetch Messages request, response code: {groups_req.status_code}\n{groups_req.text}") - input('\n Press Enter to attempt to continue ..') - - + print_download_info(config) -## starting here: download_mode = Timeline -if any(['Timeline' in download_mode, 'Normal' in download_mode]): - output(1,'\n Info','', f"Executing Timeline functionality. Anticipate remarkable outcomes!") + get_creator_account_info(config, state) - # this has to be up here so it doesn't get looped - generate_base_dir(config_username, module_requested_by = 'Timeline') + # Download mode: + # Normal: Downloads Timeline + Messages one after another. + # Timeline: Scrapes only the creator's timeline content. + # Messages: Scrapes only the creator's messages content. + # Single: Fetch a single post by the post's ID. Click on a post to see its ID in the url bar e.g. ../post/1283493240234 + # Collection: Download all content listed within the "Purchased Media Collection" - timeline_cursor = 0 - while True: - if timeline_cursor == 0: - output(1, '\n Info', '', "Inspecting most recent Timeline cursor") - else: - output(1, '\n Info', '', f"Inspecting Timeline cursor: {timeline_cursor}") - - # Simple attempt to deal with rate limiting - for itera in range(9999): - try: - # People with a high enough internet download speed & hardware specification will manage to hit a rate limit here - endpoint = "timelinenew" if itera == 0 else "timeline" - timeline_req = sess.get(f"https://apiv3.fansly.com/api/v1/{endpoint}/{creator_id}?before={timeline_cursor}&after=0&wallId=&contentSearch=&ngsw-bypass=true", headers=headers) - break # break if no errors happened; which means we will try parsing & downloading contents of that timeline_cursor - except Exception: - if itera == 0: - continue - elif itera == 1: - output(2, '\n WARNING', '', f"Uhm, looks like we\'ve hit a rate limit ..\ - \n{20 * ' '}Using a VPN might fix this issue entirely.\ - \n{20 * ' '}Regardless, will now try to continue the download infinitely, every 15 seconds.\ - \n{20 * ' '}Let me know if this logic worked out at any point in time\ - \n{20 * ' '}by opening an issue ticket, please!") - print('\n' + traceback.format_exc()) - else: - print(f"Attempt {itera} ...") - s(15) - - try: - if timeline_req.status_code == 200: - accessible_media = None - contained_posts = [] + print_info(f'Download mode is: {config.download_mode_str()}') + print() - post_object = timeline_req.json()['response'] - - # parse relevant details about the post - if not accessible_media: - # loop through the list of dictionaries and find the highest quality media URL for each one - for obj in post_object['accountMedia']: - try: - # add details into a list - contained_posts += [parse_media_info(obj)] - except Exception: - output(2,'\n [32]ERROR','', f"Unexpected error during parsing Timeline content; \n{traceback.format_exc()}") - input('\n Press Enter to attempt to continue ..') - - # summarise all scrapable & wanted media - accessible_media = [item for item in contained_posts if item.get('download_url') and (item.get('is_preview') == download_media_previews or not item.get('is_preview'))] - - # at this point we have already parsed the whole post object and determined what is scrapable with the code above - output(1,' Info','', f"Amount of Media in current cursor: {len(post_object['accountMedia'])} (scrapable: {len(accessible_media)})") + if config.download_mode == DownloadMode.SINGLE: + download_single_post(config, state) - try: - # download it - sort_download(accessible_media) - except DuplicateCountError: - output(1,' Info','', f"Already downloaded all possible Timeline content! [Duplicate threshold exceeded {DUPLICATE_THRESHOLD}]") - break - except Exception: - output(2,'\n [33]ERROR','', f"Unexpected error during sorting Timeline download: \n{traceback.format_exc()}") - input('\n Press Enter to attempt to continue ..') + elif config.download_mode == DownloadMode.COLLECTION: + download_collections(config, state) - # get next timeline_cursor - try: - timeline_cursor = post_object['posts'][-1]['id'] - except IndexError: - break # break the whole while loop, if end is reached - except Exception: - print('\n'+traceback.format_exc()) - output(2,'\n [34]ERROR','', 'Please copy & paste this on GitHub > Issues & provide a short explanation.') - input('\nPress Enter to close ...') - exit() + else: + if any([config.download_mode == DownloadMode.MESSAGES, config.download_mode == DownloadMode.NORMAL]): + download_messages(config, state) - except KeyError: - output(2,'\n [35]ERROR','', "Couldn\'t find any scrapable media at all!\ - \n This most likely happend because you\'re not following the creator, your authorisation token is wrong\ - \n or the creator is not providing unlocked content.") - input('\n Press Enter to attempt to continue ..') - except Exception: - output(2,'\n [36]ERROR','', f"Unexpected error during Timeline download: \n{traceback.format_exc()}") - input('\n Press Enter to attempt to continue ..') + if any([config.download_mode == DownloadMode.TIMELINE, config.download_mode == DownloadMode.NORMAL]): + download_timeline(config, state) - # check if atleast 20% of timeline was scraped; exluding the case when all the media was declined as duplicates - print('') # intentional empty print - issue = False - if pic_count <= total_timeline_pictures * 0.2 and duplicate_count <= total_timeline_pictures * 0.2: - output(3,'\n WARNING','', f"Low amount of Pictures scraped. Creators total Pictures: {total_timeline_pictures} | Downloaded: {pic_count}") - issue = True - - if vid_count <= total_timeline_videos * 0.2 and duplicate_count <= total_timeline_videos * 0.2: - output(3,'\n WARNING','', f"Low amount of Videos scraped. Creators total Videos: {total_timeline_videos} | Downloaded: {vid_count}") - issue = True - - if issue: - if not following: - print(f"{20*' '}Follow the creator; to be able to scrape more media!") - - if not subscribed: - print(f"{20*' '}Subscribe to the creator; if you would like to get the entire content.") - - if not download_media_previews: - print(f"{20*' '}Try setting download_media_previews to True in the config.ini file. Doing so, will help if the creator has marked all his content as previews.") - print('') + print_statistics(config, state) + # open download folder + if state.base_path is not None: + open_location(state.base_path, config.open_folder_when_finished, config.interactive) -# BASE_DIR_NAME doesn't always have to be set; e.g. user tried scraping Messages of someone, that never direct messaged him content before -if BASE_DIR_NAME: - # hacky overwrite for BASE_DIR_NAME so it doesn't point to the sub-directories e.g. /Timeline - BASE_DIR_NAME = BASE_DIR_NAME.partition('_fansly')[0] + '_fansly' + # Still continue if one creator failed + except ApiAccountInfoError as e: + print_error(str(e)) + input_enter_continue(config.interactive) + exit_code = SOME_USERS_FAILED - print(f"\n╔═\n Finished {download_mode} type, download of {pic_count} pictures & {vid_count} videos! Declined duplicates: {duplicate_count}\ - \n Saved content in directory: \'{BASE_DIR_NAME}\'\ - \n ✶ Please leave a Star on the GitHub Repository, if you are satisfied! ✶\n{74*' '}═╝") + return exit_code - # open download folder - if open_folder_when_finished: - open_location(BASE_DIR_NAME) +if __name__ == '__main__': + config = FanslyConfig(program_version=__version__) + exit_code = EXIT_SUCCESS -input('\n Press Enter to close ..') -exit() + try: + exit_code = main(config) + + except KeyboardInterrupt: + # TODO: Should there be any clean-up or in-program handling during Ctrl+C? + print() + print_warning('Program aborted.') + exit_code = EXIT_ABORT + + except ApiError as e: + print() + print_error(str(e)) + exit_code = API_ERROR + + except ConfigError as e: + print() + print_error(str(e)) + exit_code = CONFIG_ERROR + + except DownloadError as e: + print() + print_error(str(e)) + exit_code = DOWNLOAD_ERROR + + except Exception as e: + print() + print_error(f'An unexpected error occurred: {e}\n{traceback.format_exc()}') + exit_code = UNEXPECTED_ERROR + + input_enter_close(config.prompt_on_exit) + exit(exit_code) diff --git a/fileio/dedupe.py b/fileio/dedupe.py new file mode 100644 index 0000000..d02bce8 --- /dev/null +++ b/fileio/dedupe.py @@ -0,0 +1,115 @@ +"""Item Deduplication""" + + +import hashlib +import imagehash +import io + +from PIL import Image, ImageFile +from random import randint + +from fileio.fnmanip import add_hash_to_folder_items + +from config import FanslyConfig +from download.downloadstate import DownloadState +from pathio import set_create_directory_for_download +from textio import print_info, print_warning + + +# tell PIL to be tolerant of files that are truncated +ImageFile.LOAD_TRUNCATED_IMAGES = True + +# turn off for our purpose unnecessary PIL safety features +Image.MAX_IMAGE_PIXELS = None + + +def dedupe_init(config: FanslyConfig, state: DownloadState): + """Deduplicates (hashes) all existing media files in the + target directory structure. + + Downloads can then be filtered for pre-existing files. + """ + # This will create the base user path download_directory/creator_name + set_create_directory_for_download(config, state) + + if state.download_path and state.download_path.is_dir(): + print_info(f"Deduplication is automatically enabled for:\n{17*' '}{state.download_path}") + + add_hash_to_folder_items(config, state) + + print_info( + f"Deduplication process is complete! Each new download will now be compared" + f"\n{17*' '}against a total of {len(state.recent_photo_hashes)} photo & {len(state.recent_video_hashes)} " + "video hashes and corresponding media IDs." + ) + + # print("Recent Photo Hashes:", state.recent_photo_hashes) + # print("Recent Photo Media IDs:", state.recent_photo_media_ids) + # print("Recent Video Hashes:", state.recent_video_hashes) + # print("Recent Video Media IDs:", state.recent_video_media_ids) + + if randint(1, 100) <= 19: + print_warning( + f"Reminder: If you remove id_NUMBERS or hash_STRING from filenames of previously downloaded files" + f"\n{20*' '}they will no longer be compatible with Fansly Downloader's deduplication algorithm!" + ) + + # because adding information as metadata; requires specific + # configuration for each file type through PIL and that's too complex + # due to file types. maybe in the future I might decide to just save + # every image as .png and every video as .mp4 and add/read it as + # metadata or if someone contributes a function actually perfectly + # adding metadata to all common file types, that would be nice. + + +def dedupe_media_content(state: DownloadState, content: bytearray, mimetype: str, filename: str) -> str | None: + """Hashes binary media data and checks wheter it is a duplicate or not. + + The hash will be added to the respective set of hashes if it is not + a duplicate. + Returns the content hash or None if the media content is a duplicate. + + :param DownloadState state: The current download state, for statistics and + to populate the respective set of hashes. + :param bytearray content: The binary content of the media item. + :param str mimetype: The MIME type of the media item. + :param str filename: The file name to be used for saving, for + informational purposes. + + :return: The content hash or None if it is a duplicate. + :rtype: str | None + """ + file_hash = None + hashlist = None + + # Use specific hashing for images + if 'image' in mimetype: + # open the image + with Image.open(io.BytesIO(content)) as image: + + # calculate the hash of the resized image + file_hash = str(imagehash.phash(image, hash_size = 16)) + + hashlist = state.recent_photo_hashes + + else: + file_hash = hashlib.md5(content).hexdigest() + + if 'audio' in mimetype: + hashlist = state.recent_audio_hashes + + elif 'video' in mimetype: + hashlist = state.recent_video_hashes + + else: + raise RuntimeError('Internal error during media deduplication - invalid MIME type passed.') + + # Deduplication - part 2.1: decide if this media is even worth further processing; by hashing + if file_hash in hashlist: + print_info(f"Deduplication [Hashing]: {mimetype.split('/')[-2]} '{filename}' → declined") + state.duplicate_count += 1 + return None + + else: + hashlist.add(file_hash) + return file_hash diff --git a/fileio/fnmanip.py b/fileio/fnmanip.py new file mode 100644 index 0000000..05c760e --- /dev/null +++ b/fileio/fnmanip.py @@ -0,0 +1,197 @@ +"""File Name Manipulation Functions""" + + +import concurrent.futures +import hashlib +import mimetypes +import imagehash +import os +import re +import traceback + +from pathlib import Path +from PIL import Image + +from config import FanslyConfig +from download.downloadstate import DownloadState +from textio import print_debug, print_error + + +# turn off for our purpose unnecessary PIL safety features +Image.MAX_IMAGE_PIXELS = None + + +def extract_media_id(filename: str) -> int | None: + """Extracts the media_id from an existing file's name.""" + match = re.search(r'_id_(\d+)', filename) + + if match: + return int(match.group(1)) + + return None + + +def extract_hash_from_filename(filename: str) -> str | None: + """Extracts the hash from an existing file's name.""" + match = re.search(r'_hash_([a-fA-F0-9]+)', filename) + + if match: + return match.group(1) + + return None + + +def add_hash_to_filename(filename: Path, file_hash: str) -> str: + """Adds a hash to an existing file's name.""" + base_name, extension = str(filename.parent / filename.stem), filename.suffix + hash_suffix = f"_hash_{file_hash}{extension}" + + # adjust filename for 255 bytes filename limit, on all common operating systems + max_length = 250 + + if len(base_name) + len(hash_suffix) > max_length: + base_name = base_name[:max_length - len(hash_suffix)] + + return f"{base_name}{hash_suffix}" + + +def add_hash_to_image(state: DownloadState, filepath: Path): + """Hashes existing images in download directories.""" + try: + filename = filepath.name + + media_id = extract_media_id(filename) + + if media_id: + state.recent_photo_media_ids.add(media_id) + + existing_hash = extract_hash_from_filename(filename) + + if existing_hash: + state.recent_photo_hashes.add(existing_hash) + + else: + with Image.open(filepath) as img: + + file_hash = str(imagehash.phash(img, hash_size = 16)) + + state.recent_photo_hashes.add(file_hash) + + new_filename = add_hash_to_filename(Path(filename), file_hash) + new_filepath = filepath.parent / new_filename + + filepath = filepath.rename(new_filepath) + + except FileExistsError: + filepath.unlink() + + except Exception: + print_error(f"\nError processing image '{filepath}': {traceback.format_exc()}", 15) + + +def add_hash_to_other_content(state: DownloadState, filepath: Path, content_format: str): + """Hashes audio and video files in download directories.""" + + try: + filename = filepath.name + + media_id = extract_media_id(filename) + + if media_id: + + if content_format == 'video': + state.recent_video_media_ids.add(media_id) + + elif content_format == 'audio': + state.recent_audio_media_ids.add(media_id) + + existing_hash = extract_hash_from_filename(filename) + + if existing_hash: + + if content_format == 'video': + state.recent_video_hashes.add(existing_hash) + + elif content_format == 'audio': + state.recent_audio_hashes.add(existing_hash) + + else: + h = hashlib.md5() + + with open(filepath, 'rb') as f: + while (part := f.read(1_048_576)): + h.update(part) + + file_hash = h.hexdigest() + + if content_format == 'video': + state.recent_video_hashes.add(file_hash) + + elif content_format == 'audio': + state.recent_audio_hashes.add(file_hash) + + new_filename = add_hash_to_filename(Path(filename), file_hash) + new_filepath = filepath.parent / new_filename + + filepath = filepath.rename(new_filepath) + + except FileExistsError: + filepath.unlink() + + except Exception: + print_error(f"\nError processing {content_format} '{filepath}': {traceback.format_exc()}", 16) + + +def add_hash_to_file(config: FanslyConfig, state: DownloadState, file_path: Path) -> None: + """Hashes a file according to it's file type.""" + + mimetype, _ = mimetypes.guess_type(file_path) + + if config.debug: + print_debug(f"Hashing file of type '{mimetype}' at location '{file_path}' ...") + + if mimetype is not None: + + if mimetype.startswith('image'): + add_hash_to_image(state, file_path) + + elif mimetype.startswith('video'): + add_hash_to_other_content(state, file_path, content_format='video') + + elif mimetype.startswith('audio'): + add_hash_to_other_content(state, file_path, content_format='audio') + + +def add_hash_to_folder_items(config: FanslyConfig, state: DownloadState) -> None: + """Recursively adds hashes to all media files in the folder and + it's sub-folders. + """ + + if state.download_path is None: + raise RuntimeError('Internal error hashing media files - download path not set.') + + # Beware - thread pools may silently swallow exceptions! + # https://docs.python.org/3/library/concurrent.futures.html + with concurrent.futures.ThreadPoolExecutor() as executor: + + for root, _, files in os.walk(state.download_path): + + if config.debug: + print_debug(f"OS walk: '{root}', {files}") + print() + + if len(files) > 0: + futures: list[concurrent.futures.Future] = [] + + for file in files: + # map() doesn't cut it, or at least I couldn't get it to + # work with functions requiring multiple arguments. + future = executor.submit(add_hash_to_file, config, state, Path(root) / file) + futures.append(future) + + # Iterate over the future results so exceptions will be thrown + for future in futures: + future.result() + + if config.debug: + print() diff --git a/media/__init__.py b/media/__init__.py new file mode 100644 index 0000000..7909f51 --- /dev/null +++ b/media/__init__.py @@ -0,0 +1,14 @@ +"""Media Management Module""" + + +from .mediaitem import MediaItem +from .media import parse_media_info, parse_variant_metadata, parse_variants, simplify_mimetype + + +__all__ = [ + 'MediaItem', + 'simplify_mimetype', + 'parse_media_info', + 'parse_variant_metadata', + 'parse_variants', +] diff --git a/media/media.py b/media/media.py new file mode 100644 index 0000000..8b6ae35 --- /dev/null +++ b/media/media.py @@ -0,0 +1,207 @@ +"""Media and Fansly Related Utility Functions""" + + +import json + +from . import MediaItem + +from download.downloadstate import DownloadState +from textio import print_error + + +def simplify_mimetype(mimetype: str): + """Simplify (normalize) the MIME types in Fansly replies + to usable standards. + """ + if mimetype == 'application/vnd.apple.mpegurl': + mimetype = 'video/mp4' + + elif mimetype == 'audio/mp4': # another bug in fansly api, where audio is served as mp4 filetype .. + mimetype = 'audio/mp3' # i am aware that the correct mimetype would be "audio/mpeg", but we just simplify it + + return mimetype + + +def parse_variant_metadata(variant_metadata_json: str): + """Fixes Fansly API's current_variant_resolution height bug.""" + + variant_metadata = json.loads(variant_metadata_json) + + max_variant = max(variant_metadata['variants'], key=lambda variant: variant['h'], default=None) + + # if a highest height is not found, we just hope 1080p is available + if not max_variant: + return 1080 + + # else parse through variants and find highest height + if max_variant['w'] < max_variant['h']: + max_variant['w'], max_variant['h'] = max_variant['h'], max_variant['w'] + + return max_variant['h'] + + +# TODO: Enums in Python for content_type? +def parse_variants(item: MediaItem, content: dict, content_type: str, media_info: dict): # content_type: media / preview + """Parse metadata and resolution variants of a Fansly media item. + + :param MediaItem item: The media to parse and correct. + :param dict content: ??? + :param str content_type: "media" or "preview" + :param dict media_info: ??? + + :return: None. + """ + + if content.get('locations'): + location_url: str = content['locations'][0]['location'] + + current_variant_resolution = (content['width'] or 0) * (content['height'] or 0) + + if current_variant_resolution > item.highest_variants_resolution \ + and item.default_normal_mimetype == simplify_mimetype(content['mimetype']): + + item.highest_variants_resolution = current_variant_resolution + item.highest_variants_resolution_height = content['height'] or 0 + item.highest_variants_resolution_url = location_url + + item.media_id = int(content['id']) + item.mimetype = simplify_mimetype(content['mimetype']) + + # if key-pair-id is not in there we'll know it's the new .m3u8 format, so we construct a generalised url, which we can pass relevant auth strings with + # note: this url won't actually work, its purpose is to just pass the strings through the download_url variable + if not 'Key-Pair-Id' in item.highest_variants_resolution_url: + try: + # use very specific metadata, bound to the specific media to get auth info + item.metadata = content['locations'][0]['metadata'] + + item.highest_variants_resolution_url = \ + f"{item.highest_variants_resolution_url.split('.m3u8')[0]}_{parse_variant_metadata(content['metadata'])}.m3u8?ngsw-bypass=true&Policy={item.metadata['Policy']}&Key-Pair-Id={item.metadata['Key-Pair-Id']}&Signature={item.metadata['Signature']}" + + except KeyError: + # we pass here and catch below + pass + + """ + it seems like the date parsed here is actually the correct date, + which is directly attached to the content. but posts that could be uploaded + 8 hours ago, can contain images from 3 months ago. so the date we are parsing here, + might be the date, that the fansly CDN has first seen that specific content and the + content creator, just attaches that old content to a public post after e.g. 3 months. + + or createdAt & updatedAt are also just bugged out idk.. + note: images would be overwriting each other by filename, if hashing didnt provide uniqueness + else we would be forced to add randint(-1800, 1800) to epoch timestamps + """ + try: + item.created_at = int(content['updatedAt']) + + except Exception: + item.created_at = int(media_info[content_type]['createdAt']) + + item.download_url = item.highest_variants_resolution_url + + +def parse_media_info( + state: DownloadState, + media_info: dict, + post_id: str | None=None, + ) -> MediaItem: + """Parse media JSON reply from Fansly API.""" + + # initialize variables + #highest_variants_resolution_url, download_url, file_extension, metadata, default_normal_locations, default_normal_mimetype, mimetype = None, None, None, None, None, None, None + #created_at, media_id, highest_variants_resolution, highest_variants_resolution_height, default_normal_height = 0, 0, 0, 0, 0 + item = MediaItem() + + # check if media is a preview + item.is_preview = media_info['previewId'] is not None + + # fix rare bug, of free / paid content being counted as preview + if item.is_preview: + if media_info['access']: + item.is_preview = False + + # variables in api "media" = "default_" & "preview" = "preview" in our code + # parse normal basic (paid/free) media from the default location, before parsing its variants + # (later on we compare heights, to determine which one we want) + if not item.is_preview: + default_details = media_info['media'] + + item.default_normal_locations = media_info['media']['locations'] + item.default_normal_id = int(default_details['id']) + item.default_normal_created_at = int(default_details['createdAt']) + item.default_normal_mimetype = simplify_mimetype(default_details['mimetype']) + item.default_normal_height = default_details['height'] or 0 + + # if its a preview, we take the default preview media instead + else: + default_details = media_info['preview'] + + item.default_normal_locations = media_info['preview']['locations'] + item.default_normal_id = int(media_info['preview']['id']) + item.default_normal_created_at = int(default_details['createdAt']) + item.default_normal_mimetype = simplify_mimetype(default_details['mimetype']) + item.default_normal_height = default_details['height'] or 0 + + if default_details['locations']: + item.default_normal_locations = default_details['locations'][0]['location'] + + # Variants functions extracted here + + # somehow unlocked / paid media: get download url from media location + if 'location' in media_info['media']: + variants = media_info['media']['variants'] + + for content in variants: + # TODO: Check for pass by value/reference error, should this return? + parse_variants(item, content=content, content_type='media', media_info=media_info) + + # previews: if media location is not found, we work with the preview media info instead + if not item.download_url and 'preview' in media_info: + variants = media_info['preview']['variants'] + + for content in variants: + # TODO: Check for pass by value/reference error, should this return? + parse_variants(item, content=content, content_type='preview', media_info=media_info) + + """ + so the way this works is; we have these 4 base variables defined all over this function. + parse_variants() will initially overwrite them with values from each contents variants above. + then right below, we will compare the values and decide which media has the higher resolution. (default populated content vs content from variants) + or if variants didn't provide a higher resolution at all, we just fall back to the default content + """ + if \ + all( + [ + item.default_normal_height, + item.default_normal_locations, + item.highest_variants_resolution_height, + item.highest_variants_resolution_url, + ] + ) and all( + [ + item.default_normal_height > item.highest_variants_resolution_height, + item.default_normal_mimetype == item.mimetype, + ] + ) or not item.download_url: + # overwrite default variable values, which we will finally return; with the ones from the default media + item.media_id = item.default_normal_id + item.created_at = item.default_normal_created_at + item.mimetype = item.default_normal_mimetype + item.download_url = item.default_normal_locations + + # due to fansly may 2023 update + if item.download_url: + # parse file extension separately + item.file_extension = item.get_download_url_file_extension() + + if item.file_extension == 'mp4' and item.mimetype == 'audio/mp3': + item.file_extension = 'mp3' + + # if metadata didn't exist we need the user to notify us through github, because that would be detrimental + if not 'Key-Pair-Id' in item.download_url and not item.metadata: + print_error(f"Failed downloading a video! Please open a GitHub issue ticket called 'Metadata missing' and copy paste this:\n\ + \n\tMetadata Missing\n\tpost_id: {post_id} & media_id: {item.media_id} & creator username: {state.creator_name}\n", 14) + input('Press Enter to attempt continue downloading ...') + + return item diff --git a/media/mediaitem.py b/media/mediaitem.py new file mode 100644 index 0000000..7d83881 --- /dev/null +++ b/media/mediaitem.py @@ -0,0 +1,55 @@ +"""Class to Represent Media Items""" + + +from dataclasses import dataclass +from typing import Any + +from utils.datetime import get_adjusted_datetime + + +@dataclass +class MediaItem(object): + """Represents a media item published on Fansly + eg. a picture or video. + """ + default_normal_id: int = 0 + default_normal_created_at: int = 0 + default_normal_locations: str | None = None + default_normal_mimetype: str | None = None + default_normal_height: int = 0 + + media_id: int = 0 + metadata: dict[str, Any] | None = None + mimetype: str | None = None + created_at: int = 0 + download_url: str | None = None + file_extension: str | None = None + + highest_variants_resolution: int = 0 + highest_variants_resolution_height: int = 0 + highest_variants_resolution_url: str | None = None + + is_preview: bool = False + + + def created_at_str(self) -> str: + return get_adjusted_datetime(self.created_at) + + + def get_download_url_file_extension(self) -> str | None: + if self.download_url: + return self.download_url.split('/')[-1].split('.')[-1].split('?')[0] + else: + return None + + + def get_file_name(self) -> str: + """General filename construction & if content is a preview; + add that into it's filename. + """ + id = 'id' + + if self.is_preview: + id = 'preview_id' + + return f"{self.created_at_str()}_{id}_{self.media_id}.{self.file_extension}" diff --git a/pathio/__init__.py b/pathio/__init__.py new file mode 100644 index 0000000..7c9cc95 --- /dev/null +++ b/pathio/__init__.py @@ -0,0 +1,10 @@ +"""Diretory/Folder Utility Module""" + + +from .pathio import ask_correct_dir, set_create_directory_for_download + + +__all__ = [ + 'ask_correct_dir', + 'set_create_directory_for_download', +] diff --git a/pathio/pathio.py b/pathio/pathio.py new file mode 100644 index 0000000..995dc1c --- /dev/null +++ b/pathio/pathio.py @@ -0,0 +1,108 @@ +"""Work Directory Manipulation""" + + +import traceback + +from pathlib import Path +from tkinter import Tk, filedialog + +from config import FanslyConfig +from download.downloadstate import DownloadState +from download.types import DownloadType +from errors import ConfigError +from textio import print_info, print_warning, print_error + + +# if the users custom provided filepath is invalid; a tkinter dialog will open during runtime, asking to adjust download path +def ask_correct_dir() -> Path: + root = Tk() + root.withdraw() + + while True: + directory_name = filedialog.askdirectory() + + if Path(directory_name).is_dir(): + print_info(f"Folder path chosen: {directory_name}") + return Path(directory_name) + + print_error(f"You did not choose a valid folder. Please try again!", 5) + + +def set_create_directory_for_download(config: FanslyConfig, state: DownloadState) -> Path: + """Sets and creates the appropriate download directory according to + download type for storing media from a distinct creator. + + :param FanslyConfig config: The current download session's + configuration object. download_directory will be taken as base path. + + :param DownloadState state: The current download session's state. + This function will modify base_path (based on creator) and + save_path (full path based on download type) accordingly. + + :return Path: The (created) path current media downloads. + """ + if config.download_directory is None: + message = 'Internal error during directory creation - download directory not set.' + raise RuntimeError(message) + + else: + + suffix = '' + + if config.use_folder_suffix: + suffix = '_fansly' + + user_base_path = config.download_directory / f'{state.creator_name}{suffix}' + + user_base_path.mkdir(exist_ok=True) + + # Default directory if download types don't match in check below + download_directory = user_base_path + + if state.download_type == DownloadType.COLLECTIONS: + download_directory = config.download_directory / 'Collections' + + elif state.download_type == DownloadType.MESSAGES and config.separate_messages: + download_directory = user_base_path / 'Messages' + + elif state.download_type == DownloadType.TIMELINE and config.separate_timeline: + download_directory = user_base_path / 'Timeline' + + elif state.download_type == DownloadType.SINGLE: + # TODO: Maybe for "Single" we should use the post_id as subdirectory? + pass + + # If current download folder wasn't created with content separation, disable it for this download session too + is_file_hierarchy_correct = True + + if user_base_path.is_dir(): + + for directory in user_base_path.iterdir(): + + if (user_base_path / directory).is_dir(): + + if 'Pictures' in str(directory) and any([config.separate_messages, config.separate_timeline]): + is_file_hierarchy_correct = False + + if 'Videos' in str(directory) and any([config.separate_messages, config.separate_timeline]): + is_file_hierarchy_correct = False + + if not is_file_hierarchy_correct: + print_warning( + f"Due to the presence of 'Pictures' and 'Videos' sub-directories in the current download folder" + f"\n{20*' '}content separation will remain disabled throughout this download session." + ) + + config.separate_messages, config.separate_timeline = False, False + + # utilize recursion to fix BASE_DIR_NAME generation + return set_create_directory_for_download(config, state) + + # Save state + state.base_path = user_base_path + state.download_path = download_directory + + # Create the directory + download_directory.mkdir(exist_ok=True) + + return download_directory diff --git a/requirements-dev.txt b/requirements-dev.txt index f0aa93a..cbc8c97 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1 +1,4 @@ mypy +types-python-dateutil +types-requests +types-setuptools diff --git a/textio/__init__.py b/textio/__init__.py new file mode 100644 index 0000000..7b31a6b --- /dev/null +++ b/textio/__init__.py @@ -0,0 +1,25 @@ +"""Console Output""" + + +# Re-exports +from .textio import LOG_FILE_NAME +from .textio import print_config, print_debug, print_error, print_info, print_info_highlight, print_update, print_warning +from .textio import input_enter_close, input_enter_continue +from .textio import clear_terminal, set_window_title + + +# from textio import * +__all__ = [ + 'LOG_FILE_NAME', + 'print_config', + 'print_debug', + 'print_error', + 'print_info', + 'print_info_highlight', + 'print_update', + 'print_warning', + 'input_enter_close', + 'input_enter_continue', + 'clear_terminal', + 'set_window_title', +] diff --git a/textio/textio.py b/textio/textio.py new file mode 100644 index 0000000..16b4552 --- /dev/null +++ b/textio/textio.py @@ -0,0 +1,124 @@ +"""Console Output""" + + +import os +import platform +import subprocess +import sys + +from functools import partialmethod +from time import sleep +from loguru import logger +from pathlib import Path + + +LOG_FILE_NAME: str = 'fansly_downloader.log' + + +# most of the time, we utilize this to display colored output rather than logging or prints +def output(level: int, log_type: str, color: str, message: str) -> None: + try: + logger.level(log_type, no = level, color = color) + + except TypeError: + # level failsafe + pass + + logger.__class__.type = partialmethod(logger.__class__.log, log_type) + + logger.remove() + + logger.add( + sys.stdout, + format="{level} | {time:HH:mm} || {message}", + level=log_type, + ) + logger.add( + Path.cwd() / LOG_FILE_NAME, + encoding='utf-8', + format="[{level} ] [{time:YYYY-MM-DD} | {time:HH:mm}]: {message}", + level=log_type, + rotation='1MB', + retention=5, + ) + + logger.type(message) + + +def print_config(message: str) -> None: + output(5, ' Config', '', message) + + +def print_debug(message: str) -> None: + output(7,' DEBUG', '', message) + + +def print_error(message: str, number: int=-1) -> None: + if number >= 0: + output(2, f' [{number}]ERROR', '', message) + else: + output(2, ' ERROR', '', message) + + +def print_info(message: str) -> None: + output(1, ' Info', '', message) + + +def print_info_highlight(message: str) -> None: + output(4, ' lnfo', '', message) + + +def print_update(message: str) -> None: + output(6,' Updater', '', message) + + +def print_warning(message: str) -> None: + output(3, ' WARNING', '', message) + + +def input_enter_close(interactive: bool=True) -> None: + """Asks user for to close and exits the program. + In non-interactive mode sleeps instead, then exits. + """ + if interactive: + input('\nPress to close ...') + + else: + print('\nExiting in 15 seconds ...') + sleep(15) + + from utils.common import exit + exit() + + +def input_enter_continue(interactive: bool=True) -> None: + """Asks user for to continue. + In non-interactive mode sleeps instead. + """ + if interactive: + input('\nPress to attempt to continue ...') + else: + print('\nContinuing in 15 seconds ...') + sleep(15) + + +# clear the terminal based on the operating system +def clear_terminal() -> None: + system = platform.system() + + if system == 'Windows': + os.system('cls') + + else: # Linux & macOS + os.system('clear') + + +# cross-platform compatible, re-name downloaders terminal output window title +def set_window_title(title) -> None: + current_platform = platform.system() + + if current_platform == 'Windows': + subprocess.call('title {}'.format(title), shell=True) + + elif current_platform == 'Linux' or current_platform == 'Darwin': + subprocess.call(['printf', r'\33]0;{}\a'.format(title)]) diff --git a/updater/__init__.py b/updater/__init__.py new file mode 100644 index 0000000..06c104f --- /dev/null +++ b/updater/__init__.py @@ -0,0 +1,65 @@ +"""Self-Updating Functionality""" + + +import sys + +from utils.web import get_release_info_from_github + +from .utils import check_for_update, delete_deprecated_files, post_update_steps + +from config import FanslyConfig, copy_old_config_values +from textio import print_warning +from utils.common import save_config_or_raise + + +def self_update(config: FanslyConfig): + """Performs self-updating if necessary.""" + + release_info = get_release_info_from_github(config.program_version) + + # Regular start, not after update + if config.updated_to is None: + # check if a new version is available + check_for_update(config) + + # if started with --updated-to start argument + else: + + # config.ini backwards compatibility fix (≤ v0.4) -> fix spelling mistake "seperate" to "separate" + if 'seperate_messages' in config._parser['Options']: + config.separate_messages = \ + config._parser.getboolean('Options', 'seperate_messages') + config._parser.remove_option('Options', 'seperate_messages') + + if 'seperate_previews' in config._parser['Options']: + config.separate_previews = \ + config._parser.getboolean('Options', 'seperate_previews') + config._parser.remove_option('Options', 'seperate_previews') + + # config.ini backwards compatibility fix (≤ v0.4) -> config option "naming_convention" & "update_recent_download" removed entirely + options_to_remove = ['naming_convention', 'update_recent_download'] + + for option in options_to_remove: + + if option in config._parser['Options']: + config._parser.remove_option('Options', option) + + print_warning( + f"Just removed '{option}' from the config.ini file as the whole option" + f"\n{20*' '}is no longer supported after version 0.3.5." + ) + + # Just re-save the config anyway, regardless of changes + save_config_or_raise(config) + + # check if old config.ini exists, compare each pre-existing value of it and apply it to new config.ini + copy_old_config_values() + + # temporary: delete deprecated files + delete_deprecated_files() + + # get release notes and if existent display it in terminal + post_update_steps(config.program_version, release_info) + + # read the config.ini file for a last time + config._load_raw_config() diff --git a/updater/utils.py b/updater/utils.py new file mode 100644 index 0000000..ef41033 --- /dev/null +++ b/updater/utils.py @@ -0,0 +1,274 @@ +"""Self-Update Utility Functions""" + + +import dateutil.parser +import os +import platform +import re +import requests +import subprocess +import sys + +import errors + +from pathlib import Path +from pkg_resources._vendor.packaging.version import parse as parse_version +from shutil import unpack_archive + +from config import FanslyConfig +from textio import clear_terminal, print_error, print_info, print_update, print_warning +from utils.web import get_release_info_from_github + + +def delete_deprecated_files() -> None: + """Deletes deprecated files after an update.""" + old_files = [ + "old_updater", + "updater", + "Automatic Configurator", + "Fansly Scraper", + "deprecated_version", + "old_config" + ] + + directory = Path.cwd() + + for root, _, files in os.walk(directory): + for file in files: + + file_object = Path(file) + + if file_object.suffix.lower() != '.py' and file_object.stem in old_files: + + file_path = Path(root) / file + + if file_path.exists(): + file_path.unlink() + + +def display_release_notes(program_version: str, release_notes: str) -> None: + """Displays the release notes of a Fansly Downloader version. + + :param str program_version: The Fansly Downloader version. + :param str release_notes: The corresponding release notes. + """ + print_update(f"Successfully updated to version {program_version}!\n\n► Release Notes:\n{release_notes}") + print() + input('Press to start Fansly Downloader ...') + + clear_terminal() + + +def parse_release_notes(release_info: dict) -> str | None: + """Parse the release notes from the release info dictionary + obtained from GitHub. + + :param dict release_info: Program release information from GitHub. + + :return: The release notes or None if there was empty content or + a parsing error. + :rtype: str | None + """ + release_body = release_info.get("body") + + if not release_body: + return None + + body_match = re.search( + r"```(.*)```", + release_body, + re.DOTALL | re.MULTILINE + ) + + if not body_match: + return None + + release_notes = body_match[1] + + if not release_notes: + return None + + return release_notes + + +def perform_update(program_version: str, release_info: dict) -> bool: + """Performs a self-update of Fansly Downloader. + + :param str program_version: The current program version. + :param dict releas_info: Release information from GitHub. + + :return: True if successful or False otherwise. + :rtype: bool + """ + print_warning(f"A new version of fansly downloader has been found on GitHub - update required!") + + print_info(f"Latest Build:\n{18*' '}Version: {release_info['release_version']}\n{18*' '}Published: {release_info['created_at']}\n{18*' '}Download count: {release_info['download_count']}\n\n{17*' '}Your version: {program_version} is outdated!") + + # if current environment is pure python, prompt user to update fansly downloader himself + if not getattr(sys, 'frozen', False): + print_warning(f"To update Fansly Downloader, please download the latest version from the GitHub repository.\n{20*' '}Only executable versions of the downloader receive & apply updates automatically.\n") + # but we don't care if user updates or just wants to see this prompt on every execution further on + return False + + # if in executable environment, allow self-update + print_update('Please be patient, automatic update initialized ...') + + # download new release + release_download = requests.get( + release_info['download_url'], + allow_redirects=True, + headers = { + 'user-agent': f'Fansly Downloader {program_version}', + 'accept-language': 'en-US,en;q=0.9' + } + ) + + if release_download.status_code != 200: + print_error(f"Failed downloading latest build. Status code: {release_download.status_code} | Body: \n{release_download.text}") + return False + + # re-name current executable, so that the new version can delete it + try: + downloader_name = 'Fansly Downloader' + new_name = 'deprecated_version' + suffix = '' + + if platform.system() == 'Windows': + suffix = '.exe' + + downloader_path = Path.cwd() / f'{downloader_name}{suffix}' + downloader_path.rename(downloader_path.parent / f'{new_name}{suffix}') + + except FileNotFoundError: + pass + + # re-name old config ini, so new executable can read, compare and delete old one + try: + config_file = Path.cwd() / 'config.ini' + config_file.rename(config_file.parent / 'old_config.ini') + + except Exception: + pass + + # declare new release filepath + new_release_archive = Path.cwd() / release_info['release_name'] + + # write to disk + with open(new_release_archive, 'wb') as f: + f.write(release_download.content) + + # unpack if possible; for macOS .dmg this won't work though + try: + # must be a common archive format (.zip, .tar, .tar.gz, .tar.bz2, etc.) + unpack_archive(new_release_archive) + # remove .zip leftovers + new_release_archive.unlink() + + except Exception: + pass + + # start executable from just downloaded latest platform compatible release, with a start argument + # which instructs it to delete old executable & display release notes for newest version + current_platform = platform.system() + # from now on executable will be called Fansly Downloader + filename = 'Fansly Downloader' + + if current_platform == 'Windows': + filename = filename + '.exe' + + filepath = Path.cwd() / filename + + # Carry command-line arguments over + additional_arguments = ['--updated-to', release_info['release_version']] + arguments = sys.argv[1:] + additional_arguments + + if current_platform == 'Windows': + # i'm open for improvement suggestions, which will be insensitive to file paths & succeed passing start arguments to compiled executables + subprocess.run(['powershell', '-Command', f"Start-Process -FilePath '{filepath}' -ArgumentList {', '.join(arguments)}"], shell=True) + + elif current_platform == 'Linux': + # still sensitive to file paths? + subprocess.run([filepath, *arguments], shell=True) + + elif current_platform == 'Darwin': + # still sensitive to file paths? + subprocess.run(['open', filepath, *arguments], shell=False) + + else: + input(f"Platform {current_platform} not supported for auto-update, please update manually instead.") + os._exit(errors.UPDATE_MANUALLY) + + os._exit(errors.UPDATE_SUCCESS) + + +def post_update_steps(program_version: str, release_info: dict | None) -> None: + """Performs necessary steps after a self-update. + + :param str program_version: The program version updated to. + :param dict release_info: The version's release info from GitHub. + """ + if release_info is not None: + release_notes = parse_release_notes(release_info) + + if release_notes is not None: + display_release_notes(program_version, release_notes) + + +def check_for_update(config: FanslyConfig) -> bool: + """Checks for an updated program version. + + :param FanslyConfig config: The program configuration including the + current version number. + + :return: False if anything went wrong (network errors, ...) + or True otherwise. + :rtype: bool + """ + release_info = get_release_info_from_github(config.program_version) + + if release_info is None: + return False + + else: + # we don't want to ship drafts or pre-releases + if release_info["draft"] or release_info["prerelease"]: + return False + + # remove the string "v" from the version tag + new_version = release_info["tag_name"].split('v')[1] + + # we do only want current platform compatible updates + new_release = None + current_platform = 'macOS' if platform.system() == 'Darwin' else platform.system() + + for new_release in release_info['assets']: + if current_platform in new_release['name']: + d = dateutil.parser.isoparse(new_release['created_at']).replace(tzinfo=None) + + parsed_date = f"{d.strftime('%d')} {d.strftime('%B')[:3]} {d.strftime('%Y')}" + + new_release = { + 'release_name': new_release['name'], + 'release_version': new_version, + 'created_at': parsed_date, + 'download_count': new_release['download_count'], + 'download_url': new_release['browser_download_url'] + } + + if new_release is None: + return False + + empty_values = [ + value is None for key, value in new_release.items() + if key != 'download_count' + ] + + if any(empty_values): + return False + + # just return if our current version is still sufficient + if parse_version(config.program_version) >= parse_version(new_version): + return True + + else: + return perform_update(config.program_version, release_info) diff --git a/utils/common.py b/utils/common.py new file mode 100644 index 0000000..c74d054 --- /dev/null +++ b/utils/common.py @@ -0,0 +1,109 @@ +"""Common Utility Functions""" + + +import os +import platform +import subprocess + +from pathlib import Path + +from config.fanslyconfig import FanslyConfig +from errors import ConfigError + + +def exit(status: int=0) -> None: + """Exits the program. + + This function overwrites the default exit() function with a + pyinstaller compatible one. + + :param status: The exit code of the program. + :type status: int + """ + os._exit(status) + + +def save_config_or_raise(config: FanslyConfig) -> bool: + """Tries to save the configuration to `config.ini` or + raises a `ConfigError` otherwise. + + :param config: The program configuration. + :type config: FanslyConfig + + :return: True if configuration was successfully written. + :rtype: bool + + :raises ConfigError: When the configuration file could not be saved. + This may be due to invalid path issues or permission/security + software problems. + """ + if not config._save_config(): + raise ConfigError( + f"Internal error: Configuration data could not be saved to '{config.config_path}'. " + "Invalid path or permission/security software problem." + ) + else: + return True + + +def is_valid_post_id(post_id: str) -> bool: + """Validates a Fansly post ID. + + Valid post IDs must: + + - only contain digits + - be longer or equal to 10 characters + - not contain spaces + + :param post_id: The post ID string to validate. + :type post_id: str + + :return: True or False. + :rtype: bool + """ + return all( + [ + post_id.isdigit(), + len(post_id) >= 10, + not any(char.isspace() for char in post_id), + ] + ) + + +def open_location(filepath: Path, open_folder_when_finished: bool, interactive: bool) -> bool: + """Opens the download directory in the platform's respective + file manager application once the download process has finished. + + :param filepath: The base path of all downloads. + :type filepath: Path + :param open_folder_when_finished: Open the folder or do nothing. + :type open_folder_when_finished: bool + :param interactive: Running interactively or not. + Folder will not be opened when set to False. + :type interactive: bool + + :return: True when the folder was opened or False otherwise. + :rtype: bool + """ + plat = platform.system() + + if not open_folder_when_finished or not interactive: + return False + + if not os.path.isfile(filepath) and not os.path.isdir(filepath): + return False + + # tested below and they work to open folder locations + if plat == 'Windows': + # verified works + os.startfile(filepath) + + elif plat == 'Linux': + # verified works + subprocess.run(['xdg-open', filepath], shell=False) + + elif plat == 'Darwin': + # verified works + subprocess.run(['open', filepath], shell=False) + + return True diff --git a/utils/config_util.py b/utils/config_util.py deleted file mode 100644 index 003866e..0000000 --- a/utils/config_util.py +++ /dev/null @@ -1,209 +0,0 @@ -import os, plyvel, json, requests, traceback, psutil, platform, sqlite3, sys -from functools import partialmethod -from loguru import logger as log -from os.path import join -from time import sleep as s - -# overwrite default exit, with a pyinstaller compatible one -def exit(): - os._exit(0) - -def output(level: int, log_type: str, color: str, mytext: str): - try: - log.level(log_type, no = level, color = color) - except TypeError: - pass # level failsafe - log.__class__.type = partialmethod(log.__class__.log, log_type) - log.remove() - log.add(sys.stdout, format = "{level} | {time:HH:mm} || {message}", level=log_type) - log.type(mytext) - -# Function to recursively search for "storage" folders and process SQLite files -def process_storage_folders(directory): - for root, _, files in os.walk(directory): - if "storage" in root: - for file in files: - if file.endswith(".sqlite"): - sqlite_file = join(root, file) - session_active_session = process_sqlite_file(sqlite_file) - if session_active_session: - return session_active_session - - -# Function to read SQLite file and retrieve key-value pairs -def process_sqlite_file(sqlite_file): - session_active_session = None - try: - conn = sqlite3.connect(sqlite_file) - cursor = conn.cursor() - - # Get all table names in the SQLite database - cursor.execute("SELECT name FROM sqlite_master WHERE type='table';") - tables = cursor.fetchall() - - for table in tables: - table_name = table[0] - cursor.execute(f"SELECT * FROM {table_name};") - rows = cursor.fetchall() - - for row in rows: - if row[0] == 'session_active_session': - session_active_session = json.loads(row[5].decode('utf-8'))['token'] - break - - conn.close() - - return session_active_session - - except sqlite3.Error as e: - sqlite_error = str(e) - if 'locked' in sqlite_error and 'irefox' in sqlite_file: - output(5,'\n Config','', f"Firefox browser is open, but it needs to be closed for automatic configurator\n\ - {11*' '}to search your fansly account in the browsers storage.\n\ - {11*' '}Please save any important work within the browser & close the browser yourself,\n\ - {11*' '}else press Enter to close it programmatically and continue configuration.") - input(f"\n{19*' '} ► Press Enter to continue! ") - close_browser_by_name('firefox') - return process_sqlite_file(sqlite_file) # recursively restart function - else: - print(f"Unexpected Error processing SQLite file: {traceback.format_exc()}") - except Exception: - print(f'Unexpected Error, parsing out of firefox SQLite {traceback.format_exc()}') - return None - - -def get_browser_paths(): - if platform.system() == 'Windows': - local_appdata = os.getenv('localappdata') - appdata = os.getenv('appdata') - browser_paths = [ - join(local_appdata, 'Google', 'Chrome', 'User Data'), - join(local_appdata, 'Microsoft', 'Edge', 'User Data'), - join(appdata, 'Mozilla', 'Firefox', 'Profiles'), - join(appdata, 'Opera Software', 'Opera Stable'), - join(appdata, 'Opera Software', 'Opera GX Stable'), - join(local_appdata, 'BraveSoftware', 'Brave-Browser', 'User Data'), - ] - elif platform.system() == 'Darwin': # macOS - home = os.path.expanduser("~") - # regarding safari comp: https://stackoverflow.com/questions/58479686/permissionerror-errno-1-operation-not-permitted-after-macos-catalina-update - browser_paths = [ - join(home, 'Library', 'Application Support', 'Google', 'Chrome'), - join(home, 'Library', 'Application Support', 'Microsoft Edge'), - join(home, 'Library', 'Application Support', 'Firefox', 'Profiles'), - join(home, 'Library', 'Application Support', 'com.operasoftware.Opera'), - join(home, 'Library', 'Application Support', 'com.operasoftware.OperaGX'), - join(home, 'Library', 'Application Support', 'BraveSoftware'), - ] - elif platform.system() == 'Linux': - home = os.path.expanduser("~") - browser_paths = [ - join(home, '.config', 'google-chrome', 'Default'), - join(home, '.mozilla', 'firefox'), # firefox non-snap (couldn't verify with ubuntu) - join(home, 'snap', 'firefox', 'common', '.mozilla', 'firefox'), # firefox snap - join(home, '.config', 'opera'), # btw opera gx, does not exist for linux - join(home, '.config', 'BraveSoftware', 'Brave-Browser', 'Default'), - ] - return browser_paths - - -def find_leveldb_folders(root_path): - leveldb_folders = set() - for root, dirs, files in os.walk(root_path): - for dir_name in dirs: - if 'leveldb' in dir_name.lower(): - leveldb_folders.add(join(root, dir_name)) - break - for file in files: - if file.endswith('.ldb'): - leveldb_folders.add(root) - break - return leveldb_folders - - -def close_browser_by_name(browser_name): - # microsoft edge names its process msedge - if browser_name == 'Microsoft Edge': - browser_name = 'msedge' - # opera gx just names its process opera - elif browser_name == 'Opera Gx': - browser_name = 'opera' - - browser_processes = [proc for proc in psutil.process_iter(attrs=['name']) if browser_name.lower() in proc.info['name'].lower()] - closed = False # Flag to track if any process was closed - if platform.system() == 'Windows': - for proc in browser_processes: - proc.terminate() - closed = True - elif platform.system() == 'Darwin' or platform.system() == 'Linux': - for proc in browser_processes: - proc.kill() - closed = True - - if closed: - output(5,'\n Config','', f"Succesfully closed {browser_name} browser.") - s(3) # give browser time to close its children processes - -def parse_browser_from_string(string): - compatible = ['Firefox', 'Brave', 'Opera GX', 'Opera', 'Chrome', 'Edge'] - for browser in compatible: - if browser.lower() in string.lower(): - if browser.lower() == 'edge' and 'microsoft' in string.lower(): - return 'Microsoft Edge' - else: - return browser - return "Unknown" - -def get_auth_token_from_leveldb_folder(leveldb_folder): - try: - db = plyvel.DB(leveldb_folder, compression='snappy') - - key = b'_https://fansly.com\x00\x01session_active_session' - value = db.get(key) - - if value: - session_active_session = value.decode('utf-8').replace('\x00', '').replace('\x01', '') - auth_token = json.loads(session_active_session).get('token') - db.close() - return auth_token - else: - db.close() - return None - except plyvel._plyvel.IOError as e: - error_message = str(e) - used_browser = parse_browser_from_string(error_message) - output(5,'\n Config','', f"{used_browser} browser is open, but it needs to be closed for automatic configurator\n\ - {11*' '}to search your fansly account in the browsers storage.\n\ - {11*' '}Please save any important work within the browser & close the browser yourself,\n\ - {11*' '}else press Enter to close it programmatically and continue configuration.") - input(f"\n{19*' '} ► Press Enter to continue! ") - close_browser_by_name(used_browser) - return get_auth_token_from_leveldb_folder(leveldb_folder) # recursively restart function - except Exception: - return None - - -def link_fansly_downloader_to_account(auth_token): - headers = { - 'authority': 'apiv3.fansly.com', - 'accept': 'application/json, text/plain, */*', - 'accept-language': 'en;q=0.8,en-US;q=0.7', - 'authorization': auth_token, - 'origin': 'https://fansly.com', - 'referer': 'https://fansly.com/', - 'sec-ch-ua': '"Not.A/Brand";v="8", "Chromium";v="114", "Google Chrome";v="114"', - 'sec-ch-ua-mobile': '?0', - 'sec-ch-ua-platform': '"Windows"', - 'sec-fetch-dest': 'empty', - 'sec-fetch-mode': 'cors', - 'sec-fetch-site': 'same-site', - 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36', - } - - me_req = requests.get('https://apiv3.fansly.com/api/v1/account/me', params={'ngsw-bypass': 'true'}, headers=headers) - if me_req.status_code == 200: - me_req = me_req.json()['response']['account'] - account_username = me_req['username'] - if account_username: - return account_username - return None diff --git a/utils/datetime.py b/utils/datetime.py new file mode 100644 index 0000000..f7ec316 --- /dev/null +++ b/utils/datetime.py @@ -0,0 +1,44 @@ +"""Time Manipulation""" + + +import time + + +def get_time_format() -> int: + """Detect and return 12 vs 24 hour time format usage. + + :return: 12 or 24 + :rtype: int + """ + return 12 if ('AM' in time.strftime('%X') or 'PM' in time.strftime('%X')) else 24 + + +def get_timezone_offset(): + """Returns the local timezone offset from UTC. + + :return: The tuple (diff_from_utc, hours_in_seconds) + :rtype: Tuple[int, int] + """ + offset = time.timezone if (time.localtime().tm_isdst == 0) else time.altzone + + diff_from_utc = int(offset / 60 / 60 * -1) + hours_in_seconds = diff_from_utc * 3600 * -1 + + return diff_from_utc, hours_in_seconds + + +def get_adjusted_datetime(epoch_timestamp: int): + """Converts an epoch timestamp to the time of the + local computers' timezone. + """ + diff_from_utc, hours_in_seconds = get_timezone_offset() + + adjusted_timestamp = epoch_timestamp + diff_from_utc * 3600 + adjusted_timestamp += hours_in_seconds + + # start of strings are ISO 8601; so that they're sortable by Name after download + if get_time_format() == 24: + return time.strftime("%Y-%m-%d_at_%H-%M", time.localtime(adjusted_timestamp)) + + else: + return time.strftime("%Y-%m-%d_at_%I-%M-%p", time.localtime(adjusted_timestamp)) diff --git a/utils/update_util.py b/utils/update_util.py deleted file mode 100644 index 103d49f..0000000 --- a/utils/update_util.py +++ /dev/null @@ -1,207 +0,0 @@ -# have to eventually remove dateutil requirement -import os, requests, re, platform, sys, subprocess -from os.path import join -from os import getcwd -from loguru import logger as log -from functools import partialmethod -import dateutil.parser as dp -from shutil import unpack_archive -from configparser import RawConfigParser - - -# most of the time, we utilize this to display colored output rather than logging or prints -def output(level: int, log_type: str, color: str, mytext: str): - try: - log.level(log_type, no = level, color = color) - except TypeError: - pass # level failsafe - log.__class__.type = partialmethod(log.__class__.log, log_type) - log.remove() - log.add(sys.stdout, format = "{level} | {time:HH:mm} || {message}", level=log_type) - log.type(mytext) - - -# clear the terminal based on the operating system -def clear_terminal(): - system = platform.system() - if system == 'Windows': - os.system('cls') - else: # Linux & macOS - os.system('clear') - - -def apply_old_config_values(): - current_directory = getcwd() - old_config_path = join(current_directory, 'old_config.ini') - new_config_path = join(current_directory, 'config.ini') - - if os.path.isfile(old_config_path) and os.path.isfile(new_config_path): - old_config = RawConfigParser() - old_config.read(old_config_path) - - new_config = RawConfigParser() - new_config.read(new_config_path) - - # iterate over each section in the old config - for section in old_config.sections(): - # check if the section exists in the new config - if new_config.has_section(section): - # iterate over each option in the section - for option in old_config.options(section): - # check if the option exists in the new config - if new_config.has_option(section, option): - # get the value from the old config and set it in the new config - value = old_config.get(section, option) - - # skip overwriting the version value - if section == 'Other' and option == 'version': - continue - - new_config.set(section, option, value) - - # save the updated new config - with open(new_config_path, 'w') as config_file: - new_config.write(config_file) - - -def delete_deprecated_files(): - executables = ["old_updater", "updater", "Automatic Configurator", "Fansly Scraper", "deprecated_version", "old_config"] - directory = getcwd() - - for root, dirs, files in os.walk(directory): - for file in files: - file_name, file_extension = os.path.splitext(file) - if file_extension.lower() != '.py' and file_name in executables: - file_path = join(root, file) - if os.path.exists(file_path): - os.remove(file_path) - - -def display_release_notes(version_string: str, code_contents: str): - output(6,'\n Updater', '', f"Successfully updated to version {version_string}\n\n ► Release Notes:{code_contents}") - - input('Press Enter to start Fansly Downloader ...') - - clear_terminal() - - -def get_release_description(version_string, response_json): - release_body = response_json.get("body") - if not release_body: - return None - - code_contents = re.search(r"```(.*)```", release_body, re.DOTALL | re.MULTILINE)[1] - if not code_contents: - return None - - display_release_notes(version_string, code_contents) - - -def handle_update(current_version: str, release: dict): - output(3, '\n WARNING', '', f"A new version of fansly downloader has been found on GitHub; update required!") - - output(1, '\n info', '', f"Latest Build:\n{18*' '}Version: {release['release_version']}\n{18*' '}Published: {release['created_at']}\n{18*' '}Download count: {release['download_count']}\n\n{17*' '}Your version: {current_version} is outdated!") - - # if current environment is pure python, prompt user to update fansly downloader himself - if not getattr(sys, 'frozen', False): - output(3, '\n WARNING', '', f"To update Fansly Downloader, please download the latest version from the GitHub repository.\n{20*' '}Only executable versions of the downloader, receive & apply updates automatically.\n") - return False # but we don't care if user updates or just wants to see this prompt on every execution further on - - # if in executable environment, allow self-update - output(6,'\n Updater', '', 'Please be patient, automatic update initialized ...') - - # download new release - release_download = requests.get(release['download_url'], allow_redirects = True, headers = {'user-agent': f'Fansly Downloader {current_version}', 'accept-language': 'en-US,en;q=0.9'}) - if not release_download.ok: - output(2,'\n ERROR', '', f"Failed downloading latest build. Release request status code: {release_download.status_code} | Body: \n{release_download.text}") - return False - - # re-name current executable, so that the new version can delete it - try: - if platform.system() == 'Windows': - os.rename(join(getcwd(), 'Fansly Downloader.exe'), join(getcwd(), 'deprecated_version.exe')) - else: - os.rename(join(getcwd(), 'Fansly Downloader'), join(getcwd(), 'deprecated_version')) - except FileNotFoundError: - pass - - # re-name old config ini, so new executable can read, compare and delete old one - try: - os.rename(join(getcwd(), 'config.ini'), join(getcwd(), 'old_config.ini')) - except Exception: - pass - - # declare new release filepath - new_release_filepath = join(getcwd(), release['release_name']) - - # write to disk - with open(new_release_filepath, 'wb') as f: - f.write(release_download.content) - - # unpack if possible; for macOS .dmg this won't work though - try: - unpack_archive(new_release_filepath) # must be a common archive format (.zip, .tar, .tar.gz, .tar.bz2, etc.) - os.remove(new_release_filepath) # remove .zip leftovers - except Exception: - pass - - # start executable from just downloaded latest platform compatible release, with a start argument - # which instructs it to delete old executable & display release notes for newest version - plat = platform.system() - filename = 'Fansly Downloader' # from now on; executable always has to be called Fansly Downloader - if plat == 'Windows': - filename = filename+'.exe' - filepath = join(getcwd(), filename) - - if plat == 'Windows': - arguments = ['--update', release['release_version']] # i'm open for improvement suggestions, which will be insensitive to file paths & succeed passing start arguments to compiled executables - subprocess.run(['powershell', '-Command', f"Start-Process -FilePath \'{filepath}\' -ArgumentList {', '.join(arguments)}"], shell=True) - elif plat == 'Linux': - subprocess.run([filepath, '--update', release['release_version']], shell=True) # still sensitive to file paths? - elif plat == 'Darwin': - subprocess.run(['open', filepath, '--update', release['release_version']], shell=False) # still sensitive to file paths? - else: - input(f"Platform {plat} not supported for auto update, please manually update instead.") - - os._exit(0) - - -def check_latest_release(update_version: str = 0, current_version: str = 0, intend: str = None): # intend: update / check - try: - url = f"https://api.github.com/repos/avnsx/fansly-downloader/releases/latest" - response = requests.get(url, allow_redirects = True, headers={'user-agent': f'Fansly Downloader {update_version if update_version is not None else current_version}', 'accept-language': 'en-US,en;q=0.9'}) - response.raise_for_status() - except Exception: - return False - - if not response.ok: - return False - - response_json = response.json() - if intend == 'update': - get_release_description(update_version, response_json) - elif intend == 'check': - # we don't want to ship drafts or pre-releases - if response_json["draft"] or response_json["prerelease"]: - return False - - # remove the string "v" from the version tag - if not update_version: - update_version = response_json["tag_name"].split('v')[1] - - # we do only want current platform compatible updates - release = None - current_platform = 'macOS' if platform.system() == 'Darwin' else platform.system() - for release in response_json['assets']: - if current_platform in release['name']: - d=dp.isoparse(release['created_at']).replace(tzinfo=None) - parsed_date = f"{d.strftime('%d')} {d.strftime('%B')[:3]} {d.strftime('%Y')}" - release = {'release_name': release['name'], 'release_version': update_version, 'created_at': parsed_date, 'download_count': release['download_count'], 'download_url': release['browser_download_url']} - if not release or any(value is None for key, value in release.items() if key != 'download_count'): - return False - - # just return if our current version is still sufficient - if current_version >= update_version: - return - else: - handle_update(current_version, release) diff --git a/utils/web.py b/utils/web.py new file mode 100644 index 0000000..b1494b5 --- /dev/null +++ b/utils/web.py @@ -0,0 +1,200 @@ +"""Web Utilities""" + + +import platform +import re +import requests +import traceback + +from time import sleep + +from config.fanslyconfig import FanslyConfig +from textio import print_error, print_info_highlight, print_warning + + +# mostly used to attempt to open fansly downloaders documentation +def open_url(url_to_open: str) -> None: + """Opens an URL in a browser window. + + :param str url_to_open: The URL to open in the browser. + """ + sleep(10) + + try: + import webbrowser + webbrowser.open(url_to_open, new=0, autoraise=True) + + except Exception: + pass + + +def open_get_started_url() -> None: + open_url('https://github.com/Avnsx/fansly-downloader/wiki/Get-Started') + + +def get_fansly_account_for_token(auth_token: str) -> str | None: + """Fetches user account information for a particular authorization token. + + :param auth_token: The Fansly authorization token. + :type auth_token: str + + :return: The account user name or None. + :rtype: str | None + """ + headers = { + 'authority': 'apiv3.fansly.com', + 'accept': 'application/json, text/plain, */*', + 'accept-language': 'en;q=0.8,en-US;q=0.7', + 'authorization': auth_token, + 'origin': 'https://fansly.com', + 'referer': 'https://fansly.com/', + 'sec-ch-ua': '"Not.A/Brand";v="8", "Chromium";v="114", "Google Chrome";v="114"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"Windows"', + 'sec-fetch-dest': 'empty', + 'sec-fetch-mode': 'cors', + 'sec-fetch-site': 'same-site', + 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36', + } + + me_req = requests.get( + 'https://apiv3.fansly.com/api/v1/account/me', + params={'ngsw-bypass': 'true'}, + headers=headers + ) + + if me_req.status_code == 200: + me_req = me_req.json()['response']['account'] + account_username = me_req['username'] + + if account_username: + return account_username + + return None + + +def guess_user_agent(user_agents: dict, based_on_browser: str, default_ua: str) -> str: + """Returns the guessed browser's user agent or a default one.""" + + if based_on_browser == 'Microsoft Edge': + based_on_browser = 'Edg' # msedge only reports "Edg" as its identifier + + # could do the same for opera, opera gx, brave. but those are not supported by @jnrbsn's repo. so we just return chrome ua + # in general his repo, does not provide the most accurate latest user-agents, if I am borred some time in the future, + # I might just write my own similar repo and use that instead + + os_name = platform.system() + + try: + if os_name == "Windows": + for user_agent in user_agents: + if based_on_browser in user_agent and "Windows" in user_agent: + match = re.search(r'Windows NT ([\d.]+)', user_agent) + if match: + os_version = match.group(1) + if os_version in user_agent: + return user_agent + + elif os_name == "Darwin": # macOS + for user_agent in user_agents: + if based_on_browser in user_agent and "Macintosh" in user_agent: + match = re.search(r'Mac OS X ([\d_.]+)', user_agent) + if match: + os_version = match.group(1).replace('_', '.') + if os_version in user_agent: + return user_agent + + elif os_name == "Linux": + for user_agent in user_agents: + if based_on_browser in user_agent and "Linux" in user_agent: + match = re.search(r'Linux ([\d.]+)', user_agent) + if match: + os_version = match.group(1) + if os_version in user_agent: + return user_agent + + except Exception: + print_error(f'Regexing user-agent from online source failed: {traceback.format_exc()}', 4) + + print_warning(f"Missing user-agent for {based_on_browser} & OS: {os_name}. Chrome & Windows UA will be used instead.") + + return default_ua + + +def get_release_info_from_github(current_program_version: str) -> dict | None: + """Fetches and parses the Fansly Downloader release info JSON from GitHub. + + :param str current_program_version: The current program version to be + used in the user agent of web requests. + + :return: The release info from GitHub as dictionary or + None if there where any complications eg. network error. + :rtype: dict | None + """ + try: + url = f"https://api.github.com/repos/avnsx/fansly-downloader/releases/latest" + + response = requests.get( + url, + allow_redirects=True, + headers={ + 'user-agent': f'Fansly Downloader {current_program_version}', + 'accept-language': 'en-US,en;q=0.9' + } + ) + + response.raise_for_status() + + except Exception: + return None + + if response.status_code != 200: + return None + + return response.json() + + +def remind_stargazing(config: FanslyConfig) -> bool: + """Reminds the user to star the repository.""" + + import requests + + stargazers_count, total_downloads = 0, 0 + + # depends on global variable current_version + stats_headers = {'user-agent': f"Avnsx/Fansly Downloader {config.program_version}", + 'referer': f"Avnsx/Fansly Downloader {config.program_version}", + 'accept-language': 'en-US,en;q=0.9'} + + # get total_downloads count + stargazers_check_request = requests.get('https://api.github.com/repos/avnsx/fansly-downloader/releases', allow_redirects = True, headers = stats_headers) + if stargazers_check_request.status_code != 200: + return False + + stargazers_check_request = stargazers_check_request.json() + + for x in stargazers_check_request: + total_downloads += x['assets'][0]['download_count'] or 0 + + # get stargazers_count + downloads_check_request = requests.get('https://api.github.com/repos/avnsx/fansly-downloader', allow_redirects = True, headers = stats_headers) + + if downloads_check_request.status_code != 200: + return False + + downloads_check_request = downloads_check_request.json() + stargazers_count = downloads_check_request['stargazers_count'] or 0 + + percentual_stars = round(stargazers_count / total_downloads * 100, 2) + + # display message (intentionally "lnfo" with lvl 4) + print_info_highlight( + f"Fansly Downloader was downloaded {total_downloads} times, but only {percentual_stars} % of you (!) have starred it." + f"\n{6*' '}Stars directly influence my willingness to continue maintaining the project." + f"\n{5*' '}Help the repository grow today, by leaving a star on it and sharing it to others online!" + ) + print() + + sleep(15) + + return True From 8b700dc8260b8f2dbd8b5e79912914c1e07f84e8 Mon Sep 17 00:00:00 2001 From: prof79 Date: Wed, 30 Aug 2023 22:17:10 +0200 Subject: [PATCH 06/19] The new sample config.ini with all options. --- config.ini | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/config.ini b/config.ini index 31d2057..058665a 100644 --- a/config.ini +++ b/config.ini @@ -6,15 +6,16 @@ authorization_token = ReplaceMe user_agent = ReplaceMe [Options] +download_directory = Local_directory download_mode = Normal show_downloads = True download_media_previews = True open_folder_when_finished = True -download_directory = Local_directory separate_messages = True separate_previews = False separate_timeline = True -utilise_duplicate_threshold = False +use_duplicate_threshold = False +use_folder_suffix = True +interactive = True +prompt_on_exit = True -[Other] -version = 0.4.1 From e81155ff5459204a4f2cf2f45ba24dddff745853 Mon Sep 17 00:00:00 2001 From: prof79 Date: Wed, 30 Aug 2023 23:06:38 +0200 Subject: [PATCH 07/19] Forgot to remove some old TODOs. --- fansly_downloader.py | 1 - media/media.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/fansly_downloader.py b/fansly_downloader.py index fc0b37a..10da198 100644 --- a/fansly_downloader.py +++ b/fansly_downloader.py @@ -10,7 +10,6 @@ __credits__: list[str] = [] # TODO: Fix in future: audio needs to be properly transcoded from mp4 to mp3, instead of just saved as -# TODO: Maybe write a log file? import base64 diff --git a/media/media.py b/media/media.py index 8b6ae35..5896f65 100644 --- a/media/media.py +++ b/media/media.py @@ -153,7 +153,6 @@ def parse_media_info( variants = media_info['media']['variants'] for content in variants: - # TODO: Check for pass by value/reference error, should this return? parse_variants(item, content=content, content_type='media', media_info=media_info) # previews: if media location is not found, we work with the preview media info instead @@ -161,7 +160,6 @@ def parse_media_info( variants = media_info['preview']['variants'] for content in variants: - # TODO: Check for pass by value/reference error, should this return? parse_variants(item, content=content, content_type='preview', media_info=media_info) """ From c97c2641c5830a61f2a04ed2ea9e0e2b2ada69cd Mon Sep 17 00:00:00 2001 From: prof79 Date: Sat, 2 Sep 2023 14:24:55 +0200 Subject: [PATCH 08/19] Added MetadataManager() and the required config infrastructure. --- config.ini | 2 +- config/args.py | 22 ++++++ config/config.py | 5 ++ config/fanslyconfig.py | 9 +++ config/metadatahandling.py | 10 +++ requirements.txt | 2 + utils/metadata_manager.py | 152 +++++++++++++++++++++++++++++++++++++ 7 files changed, 201 insertions(+), 1 deletion(-) create mode 100644 config/metadatahandling.py create mode 100644 utils/metadata_manager.py diff --git a/config.ini b/config.ini index 058665a..0ef5b00 100644 --- a/config.ini +++ b/config.ini @@ -18,4 +18,4 @@ use_duplicate_threshold = False use_folder_suffix = True interactive = True prompt_on_exit = True - +metadata_handling = Advanced diff --git a/config/args.py b/config/args.py index 3f2f640..db3c91f 100644 --- a/config/args.py +++ b/config/args.py @@ -8,6 +8,7 @@ from .config import parse_items_from_line, sanitize_creator_names from .fanslyconfig import FanslyConfig +from .metadatahandling import MetadataHandling from .modes import DownloadMode from errors import ConfigError @@ -205,6 +206,15 @@ def parse_args() -> argparse.Namespace: help="Use an internal de-deduplication threshold to not download " "already downloaded media again.", ) + parser.add_argument( + '-mh', '--metadata-handling', + required=False, + default=None, + type=str, + dest='metadata_handling', + help="How to handle media EXIF metadata. " + "Supported strategies: Advanced (Default), Simple", + ) #endregion @@ -325,6 +335,18 @@ def map_args_to_config(args: argparse.Namespace, config: FanslyConfig) -> None: config.post_id = post_id config_overridden = True + if args.metadata_handling is not None: + handling = args.metadata_handling.strip().lower() + + try: + config.metadata_handling = MetadataHandling(handling) + config_overridden = True + + except ValueError: + raise ConfigError( + f"Argument error - '{handling}' is not a valid metadata handling strategy." + ) + # The code following avoids code duplication of checking an # argument and setting the override flag for each argument. # On the other hand, this certainly not refactoring/renaming friendly. diff --git a/config/config.py b/config/config.py index 9698df1..814696c 100644 --- a/config/config.py +++ b/config/config.py @@ -10,6 +10,7 @@ from pathlib import Path from .fanslyconfig import FanslyConfig +from .metadatahandling import MetadataHandling from .modes import DownloadMode from errors import ConfigError @@ -208,6 +209,10 @@ def load_config(config: FanslyConfig) -> None: download_mode = config._parser.get(options_section, 'download_mode', fallback='Normal') config.download_mode = DownloadMode(download_mode.lower()) + # Advanced, Simple -> str + metadata_handling = config._parser.get(options_section, 'metadata_handling', fallback='Advanced') + config.metadata_handling = MetadataHandling(metadata_handling.lower()) + config.download_media_previews = config._parser.getboolean(options_section, 'download_media_previews', fallback=True) config.open_folder_when_finished = config._parser.getboolean(options_section, 'open_folder_when_finished', fallback=True) config.separate_messages = config._parser.getboolean(options_section, 'separate_messages', fallback=True) diff --git a/config/fanslyconfig.py b/config/fanslyconfig.py index 1f8c67c..7e21ae4 100644 --- a/config/fanslyconfig.py +++ b/config/fanslyconfig.py @@ -7,6 +7,7 @@ from dataclasses import dataclass from pathlib import Path +from .metadatahandling import MetadataHandling from .modes import DownloadMode @@ -55,6 +56,8 @@ class FanslyConfig(object): download_mode: DownloadMode = DownloadMode.NORMAL download_directory: (None | Path) = None download_media_previews: bool = True + # "Advanced" | "Simple" + metadata_handling: MetadataHandling = MetadataHandling.ADVANCED open_folder_when_finished: bool = True separate_messages: bool = True separate_previews: bool = False @@ -94,6 +97,11 @@ def download_mode_str(self) -> str: """Gets `download_mod` as a string representation.""" return str(self.download_mode).capitalize() + + def metadata_handling_str(self) -> str: + """Gets the string representation of `metadata_handling`.""" + return str(self.metadata_handling).capitalize() + def _sync_settings(self) -> None: """Syncs the settings of the config object @@ -112,6 +120,7 @@ def _sync_settings(self) -> None: self._parser.set('Options', 'download_directory', str(self.download_directory)) self._parser.set('Options', 'download_mode', self.download_mode_str()) + self._parser.set('Options', 'metadata_handling', self.metadata_handling_str()) # Booleans self._parser.set('Options', 'show_downloads', str(self.show_downloads)) diff --git a/config/metadatahandling.py b/config/metadatahandling.py new file mode 100644 index 0000000..1743ee2 --- /dev/null +++ b/config/metadatahandling.py @@ -0,0 +1,10 @@ +"""Metadata Handling""" + + +from enum import StrEnum, auto + + +class MetadataHandling(StrEnum): + NOTSET = auto() + ADVANCED = auto() + SIMPLE = auto() diff --git a/requirements.txt b/requirements.txt index 314f414..4dea868 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,9 +2,11 @@ av>=9.0.0 imagehash>=4.2.1 loguru>=0.5.3 m3u8>=3.0.0 +mutagen>=1.46.0 pillow>=8.4.0 plyvel-ci>=1.5.0 psutil>=5.9.0 +pyexiv2>=2.8.2 python-dateutil>=2.8.2 requests>=2.26.0 rich>=13.0.0 diff --git a/utils/metadata_manager.py b/utils/metadata_manager.py new file mode 100644 index 0000000..027a271 --- /dev/null +++ b/utils/metadata_manager.py @@ -0,0 +1,152 @@ +import pyexiv2 +from mutagen.mp4 import MP4 +from mutagen.id3 import ID3, TXXX + + +class InvalidKeyError(Exception): + pass + +class MetadataManager: + """ + What is this? + This class utilizes mutagen & pyexiv2 to provide Exif metadata support, most importantly to the mp4, mp3, png, jpg and jpeg file formats. + While not focused on perfect integration, it achieves the metadata addition, cross-platform compatible, to supported formats in a timely manner. + The resulting cleaned metadata can be accessed as dict through .formatted_metadata() or unformatted with .raw_metadata + Only the following custom_key names are permissible: HSH (representing Hash) and ID (representing MediaID). + + Limitations: + - Inability to add metadata to all images over 1 GB in size, due to pyexiv2. + - Inability to read metadata from images, over 2 GB in filesize, due to pyexiv2. + - Lack of thread safety due to pyexiv2's global variables in C++. + - Incomplete support for ARM platform with pyexiv2. + - In line with GIFs general lack of Exif support, this class also doesn't cover GIFs. + + Usage: + filepath = '[filename].[fileformat]' + + Add metadata: + metadata_manager = MetadataManager() + metadata_manager.is_file_supported(file_extension) # e.g. use this as conditional, returns boolean + metadata_manager.set_filepath(filepath) + metadata_manager.set_custom_metadata("ID", '305462832970526416') + metadata_manager.set_custom_metadata("HSH", '10ej3e691af63ae66843218c42d5d0b3') + metadata_manager.add_metadata() + metadata_manager.save() + + Read metadata: + metadata_manager = MetadataManager() + metadata_manager.read_metadata(filepath) + print(metadata_manager.formatted_metadata()) + print(metadata_manager.raw_metadata) + """ + def __init__(self, filepath=None): + self.filepath = filepath + self.custom_metadata = {} + self.filetype = None if filepath is None else filepath.split('.')[-1].lower() + self.raw_metadata = {} + self.image_filetypes = [ + 'jpeg', 'jpg', 'png', + 'exv', 'cr2', 'crw', 'tiff', 'webp', 'dng', 'nef', 'pef', + 'srw', 'orf', 'pgf', 'raf', 'xmp', 'psd', 'jp2' + ] + + def is_file_supported(self, filetype=None): + filetype = self.filetype if filetype is None else filetype + return filetype in ['mp4', 'mp3'] or filetype in self.image_filetypes + + def set_filepath(self, filepath): + self.filepath = filepath + self.filetype = filepath.split('.')[-1].lower() + + # initial temporary storage in-case multiple keys shall be added, in one run + def set_custom_metadata(self, custom_key: str, custom_value: str): + if not any([custom_key, custom_value]): + return + if custom_key not in ["HSH", "ID"]: + raise InvalidKeyError(f"Received custom_key \'{custom_key}\', but MetadataManager only supports custom keys named \'HSH\' or \'ID\'") + self.custom_metadata[custom_key] = custom_value + + # return formatted metadata + def formatted_metadata(self): + self.read_metadata() + result = {} + if self.filetype == 'mp3': + if 'TXXX:HSH' in self.raw_metadata: + value = self.raw_metadata['TXXX:HSH'].text[0] + result['HSH'] = int(value) if value.isdigit() else value + if 'TXXX:ID' in self.raw_metadata: + value = self.raw_metadata['TXXX:ID'].text[0] + result['ID'] = int(value) if value.isdigit() else value + elif self.filetype == 'mp4': + for key, value in self.raw_metadata.items(): + clean_key = key.replace('_', '') + if clean_key in ['HSH', 'ID']: + result[clean_key] = int(value[0]) if value[0].isdigit() else value[0] + elif self.filetype in self.image_filetypes: + custom_tag_mapping = { + 'Exif.Image.Software': 'ID', + 'Exif.Image.DateTime': 'HSH' + } + for key, value in self.raw_metadata.items(): + if key in custom_tag_mapping: + result[custom_tag_mapping[key]] = int(value) if value.isdigit() else value + return result + + # read metadata + def read_metadata(self, filepath=None): + if not self.filepath and filepath: + self.filepath = filepath + self.filetype = filepath.rsplit('.')[1] + if self.filetype in ['mp4', 'mp3']: + self.read_audio_video_metadata() + elif self.filetype in self.image_filetypes: + self.read_image_metadata() + + def read_audio_video_metadata(self): + if self.filetype == 'mp3': + self.raw_metadata = ID3(self.filepath) + elif self.filetype == 'mp4': + self.raw_metadata = MP4(self.filepath) + + def read_image_metadata(self): + with pyexiv2.Image(self.filepath) as image: + self.raw_metadata = image.read_exif() + + # add metadata + def add_metadata(self): + for key, value in self.custom_metadata.items(): + if self.filetype == 'mp3': + self.add_mp3_metadata(key, value) + elif self.filetype == 'mp4': + self.add_mp4_metadata(key, value) + elif self.filetype in self.image_filetypes: + self.add_image_metadata(key, value) + + def add_mp3_metadata(self, key, value): + txxx_frame = TXXX(encoding=3, desc=key, text=value) + self.raw_metadata.add(txxx_frame) + + def add_mp4_metadata(self, key, value): + if not isinstance(self.raw_metadata, MP4): + self.read_audio_video_metadata() + if len(key) < 4: + key = key + '_' * (4 - len(key)) + elif len(key) > 4: + key = key[:4] + self.raw_metadata[key] = str(value) + + def add_image_metadata(self, key, value): + custom_tag_mapping = { + 'ID': 'Exif.Image.Software', + 'HSH': 'Exif.Image.DateTime' + } + if key in custom_tag_mapping: + key = custom_tag_mapping[key] + self.raw_metadata[key] = value + + def save(self): + if self.filetype in self.image_filetypes: + with pyexiv2.Image(self.filepath) as image: + image.modify_exif(self.raw_metadata) + else: + self.raw_metadata.save(self.filepath) From 3dedc80815d3d6941ea1746949f6b93727fce590 Mon Sep 17 00:00:00 2001 From: prof79 Date: Sat, 2 Sep 2023 14:25:29 +0200 Subject: [PATCH 09/19] Fixed typo/wording. --- config/fanslyconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/fanslyconfig.py b/config/fanslyconfig.py index 7e21ae4..030af8b 100644 --- a/config/fanslyconfig.py +++ b/config/fanslyconfig.py @@ -94,7 +94,7 @@ def user_names_str(self) -> str | None: def download_mode_str(self) -> str: - """Gets `download_mod` as a string representation.""" + """Gets the string representation of `download_mode`.""" return str(self.download_mode).capitalize() From 46f0bddc70ed865c205b8849c478ae0663cdae1a Mon Sep 17 00:00:00 2001 From: prof79 Date: Sat, 2 Sep 2023 14:28:26 +0200 Subject: [PATCH 10/19] Updated UA detection failure string. --- config/validation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/validation.py b/config/validation.py index 53c8669..eb30f30 100644 --- a/config/validation.py +++ b/config/validation.py @@ -245,7 +245,7 @@ def validate_adjust_user_agent(config: FanslyConfig) -> None: """ # if no matches / error just set random UA - ua_if_failed = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36' + ua_if_failed = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36' based_on_browser = config.token_from_browser_name or 'Chrome' From ea00870780fb88906aeb358c17c66cb7348b23c5 Mon Sep 17 00:00:00 2001 From: prof79 Date: Sat, 2 Sep 2023 15:15:26 +0200 Subject: [PATCH 11/19] Beautified imports. --- download/media.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/download/media.py b/download/media.py index 4613e3b..374dfa8 100644 --- a/download/media.py +++ b/download/media.py @@ -1,23 +1,21 @@ """Fansly Download Functionality""" -from pathlib import Path +from pathlib import Path +from PIL import Image, ImageFile from rich.progress import Progress, BarColumn, TextColumn from rich.table import Column -from PIL import Image, ImageFile from .downloadstate import DownloadState from .m3u8 import download_m3u8 from .types import DownloadType from config import FanslyConfig -from errors import DownloadError +from errors import DownloadError, DuplicateCountError, MediaError from fileio.dedupe import dedupe_media_content from media import MediaItem from pathio import set_create_directory_for_download -from textio import input_enter_close, print_info, print_error, print_warning -from utils.common import exit -from errors import DuplicateCountError, MediaError +from textio import print_info, print_warning # tell PIL to be tolerant of files that are truncated From 7c4b71f8fb7221d2f362cd2853f3769af0e07af3 Mon Sep 17 00:00:00 2001 From: prof79 Date: Sat, 2 Sep 2023 15:16:18 +0200 Subject: [PATCH 12/19] Fixes for Fansly rate-limiting introduced in late August 2023. --- download/timeline.py | 43 ++++++++----------------------------------- 1 file changed, 8 insertions(+), 35 deletions(-) diff --git a/download/timeline.py b/download/timeline.py index 8747502..8ab7d74 100644 --- a/download/timeline.py +++ b/download/timeline.py @@ -1,6 +1,7 @@ """Timeline Downloads""" +import random import traceback from requests import Response @@ -34,43 +35,13 @@ def download_timeline(config: FanslyConfig, state: DownloadState) -> None: print_info(f"Inspecting Timeline cursor: {timeline_cursor}") timeline_response = Response() - - # Simple attempt to deal with rate limiting - for attempt in range(9999): - try: - # People with a high enough internet download speed & hardware specification will manage to hit a rate limit here - endpoint = "timelinenew" if attempt == 0 else "timeline" - - if config.debug: - print_debug(f'HTTP headers: {config.http_headers()}') - - timeline_response = config.http_session.get( - f"https://apiv3.fansly.com/api/v1/{endpoint}/{state.creator_id}?before={timeline_cursor}&after=0&wallId=&contentSearch=&ngsw-bypass=true", - headers=config.http_headers() - ) - - break # break if no errors happened; which means we will try parsing & downloading contents of that timeline_cursor - - except Exception: - if attempt == 0: - continue - - elif attempt == 1: - print_warning( - f"Uhm, looks like we've hit a rate limit ..." - f"\n{20 * ' '}Using a VPN might fix this issue entirely." - f"\n{20 * ' '}Regardless, will now try to continue the download infinitely, every 15 seconds." - f"\n{20 * ' '}Let me know if this logic worked out at any point in time" - f"\n{20 * ' '}by opening an issue ticket, please!" - ) - print('\n' + traceback.format_exc()) - - else: - print(f"Attempt {attempt} ...") - - sleep(15) try: + timeline_response = config.http_session.get( + f"https://apiv3.fansly.com/api/v1/timeline/{state.creator_id}?before={timeline_cursor}&after=0&wallId=&contentSearch=&ngsw-bypass=true", + headers=config.http_headers(), + ) + timeline_response.raise_for_status() if timeline_response.status_code == 200: @@ -88,6 +59,8 @@ def download_timeline(config: FanslyConfig, state: DownloadState) -> None: # get next timeline_cursor try: + # Slow down to avoid the Fansly rate-limit which was introduced in late August 2023 + sleep(random.uniform(2, 4)) timeline_cursor = post_object['posts'][-1]['id'] except IndexError: From 0302809d50a568591db45882f12395fbb2bea51c Mon Sep 17 00:00:00 2001 From: prof79 Date: Sat, 2 Sep 2023 15:21:48 +0200 Subject: [PATCH 13/19] Added upstream Read Me changes. --- README.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index b657934..619d8d0 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Latest Release - + Commits since latest release @@ -123,22 +123,22 @@ Fansly Downloader is the go-to app for all your bulk media downloading needs. Do On windows you can just install the [Executable version](https://github.com/Avnsx/fansly-downloader/releases/latest), skip the entire set up section & go to [Quick Start](https://github.com/Avnsx/fansly-downloader#-quick-start) #### Python Version Requirements -If your operating system is not compatible with **executable versions** of Fansly Downloader (only Windows supported for ``.exe``) or you just generally intend to use the Python source directly, please [download the repository](https://github.com/Avnsx/fansly-downloader/archive/refs/heads/main.zip), extract the files from the folder and ensure that [Python is installed](https://www.python.org/downloads/) on your system. Once Python is installed, you can proceed by installing the following requirements using [Python's package manager](https://realpython.com/what-is-pip/) (``"pip"``), within your systems terminal copy & paste: +If your operating system is not compatible with **executable versions** of Fansly Downloader (only Windows supported for ``.exe``) or you just generally intend to use the Python source directly, please [download the repository](https://github.com/Avnsx/fansly-downloader/archive/refs/heads/master.zip), extract the files from the folder and ensure that [Python is installed](https://www.python.org/downloads/) on your system. Once Python is installed, you can proceed by installing the following requirements using [Python's package manager](https://realpython.com/what-is-pip/) (``"pip"``), within your systems terminal copy & paste: - pip3 install requests loguru python-dateutil plyvel-ci psutil imagehash m3u8 av pillow rich -Alternatively you can use [``requirements.txt``](https://github.com/Avnsx/fansly-downloader/blob/main/requirements.txt) through opening your system's terminal (e.g.: ``cmd.exe`` on windows), [navigating to the project's download folder](https://youtu.be/8-mYKkNzjU4?t=5) and executing the following command: ``pip3 install --user -r requirements.txt`` + pip3 install requests loguru python-dateutil plyvel-ci psutil imagehash m3u8 av pillow rich pyexiv2 mutagen +Alternatively you can use [``requirements.txt``](https://github.com/Avnsx/fansly-downloader/blob/master/requirements.txt) through opening your system's terminal (e.g.: ``cmd.exe`` on windows), [navigating to the project's download folder](https://youtu.be/8-mYKkNzjU4?t=5) and executing the following command: ``pip3 install --user -r requirements.txt`` For Linux operating systems, you may need to install the Python Tkinter module separately by using the command ``sudo apt-get install python3-tk``. On Windows and macOS, the Tkinter module is typically included in the [Python installer itself](https://youtu.be/O2PzLeiBEuE?t=38). After all requirements are installed into your python environment; click on *fansly_downloader.py* and it'll open up & [behave similar](https://github.com/Avnsx/fansly-downloader#-quick-start) to how the executable version would. -Raw python code versions of Fansly Downloader do not receive automatic updates. If an update is available, you will be notified, but will need to manually [download the repository](https://github.com/Avnsx/fansly-downloader/archive/refs/heads/main.zip) as zip again, extract files and set-up the latest version of fansly downloader yourself. +Raw python code versions of Fansly Downloader do not receive automatic updates. If an update is available, you will be notified, but will need to manually [download the repository](https://github.com/Avnsx/fansly-downloader/archive/refs/heads/master.zip) as zip again, extract files and set-up the latest version of fansly downloader yourself. ## 🚀 Quick Start To quickly get started with either the [python](https://github.com/Avnsx/fansly-downloader#python-version-requirements) or the [executable](https://github.com/Avnsx/fansly-downloader/releases/latest) version of Fansly Downloader, follow these steps: 1. Download the latest version of Fansly Downloader by choosing one of the options below: - - [Windows exclusive executable version](https://github.com/Avnsx/fansly-downloader/releases/latest) - "*Fansly_Downloader.exe*" + - [Windows exclusive executable version](https://github.com/Avnsx/fansly-downloader/releases/latest) - "*Fansly Downloader.exe*" - [Python code version](https://github.com/Avnsx/fansly-downloader#python-version-requirements) - "*fansly_downloader.py*" and extract the files from the zip folder. @@ -177,8 +177,6 @@ If you still need help with something open up a [New Discussion](https://github. ## 🤝 Contributing to `Fansly Downloader` Any kind of positive contribution is welcome! Please help the project improve by [opening a pull request](https://github.com/Avnsx/fansly-downloader/pulls) with your suggested changes! -Currently greatly appreciated would be the integration of a cross-platform compatible download progress bar or some kind of visual display for monitoring the current download speed in Mb/s within the terminal or in another concise visually appealing way. Furthermore propper transcoding of mp4 audio to the mp3 format using [pyav](https://github.com/PyAV-Org/PyAV), similar of how it is handled with m3u8 to mp4 within fansly-downloader already, would be a required addition to future versions of fansly downloader. - ### Special Thanks A heartfelt thank you goes out to [@liviaerxin](https://github.com/liviaerxin) for their invaluable contribution in providing cross-platform [plyvel](https://github.com/wbolster/plyvel) (python module) builds. It is due to [these builds](https://github.com/liviaerxin/plyvel/releases/latest) that fansly downloaders initial interactive set-up configuration functionality, has become a cross-platform reality. From e3325d35f2750bdb24dec75ed59dde080e31fc1c Mon Sep 17 00:00:00 2001 From: prof79 Date: Sat, 2 Sep 2023 15:32:01 +0200 Subject: [PATCH 14/19] Added rate-limiting fix to messages, just to be sure. --- download/messages.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/download/messages.py b/download/messages.py index eecf006..f7b0c5f 100644 --- a/download/messages.py +++ b/download/messages.py @@ -1,6 +1,10 @@ """Message Downloading""" +import random + +from time import sleep + from .common import process_download_accessible_media from .downloadstate import DownloadState from .types import DownloadType @@ -63,6 +67,9 @@ def download_messages(config: FanslyConfig, state: DownloadState): # get next cursor try: + # Fansly rate-limiting fix + # (don't know if messages were affected at all) + sleep(random.uniform(2, 4)) msg_cursor = post_object['messages'][-1]['id'] except IndexError: From 541a87b3abc692bc1fe66fdfc8e17fd96f097a8d Mon Sep 17 00:00:00 2001 From: prof79 Date: Sat, 2 Sep 2023 18:55:54 +0200 Subject: [PATCH 15/19] Added PyInstaller clean-up. --- fansly_downloader.py | 6 ++++-- pathio/__init__.py | 7 ++++++- pathio/pathio.py | 41 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 49 insertions(+), 5 deletions(-) diff --git a/fansly_downloader.py b/fansly_downloader.py index 10da198..ae7fa2a 100644 --- a/fansly_downloader.py +++ b/fansly_downloader.py @@ -2,8 +2,8 @@ """Fansly Downloader""" -__version__ = '0.5.0' -__date__ = '2023-08-30T21:24:00+02' +__version__ = '0.5.1' +__date__ = '2023-09-02T16:20:00+02' __maintainer__ = 'Avnsx (Mika C.)' __copyright__ = f'Copyright (C) 2021-2023 by {__maintainer__}' __authors__: list[str] = [] @@ -24,6 +24,7 @@ from download.core import * from errors import * from fileio.dedupe import dedupe_init +from pathio import delete_temporary_pyinstaller_files from textio import ( input_enter_close, input_enter_continue, @@ -72,6 +73,7 @@ def main(config: FanslyConfig) -> int: # base64 code to display logo in console print(base64.b64decode('CiAg4paI4paI4paI4paI4paI4paI4paI4pWXIOKWiOKWiOKWiOKWiOKWiOKVlyDilojilojilojilZcgICDilojilojilZfilojilojilojilojilojilojilojilZfilojilojilZcgIOKWiOKWiOKVlyAgIOKWiOKWiOKVlyAgICDilojilojilojilojilojilojilZcg4paI4paI4pWXICAgICAgICAgIOKWiOKWiOKWiOKWiOKWiOKVlyDilojilojilojilojilojilojilZcg4paI4paI4paI4paI4paI4paI4pWXIAogIOKWiOKWiOKVlOKVkOKVkOKVkOKVkOKVneKWiOKWiOKVlOKVkOKVkOKWiOKWiOKVl+KWiOKWiOKWiOKWiOKVlyAg4paI4paI4pWR4paI4paI4pWU4pWQ4pWQ4pWQ4pWQ4pWd4paI4paI4pWRICDilZrilojilojilZcg4paI4paI4pWU4pWdICAgIOKWiOKWiOKVlOKVkOKVkOKWiOKWiOKVl+KWiOKWiOKVkSAgICAgICAgIOKWiOKWiOKVlOKVkOKVkOKWiOKWiOKVl+KWiOKWiOKVlOKVkOKVkOKWiOKWiOKVl+KWiOKWiOKVlOKVkOKVkOKWiOKWiOKVlwogIOKWiOKWiOKWiOKWiOKWiOKVlyAg4paI4paI4paI4paI4paI4paI4paI4pWR4paI4paI4pWU4paI4paI4pWXIOKWiOKWiOKVkeKWiOKWiOKWiOKWiOKWiOKWiOKWiOKVl+KWiOKWiOKVkSAgIOKVmuKWiOKWiOKWiOKWiOKVlOKVnSAgICAg4paI4paI4pWRICDilojilojilZHilojilojilZEgICAgICAgICDilojilojilojilojilojilojilojilZHilojilojilojilojilojilojilZTilZ3ilojilojilojilojilojilojilZTilZ0KICDilojilojilZTilZDilZDilZ0gIOKWiOKWiOKVlOKVkOKVkOKWiOKWiOKVkeKWiOKWiOKVkeKVmuKWiOKWiOKVl+KWiOKWiOKVkeKVmuKVkOKVkOKVkOKVkOKWiOKWiOKVkeKWiOKWiOKVkSAgICDilZrilojilojilZTilZ0gICAgICDilojilojilZEgIOKWiOKWiOKVkeKWiOKWiOKVkSAgICAgICAgIOKWiOKWiOKVlOKVkOKVkOKWiOKWiOKVkeKWiOKWiOKVlOKVkOKVkOKVkOKVnSDilojilojilZTilZDilZDilZDilZ0gCiAg4paI4paI4pWRICAgICDilojilojilZEgIOKWiOKWiOKVkeKWiOKWiOKVkSDilZrilojilojilojilojilZHilojilojilojilojilojilojilojilZHilojilojilojilojilojilojilojilZfilojilojilZEgICAgICAg4paI4paI4paI4paI4paI4paI4pWU4pWd4paI4paI4paI4paI4paI4paI4paI4pWXICAgIOKWiOKWiOKVkSAg4paI4paI4pWR4paI4paI4pWRICAgICDilojilojilZEgICAgIAogIOKVmuKVkOKVnSAgICAg4pWa4pWQ4pWdICDilZrilZDilZ3ilZrilZDilZ0gIOKVmuKVkOKVkOKVkOKVneKVmuKVkOKVkOKVkOKVkOKVkOKVkOKVneKVmuKVkOKVkOKVkOKVkOKVkOKVkOKVneKVmuKVkOKVnSAgICAgICDilZrilZDilZDilZDilZDilZDilZ0g4pWa4pWQ4pWQ4pWQ4pWQ4pWQ4pWQ4pWdICAgIOKVmuKVkOKVnSAg4pWa4pWQ4pWd4pWa4pWQ4pWdICAgICDilZrilZDilZ0gICAgIAogICAgICAgICAgICAgICAgICAgICAgICBkZXZlbG9wZWQgb24gZ2l0aHViLmNvbS9Bdm5zeC9mYW5zbHktZG93bmxvYWRlcgo=').decode('utf-8')) + delete_temporary_pyinstaller_files() load_config(config) args = parse_args() diff --git a/pathio/__init__.py b/pathio/__init__.py index 7c9cc95..17936a3 100644 --- a/pathio/__init__.py +++ b/pathio/__init__.py @@ -1,10 +1,15 @@ """Diretory/Folder Utility Module""" -from .pathio import ask_correct_dir, set_create_directory_for_download +from .pathio import ( + ask_correct_dir, + set_create_directory_for_download, + delete_temporary_pyinstaller_files +) __all__ = [ 'ask_correct_dir', 'set_create_directory_for_download', + 'delete_temporary_pyinstaller_files', ] diff --git a/pathio/pathio.py b/pathio/pathio.py index 995dc1c..ebda9ee 100644 --- a/pathio/pathio.py +++ b/pathio/pathio.py @@ -1,7 +1,9 @@ """Work Directory Manipulation""" -import traceback +import os +import sys +import time from pathlib import Path from tkinter import Tk, filedialog @@ -9,7 +11,6 @@ from config import FanslyConfig from download.downloadstate import DownloadState from download.types import DownloadType -from errors import ConfigError from textio import print_info, print_warning, print_error @@ -106,3 +107,39 @@ def set_create_directory_for_download(config: FanslyConfig, state: DownloadState download_directory.mkdir(exist_ok=True) return download_directory + + +def delete_temporary_pyinstaller_files(): + """Delete old files from the PyInstaller temporary folder. + + Files older than an hour will be deleted. + """ + try: + base_path = sys._MEIPASS + + except Exception: + return + + temp_dir = os.path.abspath(os.path.join(base_path, '..')) + current_time = time.time() + + for folder in os.listdir(temp_dir): + try: + item = os.path.join(temp_dir, folder) + + if folder.startswith('_MEI') \ + and os.path.isdir(item) \ + and (current_time - os.path.getctime(item)) > 3600: + + for root, dirs, files in os.walk(item, topdown=False): + + for file in files: + os.remove(os.path.join(root, file)) + + for dir in dirs: + os.rmdir(os.path.join(root, dir)) + + os.rmdir(item) + + except Exception: + pass From 9a34cc89821cd48c25e698fc055b61a48c6fc3ec Mon Sep 17 00:00:00 2001 From: prof79 Date: Sun, 3 Sep 2023 14:56:06 +0200 Subject: [PATCH 16/19] Directory structure changes requested by Avnsx. --- fansly_downloader.py | 26 +++++++++---------- fansly_downloader/__init__.py | 0 .../config}/__init__.py | 0 {config => fansly_downloader/config}/args.py | 6 ++--- .../config}/browser.py | 2 +- .../config}/config.py | 8 +++--- .../config}/fanslyconfig.py | 0 .../config}/metadatahandling.py | 0 {config => fansly_downloader/config}/modes.py | 0 .../config}/validation.py | 14 +++++----- .../download}/__init__.py | 0 .../download}/account.py | 8 +++--- .../download}/collections.py | 4 +-- .../download}/common.py | 10 +++---- .../download}/core.py | 0 .../download}/downloadstate.py | 0 .../download}/m3u8.py | 4 +-- .../download}/media.py | 12 ++++----- .../download}/messages.py | 4 +-- .../download}/single.py | 8 +++--- .../download}/timeline.py | 6 ++--- .../download}/types.py | 0 .../errors}/__init__.py | 0 .../fileio}/dedupe.py | 11 ++++---- .../fileio}/fnmanip.py | 6 ++--- .../media}/__init__.py | 0 {media => fansly_downloader/media}/media.py | 4 +-- .../media}/mediaitem.py | 2 +- .../pathio}/__init__.py | 0 .../pathio}/pathio.py | 8 +++--- .../textio}/__init__.py | 0 .../textio}/textio.py | 0 .../updater}/__init__.py | 11 +++----- .../updater}/utils.py | 8 +++--- {utils => fansly_downloader/utils}/common.py | 4 +-- .../utils}/datetime.py | 0 .../utils}/metadata_manager.py | 0 {utils => fansly_downloader/utils}/web.py | 4 +-- 38 files changed, 83 insertions(+), 87 deletions(-) create mode 100644 fansly_downloader/__init__.py rename {config => fansly_downloader/config}/__init__.py (100%) rename {config => fansly_downloader/config}/args.py (98%) rename {config => fansly_downloader/config}/browser.py (99%) rename {config => fansly_downloader/config}/config.py (97%) rename {config => fansly_downloader/config}/fanslyconfig.py (100%) rename {config => fansly_downloader/config}/metadatahandling.py (100%) rename {config => fansly_downloader/config}/modes.py (100%) rename {config => fansly_downloader/config}/validation.py (96%) rename {download => fansly_downloader/download}/__init__.py (100%) rename {download => fansly_downloader/download}/account.py (92%) rename {download => fansly_downloader/download}/collections.py (93%) rename {download => fansly_downloader/download}/common.py (92%) rename {download => fansly_downloader/download}/core.py (100%) rename {download => fansly_downloader/download}/downloadstate.py (100%) rename {download => fansly_downloader/download}/m3u8.py (97%) rename {download => fansly_downloader/download}/media.py (94%) rename {download => fansly_downloader/download}/messages.py (95%) rename {download => fansly_downloader/download}/single.py (93%) rename {download => fansly_downloader/download}/timeline.py (95%) rename {download => fansly_downloader/download}/types.py (100%) rename {errors => fansly_downloader/errors}/__init__.py (100%) rename {fileio => fansly_downloader/fileio}/dedupe.py (93%) rename {fileio => fansly_downloader/fileio}/fnmanip.py (97%) rename {media => fansly_downloader/media}/__init__.py (100%) rename {media => fansly_downloader/media}/media.py (98%) rename {media => fansly_downloader/media}/mediaitem.py (95%) rename {pathio => fansly_downloader/pathio}/__init__.py (100%) rename {pathio => fansly_downloader/pathio}/pathio.py (95%) rename {textio => fansly_downloader/textio}/__init__.py (100%) rename {textio => fansly_downloader/textio}/textio.py (100%) rename {updater => fansly_downloader/updater}/__init__.py (89%) rename {updater => fansly_downloader/updater}/utils.py (97%) rename {utils => fansly_downloader/utils}/common.py (96%) rename {utils => fansly_downloader/utils}/datetime.py (100%) rename {utils => fansly_downloader/utils}/metadata_manager.py (100%) rename {utils => fansly_downloader/utils}/web.py (97%) diff --git a/fansly_downloader.py b/fansly_downloader.py index ae7fa2a..d23e016 100644 --- a/fansly_downloader.py +++ b/fansly_downloader.py @@ -2,8 +2,8 @@ """Fansly Downloader""" -__version__ = '0.5.1' -__date__ = '2023-09-02T16:20:00+02' +__version__ = '0.5.2' +__date__ = '2023-09-03T14:40:00+02' __maintainer__ = 'Avnsx (Mika C.)' __copyright__ = f'Copyright (C) 2021-2023 by {__maintainer__}' __authors__: list[str] = [] @@ -18,14 +18,14 @@ from random import randint from time import sleep -from config import FanslyConfig, load_config, validate_adjust_config -from config.args import parse_args, map_args_to_config -from config.modes import DownloadMode -from download.core import * -from errors import * -from fileio.dedupe import dedupe_init -from pathio import delete_temporary_pyinstaller_files -from textio import ( +from fansly_downloader.config import FanslyConfig, load_config, validate_adjust_config +from fansly_downloader.config.args import parse_args, map_args_to_config +from fansly_downloader.config.modes import DownloadMode +from fansly_downloader.download.core import * +from fansly_downloader.errors import * +from fansly_downloader.fileio.dedupe import dedupe_init +from fansly_downloader.pathio import delete_temporary_pyinstaller_files +from fansly_downloader.textio import ( input_enter_close, input_enter_continue, print_error, @@ -33,9 +33,9 @@ print_warning, set_window_title, ) -from updater import self_update -from utils.common import exit, open_location -from utils.web import remind_stargazing +from fansly_downloader.updater import self_update +from fansly_downloader.utils.common import exit, open_location +from fansly_downloader.utils.web import remind_stargazing # tell PIL to be tolerant of files that are truncated diff --git a/fansly_downloader/__init__.py b/fansly_downloader/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/config/__init__.py b/fansly_downloader/config/__init__.py similarity index 100% rename from config/__init__.py rename to fansly_downloader/config/__init__.py diff --git a/config/args.py b/fansly_downloader/config/args.py similarity index 98% rename from config/args.py rename to fansly_downloader/config/args.py index db3c91f..7e4cfbe 100644 --- a/config/args.py +++ b/fansly_downloader/config/args.py @@ -11,9 +11,9 @@ from .metadatahandling import MetadataHandling from .modes import DownloadMode -from errors import ConfigError -from textio import print_debug, print_warning -from utils.common import is_valid_post_id, save_config_or_raise +from fansly_downloader.errors import ConfigError +from fansly_downloader.textio import print_debug, print_warning +from fansly_downloader.utils.common import is_valid_post_id, save_config_or_raise def parse_args() -> argparse.Namespace: diff --git a/config/browser.py b/fansly_downloader/config/browser.py similarity index 99% rename from config/browser.py rename to fansly_downloader/config/browser.py index 230548f..fbbb32f 100644 --- a/config/browser.py +++ b/fansly_downloader/config/browser.py @@ -12,7 +12,7 @@ from time import sleep -from textio import print_config +from fansly_downloader.textio import print_config # Function to recursively search for "storage" folders and process SQLite files diff --git a/config/config.py b/fansly_downloader/config/config.py similarity index 97% rename from config/config.py rename to fansly_downloader/config/config.py index 814696c..0e03a04 100644 --- a/config/config.py +++ b/fansly_downloader/config/config.py @@ -13,10 +13,10 @@ from .metadatahandling import MetadataHandling from .modes import DownloadMode -from errors import ConfigError -from textio import print_info, print_config, print_warning -from utils.common import save_config_or_raise -from utils.web import open_url +from fansly_downloader.errors import ConfigError +from fansly_downloader.textio import print_info, print_config, print_warning +from fansly_downloader.utils.common import save_config_or_raise +from fansly_downloader.utils.web import open_url def parse_items_from_line(line: str) -> list[str]: diff --git a/config/fanslyconfig.py b/fansly_downloader/config/fanslyconfig.py similarity index 100% rename from config/fanslyconfig.py rename to fansly_downloader/config/fanslyconfig.py diff --git a/config/metadatahandling.py b/fansly_downloader/config/metadatahandling.py similarity index 100% rename from config/metadatahandling.py rename to fansly_downloader/config/metadatahandling.py diff --git a/config/modes.py b/fansly_downloader/config/modes.py similarity index 100% rename from config/modes.py rename to fansly_downloader/config/modes.py diff --git a/config/validation.py b/fansly_downloader/config/validation.py similarity index 96% rename from config/validation.py rename to fansly_downloader/config/validation.py index eb30f30..adfea8a 100644 --- a/config/validation.py +++ b/fansly_downloader/config/validation.py @@ -8,11 +8,11 @@ from .config import username_has_valid_chars, username_has_valid_length from .fanslyconfig import FanslyConfig -from errors import ConfigError -from pathio.pathio import ask_correct_dir -from textio import print_config, print_error, print_info, print_warning -from utils.common import save_config_or_raise -from utils.web import guess_user_agent, open_get_started_url +from fansly_downloader.errors import ConfigError +from fansly_downloader.pathio.pathio import ask_correct_dir +from fansly_downloader.textio import print_config, print_error, print_info, print_warning +from fansly_downloader.utils.common import save_config_or_raise +from fansly_downloader.utils.web import guess_user_agent, open_get_started_url def validate_creator_names(config: FanslyConfig) -> bool: @@ -139,7 +139,7 @@ def validate_adjust_token(config: FanslyConfig) -> None: if plyvel_installed and not config.token_is_valid(): # fansly-downloader plyvel dependant package imports - from config.browser import ( + from fansly_downloader.config.browser import ( find_leveldb_folders, get_auth_token_from_leveldb_folder, get_browser_config_paths, @@ -147,7 +147,7 @@ def validate_adjust_token(config: FanslyConfig) -> None: parse_browser_from_string, ) - from utils.web import get_fansly_account_for_token + from fansly_downloader.utils.web import get_fansly_account_for_token print_warning( f"Authorization token '{config.token}' is unmodified, missing or malformed" diff --git a/download/__init__.py b/fansly_downloader/download/__init__.py similarity index 100% rename from download/__init__.py rename to fansly_downloader/download/__init__.py diff --git a/download/account.py b/fansly_downloader/download/account.py similarity index 92% rename from download/account.py rename to fansly_downloader/download/account.py index 6c45536..d88ede3 100644 --- a/download/account.py +++ b/fansly_downloader/download/account.py @@ -7,10 +7,10 @@ from .downloadstate import DownloadState -from config import FanslyConfig -from config.modes import DownloadMode -from errors import ApiAccountInfoError, ApiAuthenticationError, ApiError -from textio import print_info +from fansly_downloader.config.fanslyconfig import FanslyConfig +from fansly_downloader.config.modes import DownloadMode +from fansly_downloader.errors import ApiAccountInfoError, ApiAuthenticationError, ApiError +from fansly_downloader.textio import print_info def get_creator_account_info(config: FanslyConfig, state: DownloadState) -> None: diff --git a/download/collections.py b/fansly_downloader/download/collections.py similarity index 93% rename from download/collections.py rename to fansly_downloader/download/collections.py index f62cc02..c9c25f2 100644 --- a/download/collections.py +++ b/fansly_downloader/download/collections.py @@ -5,8 +5,8 @@ from .downloadstate import DownloadState from .types import DownloadType -from config import FanslyConfig -from textio import input_enter_continue, print_error, print_info +from fansly_downloader.config import FanslyConfig +from fansly_downloader.textio import input_enter_continue, print_error, print_info def download_collections(config: FanslyConfig, state: DownloadState): diff --git a/download/common.py b/fansly_downloader/download/common.py similarity index 92% rename from download/common.py rename to fansly_downloader/download/common.py index 7ffaa1c..9370b69 100644 --- a/download/common.py +++ b/fansly_downloader/download/common.py @@ -7,11 +7,11 @@ from .media import download_media from .types import DownloadType -from config import FanslyConfig -from errors import DuplicateCountError -from media import MediaItem, parse_media_info -from pathio import set_create_directory_for_download -from textio import print_error, print_info, print_warning, input_enter_continue +from fansly_downloader.config import FanslyConfig +from fansly_downloader.errors import DuplicateCountError +from fansly_downloader.media import MediaItem, parse_media_info +from fansly_downloader.pathio import set_create_directory_for_download +from fansly_downloader.textio import print_error, print_info, print_warning, input_enter_continue def print_download_info(config: FanslyConfig) -> None: diff --git a/download/core.py b/fansly_downloader/download/core.py similarity index 100% rename from download/core.py rename to fansly_downloader/download/core.py diff --git a/download/downloadstate.py b/fansly_downloader/download/downloadstate.py similarity index 100% rename from download/downloadstate.py rename to fansly_downloader/download/downloadstate.py diff --git a/download/m3u8.py b/fansly_downloader/download/m3u8.py similarity index 97% rename from download/m3u8.py rename to fansly_downloader/download/m3u8.py index 1798de3..d709f21 100644 --- a/download/m3u8.py +++ b/fansly_downloader/download/m3u8.py @@ -11,8 +11,8 @@ from rich.table import Column from rich.progress import BarColumn, TextColumn, Progress -from config.fanslyconfig import FanslyConfig -from textio import print_error +from fansly_downloader.config.fanslyconfig import FanslyConfig +from fansly_downloader.textio import print_error def download_m3u8(config: FanslyConfig, m3u8_url: str, save_path: Path) -> bool: diff --git a/download/media.py b/fansly_downloader/download/media.py similarity index 94% rename from download/media.py rename to fansly_downloader/download/media.py index 374dfa8..42109de 100644 --- a/download/media.py +++ b/fansly_downloader/download/media.py @@ -10,12 +10,12 @@ from .m3u8 import download_m3u8 from .types import DownloadType -from config import FanslyConfig -from errors import DownloadError, DuplicateCountError, MediaError -from fileio.dedupe import dedupe_media_content -from media import MediaItem -from pathio import set_create_directory_for_download -from textio import print_info, print_warning +from fansly_downloader.config import FanslyConfig +from fansly_downloader.errors import DownloadError, DuplicateCountError, MediaError +from fansly_downloader.fileio.dedupe import dedupe_media_content +from fansly_downloader.media import MediaItem +from fansly_downloader.pathio import set_create_directory_for_download +from fansly_downloader.textio import print_info, print_warning # tell PIL to be tolerant of files that are truncated diff --git a/download/messages.py b/fansly_downloader/download/messages.py similarity index 95% rename from download/messages.py rename to fansly_downloader/download/messages.py index f7b0c5f..32bcd61 100644 --- a/download/messages.py +++ b/fansly_downloader/download/messages.py @@ -9,8 +9,8 @@ from .downloadstate import DownloadState from .types import DownloadType -from config import FanslyConfig -from textio import input_enter_continue, print_error, print_info, print_warning +from fansly_downloader.config import FanslyConfig +from fansly_downloader.textio import input_enter_continue, print_error, print_info, print_warning def download_messages(config: FanslyConfig, state: DownloadState): diff --git a/download/single.py b/fansly_downloader/download/single.py similarity index 93% rename from download/single.py rename to fansly_downloader/download/single.py index 6c2dbbc..fb91573 100644 --- a/download/single.py +++ b/fansly_downloader/download/single.py @@ -1,14 +1,14 @@ """Single Post Downloading""" -from fileio.dedupe import dedupe_init from .common import process_download_accessible_media from .core import DownloadState from .types import DownloadType -from config import FanslyConfig -from textio import input_enter_continue, print_error, print_info, print_warning -from utils.common import is_valid_post_id +from fansly_downloader.config import FanslyConfig +from fansly_downloader.fileio.dedupe import dedupe_init +from fansly_downloader.textio import input_enter_continue, print_error, print_info, print_warning +from fansly_downloader.utils.common import is_valid_post_id def download_single_post(config: FanslyConfig, state: DownloadState): diff --git a/download/timeline.py b/fansly_downloader/download/timeline.py similarity index 95% rename from download/timeline.py rename to fansly_downloader/download/timeline.py index 8ab7d74..90613c9 100644 --- a/download/timeline.py +++ b/fansly_downloader/download/timeline.py @@ -11,9 +11,9 @@ from .core import DownloadState from .types import DownloadType -from config import FanslyConfig -from errors import ApiError -from textio import input_enter_continue, print_debug, print_error, print_info, print_warning +from fansly_downloader.config import FanslyConfig +from fansly_downloader.errors import ApiError +from fansly_downloader.textio import input_enter_continue, print_debug, print_error, print_info, print_warning def download_timeline(config: FanslyConfig, state: DownloadState) -> None: diff --git a/download/types.py b/fansly_downloader/download/types.py similarity index 100% rename from download/types.py rename to fansly_downloader/download/types.py diff --git a/errors/__init__.py b/fansly_downloader/errors/__init__.py similarity index 100% rename from errors/__init__.py rename to fansly_downloader/errors/__init__.py diff --git a/fileio/dedupe.py b/fansly_downloader/fileio/dedupe.py similarity index 93% rename from fileio/dedupe.py rename to fansly_downloader/fileio/dedupe.py index d02bce8..20f3e0c 100644 --- a/fileio/dedupe.py +++ b/fansly_downloader/fileio/dedupe.py @@ -8,12 +8,11 @@ from PIL import Image, ImageFile from random import randint -from fileio.fnmanip import add_hash_to_folder_items - -from config import FanslyConfig -from download.downloadstate import DownloadState -from pathio import set_create_directory_for_download -from textio import print_info, print_warning +from .fnmanip import add_hash_to_folder_items +from fansly_downloader.config import FanslyConfig +from fansly_downloader.download.downloadstate import DownloadState +from fansly_downloader.pathio import set_create_directory_for_download +from fansly_downloader.textio import print_info, print_warning # tell PIL to be tolerant of files that are truncated diff --git a/fileio/fnmanip.py b/fansly_downloader/fileio/fnmanip.py similarity index 97% rename from fileio/fnmanip.py rename to fansly_downloader/fileio/fnmanip.py index 05c760e..45bd175 100644 --- a/fileio/fnmanip.py +++ b/fansly_downloader/fileio/fnmanip.py @@ -12,9 +12,9 @@ from pathlib import Path from PIL import Image -from config import FanslyConfig -from download.downloadstate import DownloadState -from textio import print_debug, print_error +from fansly_downloader.config import FanslyConfig +from fansly_downloader.download.downloadstate import DownloadState +from fansly_downloader.textio import print_debug, print_error # turn off for our purpose unnecessary PIL safety features diff --git a/media/__init__.py b/fansly_downloader/media/__init__.py similarity index 100% rename from media/__init__.py rename to fansly_downloader/media/__init__.py diff --git a/media/media.py b/fansly_downloader/media/media.py similarity index 98% rename from media/media.py rename to fansly_downloader/media/media.py index 5896f65..55a1944 100644 --- a/media/media.py +++ b/fansly_downloader/media/media.py @@ -5,8 +5,8 @@ from . import MediaItem -from download.downloadstate import DownloadState -from textio import print_error +from fansly_downloader.download.downloadstate import DownloadState +from fansly_downloader.textio import print_error def simplify_mimetype(mimetype: str): diff --git a/media/mediaitem.py b/fansly_downloader/media/mediaitem.py similarity index 95% rename from media/mediaitem.py rename to fansly_downloader/media/mediaitem.py index 7d83881..e9eb9cb 100644 --- a/media/mediaitem.py +++ b/fansly_downloader/media/mediaitem.py @@ -4,7 +4,7 @@ from dataclasses import dataclass from typing import Any -from utils.datetime import get_adjusted_datetime +from fansly_downloader.utils.datetime import get_adjusted_datetime @dataclass diff --git a/pathio/__init__.py b/fansly_downloader/pathio/__init__.py similarity index 100% rename from pathio/__init__.py rename to fansly_downloader/pathio/__init__.py diff --git a/pathio/pathio.py b/fansly_downloader/pathio/pathio.py similarity index 95% rename from pathio/pathio.py rename to fansly_downloader/pathio/pathio.py index ebda9ee..c7a1952 100644 --- a/pathio/pathio.py +++ b/fansly_downloader/pathio/pathio.py @@ -8,10 +8,10 @@ from pathlib import Path from tkinter import Tk, filedialog -from config import FanslyConfig -from download.downloadstate import DownloadState -from download.types import DownloadType -from textio import print_info, print_warning, print_error +from fansly_downloader.config import FanslyConfig +from fansly_downloader.download.downloadstate import DownloadState +from fansly_downloader.download.types import DownloadType +from fansly_downloader.textio import print_info, print_warning, print_error # if the users custom provided filepath is invalid; a tkinter dialog will open during runtime, asking to adjust download path diff --git a/textio/__init__.py b/fansly_downloader/textio/__init__.py similarity index 100% rename from textio/__init__.py rename to fansly_downloader/textio/__init__.py diff --git a/textio/textio.py b/fansly_downloader/textio/textio.py similarity index 100% rename from textio/textio.py rename to fansly_downloader/textio/textio.py diff --git a/updater/__init__.py b/fansly_downloader/updater/__init__.py similarity index 89% rename from updater/__init__.py rename to fansly_downloader/updater/__init__.py index 06c104f..f859bfe 100644 --- a/updater/__init__.py +++ b/fansly_downloader/updater/__init__.py @@ -1,15 +1,12 @@ """Self-Updating Functionality""" -import sys - -from utils.web import get_release_info_from_github - from .utils import check_for_update, delete_deprecated_files, post_update_steps -from config import FanslyConfig, copy_old_config_values -from textio import print_warning -from utils.common import save_config_or_raise +from fansly_downloader.config import FanslyConfig, copy_old_config_values +from fansly_downloader.textio import print_warning +from fansly_downloader.utils.common import save_config_or_raise +from fansly_downloader.utils.web import get_release_info_from_github def self_update(config: FanslyConfig): diff --git a/updater/utils.py b/fansly_downloader/updater/utils.py similarity index 97% rename from updater/utils.py rename to fansly_downloader/updater/utils.py index ef41033..96465e2 100644 --- a/updater/utils.py +++ b/fansly_downloader/updater/utils.py @@ -9,15 +9,15 @@ import subprocess import sys -import errors +import fansly_downloader.errors as errors from pathlib import Path from pkg_resources._vendor.packaging.version import parse as parse_version from shutil import unpack_archive -from config import FanslyConfig -from textio import clear_terminal, print_error, print_info, print_update, print_warning -from utils.web import get_release_info_from_github +from fansly_downloader.config import FanslyConfig +from fansly_downloader.textio import clear_terminal, print_error, print_info, print_update, print_warning +from fansly_downloader.utils.web import get_release_info_from_github def delete_deprecated_files() -> None: diff --git a/utils/common.py b/fansly_downloader/utils/common.py similarity index 96% rename from utils/common.py rename to fansly_downloader/utils/common.py index c74d054..08d0ac8 100644 --- a/utils/common.py +++ b/fansly_downloader/utils/common.py @@ -7,8 +7,8 @@ from pathlib import Path -from config.fanslyconfig import FanslyConfig -from errors import ConfigError +from fansly_downloader.config.fanslyconfig import FanslyConfig +from fansly_downloader.errors import ConfigError def exit(status: int=0) -> None: diff --git a/utils/datetime.py b/fansly_downloader/utils/datetime.py similarity index 100% rename from utils/datetime.py rename to fansly_downloader/utils/datetime.py diff --git a/utils/metadata_manager.py b/fansly_downloader/utils/metadata_manager.py similarity index 100% rename from utils/metadata_manager.py rename to fansly_downloader/utils/metadata_manager.py diff --git a/utils/web.py b/fansly_downloader/utils/web.py similarity index 97% rename from utils/web.py rename to fansly_downloader/utils/web.py index b1494b5..0ed66d8 100644 --- a/utils/web.py +++ b/fansly_downloader/utils/web.py @@ -8,8 +8,8 @@ from time import sleep -from config.fanslyconfig import FanslyConfig -from textio import print_error, print_info_highlight, print_warning +from fansly_downloader.config.fanslyconfig import FanslyConfig +from fansly_downloader.textio import print_error, print_info_highlight, print_warning # mostly used to attempt to open fansly downloaders documentation From 2d12f6b0cfcb4b178a816dc90c02b60216960b4d Mon Sep 17 00:00:00 2001 From: prof79 Date: Sun, 3 Sep 2023 14:57:07 +0200 Subject: [PATCH 17/19] .gitignore removal requested by Avnsx. --- .gitignore | 164 ----------------------------------------------------- 1 file changed, 164 deletions(-) delete mode 100644 .gitignore diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 21776ad..0000000 --- a/.gitignore +++ /dev/null @@ -1,164 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -#poetry.lock - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -#pdm.lock -# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it -# in version control. -# https://pdm.fming.dev/#use-with-ide -.pdm.toml - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ - -# User-specific -*_fansly/ -config_args.ini From cc646ea54bfec854577466ae596ab53444181449 Mon Sep 17 00:00:00 2001 From: prof79 Date: Sun, 3 Sep 2023 15:18:16 +0200 Subject: [PATCH 18/19] Removed custom exit() as not required any longer according to Avnsx. --- fansly_downloader.py | 6 +++--- fansly_downloader/textio/textio.py | 1 - fansly_downloader/utils/common.py | 12 ------------ 3 files changed, 3 insertions(+), 16 deletions(-) diff --git a/fansly_downloader.py b/fansly_downloader.py index d23e016..b3369f1 100644 --- a/fansly_downloader.py +++ b/fansly_downloader.py @@ -2,8 +2,8 @@ """Fansly Downloader""" -__version__ = '0.5.2' -__date__ = '2023-09-03T14:40:00+02' +__version__ = '0.5.3' +__date__ = '2023-09-03T15:17:00+02' __maintainer__ = 'Avnsx (Mika C.)' __copyright__ = f'Copyright (C) 2021-2023 by {__maintainer__}' __authors__: list[str] = [] @@ -34,7 +34,7 @@ set_window_title, ) from fansly_downloader.updater import self_update -from fansly_downloader.utils.common import exit, open_location +from fansly_downloader.utils.common import open_location from fansly_downloader.utils.web import remind_stargazing diff --git a/fansly_downloader/textio/textio.py b/fansly_downloader/textio/textio.py index 16b4552..8897d61 100644 --- a/fansly_downloader/textio/textio.py +++ b/fansly_downloader/textio/textio.py @@ -87,7 +87,6 @@ def input_enter_close(interactive: bool=True) -> None: print('\nExiting in 15 seconds ...') sleep(15) - from utils.common import exit exit() diff --git a/fansly_downloader/utils/common.py b/fansly_downloader/utils/common.py index 08d0ac8..f9026b5 100644 --- a/fansly_downloader/utils/common.py +++ b/fansly_downloader/utils/common.py @@ -11,18 +11,6 @@ from fansly_downloader.errors import ConfigError -def exit(status: int=0) -> None: - """Exits the program. - - This function overwrites the default exit() function with a - pyinstaller compatible one. - - :param status: The exit code of the program. - :type status: int - """ - os._exit(status) - - def save_config_or_raise(config: FanslyConfig) -> bool: """Tries to save the configuration to `config.ini` or raises a `ConfigError` otherwise. From 4a6ba2cb0ca638bd30d9f54e3c11674044fc63a5 Mon Sep 17 00:00:00 2001 From: prof79 Date: Sun, 3 Sep 2023 16:59:40 +0200 Subject: [PATCH 19/19] Fix exit() call after custom code removal. --- fansly_downloader/textio/textio.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fansly_downloader/textio/textio.py b/fansly_downloader/textio/textio.py index 8897d61..acca6c4 100644 --- a/fansly_downloader/textio/textio.py +++ b/fansly_downloader/textio/textio.py @@ -7,9 +7,9 @@ import sys from functools import partialmethod -from time import sleep from loguru import logger from pathlib import Path +from time import sleep LOG_FILE_NAME: str = 'fansly_downloader.log' @@ -87,7 +87,7 @@ def input_enter_close(interactive: bool=True) -> None: print('\nExiting in 15 seconds ...') sleep(15) - exit() + sys.exit() def input_enter_continue(interactive: bool=True) -> None: