diff --git a/src/prep.py b/src/prep.py index 1ebed8b11..e991ba21e 100644 --- a/src/prep.py +++ b/src/prep.py @@ -34,6 +34,7 @@ import tmdbsimple as tmdb from datetime import datetime, date from difflib import SequenceMatcher + import torf from torf import Torrent import base64 import time @@ -2079,19 +2080,72 @@ def get_edition(self, video, bdinfo, filelist, manual_edition): return edition, repack + """ + Create Torrent + """ + class CustomTorrent(torf.Torrent): + # Ensure the piece size is within the desired limits + torf.Torrent.piece_size_min = 16384 # 16 KiB + torf.Torrent.piece_size_max = 67108864 # 64 MiB + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # Calculate and set the piece size + total_size = self._calculate_total_size() + piece_size = self.calculate_piece_size(total_size, self.piece_size_min, self.piece_size_max) + self.piece_size = piece_size + + @property + def piece_size(self): + return self._piece_size + + @piece_size.setter + def piece_size(self, value): + if value is None: + total_size = self._calculate_total_size() + value = self.calculate_piece_size(total_size, self.piece_size_min, self.piece_size_max) + self._piece_size = value + self.metainfo['info']['piece length'] = value # Ensure 'piece length' is set + + @classmethod + def calculate_piece_size(cls, total_size, min_size, max_size): + our_min_size = 16384 + our_max_size = 67108864 + # Start with a piece size of 4 MiB + piece_size = 4194304 # 4 MiB in bytes + num_pieces = math.ceil(total_size / piece_size) + torrent_file_size = 20 + (num_pieces * 20) # Approximate .torrent size: 20 bytes header + 20 bytes per piece + + # Adjust the piece size to fit within the constraints + while not (1000 <= num_pieces <= 2000 and torrent_file_size <= 102400): # 100 KiB .torrent size limit + if num_pieces < 1000: + piece_size //= 2 + if piece_size < our_min_size: + piece_size = our_min_size + break + elif num_pieces > 2000 or torrent_file_size > 102400: + piece_size *= 2 + if piece_size > our_max_size: + piece_size = our_max_size + break + num_pieces = math.ceil(total_size / piece_size) + torrent_file_size = 20 + (num_pieces * 20) + return piece_size + def _calculate_total_size(self): + return sum(file.size for file in self.files) + def validate_piece_size(self): + if not hasattr(self, '_piece_size') or self._piece_size is None: + self.piece_size = self.calculate_piece_size(self._calculate_total_size(), self.piece_size_min, self.piece_size_max) + self.metainfo['info']['piece length'] = self.piece_size # Ensure 'piece length' is set - """ - Create Torrent - """ - def create_torrent(self, meta, path, output_filename, piece_size_max): - piece_size_max = int(piece_size_max) if piece_size_max is not None else 0 - if meta['isdir'] == True: + def create_torrent(self, meta, path, output_filename): + # Handle directories and file inclusion logic + if meta['isdir']: if meta['keep_folder']: cli_ui.info('--keep-folder was specified. Using complete folder for torrent creation.') - path = path else: os.chdir(path) globs = glob.glob1(path, "*.mkv") + glob.glob1(path, "*.mp4") + glob.glob1(path, "*.ts") @@ -2106,66 +2160,30 @@ def create_torrent(self, meta, path, output_filename, piece_size_max): else: exclude = ["*.*", "*sample.mkv", "!sample*.*"] include = ["*.mkv", "*.mp4", "*.ts"] - torrent = Torrent(path, - trackers = ["https://fake.tracker"], - source = "L4G", - private = True, - exclude_globs = exclude or [], - include_globs = include or [], - creation_date = datetime.now(), - comment = "Created by L4G's Upload Assistant", - created_by = "L4G's Upload Assistant") - file_size = torrent.size - if file_size < 268435456: # 256 MiB File / 256 KiB Piece Size - piece_size = 18 - piece_size_text = "256KiB" - elif file_size < 1073741824: # 1 GiB File/512 KiB Piece Size - piece_size = 19 - piece_size_text = "512KiB" - elif file_size < 2147483648 or piece_size_max == 1: # 2 GiB File/1 MiB Piece Size - piece_size = 20 - piece_size_text = "1MiB" - elif file_size < 4294967296 or piece_size_max == 2: # 4 GiB File/2 MiB Piece Size - piece_size = 21 - piece_size_text = "2MiB" - elif file_size < 8589934592 or piece_size_max == 4: # 8 GiB File/4 MiB Piece Size - piece_size = 22 - piece_size_text = "4MiB" - elif file_size < 17179869184 or piece_size_max == 8: # 16 GiB File/8 MiB Piece Size - piece_size = 23 - piece_size_text = "8MiB" - else: # 16MiB Piece Size - piece_size = 24 - piece_size_text = "16MiB" - console.print(f"[bold yellow]Creating .torrent with a piece size of {piece_size_text}... (No valid --torrenthash was provided to reuse)") - if meta.get('torrent_creation') != None: - torrent_creation = meta['torrent_creation'] - else: - torrent_creation = self.config['DEFAULT'].get('torrent_creation', 'torf') - if torrent_creation == 'torrenttools': - args = ['torrenttools', 'create', '-a', 'https://fake.tracker', '--private', 'on', '--piece-size', str(2**piece_size), '--created-by', "L4G's Upload Assistant", '--no-cross-seed','-o', f"{meta['base_dir']}/tmp/{meta['uuid']}/{output_filename}.torrent"] - if not meta['is_disc']: - args.extend(['--include', '^.*\.(mkv|mp4|ts)$']) - args.append(path) - err = subprocess.call(args) - if err != 0: - args[3] = "OMITTED" - console.print(f"[bold red]Process execution {args} returned with error code {err}.") - elif torrent_creation == 'mktorrent': - args = ['mktorrent', '-a', 'https://fake.tracker', '-p', f'-l {piece_size}', '-o', f"{meta['base_dir']}/tmp/{meta['uuid']}/{output_filename}.torrent", path] - err = subprocess.call(args) - if err != 0: - args[2] = "OMITTED" - console.print(f"[bold red]Process execution {args} returned with error code {err}.") - else: - torrent.piece_size = 2**piece_size - torrent.piece_size_max = 16777216 - torrent.generate(callback=self.torf_cb, interval=5) - torrent.write(f"{meta['base_dir']}/tmp/{meta['uuid']}/{output_filename}.torrent", overwrite=True) - torrent.verify_filesize(path) + + # Create and write the new torrent using the CustomTorrent class + torrent = self.CustomTorrent( + path=path, + trackers=["https://fake.tracker"], + source="L4G", + private=True, + exclude_globs=exclude or [], + include_globs=include or [], + creation_date=datetime.now(), + comment="Created by L4G's Upload Assistant", + created_by="L4G's Upload Assistant" + ) + + # Ensure piece size is validated before writing + torrent.validate_piece_size() + + # Generate and write the new torrent + torrent.generate(callback=self.torf_cb, interval=5) + torrent.write(f"{meta['base_dir']}/tmp/{meta['uuid']}/{output_filename}.torrent", overwrite=True) + torrent.verify_filesize(path) + console.print("[bold green].torrent created", end="\r") return torrent - def torf_cb(self, torrent, filepath, pieces_done, pieces_total): # print(f'{pieces_done/pieces_total*100:3.0f} % done') @@ -2197,8 +2215,6 @@ def create_base_from_existing_torrent(self, torrentpath, base_dir, uuid): Torrent.copy(base_torrent).write(f"{base_dir}/tmp/{uuid}/BASE.torrent", overwrite=True) - - """ Upload Screenshots """ @@ -3193,4 +3209,4 @@ async def search_tvmaze(self, filename, year, imdbID, tvdbID): if int(tvdbID) == 0: if show.get('externals', {}).get('tvdb', '0') != None: tvdbID = show.get('externals', {}).get('tvdb', '0') - return tvmazeID, imdbID, tvdbID + return tvmazeID, imdbID, tvdbID \ No newline at end of file diff --git a/src/trackers/ANT.py b/src/trackers/ANT.py index 670d63595..cd1cd7690 100644 --- a/src/trackers/ANT.py +++ b/src/trackers/ANT.py @@ -9,11 +9,9 @@ import math from torf import Torrent from pathlib import Path - from src.trackers.COMMON import COMMON from src.console import console - class ANT(): """ Edit for Tracker: @@ -76,32 +74,27 @@ async def upload(self, meta): torrent_filename = "BASE" torrent = Torrent.read(f"{meta['base_dir']}/tmp/{meta['uuid']}/BASE.torrent") total_size = sum(file.size for file in torrent.files) + + # Calculate the number of pieces and the torrent file size based on the current piece size def calculate_pieces_and_file_size(total_size, piece_size): num_pieces = math.ceil(total_size / piece_size) torrent_file_size = 20 + (num_pieces * 20) # Approximate size: 20 bytes header + 20 bytes per piece return num_pieces, torrent_file_size - # Start with 4 MiB piece size and adjust if necessary - piece_size = 4194304 # 4 MiB - num_pieces, torrent_file_size = calculate_pieces_and_file_size(total_size, piece_size) - while not (1000 <= num_pieces <= 2000 and torrent_file_size <= 81920): # 80 KiB = 81920 bytes - if num_pieces < 1000: - piece_size //= 2 - if piece_size < 16384: # 16 KiB is the smallest allowed by the BitTorrent spec - piece_size = 16384 - break - elif num_pieces > 2000 or torrent_file_size > 81920: - piece_size *= 2 - num_pieces, torrent_file_size = calculate_pieces_and_file_size(total_size, piece_size) + # Check if the existing torrent fits within the constraints + num_pieces, torrent_file_size = calculate_pieces_and_file_size(total_size, torrent.piece_size) - if not (1000 <= num_pieces <= 2000): - console.print("[red]Unable to generate a .torrent with the required number of pieces and file size constraints") - else: - console.print("[yellow]Regenerating torrent to fit within 1000-2000 pieces and 80 KiB size limit.") + # If the torrent doesn't meet the constraints, regenerate it + if not (1000 <= num_pieces <= 2000) or torrent_file_size > 102400: + console.print("[yellow]Regenerating torrent to fit within 1000-2000 pieces and 100 KiB .torrent size limit needed for ANT.") from src.prep import Prep prep = Prep(screens=meta['screens'], img_host=meta['imghost'], config=self.config) - prep.create_torrent(meta, Path(meta['path']), "ANT", piece_size_max=piece_size) + + # Call create_torrent with the default piece size calculation + prep.create_torrent(meta, Path(meta['path']), "ANT") torrent_filename = "ANT" + else: + console.print("[green]Existing torrent meets the constraints.") await common.edit_torrent(meta, self.tracker, self.source_flag, torrent_filename=torrent_filename) flags = await self.get_flags(meta) @@ -190,4 +183,4 @@ async def search_existing(self, meta): console.print('[bold red]Unable to search for existing torrents on site. Either the site is down or your API key is incorrect') await asyncio.sleep(5) - return dupes + return dupes \ No newline at end of file diff --git a/src/trackers/MTV.py b/src/trackers/MTV.py index de14e5dc9..f5d3069a2 100644 --- a/src/trackers/MTV.py +++ b/src/trackers/MTV.py @@ -12,6 +12,7 @@ from pathlib import Path from str2bool import str2bool from src.trackers.COMMON import COMMON +from datetime import datetime, date class MTV(): """ @@ -66,9 +67,32 @@ async def upload_with_retry(self, meta, cookiefile, common, img_host_index=1): from src.prep import Prep prep = Prep(screens=meta['screens'], img_host=meta['imghost'], config=self.config) - if torrent.piece_size > 8388608: + # Check if the piece size exceeds 8 MiB and regenerate the torrent if needed + if torrent.piece_size > 8388608: # 8 MiB in bytes console.print("[red]Piece size is OVER 8M and does not work on MTV. Generating a new .torrent") - prep.create_torrent(meta, Path(meta['path']), "MTV", piece_size_max=8) + + # Create a new torrent with piece size explicitly set to 8 MiB + new_torrent = prep.CustomTorrent( + path=Path(meta['path']), + trackers=["https://fake.tracker"], + source="L4G", + private=True, + exclude_globs=["*.*", "*sample.mkv", "!sample*.*"], + include_globs=["*.mkv", "*.mp4", "*.ts"], + creation_date=datetime.now(), + comment="Created by L4G's Upload Assistant", + created_by="L4G's Upload Assistant" + ) + + # Explicitly set the piece size and update metainfo + new_torrent.piece_size = 8388608 # 8 MiB in bytes + new_torrent.metainfo['info']['piece length'] = 8388608 # Ensure 'piece length' is set + + # Validate and write the new torrent + new_torrent.validate_piece_size() + new_torrent.generate(callback=prep.torf_cb, interval=5) + new_torrent.write(f"{meta['base_dir']}/tmp/{meta['uuid']}/MTV.torrent", overwrite=True) + torrent_filename = "MTV" await common.edit_torrent(meta, self.tracker, self.source_flag, torrent_filename=torrent_filename) @@ -207,7 +231,6 @@ async def edit_group_desc(self, meta): return description - async def edit_name(self, meta): mtv_name = meta['uuid'] # Try to use original filename if possible @@ -232,47 +255,6 @@ async def edit_name(self, meta): mtv_name = mtv_name.replace(' ', '.').replace('..', '.') return mtv_name - - # Not needed as its optional - # async def get_poster(self, meta): - # if 'poster_image' in meta: - # return meta['poster_image'] - # else: - # if meta['poster'] is not None: - # poster = meta['poster'] - # else: - # if 'cover' in meta['imdb_info'] and meta['imdb_info']['cover'] is not None: - # poster = meta['imdb_info']['cover'] - # else: - # console.print(f'[red]No poster can be found for this EXITING!!') - # return - # with requests.get(url=poster, stream=True) as r: - # with open(f"{meta['base_dir']}/tmp/{meta['uuid']}/{meta['clean_name']}-poster.jpg", - # 'wb') as f: - # shutil.copyfileobj(r.raw, f) - # - # url = "https://api.imgbb.com/1/upload" - # data = { - # 'key': self.config['DEFAULT']['imgbb_api'], - # 'image': base64.b64encode(open(f"{meta['base_dir']}/tmp/{meta['uuid']}/{meta['clean_name']}-poster.jpg", "rb").read()).decode('utf8') - # } - # try: - # console.print("[yellow]uploading poster to imgbb") - # response = requests.post(url, data=data) - # response = response.json() - # if response.get('success') != True: - # console.print(response, 'red') - # img_url = response['data'].get('medium', response['data']['image'])['url'] - # th_url = response['data']['thumb']['url'] - # web_url = response['data']['url_viewer'] - # raw_url = response['data']['image']['url'] - # meta['poster_image'] = raw_url - # console.print(f'[green]{raw_url} ') - # except Exception: - # console.print("[yellow]imgbb failed to upload cover") - # - # return raw_url - async def get_res_id(self, resolution): resolution_id = { '8640p':'0', @@ -307,7 +289,6 @@ async def get_cat_id(self, meta): else: return 3 - async def get_source_id(self, meta): if meta['is_disc'] == 'DVD': return '1' @@ -331,7 +312,6 @@ async def get_source_id(self, meta): }.get(meta['type'], '0') return type_id - async def get_origin_id(self, meta): if meta['personalrelease']: return '4' @@ -340,8 +320,6 @@ async def get_origin_id(self, meta): # returning P2P else: return '3' - - async def get_tags(self, meta): tags = [] # Genres @@ -361,8 +339,6 @@ async def get_tags(self, meta): for each in ['remux', 'WEB.DL', 'WEBRip', 'HDTV', 'BluRay', 'DVD', 'HDDVD']: if (each.lower().replace('.', '') in meta['type'].lower()) or (each.lower().replace('-', '') in meta['source']): tags.append(each) - - # series tags if meta['category'] == "TV": if meta.get('tv_pack', 0) == 0: @@ -385,8 +361,6 @@ async def get_tags(self, meta): else: tags.append('hd.movie') - - # Audio tags audio_tag = "" for each in ['dd', 'ddp', 'aac', 'truehd', 'mp3', 'mp2', 'dts', 'dts.hd', 'dts.x']: @@ -422,8 +396,6 @@ async def get_tags(self, meta): tags = ' '.join(tags) return tags - - async def validate_credentials(self, meta): cookiefile = os.path.abspath(f"{meta['base_dir']}/data/cookies/MTV.pkl") if not os.path.exists(cookiefile): @@ -569,4 +541,4 @@ async def search_existing(self, meta): print(traceback.print_exc()) await asyncio.sleep(5) - return dupes + return dupes \ No newline at end of file diff --git a/upload.py b/upload.py index feb7ee52e..c2384a842 100644 --- a/upload.py +++ b/upload.py @@ -158,7 +158,6 @@ async def do_the_thing(base_dir): console.print(f"[red]There was an issue with your input. If you think this was not an issue, please make a report that includes the full command used.") exit() - base_meta = {k: v for k, v in meta.items()} for path in queue: meta = {k: v for k, v in base_meta.items()} @@ -207,11 +206,11 @@ async def do_the_thing(base_dir): if reuse_torrent != None: prep.create_base_from_existing_torrent(reuse_torrent, meta['base_dir'], meta['uuid']) if meta['nohash'] == False and reuse_torrent == None: - prep.create_torrent(meta, Path(meta['path']), "BASE", meta.get('piece_size_max', 0)) + prep.create_torrent(meta, Path(meta['path']), "BASE") if meta['nohash']: meta['client'] = "none" elif os.path.exists(os.path.abspath(f"{meta['base_dir']}/tmp/{meta['uuid']}/BASE.torrent")) and meta.get('rehash', False) == True and meta['nohash'] == False: - prep.create_torrent(meta, Path(meta['path']), "BASE", meta.get('piece_size_max', 0)) + prep.create_torrent(meta, Path(meta['path']), "BASE") if int(meta.get('randomized', 0)) >= 1: prep.create_random_torrents(meta['base_dir'], meta['uuid'], meta['randomized'], meta['path']) @@ -244,8 +243,6 @@ async def do_the_thing(base_dir): if meta.get('manual', False): trackers.insert(0, "MANUAL") - - #################################### ####### Upload to Trackers ####### ####################################