Skip to content

Commit

Permalink
Recalibrate torrent creation
Browse files Browse the repository at this point in the history 
This fucker should not get caught in loops any longer, using sane break points instead of insisting on strict limits.

ANT - just run with it unless .torrent > 100 KiB, then just do a thing.
  • Loading branch information
@Audionut
Audionut committed Oct 25, 2024
1 parent 086e340 commit 8906480
Show file tree
Hide file tree
Showing 2 changed files with 69 additions and 68 deletions.
77 changes: 57 additions & 20 deletions src/prep.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2482,9 +2482,9 @@ def __init__(self, meta, *args, **kwargs):
self.piece_size_max = torf.Torrent.piece_size_max

# Calculate and set the piece size
total_size = self._calculate_total_size()
piece_size = self.calculate_piece_size(total_size, self.piece_size_min, self.piece_size_max, self.files)
self.piece_size = piece_size
# total_size = self._calculate_total_size()
# piece_size = self.calculate_piece_size(total_size, self.piece_size_min, self.piece_size_max, self.files)
self.metainfo['info']['piece length'] = self._piece_size

@property
def piece_size(self):
Expand All @@ -2500,53 +2500,90 @@ def piece_size(self, value):

@classmethod
def calculate_piece_size(cls, total_size, min_size, max_size, files):
file_count = len(files)
# console.print(f"[red]Calculating piece size for {file_count} files")

our_min_size = 16384
our_max_size = max_size if max_size else 268435456 # Default to 256 MiB if max_size is None
piece_size = 268435456 # Start with 256 MiB
piece_size = 4194304 # Start with 4 MiB

num_pieces = math.ceil(total_size / piece_size)
torrent_file_size = 20 + (num_pieces * 20) + cls._calculate_pathname_bytes(files) # Approximate .torrent size

# Initial torrent_file_size calculation based on file_count
# More paths = greater error in pathname_bytes, roughly recalibrate
if file_count > 1000:
torrent_file_size = 20 + (num_pieces * 20) + int(cls._calculate_pathname_bytes(files) * 71 / 100)
elif file_count > 500:
torrent_file_size = 20 + (num_pieces * 20) + int(cls._calculate_pathname_bytes(files) * 4 / 5)
else:
torrent_file_size = 20 + (num_pieces * 20) + cls._calculate_pathname_bytes(files)

# iteration = 0 # Track the number of iterations
# print(f"Initial piece size: {piece_size} bytes")
# print(f"Initial num_pieces: {num_pieces}, Initial torrent_file_size: {torrent_file_size} bytes")

# Adjust the piece size to fit within the constraints
while not (1000 <= num_pieces <= 2000 and torrent_file_size <= 102400): # 100 KiB .torrent size limit
if num_pieces < 1000 and torrent_file_size >= 102400:
while not ((750 <= num_pieces <= 2200 or num_pieces < 750 and 40960 <= torrent_file_size <= 102400) and torrent_file_size <= 102400):
# iteration += 1
# print(f"\nIteration {iteration}:")
# print(f"Current piece_size: {piece_size} bytes")
# print(f"Current num_pieces: {num_pieces}, Current torrent_file_size: {torrent_file_size} bytes")
if num_pieces > 1000 and num_pieces < 2000 and torrent_file_size < 100000:
break
elif num_pieces < 1500 and torrent_file_size >= 102400:
piece_size *= 2
# print(f"Doubled piece_size to {piece_size} bytes (num_pieces < 1500 and torrent_file_size >= 100 KiB)")
if piece_size > our_max_size:
piece_size = our_max_size
# print(f"piece_size exceeded max_size, set to our_max_size: {our_max_size} bytes")
break
elif num_pieces < 1000:
elif num_pieces < 750:
piece_size //= 2
# print(f"Halved piece_size to {piece_size} bytes (num_pieces < 750)")
if piece_size < our_min_size:
piece_size = our_min_size
# print(f"piece_size went below min_size, set to our_min_size: {our_min_size} bytes")
break
elif piece_size > 18000000 and torrent_file_size >= 102400:
piece_size *= 2
if piece_size > our_max_size:
piece_size = our_max_size
break
elif torrent_file_size > 61440: # Break if .torrent size exceeds 60 KiB
elif 40960 < torrent_file_size < 102400:
# print(f"torrent_file_size is between 40 KiB and 100 KiB, exiting loop.")
break
elif num_pieces > 2000:
elif num_pieces > 2200:
piece_size *= 2
# print(f"Doubled piece_size to {piece_size} bytes (num_pieces > 2500)")
if piece_size > our_max_size:
piece_size = our_max_size
# print(f"piece_size exceeded max_size, set to our_max_size: {our_max_size} bytes")
break
elif torrent_file_size < 81920: # Break if .torrent size less than 80 KiB
break
elif torrent_file_size > 2048: # Break if .torrent size exceeds 2 KiB
elif torrent_file_size < 2048:
# print(f"torrent_file_size is less than 2 KiB, exiting loop.")
break
elif torrent_file_size > 102400:
piece_size *= 2
# print(f"Doubled piece_size to {piece_size} bytes (torrent_file_size > 100 KiB)")
if piece_size > our_max_size:
piece_size = our_max_size
# print(f"piece_size exceeded max_size, set to our_max_size: {our_max_size} bytes")
cli_ui.warning('WARNING: .torrent size will exceed 100 KiB!')
break

# Update num_pieces
num_pieces = math.ceil(total_size / piece_size)
torrent_file_size = 20 + (num_pieces * 20) + cls._calculate_pathname_bytes(files)

# Recalculate torrent_file_size based on file_count in each iteration
if file_count > 1000:
torrent_file_size = 20 + (num_pieces * 20) + int(cls._calculate_pathname_bytes(files) * 71 / 100)
elif file_count > 500:
torrent_file_size = 20 + (num_pieces * 20) + int(cls._calculate_pathname_bytes(files) * 4 / 5)
else:
torrent_file_size = 20 + (num_pieces * 20) + cls._calculate_pathname_bytes(files)

# print(f"\nFinal piece_size: {piece_size} bytes after {iteration} iterations.")
print(f"Final num_pieces: {num_pieces}, Final torrent_file_size: {torrent_file_size} bytes")
return piece_size

def _calculate_total_size(self):
return sum(file.size for file in self.files)
total_size = sum(file.size for file in self.files)
return total_size

@classmethod
def _calculate_pathname_bytes(cls, files):
Expand Down
60 changes: 12 additions & 48 deletions src/trackers/ANT.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,8 @@
import asyncio
import requests
import platform
import cli_ui
from str2bool import str2bool
from pymediainfo import MediaInfo
import math
from torf import Torrent
from pathlib import Path
from src.trackers.COMMON import COMMON
from src.console import console
Expand Down Expand Up @@ -66,51 +63,18 @@ async def get_flags(self, meta):
async def upload(self, meta, disctype):
common = COMMON(config=self.config)
torrent_filename = "BASE"
torrent = Torrent.read(f"{meta['base_dir']}/tmp/{meta['uuid']}/BASE.torrent")

# Calculate the total size of all files in the torrent
total_size = sum(file.size for file in torrent.files)

# Calculate the total bytes consumed by all the pathnames in the torrent
def calculate_pathname_bytes(files):
total_pathname_bytes = sum(len(str(file).encode('utf-8')) for file in files)
return total_pathname_bytes

total_pathname_bytes = calculate_pathname_bytes(torrent.files)

# Calculate the number of pieces and the torrent file size based on the current piece size
def calculate_pieces_and_file_size(total_size, pathname_bytes, piece_size):
num_pieces = math.ceil(total_size / piece_size)
# Approximate size: 20 bytes header + 20 bytes per piece + pathname bytes
torrent_file_size = 20 + (num_pieces * 20) + pathname_bytes
return num_pieces, torrent_file_size

# Check if the existing torrent fits within the constraints
num_pieces, torrent_file_size = calculate_pieces_and_file_size(total_size, total_pathname_bytes, torrent.piece_size)

# Convert torrent file size to KiB for display
torrent_file_size_kib = torrent_file_size / 1024

# If the torrent doesn't meet the constraints, ask the user if they want to regenerate it
if not (1000 <= num_pieces <= 2000) or torrent_file_size > 102400:
console.print(f"[yellow]Existing .torrent is outside of ANT preferred constraints with {num_pieces} pieces and is approximately {torrent_file_size_kib:.2f} KiB.")
regenerate = cli_ui.ask_yes_no("Do you wish to regenerate the torrent?", default=True)

if regenerate:
console.print("[yellow]Regenerating torrent to fit within 1000-2000 pieces and 100 KiB .torrent size limit needed for ANT.")
from src.prep import Prep
prep = Prep(screens=meta['screens'], img_host=meta['imghost'], config=self.config)

# Override the max piece size before regenerating the torrent
meta['max_piece_size'] = '64' # 64 MiB, the maximum piece size allowed

# Call create_torrent with the adjusted piece size
prep.create_torrent(meta, Path(meta['path']), "ANT")
torrent_filename = "ANT"
else:
console.print("[green]Using the existing torrent despite not meeting the preferred constraints.")
else:
console.print("[green]Existing torrent meets the constraints.")
torrent_path = f"{meta['base_dir']}/tmp/{meta['uuid']}/BASE.torrent"
torrent_file_size_kib = os.path.getsize(torrent_path) / 1024

# Trigger regeneration automatically if size constraints aren't met
if torrent_file_size_kib > 100: # 100 KiB
console.print("[yellow]Existing .torrent exceeds 100 KiB and will be regenerated to fit constraints.")

from src.prep import Prep
prep = Prep(screens=meta['screens'], img_host=meta['imghost'], config=self.config)
meta['max_piece_size'] = '256' # 256 MiB
prep.create_torrent(meta, Path(meta['path']), "ANT")
torrent_filename = "ANT"

await common.edit_torrent(meta, self.tracker, self.source_flag, torrent_filename=torrent_filename)
flags = await self.get_flags(meta)
Expand Down

0 comments on commit 8906480

Please sign in to comment.