From 2848894115766659edbfd8f6e07519f849dcc025 Mon Sep 17 00:00:00 2001 From: Denis <146707790+dnzbk@users.noreply.github.com> Date: Wed, 8 May 2024 12:25:05 +0300 Subject: [PATCH] Fix: encoding, path to NZBfile on Windows, code formatting, round the max failed limit (#5) - fixed a UnicodeEncodeError exception that occurs when using special characters in names; - fixed several "NoneType" errors; - fixed wrong path to NZB file on Windows; - rounded the failed limit precision value; - added tests and pipeline for tests; - formatted the code using Black. --- .github/workflows/tests.yml | 19 + .gitignore | 1 + README.md | 7 +- main.py | 1885 ++++++++++++++++++++------------ manifest.json | 5 +- test_data/listgroups_resp.json | 83 ++ test_data/nzb_filename.queued | 0 test_data/result_resp.xml | 47 + test_data/status_resp.xml | 47 + tests.py | 197 ++++ 10 files changed, 1564 insertions(+), 727 deletions(-) create mode 100644 .github/workflows/tests.yml create mode 100644 .gitignore create mode 100644 test_data/listgroups_resp.json create mode 100644 test_data/nzb_filename.queued create mode 100644 test_data/result_resp.xml create mode 100644 test_data/status_resp.xml create mode 100644 tests.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..991a1e1 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,19 @@ +name: tests + +on: + push: + branches: + - feature/* + - master + pull_request: + branches: + - main + +jobs: + tests: + uses: nzbgetcom/nzbget-extensions/.github/workflows/python-tests.yml@main + with: + python-versions: "3.3 3.4 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12" + supported-python-versions: "3.8 3.9 3.10 3.11 3.12" + test-script: tests.py + debug: true diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bee8a64 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +__pycache__ diff --git a/README.md b/README.md index 1d414f3..fbc95dc 100644 --- a/README.md +++ b/README.md @@ -3,13 +3,12 @@ If you need support for Python 2.x versions then you can get legacy version v1.1 ## NZBGet Versions -- pre-release v23+ [v3.0](https://github.com/nzbgetcom/Extension-Completion/releases/tag/v3.0) -- stable v22 [v2.0](https://github.com/nzbgetcom/Extension-Completion/releases/tag/v2.0) -- legacy v21 [v2.0](https://github.com/nzbgetcom/Extension-Completion/releases/tag/v2.0) +- stable v23+ [v3.0](https://github.com/nzbgetcom/Extension-Completion/releases/tag/v3.0) +- legacy v22 [v2.0](https://github.com/nzbgetcom/Extension-Completion/releases/tag/v2.0) # Completion -[NZBGet](https://nzbget.com) [script](https://nzbget.com/documentation/post-processing-scripts/) that checks if the data in the NZB file is sufficiently complete at your usenet provider(s), before starting the download. If incomplete it would wait for a certain period and check the completion of the NZB file again. This check is done by requesting the header status, and is in normal cases done within seconds (like 1 - 5 sec. for a 1 GB file). This method is significantly faster than when NZBGet would report a failure, after actual downloading a (part of) the files that end up incomplete. The script is typically useful for issues related to: +[NZBGet](https://nzbget.com) [extension](https://github.com/nzbgetcom/nzbget/blob/main/docs/extensions/EXTENSIONS.md) that checks if the data in the NZB file is sufficiently complete at your usenet provider(s), before starting the download. If incomplete it would wait for a certain period and check the completion of the NZB file again. This check is done by requesting the header status, and is in normal cases done within seconds (like 1 - 5 sec. for a 1 GB file). This method is significantly faster than when NZBGet would report a failure, after actual downloading a (part of) the files that end up incomplete. The script is typically useful for issues related to: - very recent posts, - failed downloads, which after a while are just ok (propagation issues), - incomplete posts, diff --git a/main.py b/main.py index 5b84df0..b67ae0c 100644 --- a/main.py +++ b/main.py @@ -1,8 +1,8 @@ -#!/usr/bin/env python # # Completion.py script for NZBGet # # Copyright (C) 2014-2017 kloaknet. +# Copyright (C) 2024 Denis # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -33,242 +33,275 @@ from xmlrpc.client import ServerProxy from operator import itemgetter -# Check if the script is called from NZBGet 15.0 or later. -# Extraction of server data uses .Retention, introduced in NZBGet 15 -if not 'NZBOP_Server1.Retention' in os.environ: # new variable since NZBGet 15.0 - print(('NZBget version: ' + os.environ['NZBOP_VERSION'])) - print('[ERROR] This script is only supported by NZBGet 15.0+, stopping') - sys.exit(0) -py_ver = sys.version_info -if py_ver < (3,8,0): - print(('Python version: ' + sys.version)) - print(('[ERROR] This script requires python 3.8.0+')) - sys.exit(0) +sys.stdout.reconfigure(encoding="utf-8") + # Defining constants -AGE_LIMIT = int(os.environ.get('NZBPO_AgeLimit', 4)) +AGE_LIMIT = int(os.environ.get("NZBPO_AgeLimit", 4)) AGE_LIMIT_SEC = 3600 * AGE_LIMIT -AGE_SORT_LIMIT = int(os.environ.get('NZBPO_AgeSortLimit', 48)) -if AGE_LIMIT > AGE_SORT_LIMIT: AGE_SORT_LIMIT = AGE_LIMIT +AGE_SORT_LIMIT = int(os.environ.get("NZBPO_AgeSortLimit", 48)) +if AGE_LIMIT > AGE_SORT_LIMIT: + AGE_SORT_LIMIT = AGE_LIMIT AGE_SORT_LIMIT_SEC = 3600 * AGE_SORT_LIMIT -CHECK_DUPES = os.environ.get('NZBPO_CheckDupes', 'No') -if CHECK_DUPES != 'No' and os.environ.get('NZBOP_DUPECHECK') == 'No': - print(('[WARNING] DupeCheck should be enabled in NZBGet, otherwise ' + - 'the CheckDupes option of this script that you have enabled ' + - 'does not work')) -FORCE_FAILURE = os.environ.get('NZBPO_ForceFailure', 'No') == 'Yes' -CATEGORIES = os.environ.get('NZBPO_Categories', '').lower().split(',') -CATEGORIES = [c.strip(' ') for c in CATEGORIES] -SERVERS = os.environ.get('NZBPO_Servers', '').lower().split(',') -SERVERS = [c.strip(' ') for c in SERVERS] -FILL_SERVERS = os.environ.get('NZBPO_FillServers', '').lower().split(',') -FILL_SERVERS = [c.strip(' ') for c in FILL_SERVERS] -MAX_FAILURE = int(os.environ.get('NZBPO_MaxFailure', 0)) -CHECK_METHOD = 'STAT' -VERBOSE = os.environ.get('NZBPO_Verbose', 'No') == 'Yes' -EXTREME = os.environ.get('NZBPO_Extreme', 'No') == 'Yes' -CHECK_LIMIT = int(os.environ.get('NZBPO_CheckLimit', 10)) -MAX_ARTICLES = int(os.environ.get('NZBPO_MaxArticles', 1000)) -MIN_ARTICLES = int(os.environ.get('NZBPO_MinArticles', 50)) -FULL_CHECK_NO_PARS = os.environ.get('NZBPO_FullCheckNoPars', 'Yes') == 'Yes' +CHECK_DUPES = os.environ.get("NZBPO_CheckDupes", "No") +if CHECK_DUPES != "No" and os.environ.get("NZBOP_DUPECHECK") == "No": + print( + "[WARNING] DupeCheck should be enabled in NZBGet, otherwise " + + "the CheckDupes option of this script that you have enabled " + + "does not work" + ) +FORCE_FAILURE = os.environ.get("NZBPO_ForceFailure", "No") == "Yes" +CATEGORIES = os.environ.get("NZBPO_Categories", "").lower().split(",") +CATEGORIES = [c.strip(" ") for c in CATEGORIES] +SERVERS = os.environ.get("NZBPO_Servers", "").lower().split(",") +SERVERS = [c.strip(" ") for c in SERVERS] +FILL_SERVERS = os.environ.get("NZBPO_FillServers", "").lower().split(",") +FILL_SERVERS = [c.strip(" ") for c in FILL_SERVERS] +MAX_FAILURE = int(os.environ.get("NZBPO_MaxFailure", 0)) +CHECK_METHOD = "STAT" +VERBOSE = os.environ.get("NZBPO_Verbose", "No") == "Yes" +EXTREME = os.environ.get("NZBPO_Extreme", "No") == "Yes" +CHECK_LIMIT = int(os.environ.get("NZBPO_CheckLimit", 10)) +MAX_ARTICLES = int(os.environ.get("NZBPO_MaxArticles", 1000)) +MIN_ARTICLES = int(os.environ.get("NZBPO_MinArticles", 50)) +FULL_CHECK_NO_PARS = os.environ.get("NZBPO_FullCheckNoPars", "Yes") == "Yes" NNTP_TIME_OUT = 2 # low, but should be sufficient for connection check SOCKET_CREATE_INTERVAL = 0.000 # optional delay to avoid handshake time outs SOCKET_LOOP_INTERVAL = 0.200 # max delay single loop on data received -HOST = os.environ['NZBOP_CONTROLIP'] # NZBGet host -if HOST == '0.0.0.0': - HOST = '127.0.0.1' # fix to localhost -PORT = os.environ['NZBOP_CONTROLPORT'] # NZBGet port -USERNAME = os.environ['NZBOP_CONTROLUSERNAME'] # NZBGet username -PASSWORD = os.environ['NZBOP_CONTROLPASSWORD'] # NZBGet password +HOST = os.environ["NZBOP_CONTROLIP"] # NZBGet host +if HOST == "0.0.0.0": + HOST = "127.0.0.1" # fix to localhost +PORT = os.environ["NZBOP_CONTROLPORT"] # NZBGet port +USERNAME = os.environ["NZBOP_CONTROLUSERNAME"] # NZBGet username +PASSWORD = os.environ["NZBOP_CONTROLPASSWORD"] # NZBGet password + def unpause_nzb(nzb_id): - ''' - resume the nzb with NZBid in the NZBGet queue via RPC-API - ''' + """ + resume the nzb with NZBid in the NZBGet queue via RPC-API + """ NZBGet = connect_to_nzbget() - NZBGet.editqueue('GroupResume', 0, '', [int(nzb_id)]) # Resume nzb - NZBGet.editqueue('GroupPauseExtraPars', 0, '', [int(nzb_id)]) # Pause pars + NZBGet.editqueue("GroupResume", 0, "", [int(nzb_id)]) # Resume nzb + NZBGet.editqueue("GroupPauseExtraPars", 0, "", [int(nzb_id)]) # Pause pars + def unpause_nzb_dupe(dupe_nzb_id, nzb_id): - ''' - resume the nzb with NZBid in the NZBGet history via RPC-API, move the - other one to history. - ''' + """ + resume the nzb with NZBid in the NZBGet history via RPC-API, move the + other one to history. + """ NZBGet = connect_to_nzbget() # Return item from history (before deleting, to avoid NZBGet automatically # returning a DUPE instead of the script). - NZBGet.editqueue('HistoryRedownload', 0, '', [int(dupe_nzb_id)]) # Return - NZBGet.editqueue('GroupResume', 0, '', [int(dupe_nzb_id)]) # Resume nzb + NZBGet.editqueue("HistoryRedownload", 0, "", [int(dupe_nzb_id)]) # Return + NZBGet.editqueue("GroupResume", 0, "", [int(dupe_nzb_id)]) # Resume nzb # Pause pars - NZBGet.editqueue('GroupPauseExtraPars', 0, '', [int(dupe_nzb_id)]) + NZBGet.editqueue("GroupPauseExtraPars", 0, "", [int(dupe_nzb_id)]) # Remove item in queue, send back to history as DUPE - NZBGet.editqueue('GroupDupeDelete', 0, '', [int(nzb_id)]) + NZBGet.editqueue("GroupDupeDelete", 0, "", [int(nzb_id)]) + def mark_bad(nzb_id): - ''' - mark the nzb with NZBid BAD in the NZBGet queue via RPC-API - ''' + """ + mark the nzb with NZBid BAD in the NZBGet queue via RPC-API + """ NZBGet = connect_to_nzbget() - NZBGet.editqueue('GroupDelete', 0, '', [int(nzb_id)]) # need to delete - NZBGet.editqueue('HistoryMarkBad', 0, '', [int(nzb_id)]) # mark bad + NZBGet.editqueue("GroupDelete", 0, "", [int(nzb_id)]) # need to delete + NZBGet.editqueue("HistoryMarkBad", 0, "", [int(nzb_id)]) # mark bad + def mark_bad_dupe(dupe_nzb_id): - ''' - mark the nzb with NZBid BAD in the NZBGet history via RPC-API, item is - already in history, so no moving. - ''' + """ + mark the nzb with NZBid BAD in the NZBGet history via RPC-API, item is + already in history, so no moving. + """ NZBGet = connect_to_nzbget() - NZBGet.editqueue('HistoryMarkBad', 0, '', [int(dupe_nzb_id)]) # mark bad + NZBGet.editqueue("HistoryMarkBad", 0, "", [int(dupe_nzb_id)]) # mark bad + def force_failure(nzb_id): - ''' - mark BAD doesn't do the trick for FailureLink, Sonarr, SickBeard - Forces failure, removes all files but one .par2 - sending deleted nzb back to queue will restore the deleted files. - When available, the smallest par and smallest other file will be kept. - ''' + """ + mark BAD doesn't do the trick for FailureLink, Sonarr, SickBeard + Forces failure, removes all files but one .par2 + sending deleted nzb back to queue will restore the deleted files. + When available, the smallest par and smallest other file will be kept. + """ if VERBOSE: - print(('[V] force_failure(nzb_id=' + str(nzb_id) + ')')) + print("[V] force_failure(nzb_id=" + str(nzb_id) + ")") NZBGet = connect_to_nzbget() data = NZBGet.listfiles(0, 0, [int(nzb_id)]) id_list = [] file_size_low = 100000000000 # 100 Gb, file size will never occur. par_size_low = 100000000000 for f in data: # find smallest par2 and file - file_name = f.get('Filename').lower() - if '.par2' not in file_name: - temp = f.get('FileSizeLo') + file_name = f.get("Filename").lower() + if ".par2" not in file_name: + temp = f.get("FileSizeLo") if temp < file_size_low: file_size_low = temp else: - temp = f.get('FileSizeLo') + temp = f.get("FileSizeLo") if temp < par_size_low: par_size_low = temp for f in data: # match files on size for deletion - temp = f.get('FileSizeLo') - if temp != file_size_low and temp !=par_size_low: - id = int(f.get('ID')) + temp = f.get("FileSizeLo") + if temp != file_size_low and temp != par_size_low: + id = int(f.get("ID")) id_list.append(id) else: if VERBOSE: - print(('[V] Leaving file: ' + str(f.get('Filename')) + - ' in the NZB file')) - print('[WARNING] Forcing failure of NZB:') + print( + "[V] Leaving file: " + str(f.get("Filename")) + " in the NZB file" + ) + print("[WARNING] Forcing failure of NZB:") sys.stdout.flush() # force message before lot of NZBGet messages time.sleep(0.1) # create time to flush # delete all listed files - NZBGet.editqueue('FileDelete', 0, '', id_list) + NZBGet.editqueue("FileDelete", 0, "", id_list) # Resume nzb in queue to download single remaining file in NZB - NZBGet.editqueue('GroupResume', 0, '', nzb_id) + NZBGet.editqueue("GroupResume", 0, "", nzb_id) + def force_failure_dupe(dupe_nzb_id): - ''' - mark BAD doesn't do the trick for FailureLink, Sonarr, SickBeard - Forces failure, removes all files but one .par2 - sending deleted nzb back to queue will restore the deleted files. - although no dupes are expected from Sonarr and the likes, maybe a RSS - feed for movies or other stuff is used that might produce dupes. - ''' + """ + mark BAD doesn't do the trick for FailureLink, Sonarr, SickBeard + Forces failure, removes all files but one .par2 + sending deleted nzb back to queue will restore the deleted files. + although no dupes are expected from Sonarr and the likes, maybe a RSS + feed for movies or other stuff is used that might produce dupes. + """ if VERBOSE: - print(('[V] force_failure_dupe(nzb_id=' + str(dupe_nzb_id) + ')')) + print("[V] force_failure_dupe(nzb_id=" + str(dupe_nzb_id) + ")") NZBGet = connect_to_nzbget() if VERBOSE: - print('[V] Pausing failed DUPE NZB before returning to queue.') + print("[V] Pausing failed DUPE NZB before returning to queue.") # pause all files before returning to queue - NZBGet.editqueue('GroupPause', 0, '', dupe_nzb_id) + NZBGet.editqueue("GroupPause", 0, "", dupe_nzb_id) if VERBOSE: - print('[V] Returning failed DUPE NZB to queue.') + print("[V] Returning failed DUPE NZB to queue.") # return item back to queue to be able to force a failure - NZBGet.editqueue('HistoryReturn', 0, '', dupe_nzb_id) + NZBGet.editqueue("HistoryReturn", 0, "", dupe_nzb_id) force_failure(dupe_nzb_id) + def connect_to_nzbget(): - ''' - Establish connection to NZBGet via RPC-API using HTTP. - ''' + """ + Establish connection to NZBGet via RPC-API using HTTP. + """ # Build an URL for XML-RPC requests: - xmlRpcUrl = 'http://%s:%s@%s:%s/xmlrpc' % (USERNAME, PASSWORD, HOST, PORT) + xmlRpcUrl = "http://%s:%s@%s:%s/xmlrpc" % (USERNAME, PASSWORD, HOST, PORT) # Create remote server object NZBGet = ServerProxy(xmlRpcUrl) return NZBGet + def call_nzbget_direct(url_command): - ''' - Connect to NZBGet and call an RPC-API-method without using of python's - XML-RPC. XML-RPC is easy to use but it is slow for large amounts of - data. - ''' + """ + Connect to NZBGet and call an RPC-API-method without using of python's + XML-RPC. XML-RPC is easy to use but it is slow for large amounts of + data. + """ # Building http-URL to call the method - http_url = 'http://%s:%s/jsonrpc/%s' % (HOST, PORT, url_command) + http_url = "http://%s:%s/jsonrpc/%s" % (HOST, PORT, url_command) request = urllib.request.Request(http_url) - base_64_string = base64.b64encode(('%s:%s' % (USERNAME, - PASSWORD)).encode()).decode().strip() + base_64_string = ( + base64.b64encode(("%s:%s" % (USERNAME, PASSWORD)).encode("utf-8")) + .decode("utf-8") + .strip() + ) request.add_header("Authorization", "Basic %s" % base_64_string) response = urllib.request.urlopen(request) # get some data from NZBGet # data is a JSON raw-string, contains ALL properties each NZB in queue - data = response.read() + data = response.read().decode("utf-8") return data + def get_nzb_filename(parameters): - ''' - get the real nzb_filename from the added parameter CnpNZBFileName - ''' + """ + get the real nzb_filename from the added parameter CnpNZBFileName or from env + """ + file_name = os.environ.get("NZBNA_QUEUEDFILE") + if file_name: + return file_name + for p in parameters: - if p['Name'] == 'CnpNZBFileName': + if p["Name"] == "CnpNZBFileName": break - return p['Value'] + return p["Value"] + + +def get_max_failed_limit(critical_health) -> float: + return round(100 - critical_health / 10.0, 1) + def get_nzb_status(nzb): - ''' - check if amount of failed articles is not too much. If too much keep - paused, if too old and too much failure mark bad / force failure, - otherwise resume. When an -1 or -2 is returned from check_nzb(), the - nzb is unpaused, hoping NZBGet can still process the file, while the - script can't. - ''' + """ + check if amount of failed articles is not too much. If too much keep + paused, if too old and too much failure mark bad / force failure, + otherwise resume. When an -1 or -2 is returned from check_nzb(), the + nzb is unpaused, hoping NZBGet can still process the file, while the + script can't. + """ if VERBOSE: - print(('[V] get_nzb_status(nzb=' + str(nzb) + ')')) - print(('Checking: "' + nzb[1] + '"')) + print("[V] get_nzb_status(nzb=" + str(nzb) + ")") + print('Checking: "' + nzb[1] + '"') # collect rar msg ids that need to be checked - rar_msg_ids = get_nzb_data(os.environ['NZBOP_NZBDIR'] + os.sep + nzb[1]) + rar_msg_ids = get_nzb_data(nzb[1]) if rar_msg_ids == -1: # no such NZB file succes = True # file send back to queue - print(('[WARNING] The NZB file ' + str(nzb[1]) + ' does not seem to ' + - 'exist, resuming NZB.')) + print( + "[WARNING] The NZB file " + + str(nzb[1]) + + " does not seem to " + + "exist, resuming NZB." + ) unpause_nzb(nzb[0]) # unpause based on NZBGet ID elif rar_msg_ids == -2: # empty NZB or no group succes = True # file send back to queue - print(('[WARNING] The NZB file ' + str(nzb[1]) + ' appears to be '+ - 'invalid, resuming NZB.')) + print( + "[WARNING] The NZB file " + + str(nzb[1]) + + " appears to be " + + "invalid, resuming NZB." + ) unpause_nzb(nzb[0]) # unpause based on NZBGet ID elif rar_msg_ids == -3: # NZB without RAR files. succes = True # file send back to queue - print(('[WARNING] The NZB file ' + str(nzb[1]) + ' does not contain ' + - 'any .rar files and has been moved back to the queue.')) + print( + "[WARNING] The NZB file " + + str(nzb[1]) + + " does not contain " + + "any .rar files and has been moved back to the queue." + ) unpause_nzb(nzb[0]) # unpause based on NZBGet ID else: - failed_limit = 100 - nzb[3] / 10.0 - print(('Maximum failed articles limit for NZB: ' - + str(failed_limit) + '%')) + failed_limit = get_max_failed_limit(nzb[3]) + print("Maximum failed articles limit for NZB: " + str(failed_limit) + "%") if MAX_FAILURE > 0: - print(('Maximum failed articles limit for highest level news server: ' - + str(MAX_FAILURE) + '%')) + print( + "Maximum failed articles limit for highest level news server: " + + str(MAX_FAILURE) + + "%" + ) failed_ratio = check_failure_status(rar_msg_ids, failed_limit, nzb[2]) if VERBOSE: - print(('[V] Total failed ratio: ' + str(round(failed_ratio,1)) + '%')) - if ((failed_ratio < failed_limit and (failed_ratio < MAX_FAILURE or - MAX_FAILURE == 0)) or failed_ratio == 0): + print("[V] Total failed ratio: " + str(round(failed_ratio, 1)) + "%") + if ( + failed_ratio < failed_limit + and (failed_ratio < MAX_FAILURE or MAX_FAILURE == 0) + ) or failed_ratio == 0: succes = True - print(('Resuming: "' + nzb[1] + '"')) + print('Resuming: "' + nzb[1] + '"') sys.stdout.flush() unpause_nzb(nzb[0]) # unpause based on NZBGet ID - elif ((failed_ratio >= failed_limit or - (failed_ratio >= MAX_FAILURE and MAX_FAILURE > 0)) and - nzb[2] < (int(time.time()) - int(AGE_LIMIT_SEC))): + elif ( + failed_ratio >= failed_limit + or (failed_ratio >= MAX_FAILURE and MAX_FAILURE > 0) + ) and nzb[2] < (int(time.time()) - int(AGE_LIMIT_SEC)): succes = False if VERBOSE: if not FORCE_FAILURE: - print(('[V] Marked as BAD: "' + nzb[1] + '"')) + print('[V] Marked as BAD: "' + nzb[1] + '"') sys.stdout.flush() # otherwise NZBGet sends message first if FORCE_FAILURE: force_failure(nzb[0]) @@ -277,74 +310,101 @@ def get_nzb_status(nzb): else: succes = False # dupekey should not be '', that would mean it is not added by RSS - if CHECK_DUPES != 'no' and nzb[4] != '': + if CHECK_DUPES != "no" and nzb[4] != "": if get_dupe_nzb_status(nzb): - print(('"' + nzb[1] + '" moved to history as DUPE, ' + - 'complete DUPE returned to queue.')) + print( + '"' + + nzb[1] + + '" moved to history as DUPE, ' + + "complete DUPE returned to queue." + ) else: - print(('[WARNING] "' + nzb[1] + - '", remains paused for next check, ' + - 'no suitable/complete DUPEs found in history')) - elif CHECK_DUPES != 'no' and nzb[4] == '' and VERBOSE: - print(('[V] ' + nzb[1] + ' is not added via RSS, therefore ' + - 'the dupekey is empty and checking for DUPEs in the history ' + - 'is skipped.')) - return succes + print( + '[WARNING] "' + + nzb[1] + + '", remains paused for next check, ' + + "no suitable/complete DUPEs found in history" + ) + elif CHECK_DUPES != "no" and nzb[4] == "" and VERBOSE: + print( + "[V] " + + nzb[1] + + " is not added via RSS, therefore " + + "the dupekey is empty and checking for DUPEs in the history " + + "is skipped." + ) + return succes + def get_dupe_nzb_status(nzb): - ''' - check dupes in the history on their possible completion when the item - in the queue is not yet complete. When complete DUPE item, move it - back into the queue, and move the otherone to history. - ''' + """ + check dupes in the history on their possible completion when the item + in the queue is not yet complete. When complete DUPE item, move it + back into the queue, and move the otherone to history. + """ if VERBOSE: - print(('[V] get_dupe_nzb_status(nzb=' + str(nzb) + ')')) + print("[V] get_dupe_nzb_status(nzb=" + str(nzb) + ")") # get the data from the active history - data = call_nzbget_direct('history') + data = call_nzbget_direct("history") jobs = json.loads(data) duplicate = False num_duplicates = 0 list_duplicates = [] - for job in jobs['result']: - if (job['Status'] == 'DELETED/DUPE' and job['DupeKey'] == nzb[4] and - 'CnpNZBFileName' in str(job)): - if CHECK_DUPES == 'yes': + for job in jobs["result"]: + if ( + job["Status"] == "DELETED/DUPE" + and job["DupeKey"] == nzb[4] + and "CnpNZBFileName" in str(job) + ): + if CHECK_DUPES == "yes": duplicate = True num_duplicates += 1 list_duplicates.append(job) - elif CHECK_DUPES == 'SameScore' and job['DupeScore'] >= nzb[5]: + elif CHECK_DUPES == "SameScore" and job["DupeScore"] >= nzb[5]: duplicate = True num_duplicates += 1 list_duplicates.append(job) else: if VERBOSE: - print(('[V] DUPE NZB found with lower dupe score, ' + - 'ignored due to SameScore setting.')) + print( + "[V] DUPE NZB found with lower dupe score, " + + "ignored due to SameScore setting." + ) if duplicate: - # sort on nzb age, then on dupescore. Higher score items will be on + # sort on nzb age, then on dupescore. Higher score items will be on # top. Oldest file has lowest maxposttime. if VERBOSE: - print(('[V] ' + str(num_duplicates) + ' duplicate of ' + - nzb[1] + ' found in history')) - t = sorted(list_duplicates, key=itemgetter('MaxPostTime')) - sorted_duplicates = (sorted(t, key=itemgetter('DupeScore'), - reverse=True)) + print( + "[V] " + + str(num_duplicates) + + " duplicate of " + + nzb[1] + + " found in history" + ) + t = sorted(list_duplicates, key=itemgetter("MaxPostTime")) + sorted_duplicates = sorted(t, key=itemgetter("DupeScore"), reverse=True) i = 0 # loop through all DUPE items (with optional matching DUPEscore) for job in sorted_duplicates: i += 1 - nzb_id = job['NZBID'] - nzb_filename = get_nzb_filename(job['Parameters']) - nzb_age = job['MaxPostTime'] # nzb age - nzb_critical_health = job['CriticalHealth'] - print(('Checking DUPE: "' + nzb_filename + '" [' + str(i) + '/' + - str(num_duplicates) + ']')) - rar_msg_ids = get_nzb_data(os.environ['NZBOP_NZBDIR'] + os.sep + - nzb_filename) + nzb_id = job["NZBID"] + nzb_filename = get_nzb_filename(job["Parameters"]) + nzb_age = job["MaxPostTime"] # nzb age + nzb_critical_health = job["CriticalHealth"] + print( + 'Checking DUPE: "' + + nzb_filename + + '" [' + + str(i) + + "/" + + str(num_duplicates) + + "]" + ) + rar_msg_ids = get_nzb_data(nzb[1]) if rar_msg_ids == -1: # no such NZB file success = False # file marked BAD if VERBOSE: - print('[WARNING] [V] No such DUPE NZB file, marking BAD.') + print("[WARNING] [V] No such DUPE NZB file, marking BAD.") if FORCE_FAILURE: force_failure_dupe(nzb_id) # else: @@ -352,35 +412,39 @@ def get_dupe_nzb_status(nzb): elif rar_msg_ids == -2: # empty NZB or no group success = False # file marked BAD if VERBOSE: - print('[WARNING] [V] DUPE NZB appears invalid, marking BAD.') + print("[WARNING] [V] DUPE NZB appears invalid, marking BAD.") if FORCE_FAILURE: force_failure_dupe(nzb_id) # else: mark_bad_dupe(nzb_id) else: - failed_limit = 100 - nzb_critical_health / 10.0 - print(('[V] Maximum failed articles limit: ' + - str(failed_limit) + '%')) + failed_limit = get_max_failed_limit(nzb[3]) + print("[V] Maximum failed articles limit: " + str(failed_limit) + "%") failed_ratio = check_failure_status(rar_msg_ids, failed_limit, nzb[2]) if VERBOSE: - print(('[V] Total failed ratio: ' + - str(round(failed_ratio,1)) + '%')) - if ((failed_ratio < failed_limit and (failed_ratio < MAX_FAILURE or - MAX_FAILURE == 0)) or failed_ratio == 0): + print( + "[V] Total failed ratio: " + str(round(failed_ratio, 1)) + "%" + ) + + if ( + failed_ratio < failed_limit + and (failed_ratio < MAX_FAILURE or MAX_FAILURE == 0) + ) or failed_ratio == 0: success = True - print(('Resuming DUPE: "' + nzb_filename + '"')) + print('Resuming DUPE: "' + nzb_filename + '"') sys.stdout.flush() unpause_nzb_dupe(nzb_id, nzb[0]) # resume on NZBGet ID break - elif ((failed_ratio >= failed_limit or - (failed_ratio >= MAX_FAILURE and MAX_FAILURE > 0)) and - nzb_age < (int(time.time()) - int(AGE_LIMIT_SEC))): + elif ( + failed_ratio >= failed_limit + or (failed_ratio >= MAX_FAILURE and MAX_FAILURE > 0) + ) and nzb_age < (int(time.time()) - int(AGE_LIMIT_SEC)): success = False if VERBOSE: if not FORCE_FAILURE: - print(('[V] Marked as BAD: "' + nzb[1] + '"')) + print('[V] Marked as BAD: "' + nzb[1] + '"') else: - print(('[V] Forcing failure of: "' + nzb[1] + '"')) + print('[V] Forcing failure of: "' + nzb[1] + '"') sys.stdout.flush() if FORCE_FAILURE: force_failure_dupe(nzb_id) @@ -390,186 +454,298 @@ def get_dupe_nzb_status(nzb): success = False else: if VERBOSE: - print(('[V] No DUPE of ' + nzb[1] + ' found in history.')) + print("[V] No DUPE of " + nzb[1] + " found in history.") success = False return success + def is_number(s): - ''' - Checks if the string can be converted to a number - ''' -# if VERBOSE: -# print('[V] is_number(s= ' + str(s) + ' )') + """ + Checks if the string can be converted to a number + """ try: float(s) return True except ValueError: return False -def check_send_server_reply(sock, t, group, id, i, host, username, password): - ''' - Check NNTP server messages, send data for next recv. - After connecting, there will be a 200 message, after each message, a - reply (t) will be send to get a next message. - - More info on NNTP server responses: - The first digit of the response broadly indicates the success, - failure, or progress of the previous command: - 1xx - Informative message - 2xx - Command completed OK - 3xx - Command OK so far; send the rest of it - 4xx - Command was syntactically correct but failed for some reason - 5xx - Command unknown, unsupported, unavailable, or syntax error - The next digit in the code indicates the function response category: - x0x - Connection, setup, and miscellaneous messages - x1x - Newsgroup selection - x2x - Article selection - x3x - Distribution functions - x4x - Posting - x8x - Reserved for authentication and privacy extensions - x9x - Reserved for private use (non-standard extensions - ''' + +def check_send_server_reply( + sock, reply: str, group: str, id, i, host, username, password +): + """ + Check NNTP server messages, send data for next recv. + After connecting, there will be a 200 message, after each message, a + reply (t) will be send to get a next message. + + More info on NNTP server responses: + The first digit of the response broadly indicates the success, + failure, or progress of the previous command: + 1xx - Informative message + 2xx - Command completed OK + 3xx - Command OK so far; send the rest of it + 4xx - Command was syntactically correct but failed for some reason + 5xx - Command unknown, unsupported, unavailable, or syntax error + The next digit in the code indicates the function response category: + x0x - Connection, setup, and miscellaneous messages + x1x - Newsgroup selection + x2x - Article selection + x3x - Distribution functions + x4x - Posting + x8x - Reserved for authentication and privacy extensions + x9x - Reserved for private use (non-standard extensions + """ if EXTREME: - print('[E] check_send_server_reply(sock= ' + str(sock) + ', t= ' + str(t) + - ' ,group= ' + str(group) + ' , id= ' + str(id) + ' , i= ' + - str(i) + ' )') + print( + "[E] check_send_server_reply(sock= " + + str(sock) + + ", t= " + + reply + + " ,group= " + + str(group) + + " , id= " + + str(id) + + " , i= " + + str(i) + + " )" + ) try: id_used = False # is id used via HEAD / STAT request to NNTP server msg_id_used = None error = False - server_reply = str(t[:3]) # only first 3 chars are relevant + server_reply = str(reply[:3]) # only first 3 chars are relevant # no correct NNTP server code received, most likely still propagating? if not is_number(server_reply): if VERBOSE: - print(('[WARNING] [V] Socket: ' + str(i) + ' ' + str(host) + - ', NNTP reply incorrect:' + str(t.split()))) - server_reply = 'NNTP reply incorrect.' + print( + "[WARNING] [V] Socket: " + + str(i) + + " " + + str(host) + + ", NNTP reply incorrect:" + + str(reply.split()) + ) + server_reply = "NNTP reply incorrect." error = True # pass these vars so that next article will be sent id_used = True # pass these vars so that next article will be sent return (error, id_used, server_reply, msg_id_used) # checking NNTP server server_replies - if server_reply in ('411', '420', '423', '430'): + if server_reply in ("411", "420", "423", "430"): # 411 no such group # 420 no current article has been selected # 423 no such article number in this group # 430 no such article found if VERBOSE: - print(('[WARNING] [V] Socket: ' + str(i) + ' ' + str(host) + - ', NNTP reply: ' + str(t.split()))) + print( + "[WARNING] [V] Socket: " + + str(i) + + " " + + str(host) + + ", NNTP reply: " + + str(reply.split()) + ) error = True # article is not there - elif server_reply in ('412'): # 412 no newsgroup has been selected - text = 'GROUP ' + group + '\r\n' + elif server_reply in ("412"): # 412 no newsgroup has been selected + text = "GROUP " + group + "\r\n" if EXTREME: - print(('[E] Socket: ' + str(i) + ' ' + str(host) + - ', NNTP reply: ' + str(t.split()))) - print(('[E] Socket: ' + str(i) + ' ' + str(host) + ', Send: ' + - str(text))) - sock.send(text.encode()) - elif server_reply in ('221'): + print( + "[E] Socket: " + + str(i) + + " " + + str(host) + + ", NNTP reply: " + + str(reply.split()) + ) + print("[E] Socket: " + str(i) + " " + str(host) + ", Send: " + text) + sock.send(text.encode("utf-8")) + elif server_reply in ("221"): # 221 article retrieved - head follows (reply on HEAD) - msg_id_used = t.split()[2][1:-1] # get msg id to identify ok article + msg_id_used = reply.split()[2][1:-1] # get msg id to identify ok article if EXTREME: - print(('[E] Socket: ' + str(i) + ' ' + str(host) + - ', NNTP reply: ' + str(t.split()))) - elif server_reply in ('223'): + print( + "[E] Socket: " + + str(i) + + " " + + str(host) + + ", NNTP reply: " + + str(reply.split()) + ) + elif server_reply in ("223"): # 223 article retrieved - request text separately (reply on STAT) - msg_id_used = t.split()[2][1:-1] # get msg id to identify ok article + msg_id_used = reply.split()[2][1:-1] # get msg id to identify ok article if EXTREME: - print(('[E] Socket: ' + str(i) + ' ' + str(host) + - ', NNTP reply: ' + str(t.split()))) - elif server_reply in ('200', '201'): + print( + "[E] Socket: " + + str(i) + + " " + + str(host) + + ", NNTP reply: " + + str(reply.split()) + ) + elif server_reply in ("200", "201"): # 200 service available, posting permitted # 201 service available, posting prohibited if EXTREME: - print(('[INFO] [E] Socket: ' + str(i) + ' ' + str(host) + - ', NNTP reply: ' + str(t.split()))) - text = CHECK_METHOD + ' <' + id + '>\r\n' # STAT is faster than HEAD + print( + "[INFO] [E] Socket: " + + str(i) + + " " + + str(host) + + ", NNTP reply: " + + str(reply.split()) + ) + text = CHECK_METHOD + " <" + id + ">\r\n" # STAT is faster than HEAD if EXTREME: - print(('[E] Socket: ' + str(i) + ' ' + str(host) + ', Send: ' + - str(text))) - sock.send(text.encode()) - elif server_reply in ('381'): # 381 Password required - text = 'AUTHINFO PASS %s\r\n' % (password) + print( + "[E] Socket: " + str(i) + " " + str(host) + ", Send: " + str(text) + ) + sock.send(text.encode("utf-8")) + elif server_reply in ("381"): # 381 Password required + text = "AUTHINFO PASS %s\r\n" % (password) if EXTREME: - print(('[E] Socket: ' + str(i) + ' ' + str(host) + - ', NNTP reply: ' + str(t.split()))) - print(('[E] Socket: ' + str(i) + ' ' + str(host) + ', Send: ' + - str(text))) - sock.send(text.encode()) - elif server_reply in ('281'): # 281 Authentication accepted + print( + "[E] Socket: " + + str(i) + + " " + + str(host) + + ", NNTP reply: " + + str(reply.split()) + ) + print( + "[E] Socket: " + str(i) + " " + str(host) + ", Send: " + str(text) + ) + sock.send(text.encode("utf-8")) + elif server_reply in ("281"): # 281 Authentication accepted if EXTREME: - print(('[E] Socket: ' + str(i) + ' ' + str(host) + - ', NNTP reply: ' + str(t.split()))) - elif server_reply in ('211'): # 211 group selected (group) + print( + "[E] Socket: " + + str(i) + + " " + + str(host) + + ", NNTP reply: " + + str(reply.split()) + ) + elif server_reply in ("211"): # 211 group selected (group) if EXTREME: - print(('[INFO] [E] Socket: ' + str(i) + ' ' + str(host) + - ', NNTP reply: ' + str(t.split()))) - elif server_reply in ('480'): # 480 AUTHINFO required - text = 'AUTHINFO USER %s\r\n' % (username) + print( + "[INFO] [E] Socket: " + + str(i) + + " " + + str(host) + + ", NNTP reply: " + + str(reply.split()) + ) + elif server_reply in ("480"): # 480 AUTHINFO required + text = "AUTHINFO USER %s\r\n" % (username) if EXTREME: - print(('[E] Socket: ' + str(i) + ' ' + str(host) + - ', NNTP reply: ' + str(t.split()))) - print(('[E] Socket: ' + str(i) + ' ' + str(host) + ', Send: ' + - str(text))) - sock.send(text.encode()) - elif str(server_reply[:2]) in ('48', '50'): + print( + "[E] Socket: " + + str(i) + + " " + + str(host) + + ", NNTP reply: " + + str(reply.split()) + ) + print( + "[E] Socket: " + str(i) + " " + str(host) + ", Send: " + str(text) + ) + sock.send(text.encode("utf-8")) + elif str(server_reply[:2]) in ("48", "50"): # 48X or 50X incorrect news server account settings - print(('[ERROR] Socket: ' + str(i) + ' ' + str(host) + - ', Incorrect news server account settings: ' + str(t))) - elif server_reply in ('205'): # NNTP Service exits normally + print( + "[ERROR] Socket: " + + str(i) + + " " + + str(host) + + ", Incorrect news server account settings: " + + reply + ) + elif server_reply in ("205"): # NNTP Service exits normally sock.close() if EXTREME: - print(('[E] Socket: ' + str(i) + ' ' + str(host) + - ', NNTP reply: ' + str(t.split()))) + print( + "[E] Socket: " + + str(i) + + " " + + str(host) + + ", NNTP reply: " + + str(reply.split()) + ) if VERBOSE: - print(('[V] Socket ' + str(i) + ' closed.')) - elif server_reply in ('999'): # script code for very slow news server + print("[V] Socket " + str(i) + " closed.") + elif server_reply in ("999"): # script code for very slow news server if VERBOSE: - print(('[WARNING] [V] Socket: ' + str(i) + ' ' + str(host) + - ', NNTP reply: ' + str(t.split()))) + print( + "[WARNING] [V] Socket: " + + str(i) + + " " + + str(host) + + ", NNTP reply: " + + str(reply.split()) + ) error = True # article is assumed to be not there id_used = True else: if VERBOSE: - print(('[WARNING] [V] Socket: ' + str(i) + ' ' + str(host) + - ', Not covered NNTP server reply code: ' + str(t.split()))) + print( + "[WARNING] [V] Socket: " + + str(i) + + " " + + str(host) + + ", Not covered NNTP server reply code: " + + str(reply.split()) + ) if VERBOSE or EXTREME: sys.stdout.flush() - if end_loop == False and server_reply in ('211', '221', '223', '281', - '411', '420', '423', '430'): + if end_loop == False and server_reply in ( + "211", + "221", + "223", + "281", + "411", + "420", + "423", + "430", + ): # Send next message - text = CHECK_METHOD + ' <' + id + '>\r\n' # STAT is faster than HEAD + text = CHECK_METHOD + " <" + id + ">\r\n" # STAT is faster than HEAD id_used = True if EXTREME: - print(('[E] Socket: ' + str(i) + ' ' + str(host) + ', Send: ' + - str(text))) - sock.send(text.encode()) - elif end_loop and server_reply not in ('205'): - text = 'QUIT\r\n' - sock.send(text.encode()) + print( + "[E] Socket: " + str(i) + " " + str(host) + ", Send: " + str(text) + ) + sock.send(text.encode("utf-8")) + elif end_loop and server_reply not in ("205"): + text = "QUIT\r\n" + sock.send(text.encode("utf-8")) if EXTREME: - print(('[E] Socket: ' + str(i) + ' ' + str(host) + - ', Send: ' + str(text))) + print( + "[E] Socket: " + str(i) + " " + str(host) + ", Send: " + str(text) + ) if VERBOSE or EXTREME: sys.stdout.flush() return (error, id_used, server_reply, msg_id_used) except: - print(('Exception LINE: ' + - str(traceback.print_exc()) + ': ' + - str(sys.exc_info()[1]))) + print( + "Exception LINE: " + + str(traceback.print_exc()) + + ": " + + str(sys.exc_info()[1]) + ) return (False, False, server_reply, -1) + def fix_nzb(nzb_lines): - ''' - some nzbs may contain all data on 1 single line, to handle this - correctly in check_nzb(), the single line is splitted on the >< mark - ''' + """ + some nzbs may contain all data on 1 single line, to handle this + correctly in check_nzb(), the single line is splitted on the >< mark + """ if VERBOSE: - print(('[V] fix_nzb(nzb_lines=' + str(nzb_lines) + ')')) - print('[V] Splitting NZB data into separate lines.') + print("[V] fix_nzb(nzb_lines=" + str(nzb_lines) + ")") + print("[V] Splitting NZB data into separate lines.") sys.stdout.flush() nzb_lines = str(nzb_lines) - positions = [n for n in range(len(nzb_lines)) if nzb_lines.find('><', n) == n] + positions = [n for n in range(len(nzb_lines)) if nzb_lines.find("><", n) == n] first = 0 last = 0 corrected_lines = [] @@ -578,28 +754,29 @@ def fix_nzb(nzb_lines): corrected_lines.append(nzb_lines[first:last]) first = last if VERBOSE: - print('[V] Data in NZB splitted into separate lines.') + print("[V] Data in NZB splitted into separate lines.") sys.stdout.flush() return corrected_lines + def get_nzb_data(fname): - ''' - extract the nzb info from the NZB file, and return data set of articles - to be checked - ''' + """ + extract the nzb info from the NZB file, and return data set of articles + to be checked + """ if VERBOSE: - print(('[V] get_nzb_data(fname=' + str(fname) + ')')) + print("[V] get_nzb_data(fname=" + str(fname) + ")") sys.stdout.flush() if os.path.isfile(fname): file_exists = True - fd = open(fname) + fd = open(fname, encoding="utf-8") lines = fd.readlines() fd.close() if len(lines) == 1: # single line NZB lines = fix_nzb(lines) else: file_exists = False - print('[ERROR] No such nzb file.') + print("[ERROR] No such nzb file.") return -1 if file_exists: all_msg_ids = [] # list of message ids for NNTP server @@ -607,37 +784,37 @@ def get_nzb_data(fname): groups = None for line in lines: low_line = line.lower() - if '')[1].split('<')[0] + if "")[1].split("<")[0] ok = -1 # = no check / failed; 1,2,.. ok for server num all_msg_ids.append([subject, par, groups, message_id, ok]) - elif '')[0] - if '.par2' in low_line: + elif "")[0] + if ".par2" in low_line: par = 1 # found a par file, next msg_ids of par2s else: par = 0 # not a par file, next msg ids of files - elif '' in low_line: # set of groups + elif "" in low_line: # set of groups # new list of groups found groups = [] - elif '' in low_line: # group name - group = line.split('>')[1].split('<')[0] + elif "" in low_line: # group name + group = line.split(">")[1].split("<")[0] groups.append(group) if not group: - print('[ERROR] No group found in NZB file.') + print("[ERROR] No group found in NZB file.") if VERBOSE: - print(('[V] group: ' + str(group))) + print("[V] group: " + str(group)) return -2 if len(all_msg_ids) == 0: - print('[ERROR] No message-ids found in NZB file') + print("[ERROR] No message-ids found in NZB file") if VERBOSE: - print(('[V] all_msg_ids: ' + str(all_msg_ids))) + print("[V] all_msg_ids: " + str(all_msg_ids)) return -2 rar_msg_ids = [] par_msg_ids = [] for msg_id in all_msg_ids: # split par2 from other files if msg_id[1] == 0: - rar_msg_ids.append(msg_id) + rar_msg_ids.append(msg_id) else: par_msg_ids.append(msg_id) all_articles = len(all_msg_ids) @@ -645,33 +822,38 @@ def get_nzb_data(fname): par_articles = len(par_msg_ids) temp = len(rar_msg_ids) if temp == 0: - #No .rar articles in NZB. + # No .rar articles in NZB. return -3 # check if more than 1 pars are available or not. if FULL_CHECK_NO_PARS and par_articles < 1: each = 1 # check each article if VERBOSE: - print(('[V] No par files in release, all articles will be ' + - 'checked.')) + print("[V] No par files in release, all articles will be " + "checked.") elif FULL_CHECK_NO_PARS and par_articles == 1: each = 1 # check each article if VERBOSE: - print(('[V] 1 par file in release, all articles will be ' + - 'checked.')) + print("[V] 1 par file in release, all articles will be " + "checked.") else: each = int(100 / CHECK_LIMIT) # check each Xth article only if temp / each > MAX_ARTICLES: each = int(temp / MAX_ARTICLES) if VERBOSE: - print(('[V] Amount of to be checked articles limited to about ' + - str(MAX_ARTICLES) + ' articles.')) + print( + "[V] Amount of to be checked articles limited to about " + + str(MAX_ARTICLES) + + " articles." + ) elif temp / each < MIN_ARTICLES: each = int(temp / MIN_ARTICLES) if each == 0: each = 1 if VERBOSE: - print(('[V] Amount of to be checked articles increased to ' + - 'about ' + str(MIN_ARTICLES) + ' articles.')) + print( + "[V] Amount of to be checked articles increased to " + + "about " + + str(MIN_ARTICLES) + + " articles." + ) t = rar_msg_ids[::each] rar_msg_ids = t # parsing to be used ids, skipping subject parsing @@ -679,45 +861,65 @@ def get_nzb_data(fname): rar_msg_ids[i][3] = html.unescape(rar_msg_id[3]) articles_to_check = len(rar_msg_ids) if VERBOSE: - print(('[V] NZB contains ' + str(all_articles) + ' articles, ' + - str(rar_articles) + ' rar articles, ' + str(par_articles) + - ' par2 articles.')) - print(('[V] ' + str(articles_to_check) + ' rar articles will be checked.')) + print( + "[V] NZB contains " + + str(all_articles) + + " articles, " + + str(rar_articles) + + " rar articles, " + + str(par_articles) + + " par2 articles." + ) + print("[V] " + str(articles_to_check) + " rar articles will be checked.") sys.stdout.flush() return rar_msg_ids + def get_server_settings(nzb_age): - ''' - Get the settings for all the active news-servers in NZBGet, and store - them in a list. Filter out all but 1 server in same group. - ''' + """ + Get the settings for all the active news-servers in NZBGet, and store + them in a list. Filter out all but 1 server in same group. + """ if VERBOSE: - print(('[V] get_server_settings(nzb_age=' + str(nzb_age) + ')')) + print("[V] get_server_settings(nzb_age=" + str(nzb_age) + ")") # get news server settings for each server NZBGet = connect_to_nzbget() nzbget_status = NZBGet.status() - servers_status = nzbget_status['NewsServers'] + servers_status = nzbget_status["NewsServers"] temp = [] servers = [] i = 0 skip = False for server_status in servers_status: # extract all relevant data for each server: - s = str(server_status['ID']) # 9 - active = (os.environ['NZBOP_Server' + s + '.Active'] == 'yes') # 10 - level = os.environ['NZBOP_Server' + s + '.Level'] # 0 - group = os.environ['NZBOP_Server' + s + '.Group'] # 1 - host = os.environ['NZBOP_Server' + s + '.Host'] # 2 - port = os.environ['NZBOP_Server' + s + '.Port'] # 3 - username = os.environ['NZBOP_Server' + s + '.Username'] # 4 - password = os.environ['NZBOP_Server' + s + '.Password'] # 5 - encryption = (os.environ['NZBOP_Server' + s + '.Encryption'] == 'yes') # 6 - connections = os.environ['NZBOP_Server' + s + '.Connections']# 7 - retention = os.environ['NZBOP_Server' + s + '.Retention'] # 8 - if retention == '': + s = str(server_status["ID"]) # 9 + active = os.environ["NZBOP_Server" + s + ".Active"] == "yes" # 10 + level = os.environ["NZBOP_Server" + s + ".Level"] # 0 + group = os.environ["NZBOP_Server" + s + ".Group"] # 1 + host = os.environ["NZBOP_Server" + s + ".Host"] # 2 + port = os.environ["NZBOP_Server" + s + ".Port"] # 3 + username = os.environ["NZBOP_Server" + s + ".Username"] # 4 + password = os.environ["NZBOP_Server" + s + ".Password"] # 5 + encryption = os.environ["NZBOP_Server" + s + ".Encryption"] == "yes" # 6 + connections = os.environ["NZBOP_Server" + s + ".Connections"] # 7 + retention = os.environ["NZBOP_Server" + s + ".Retention"] # 8 + if retention == "": retention = 0 - temp.append([level, group, host, port, username, password, - encryption, connections, retention, s, active]) + temp.append( + [ + level, + group, + host, + port, + username, + password, + encryption, + connections, + retention, + s, + active, + ] + ) nzb_age_days = (int(time.time()) - nzb_age) / 3600.0 / 24.0 for server in temp: skip = False @@ -726,43 +928,75 @@ def get_server_settings(nzb_age): if server[10] == False: skip = True if VERBOSE: - print(('[V] Skipping server: ' + server[2] + ', disabled in ' + - 'NZBGet settings.')) + print( + "[V] Skipping server: " + + server[2] + + ", disabled in " + + "NZBGet settings." + ) # Server ID in SERVERS list - elif (SERVERS[0] != '' and server[9] not in SERVERS and - server[9] not in FILL_SERVERS): + elif ( + SERVERS[0] != "" + and server[9] not in SERVERS + and server[9] not in FILL_SERVERS + ): skip = True if VERBOSE: - print(('[V] Skipping server: ' + server[2] + ', not listed ' + - 'as Server or FillServer in script settings.')) + print( + "[V] Skipping server: " + + server[2] + + ", not listed " + + "as Server or FillServer in script settings." + ) # Server ID in FILL_SERVERS list and nzb older than AGE_LIMIT elif server[9] in FILL_SERVERS and nzb_age_days * 24.0 < AGE_LIMIT: skip = True if VERBOSE: - print(('[V] Skipping Fill server: ' + server[2] + - ', NZB age of ' + str(round(nzb_age_days * 24.0,1)) + - ' hours within AgeLimit of ' + str(AGE_LIMIT) + ' hours')) + print( + "[V] Skipping Fill server: " + + server[2] + + ", NZB age of " + + str(round(nzb_age_days * 24.0, 1)) + + " hours within AgeLimit of " + + str(AGE_LIMIT) + + " hours" + ) # Server retention lower than nzb age if retention < nzb_age_days and retention != 0: skip = True if VERBOSE: - print(('[V] Skipping server: ' + server[2] + ', retention of ' + - str(retention) + ' days is less than NZB age of ' + - str(round(nzb_age_days,1)) + ' days.')) + print( + "[V] Skipping server: " + + server[2] + + ", retention of " + + str(retention) + + " days is less than NZB age of " + + str(round(nzb_age_days, 1)) + + " days." + ) # removing all to be skipped servers if skip == False: servers.append(server) if VERBOSE: - print(('[V] All news servers after filtering on Active, Servers, ' + - 'FillServers + AgeLimit and Retention, ' + - ' BEFORE filtering on NZBGet ServerX.Group: ')) + print( + "[V] All news servers after filtering on Active, Servers, " + + "FillServers + AgeLimit and Retention, " + + " BEFORE filtering on NZBGet ServerX.Group: " + ) for server in servers: - print(('[V] * ' + str(server[2]) + ':' + str(server[3]) + - ', SSL: ' + str(server[6]) + ', connections: ' + - str(server[7]))) + print( + "[V] * " + + str(server[2]) + + ":" + + str(server[3]) + + ", SSL: " + + str(server[6]) + + ", connections: " + + str(server[7]) + ) # sort on groups, followed by lvl, so that all identical group numbers > 0 # can be removed - servers.sort(key=itemgetter(1,0)) + servers.sort(key=itemgetter(1, 0)) a = None c = [] # remove all identical groups from server: @@ -776,33 +1010,47 @@ def get_server_settings(nzb_age): a = b servers = c if VERBOSE: - print(('[V] All active news servers AFTER filtering and sorting ' + - 'on NZBGet ServerX.Group:')) + print( + "[V] All active news servers AFTER filtering and sorting " + + "on NZBGet ServerX.Group:" + ) for server in servers: - print(('[V] * ' + str(server[2]) + ':' + str(server[3]) + - ', SSL: ' + str(server[6]) + ', connections: ' + - str(server[7]))) + print( + "[V] * " + + str(server[2]) + + ":" + + str(server[3]) + + ", SSL: " + + str(server[6]) + + ", connections: " + + str(server[7]) + ) if servers == []: - print(('[WARNING] No news servers after filtering, marking NZB as' + - ' FAILED or BAD. May run in Verbose mode and check your settings!')) + print( + "[WARNING] No news servers after filtering, marking NZB as" + + " FAILED or BAD. May run in Verbose mode and check your settings!" + ) return servers + def create_sockets(server, articles_to_check): - ''' - create the sockets for the server that will be used to send in - check_send_server_reply() and receive in check_failure_status() - server dependent sockets, ssl / non ssl - ''' + """ + create the sockets for the server that will be used to send in + check_send_server_reply() and receive in check_failure_status() + server dependent sockets, ssl / non ssl + """ if EXTREME: - print(('[E] create_sockets(server=' + str(server) + - ',articles_to_check= ' + str(articles_to_check))) + print( + "[E] create_sockets(server=" + + str(server) + + ",articles_to_check= " + + str(articles_to_check) + ) server_no = -1 conn_err = 0 server_no += 1 host = server[2] port = int(server[3]) - username = server[4] - password = server[5] encryption = server[6] # ssl num_conn = int(server[7]) start_sock = 0 @@ -812,39 +1060,45 @@ def create_sockets(server, articles_to_check): num_conn = int(articles_to_check / 2.0 + 0.5) end_sock = num_conn if VERBOSE: - print(('[V] Limiting the number of sockets to ' + str(end_sock) + - ' to keep the number of sockets below the number of articles')) + print( + "[V] Limiting the number of sockets to " + + str(end_sock) + + " to keep the number of sockets below the number of articles" + ) sockets = [None] * num_conn failed_sockets = [-1] * num_conn if VERBOSE: - print(('[V] Creating sockets for server: ' + host)) + print("[V] Creating sockets for server: " + host) sys.stdout.flush() try: # check if we *must* use IPv6 for this host for res in sorted(socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)): - # sorted so IPv4 will be first + # sorted so IPv4 will be first af, socktype, proto, canonname, sa = res if af == socket.AF_INET or af == socket.AF_INET6: break if af == socket.AF_INET6: if VERBOSE: - print(('[V] Using IPv6 for ' + host)) + print("[V] Using IPv6 for " + host) sys.stdout.flush() else: - af = socket.AF_INET # Default to IPv4, even if unexpected response + af = socket.AF_INET # Default to IPv4, even if unexpected response if VERBOSE: - print(('[V] Using IPv4 for ' + host)) + print("[V] Using IPv4 for " + host) sys.stdout.flush() # create connections if encryption: - # build SSL socket, but without certificate requirement - context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - context.verify_mode = ssl.CERT_NONE - context.options |= ssl.OP_NO_SSLv2 - context.options |= ssl.OP_NO_SSLv3 + context = ssl.create_default_context() + for i in range(start_sock, end_sock): s = socket.socket(af, socket.SOCK_STREAM) - sockets[i] = context.wrap_socket(s) + try: + sockets[i] = context.wrap_socket(s, server_hostname=host) + except ssl.SSLError as e: + print( + f"[WARNING] Error creating SSL connection for socket {i}: {e}" + ) + failed_sockets[i] = i else: # Non SSL for i in range(start_sock, end_sock): @@ -855,16 +1109,22 @@ def create_sockets(server, articles_to_check): try: sockets[i].connect((host, port)) if VERBOSE: - print(('[V] Socket ' + str(i) + ' created.')) + print("[V] Socket " + str(i) + " created.") sys.stdout.flush() # remove time out, so socket is closed after completed message sockets[i].settimeout(0) # some minor delay to not hammer / create connection time outs time.sleep(SOCKET_CREATE_INTERVAL) except Exception as e: - print(('[WARNING] Socket: ' + str(i) + ' ' + str(e) + - ', check host, port and number of connections settings' + - ' for server ' + host)) + print( + "[WARNING] Socket: " + + str(i) + + " " + + str(e) + + ", check host, port and number of connections settings" + + " for server " + + host + ) sys.stdout.flush() failed_sockets[i] = i conn_err += 1 @@ -873,32 +1133,53 @@ def create_sockets(server, articles_to_check): req_wait = queue_time + 5 - time.time() + SOCKET_LOOP_INTERVAL if req_wait > 0: if VERBOSE: - print(('[V] Waiting ' + str(round(req_wait,2)) + ' sec '+ - 'while NZBGet closes its news server connections.')) + print( + "[V] Waiting " + + str(round(req_wait, 2)) + + " sec " + + "while NZBGet closes its news server connections." + ) sys.stdout.flush() - time.sleep(req_wait) # forum.NZBGet.net/viewtopic.php?t=2754 + time.sleep( + req_wait + ) # NZBGet sends QUIT after 5 seconds of innactivity (of a particular connection). if conn_err >= num_conn: - print(('[ERROR] Creation of all sockets for server ' + host + - ' failed.')) + print("[ERROR] Creation of all sockets for server " + host + " failed.") except: - print(('Exception LINE: ' + - str(traceback.print_exc()) + ': ' + - str(sys.exc_info()[1]))) - return sockets, failed_sockets, conn_err + print( + ( + "Exception LINE: " + + str(traceback.print_exc()) + + ": " + + str(sys.exc_info()[1]) + ) + ) + return (sockets, failed_sockets, conn_err) + def check_failure_status(rar_msg_ids, failed_limit, nzb_age): - ''' - Get the failed_ratio for each news server, if n th server failed_ratio - below failed_limit, return ok failure ratio for resuming - ''' + """ + Get the failed_ratio for each news server, if nth server failed_ratio + below failed_limit, return ok failure ratio for resuming + """ if EXTREME: - print(('[E] check_failure_status(rar_msg_ids=' + str(rar_msg_ids) + - ', failed_limit=' + str(failed_limit) + ')')) + print( + "[E] check_failure_status(rar_msg_ids=" + + str(rar_msg_ids) + + ", failed_limit=" + + str(failed_limit) + + ")" + ) articles_to_check = len(rar_msg_ids) # message on each 25 % - message_on = ([1, int(articles_to_check * failed_limit * 0.01), - int(articles_to_check * 0.25), int(articles_to_check * 0.50), - int(articles_to_check * 0.75), int(articles_to_check)]) + message_on = [ + 1, + int(articles_to_check * failed_limit * 0.01), + int(articles_to_check * 0.25), + int(articles_to_check * 0.50), + int(articles_to_check * 0.75), + int(articles_to_check), + ] servers = get_server_settings(nzb_age) # get news server provider settings if servers == []: return 100 @@ -908,7 +1189,7 @@ def check_failure_status(rar_msg_ids, failed_limit, nzb_age): failed_ratio = 0 for server in servers: if failed_ratio > MAX_FAILURE and MAX_FAILURE != 0: - print('[WARNING] failure ratio > MaxFailure.') + print("[WARNING] failure ratio > MaxFailure.") break host = server[2] username = server[4] @@ -926,13 +1207,14 @@ def check_failure_status(rar_msg_ids, failed_limit, nzb_age): socket_loop_count = [-1] * num_conn failed_wait_count = 0 loop_fail = False + chunk = 4096 start_time = time.time() - print(('Using server: ' + host)) + print("Using server: " + host) sys.stdout.flush() # build the (non) ssl sockets per server (sockets, failed_sockets, conn_err) = create_sockets(server, articles_to_check) if conn_err >= num_conn: - print(('[WARNING] Skipping server: ' + host)) + print("[WARNING] Skipping server: " + host) num_server += 1 failed_ratio = 100 continue @@ -950,10 +1232,12 @@ def check_failure_status(rar_msg_ids, failed_limit, nzb_age): # loop through all rar_msg_ids, check each one if available # if to much failed for server, skip check and move to next # send_articles has range 0 to x-1, while articles to check = x - while (send_articles <= articles_to_check - 1 and - (failed_ratio < failed_limit or failed_ratio == 0) or - send_articles <= articles_to_check - 1 and - (failed_ratio < MAX_FAILURE or failed_ratio == 0)): + while ( + send_articles <= articles_to_check - 1 + and (failed_ratio < failed_limit or failed_ratio == 0) + or send_articles <= articles_to_check - 1 + and (failed_ratio < MAX_FAILURE or failed_ratio == 0) + ): # check each connection for data receive if loop_fail: # exit while loop after looping failed_ratio = 100 @@ -964,48 +1248,74 @@ def check_failure_status(rar_msg_ids, failed_limit, nzb_age): if send_articles > articles_to_check - 1: break try: - reply = sockets[i].recv(4096) + reply = sockets[i].recv(chunk).decode("utf-8") except: # each error would trigger the same effect - # avoid continuous looping on fast machines by adding delays -# EAGAIN, EWOULDBLOCK, ssl.SSLWantReadError -# socket.timeout: # catching timeout for non SSL in 2.7.9+ -# # https://bugs.python.org/issue10272 + # avoid continuous looping on fast machines by adding delays + # EAGAIN, EWOULDBLOCK, ssl.SSLWantReadError + # socket.timeout: # catching timeout for non SSL in 2.7.9+ + # # https://bugs.python.org/issue10272 # managing all possible socket errors err = sys.exc_info() if socket_loop_count[i] < 5: if EXTREME: - print(('[E] Socket: ' + str(i) + ' ' + - str(err[0]) + ' ' + str(err[1]))) - print(('[E] Socket: ' + str(i) + ' Failed to ' + - 'get complete reply from server, waiting ' + - str(int(SOCKET_LOOP_INTERVAL * 1000 / num_conn)) + - ' ms to avoid looping.')) + print( + "[E] Socket: " + + str(i) + + " " + + str(err[0]) + + " " + + str(err[1]) + ) + print( + "[E] Socket: " + + str(i) + + " Failed to " + + "get complete reply from server, waiting " + + str(int(SOCKET_LOOP_INTERVAL * 1000 / num_conn)) + + " ms to avoid looping." + ) sys.stdout.flush() time.sleep(SOCKET_LOOP_INTERVAL / num_conn) socket_loop_count[i] += 1 continue if socket_loop_count[i] == 5: if VERBOSE: - print(('[V] Socket: ' + str(i) + ' No data ' + - 'received on 5th retry, pausing script for 2 sec.')) + print( + "[V] Socket: " + + str(i) + + " No data " + + "received on 5th retry, pausing script for 2 sec." + ) sys.stdout.flush() socket_loop_count[i] += 1 time.sleep(2) continue elif socket_loop_count[i] >= 5: if VERBOSE: - print(('[V] Socket: ' + str(i) + ' ' + - str(err[0]) + ' ' + str(err[1]))) - print(('[V] Socket: ' + str(i) + ' Still no data ' + - 'received after waiting for 1 sec, ' + - ' marking requested article as failed.')) + print( + "[V] Socket: " + + str(i) + + " " + + str(err[0]) + + " " + + str(err[1]) + ) + print( + "[V] Socket: " + + str(i) + + " Still no data " + + "received after waiting for 1 sec, " + + " marking requested article as failed." + ) sys.stdout.flush() - reply = '999 Article marked as failed by script.'.encode() + reply = "999 Article marked as failed by script." failed_wait_count += 1 if failed_wait_count >= 20: - print(('[WARNING] Skipping current server as ' + - 'it is replying very slow on header ' + - 'requests for this NZB file')) + print( + "[WARNING] Skipping current server as " + + "it is replying very slow on header " + + "requests for this NZB file" + ) loop_fail = True break # exit for i loop pass @@ -1016,35 +1326,47 @@ def check_failure_status(rar_msg_ids, failed_limit, nzb_age): if reply != None and rar_msg_ids[send_articles][4] > -1: socket_loop_count[i] = 0 # loop over ok articles on previous servers - while (send_articles < articles_to_check - 1 and - rar_msg_ids[send_articles][4] > -1): + while ( + send_articles < articles_to_check - 1 + and rar_msg_ids[send_articles][4] > -1 + ): if EXTREME: - print(('[E] Article ' + str(send_articles) + - ' already checked and available on server ' + - servers[rar_msg_ids[send_articles][4]-1][2])) + print( + "[E] Article " + + str(send_articles) + + " already checked and available on server " + + servers[rar_msg_ids[send_articles][4] - 1][2] + ) send_articles += 1 if send_articles in message_on: - print(('Requested [' + str(send_articles) + - '/' + str(articles_to_check) + - '] articles, ' + str(failed_articles) + - ' failed.')) + print( + "Requested [" + + str(send_articles) + + "/" + + str(articles_to_check) + + "] articles, " + + str(failed_articles) + + " failed." + ) sys.stdout.flush() - # msg received, and msg not checked/ok yet, and not all + # msg received, and msg not checked/ok yet, and not all # articles send: if reply != None and rar_msg_ids[send_articles][4] == -1: socket_loop_count[i] = 0 id = rar_msg_ids[send_articles][3] groups = rar_msg_ids[send_articles][2] - group = groups[0] # might not sufficient for cross posts - (error, id_used, server_reply, msg_id_used) = \ - check_send_server_reply(sockets[i], reply, group, id, - i, host, username, password) + group = groups[0] # might not sufficient for cross posts + (error, id_used, server_reply, msg_id_used) = ( + check_send_server_reply( + sockets[i], reply, group, id, i, host, username, password + ) + ) if id_used and error: # ID of missing article is not returned by server failed_articles += 1 # found ok article on server, store success: - if id_used and not error and server_reply == '223'.encode(): - # find row index for successfully send article + if id_used and not error and server_reply == "223": + # find row index for successfully send article # (with reply) for j, rar_msg_id in enumerate(rar_msg_ids): if msg_id_used == rar_msg_id[3]: @@ -1055,10 +1377,15 @@ def check_failure_status(rar_msg_ids, failed_limit, nzb_age): # rar_msg_ids starts with base 0 send_articles += 1 if send_articles in message_on: - print(('Requested [' + str(send_articles) + - '/' + str(articles_to_check) + - '] articles, ' + str(failed_articles) + - ' failed.')) + print( + "Requested [" + + str(send_articles) + + "/" + + str(articles_to_check) + + "] articles, " + + str(failed_articles) + + " failed." + ) sys.stdout.flush() failed_ratio = failed_articles * 100.0 / articles_to_check # loop through all sockets, to catch the last server replies @@ -1067,73 +1394,111 @@ def check_failure_status(rar_msg_ids, failed_limit, nzb_age): end_count = 0 # loop over all sockets to try to catch all remaining replies if EXTREME: - print('[E] Receiving remaining replies:') + print("[E] Receiving remaining replies:") # Start first loop after last socket used for receive to avoid errors m = socket_list.index(i) - for k in range(0,8): # loop multiple so all data will be received + for k in range(0, 8): # loop multiple so all data will be received for i in socket_list[m:]: # loop through ok sockets reply = None try: - reply = sockets[i].recv(4096) - # if no error, last article for this socket received + reply = sockets[i].recv(chunk).decode("utf-8") except: # managing all socket errors err = sys.exc_info() if socket_loop_count[i] < 5: if EXTREME: - print(('[E] Socket: ' + str(i) + ' ' + - str(err[0]) + ' ' + str(err[1]))) - print(('[E] Socket: ' + str(i) + ' Failed to ' + - 'get complete reply from server, waiting ' + - str(int(SOCKET_LOOP_INTERVAL * 1000 / num_conn)) + - ' ms to avoid looping.')) + print( + "[E] Socket: " + + str(i) + + " " + + str(err[0]) + + " " + + str(err[1]) + ) + print( + "[E] Socket: " + + str(i) + + " Failed to " + + "get complete reply from server, waiting " + + str(int(SOCKET_LOOP_INTERVAL * 1000 / num_conn)) + + " ms to avoid looping." + ) sys.stdout.flush() time.sleep(SOCKET_LOOP_INTERVAL / num_conn) socket_loop_count[i] += 1 continue if socket_loop_count[i] == 5: if VERBOSE: - print(('[V] Socket: ' + str(i) + ', no data ' + - 'received on 5th retry, pausing script for 1 sec.')) + print( + "[V] Socket: " + + str(i) + + ", no data " + + "received on 5th retry, pausing script for 1 sec." + ) sys.stdout.flush() socket_loop_count[i] += 1 time.sleep(1) continue elif socket_loop_count[i] >= 5: if VERBOSE: - print(('[V] Socket: ' + str(i) + ' ' + - str(err[0]) + ' ' + str(err[1]))) - print(('[V] Socket: ' + str(i) + ' Still no data ' + - 'received after waiting for 1 sec, ' + - ' marking request as failed.')) + print( + "[V] Socket: " + + str(i) + + " " + + str(err[0]) + + " " + + str(err[1]) + ) + print( + "[V] Socket: " + + str(i) + + " Still no data " + + "received after waiting for 1 sec, " + + " marking request as failed." + ) sys.stdout.flush() - reply = '999 request marked as failed by script.'.encode() + reply = "999 request marked as failed by script." pass if reply != None: socket_loop_count[i] = 0 - (error, id_used, server_reply, msg_id_used) = \ - check_send_server_reply(sockets[i], reply, group, id, - i, host, username, password) - if error and server_reply in ('411', '420', '423', '430'): + (error, id_used, server_reply, msg_id_used) = ( + check_send_server_reply( + sockets[i], + reply, + group, + id, + i, + host, + username, + password, + ) + ) + if error and server_reply in ("411", "420", "423", "430"): # ID of missing article is not returned by server failed_articles += 1 end_count += 1 if end_count >= num_conn: - print(('All requested replies received, ' + - str(failed_articles) + ' failed.')) + print( + "All requested replies received, " + + str(failed_articles) + + " failed." + ) # found ok article on server, store success: - elif not error and server_reply == '223'.encode(): - # find row index for successfully send article + elif not error and server_reply == "223": + # find row index for successfully send article # (with recv reply) for j, rar_msg_id in enumerate(rar_msg_ids): if msg_id_used == rar_msg_id[3]: # store success serv num rar_msg_ids[j][4] = num_server - break # for j loop + break # for j loop end_count += 1 if end_count >= num_conn: - print(('All requested article replies received, ' + - str(failed_articles) + ' failed.')) - elif not error and server_reply == '205'.encode(): + print( + "All requested article replies received, " + + str(failed_articles) + + " failed." + ) + elif not error and server_reply == "205": # socket closed in check_send_server_reply socket_list.remove(i) if failed_ratio != 100: @@ -1144,341 +1509,409 @@ def check_failure_status(rar_msg_ids, failed_limit, nzb_age): if len(socket_list) > 0: # kill still open sockets for i in socket_list: try: - sockets[i].send('QUIT\r\n') + sockets[i].send("QUIT\r\n") sockets[i].close except: continue - time.sleep(SOCKET_LOOP_INTERVAL) # waiting and pray the connection will be closed - print(('Failed ratio for server: ' + host + ': ' + - str(round(failed_ratio,1)) + '%. Server check completed in ' + - str(round(time.time() - start_time, 2)) + ' sec.')) + time.sleep( + SOCKET_LOOP_INTERVAL + ) # waiting and pray the connection will be closed + print( + "Failed ratio for server: " + + host + + ": " + + str(round(failed_ratio, 1)) + + "%. Server check completed in " + + str(round(time.time() - start_time, 2)) + + " sec." + ) if failed_ratio < failed_limit or failed_ratio == 0: # ok on last provider break return failed_ratio + def lock_file(): - ''' - This function checks if the .lock file is there, if it is created - before or after a restart of NZBGet. This prevents the script from - running twice at the same time. It returns True when there is a valid - .lock file, otherwise it will return false and create one. - ''' + """ + This function checks if the .lock file is there, if it is created + before or after a restart of NZBGet. This prevents the script from + running twice at the same time. It returns True when there is a valid + .lock file, otherwise it will return false and create one. + """ if VERBOSE: - print('[V] lock_file()') + print("[V] lock_file()") NZBGet = connect_to_nzbget() nzbget_status = NZBGet.status() # Get NZB status info XML-RPC - server_time = nzbget_status['ServerTime'] - up_time = nzbget_status['UpTimeSec'] - tmp_path = os.environ['NZBOP_TEMPDIR'] + os.sep + 'completion' + server_time = nzbget_status["ServerTime"] + up_time = nzbget_status["UpTimeSec"] + tmp_path = os.environ["NZBOP_TEMPDIR"] + os.sep + "completion" try: os.makedirs(tmp_path) except OSError as exc: if exc.errno == errno.EEXIST and os.path.isdir(tmp_path): pass - else: raise - f_name = tmp_path + os.sep + 'completion.lock' + else: + raise + f_name = tmp_path + os.sep + "completion.lock" file_exists = os.path.isfile(f_name) if file_exists: - fd = open(f_name) + fd = open(f_name, encoding="utf-8") time_stamp = int(fd.readline()) if VERBOSE: - print(('[V] time_stamp from completion.lock file= ' - + str(time_stamp))) ## added for debug issue reported by blackhawkpr, probably no time stamp in lock file + print( + "[V] time_stamp from completion.lock file= " + str(time_stamp) + ) ## added for debug issue reported by blackhawkpr, probably no time stamp in lock file fd.close() # Check if the .lock file was created before or after the last restart - if server_time - up_time > time_stamp: + if server_time - up_time > time_stamp: # .lock created before restart, overwrite .lock file time_stamp - fd = open(f_name,'w') + fd = open(f_name, encoding="utf-8", mode="w") fd.write(str(server_time)) fd.close() if VERBOSE: - print('[V] Old completion.lock file overwritten.') + print("[V] Old completion.lock file overwritten.") return False # Check if the .lock file is not older than 30 minutes elif server_time - 1800 > time_stamp: - print(('[ERROR] Script seems to be running for more than 30 ' + - 'minutes and has most likely crashed. Check your logs and ' - 'report the log and errors at ' + - 'http://forum.NZBGet.net/viewtopic.php?f=8&t=1736')) + print( + "[ERROR] Script seems to be running for more than 30 " + + "minutes and has most likely crashed. Check your logs and " + "report the log and errors at " + + "https://github.com/nzbgetcom/Extension-Completion/issues" + ) # overwrite .lock file time_stamp - fd = open(f_name,'w') + fd = open(f_name, encoding="utf-8", mode="w") fd.write(str(server_time)) fd.close() if VERBOSE: - print('[V] Existing completion.lock file overwritten.') + print("[V] Existing completion.lock file overwritten.") nzbget_resume() return False else: # .lock created after restart, script is running at this moment # don't start script if VERBOSE: - print('[V] Script is already running, check canceled.') + print("[V] Script is already running, check canceled.") return True else: - fd = open(f_name,'w') + fd = open(f_name, encoding="utf-8", mode="w") fd.write(str(server_time)) fd.close() if VERBOSE: ## added for debug issue reported by blackhawkpr - print(('[V] server_time= ' + str(server_time))) - print('[V] New completion.lock file created.') + print("[V] server_time= " + str(server_time)) + print("[V] New completion.lock file created.") return False + def del_lock_file(): - ''' - Delete the .lock file - ''' + """ + Delete the .lock file + """ if VERBOSE: - print('[V] del_lock_file()') - f_name = os.path.join( os.environ['NZBOP_TEMPDIR'], 'completion','completion.lock' ) + print("[V] del_lock_file()") + f_name = os.path.join(os.environ["NZBOP_TEMPDIR"], "completion", "completion.lock") os.remove(f_name) if VERBOSE: - print('[V] completion.lock file deleted') + print("[V] completion.lock file deleted") + def nzbget_paused(): - ''' - Pause NZBGet if not already paused, when paused don't start the check. - give the NZBGet sockets some time to close the connections, and avoid - 48X warnings on number of connections. - ''' + """ + Pause NZBGet if not already paused, when paused don't start the check. + give the NZBGet sockets some time to close the connections, and avoid + 48X warnings on number of connections. + """ if VERBOSE: - print('[V] nzbget_paused()') + print("[V] nzbget_paused()") NZBGet = connect_to_nzbget() nzbget_status = NZBGet.status() - nzbget_paused = nzbget_status['DownloadPaused'] + nzbget_paused = nzbget_status["DownloadPaused"] if nzbget_paused: paused = True else: paused = False nzbget_status = NZBGet.status() - download_rate = nzbget_status['DownloadRate'] + download_rate = nzbget_status["DownloadRate"] NZBGet.pausedownload() # pause downloading in NZBGet if VERBOSE: - print('[V] Waiting for NZBGet to end downloading') + print("[V] Waiting for NZBGet to end downloading") sys.stdout.flush() while download_rate > 0: # avoid double use of connections if VERBOSE: - print(('[V] Download rate: ' + - str(round(download_rate / 1000.0 ,1)) + - ' kB/s, waiting 1 sec to stop downloading')) + print( + "[V] Download rate: " + + str(round(download_rate / 1000.0, 1)) + + " kB/s, waiting 1 sec to stop downloading" + ) sys.stdout.flush() - time.sleep(1) # let the connections cool down 1 sec + time.sleep(1) # let the connections cool down 1 sec nzbget_status = NZBGet.status() - download_rate = nzbget_status['DownloadRate'] + download_rate = nzbget_status["DownloadRate"] if download_rate == 0: if VERBOSE: - print(('[V] Waiting 5 sec while NZBGet closes the news ' + - 'server connections.')) + print( + "[V] Waiting 5 sec while NZBGet closes the news " + + "server connections." + ) sys.stdout.flush() - time.sleep(5) # forum.NZBGet.net/viewtopic.php?t=2754 + time.sleep( + 5 + ) # NZBGet sends QUIT after 5 seconds of innactivity (of a particular connection). if VERBOSE: - print('[V] Downloading for NZBGet paused') + print("[V] Downloading for NZBGet paused") sys.stdout.flush() return paused + def nzbget_resume(): - ''' - Resume NZBGet - ''' + """ + Resume NZBGet + """ if VERBOSE: - print('[V] nzbget_resume()') + print("[V] nzbget_resume()") NZBGet = connect_to_nzbget() NZBGet.resumedownload() # resume downloading in NZBGet if VERBOSE: - print('[V] Downloading for NZBGet resumed') + print("[V] Downloading for NZBGet resumed") + def get_prio_nzb(jobs, paused_jobs): - ''' - Get queue data from NZBGet marked paused_jobs in scan_call, sort data based - on priority and age (oldest first, less chance of propagation, bigger - chance it will be DMCAed. Check the first item in sorted queue, if file - is incomplete, check next item etc. Only resume first succesfull file. - ''' + """ + Get queue data from NZBGet marked paused_jobs in scan_call, sort data based + on priority and age (oldest first, less chance of propagation, bigger + chance it will be DMCAed. Check the first item in sorted queue, if file + is incomplete, check next item etc. Only resume first succesfull file. + """ if EXTREME: - print('[E] get_prio_nzb(paused_jobs=') + print("[E] get_prio_nzb(paused_jobs=") for job in paused_jobs: - print(('[E] ' + str(job))) + print("[E] " + str(job)) start_time = time.time() do_check = False - max_queued_priority = -1.7976931348623157e+308 + max_queued_priority = -1.7976931348623157e308 # check if something is downloading, loop through jobs, extract max priority # of DOWNLOADING / QUEUED items for job in jobs: - if job['Status'] in ('DOWNLOADING', 'QUEUED'): - nzb_priority = job['MaxPriority'] + if job["Status"] in ("DOWNLOADING", "QUEUED"): + nzb_priority = job["MaxPriority"] if nzb_priority > max_queued_priority: max_queued_priority = nzb_priority - if VERBOSE and max_queued_priority != -1.7976931348623157e+308: - print(('[V] Maximum priority of DOWNLOADING / QUEUED NZBs = ' + - str(max_queued_priority))) + if VERBOSE and max_queued_priority != -1.7976931348623157e308: + print( + "[V] Maximum priority of DOWNLOADING / QUEUED NZBs = " + + str(max_queued_priority) + ) for job in paused_jobs: - nzb_priority = job['MaxPriority'] + nzb_priority = job["MaxPriority"] if nzb_priority > max_queued_priority: do_check = True - if VERBOSE and max_queued_priority != -1.7976931348623157e+308: - print(('[V] QUEUED / DOWNLOADING NZBs have lower priority ' + - 'than by script paused items, starting check')) + if VERBOSE and max_queued_priority != -1.7976931348623157e308: + print( + ( + "[V] QUEUED / DOWNLOADING NZBs have lower priority " + + "than by script paused items, starting check" + ) + ) break else: do_check = False - if do_check == False and max_queued_priority != -1.7976931348623157e+308: + if do_check == False and max_queued_priority != -1.7976931348623157e308: if VERBOSE: - print(('[V] QUEUED / DOWNLOADING NZBs have higher or equal ' + - 'priority than by script paused items, skipping check')) + print( + "[V] QUEUED / DOWNLOADING NZBs have higher or equal " + + "priority than by script paused items, skipping check" + ) if do_check: paused = nzbget_paused() # check if NZBGet is paused, +pause NZBGet if paused: # NZBGet is paused by user, no check if VERBOSE: - print('[V] Not started because download is paused') + print("[V] Not started because download is paused") if do_check and not paused: if VERBOSE: - print('[V] Paused UNSORTED NZBs in queue that will be processed:') + print("[V] Paused UNSORTED NZBs in queue that will be processed:") for job in paused_jobs: - nzb_filename = get_nzb_filename(job['Parameters']) - print(('[V] * ' + str(nzb_filename) + ', Age: ' + - str(round((int(time.time()) - job['MaxPostTime']) \ - / 3600.0, 1)) + ' hours, Priority: ' + - str(job['MaxPriority']))) + nzb_filename = get_nzb_filename(job["Parameters"]) + print( + "[V] * " + + str(nzb_filename) + + ", Age: " + + str(round((int(time.time()) - job["MaxPostTime"]) / 3600.0, 1)) + + " hours, Priority: " + + str(job["MaxPriority"]) + ) # sort on nzb age, but move older than max-age to bottom, then # sort of priority. Priority items will be on top. if VERBOSE: - print(('[V] Ignoring sorting priority of items older than ' + - 'AgeSortLimit of '+ str(AGE_SORT_LIMIT) + ' hours')) + print( + "[V] Ignoring sorting priority of items older than " + + "AgeSortLimit of " + + str(AGE_SORT_LIMIT) + + " hours" + ) max_age = int(time.time()) - int(AGE_SORT_LIMIT_SEC) - t1 = sorted((j for j in paused_jobs if - float(j['MaxPostTime']) >= max_age), - key=itemgetter('MaxPostTime')) + t1 = sorted( + (j for j in paused_jobs if float(j["MaxPostTime"]) >= max_age), + key=itemgetter("MaxPostTime"), + ) t2 = [] for j in paused_jobs: - if float(j['MaxPostTime']) < max_age: + if float(j["MaxPostTime"]) < max_age: t2.append(j) for t in t2: t1.append(t) - jobs_sorted = (sorted(t1, key=itemgetter('MaxPriority'), - reverse=True)) + jobs_sorted = sorted(t1, key=itemgetter("MaxPriority"), reverse=True) if VERBOSE: - print('[V] Paused and SORTED NZBs in queue that will be processed:') + print("[V] Paused and SORTED NZBs in queue that will be processed:") for job in jobs_sorted: - nzb_filename = get_nzb_filename(job['Parameters']) - print(('[V] * ' + str(nzb_filename) + ', Age: ' + - str(round((int(time.time()) - job['MaxPostTime']) \ - / 3600.0, 1)) + ' hours, Priority: ' + - str(job['MaxPriority']))) + nzb_filename = get_nzb_filename(job["Parameters"]) + print( + "[V] * " + + str(nzb_filename) + + ", Age: " + + str(round((int(time.time()) - job["MaxPostTime"]) / 3600.0, 1)) + + " hours, Priority: " + + str(job["MaxPriority"]) + ) for job in jobs_sorted: - nzb_filename = get_nzb_filename(job['Parameters']) - nzb_id = job['NZBID'] - nzb_age = job['MaxPostTime'] # nzb age - nzb_critical_health = job['CriticalHealth'] - nzb_dupe_key = job['DupeKey'] # if empty returns u'' - if nzb_dupe_key == '': - nzb_dupe_key = 'NONE' - nzb_dupe_score = job['DupeScore'] - nzb = [nzb_id, nzb_filename, nzb_age, nzb_critical_health, - nzb_dupe_key, nzb_dupe_score] + nzb_filename = get_nzb_filename(job["Parameters"]) + nzb_id = job["NZBID"] + nzb_age = job["MaxPostTime"] # nzb age + nzb_critical_health = job["CriticalHealth"] + nzb_dupe_key = job["DupeKey"] # if empty returns u'' + if nzb_dupe_key == "": + nzb_dupe_key = "NONE" + nzb_dupe_score = job["DupeScore"] + nzb = [ + nzb_id, + nzb_filename, + nzb_age, + nzb_critical_health, + nzb_dupe_key, + nzb_dupe_score, + ] # do a completion check, returns true if ok and resumed if get_nzb_status(nzb): break - print(('Overall check completed in ' + - str(round(time.time() - start_time, 2)) + ' sec.')) + print( + "Overall check completed in " + + str(round(time.time() - start_time, 2)) + + " sec." + ) nzbget_resume() + def scheduler_call(): - ''' - Script is called as scheduler script - check if files in the queue should be checked by the completion script - ''' + """ + Script is called as scheduler script + check if files in the queue should be checked by the completion script + """ global queue_time queue_time = -1 # NZBGet closes connection after 5 sec. Avoid too much conn if VERBOSE: - print('[V] scheduler_call()') + print("[V] scheduler_call()") # data contains ALL properties each NZB in queue - data = call_nzbget_direct('listgroups') + data = call_nzbget_direct("listgroups") jobs = json.loads(data) # check if nzb in queue, and check if paused by this script - if len(jobs['result']) > 0 and 'CnpNZBFileName' in str(jobs): + if len(jobs["result"]) > 0 and "CnpNZBFileName" in str(jobs): if not lock_file(): # check if script is not already running paused_jobs = [] - for job in jobs['result']: + for job in jobs["result"]: # send only nzbs paused by the script - if ('CnpNZBFileName' in str(job) and - job['Status'] in ('PAUSED')): + if "CnpNZBFileName" in str(job) and job["Status"] in ("PAUSED"): paused_jobs.append(job) if len(paused_jobs) > 0: - get_prio_nzb(jobs['result'], paused_jobs) + get_prio_nzb(jobs["result"], paused_jobs) del_lock_file() elif VERBOSE: - print('[V] Empty queue') + print("[V] Empty queue") + def queue_call(): - ''' - Script is called as queue script - check if new files in queue should be checked by the completion script - Option NZBGet EventInterval set to -1 avoids script being called each - time a part is donwloaded. - ''' + """ + Script is called as queue script + check if new files in queue should be checked by the completion script + Option NZBGet EventInterval set to -1 avoids script being called each + time a part is donwloaded. + """ global queue_time queue_time = -1 if VERBOSE: - print('[V] queue_call()') - print((os.environ['NZBNA_QUEUEDFILE'])) + print("[V] queue_call()") + print(os.environ["NZBNA_QUEUEDFILE"]) # check if NZB is added, otherwise it will call on each downloaded part - event = os.environ['NZBNA_EVENT'] - if (event == 'NZB_ADDED' or event == 'NZB_DOWNLOADED' - or event == 'NZB_DELETED' or event == 'NZB_MARKED'): - # when NZB_DOWNLOADED occurs, the NZB is still in queue, with the + event = os.environ["NZBNA_EVENT"] + if ( + event == "NZB_ADDED" + or event == "NZB_DOWNLOADED" + or event == "NZB_DELETED" + or event == "NZB_MARKED" + ): + # when NZB_DOWNLOADED occurs, the NZB is still in queue, with the # paused par2 etc. # data contains ALL properties each NZB in queue - data = call_nzbget_direct('listgroups') + data = call_nzbget_direct("listgroups") jobs = json.loads(data) # check if nzb in queue, and check if paused by this script - if len(jobs['result']) > 0 and 'CnpNZBFileName' in str(jobs): + if len(jobs["result"]) > 0 and "CnpNZBFileName" in str(jobs): if not lock_file(): # check if script is not already running paused_jobs = [] - for job in jobs['result']: + for job in jobs["result"]: # send only nzbs paused by the script - if ('CnpNZBFileName' in str(job) and - job['Status'] in ('PAUSED')): + if "CnpNZBFileName" in str(job) and job["Status"] in ("PAUSED"): paused_jobs.append(job) if len(paused_jobs) > 0: - if event == 'NZB_DOWNLOADED': + if event == "NZB_DOWNLOADED": queue_time = time.time() - get_prio_nzb(jobs['result'], paused_jobs) + get_prio_nzb(jobs["result"], paused_jobs) del_lock_file() + def scan_call(): - ''' - Script is called as scan script. This part of the script pauses the NZB - and marks the file as paused by the script. Files not paused by the - script won't be checked on completion. - NZBGet doesn't provide the actual name of the file when in the queue. - if 2 same filename items appear at the same time the 2nd file wiil be - _2.nzb.queued and if 2 items are added after eachoter, they will be - nzb.queued and nzb.2.queued. NZBGet does not provide the _2. or .2. in - e.g. queue, scheduler calls, 'listgroups' or 'history'. The scan script - adds the NZBPR_CnpNZBFileName variable to know the exact file name, and - uses it to recognize if a nzb is paused by the script. - ''' - if VERBOSE: - print('[V] scan_call()') + """ + Script is called as scan script. This part of the script pauses the NZB + and marks the file as paused by the script. Files not paused by the + script won't be checked on completion. + NZBGet doesn't provide the actual name of the file when in the queue. + if 2 same filename items appear at the same time the 2nd file wiil be + _2.nzb.queued and if 2 items are added after eachoter, they will be + nzb.queued and nzb.2.queued. NZBGet does not provide the _2. or .2. in + e.g. queue, scheduler calls, 'listgroups' or 'history'. The scan script + adds the NZBPR_CnpNZBFileName variable to know the exact file name, and + uses it to recognize if a nzb is paused by the script. + """ + if VERBOSE: + print("[V] scan_call()") # Check if NZB should be paused. - if (os.environ['NZBNP_CATEGORY'].lower() in CATEGORIES or - CATEGORIES[0] == ''): + if os.environ["NZBNP_CATEGORY"].lower() in CATEGORIES or CATEGORIES[0] == "": # NZBNP_FILENAME needs to be written to other NZBPR_var for later use. - nzb_filename = os.environ['NZBNP_FILENAME'] - nzb_dir = os.environ['NZBOP_NZBDIR'] - if nzb_dir[-1:] == '\\' or nzb_dir[-1:] == '/': - print(('[WARNING] Please correct your NZBGet PATHS Settings '+ - 'by removing the trailing "' + str(os.sep) + '"!')) - if os.sep == '\\': #windows \ - if nzb_dir.find('/') != -1: - print(('[WARNING] Please correct your NZBGet PATHS Settings '+ - 'using "' + str(os.sep) + '" only!')) + nzb_filename = os.environ["NZBNP_FILENAME"] + nzb_dir = os.environ["NZBOP_NZBDIR"] + if nzb_dir[-1:] == "\\" or nzb_dir[-1:] == "/": + print( + "[WARNING] Please correct your NZBGet PATHS Settings " + + 'by removing the trailing "' + + str(os.sep) + + '"!' + ) + if os.sep == "\\": # windows \ + if nzb_dir.find("/") != -1: + print( + "[WARNING] Please correct your NZBGet PATHS Settings " + + 'using "' + + str(os.sep) + + '" only!' + ) else: # nix - if nzb_dir.find('\\') != -1: - print('nix') - print(('[WARNING] Please correct your NZBGet PATHS Settings '+ - 'using "' + str(os.sep) + '" only!')) - nzb_filename = nzb_filename.replace(nzb_dir + os.sep, '') + if nzb_dir.find("\\") != -1: + print("nix") + print( + "[WARNING] Please correct your NZBGet PATHS Settings " + + 'using "' + + str(os.sep) + + '" only!' + ) + nzb_filename = nzb_filename.replace(nzb_dir + os.sep, "") l_nzb = len(nzb_filename) # length for file matching, with .nzb ext c = 0 dupe_list_num = [] @@ -1488,73 +1921,83 @@ def scan_call(): # count identical files c += 1 # extract possible number between .nzb. and .queued - dupe_num = file[file.rfind('.nzb.')+5:-7] + dupe_num = file[file.rfind(".nzb.") + 5 : -7] dupe_list_num.append(dupe_num) if c > 0: # already 1 file with same name in queue/history if VERBOSE: - print(('[V] Found ' + str(c) + - ' queued / history nzb with identical name: ' + - nzb_filename )) + print( + "[V] Found " + + str(c) + + " queued / history nzb with identical name: " + + nzb_filename + ) # num between .nzb. .queued is lowest num not in dupe_list_num for x in range(1, c + 1): if x == 1: - t = '' + t = "" else: t = str(x) if t not in dupe_list_num: - if t == '': - nzb_filename = nzb_filename + '.queued' + if t == "": + nzb_filename = nzb_filename + ".queued" else: - nzb_filename = nzb_filename + '.' + t + '.queued' + nzb_filename = nzb_filename + "." + t + ".queued" break elif c == 1 or c == x: t = str(x + 1) - nzb_filename = nzb_filename + '.' + t + '.queued' + nzb_filename = nzb_filename + "." + t + ".queued" # else: nothing else: # no identical file names - nzb_filename = nzb_filename + '.queued' + nzb_filename = nzb_filename + ".queued" if VERBOSE: - print(('[V] Expected queued file name: "' + nzb_filename + '"')) - print(('[NZB] NZBPR_CnpNZBFileName=' + nzb_filename)) + print('[V] Expected queued file name: "' + nzb_filename + '"') + print("[NZB] NZBPR_CnpNZBFileName=" + nzb_filename) # pausing NZB if VERBOSE: - print(('[V] Pausing: "' + os.environ['NZBNP_NZBNAME'] + '"')) - print('[NZB] PAUSED=1') + print('[V] Pausing: "' + os.environ["NZBNP_NZBNAME"] + '"') + print("[NZB] PAUSED=1") + def main(): - ''' - Check for which script type the script is called - ''' + """ + Check for which script type the script is called + """ # check if the script is called as Scheduler Script - if 'NZBSP_TASKID' in os.environ: scheduler_call() + if "NZBSP_TASKID" in os.environ: + scheduler_call() # Check if the script is called as Queue Script. - if 'NZBNA_NZBNAME' in os.environ: queue_call() + if "NZBNA_NZBNAME" in os.environ: + queue_call() # check if the script is called as Scan Script - if 'NZBNP_NZBNAME' in os.environ: scan_call() + if "NZBNP_NZBNAME" in os.environ: + scan_call() # check if the script is called via button - if 'NZBCP_COMMAND' in os.environ: + if "NZBCP_COMMAND" in os.environ: scheduler_call() sys.exit(93) + def write_to_file(input): - ''' - For testing purposes only - ''' - tmp_path = os.environ['NZBOP_TEMPDIR'] + os.sep + 'completion' + """ + For testing purposes only + """ + tmp_path = os.environ["NZBOP_TEMPDIR"] + os.sep + "completion" try: os.makedirs(tmp_path) except OSError as exc: if exc.errno == errno.EEXIST and os.path.isdir(tmp_path): pass - else: raise - f_name = tmp_path + os.sep + 'log.txt' - fd = open(f_name,'a') - fd.write(str(input) + '\n\n') + else: + raise + f_name = tmp_path + os.sep + "log.txt" + fd = open(f_name, encoding="utf-8", mode="a") + fd.write(str(input) + "\n\n") fd.close() + main() -''' +""" TODO: - User blackhawkpr had an issue related to a wrong or missing time stamp in .lock file, can not reproduce, added additinal logging for it in VERBOSE @@ -1605,4 +2048,4 @@ def write_to_file(input): - connect_to_nzbget() -> connection to get data from NZBGet - call_nzbget_direct() -> connect and get data from NZBGet -''' +""" diff --git a/manifest.json b/manifest.json index 1c8a353..56517b6 100644 --- a/manifest.json +++ b/manifest.json @@ -4,13 +4,14 @@ "homepage": "https://github.com/nzbgetcom/Extension-Completion", "kind": "SCAN/QUEUE/SCHEDULER", "displayName": "Completion", - "version": "3.0.0", + "version": "3.0", + "nzbgetMinVersion": "23.0", "author": "kloaknet", "license": "GPLv3", "about": "Verifies that enough articles are available before starting the download.", "queueEvents": "NZB_ADDED, NZB_DOWNLOADED, NZB_DELETED, NZB_MARKED", "requirements": [ - "This script requires NZBGet 15.0+ and Python 3.8.0+ to be installed on your system." + "This script requires Python 3.8.0 and above to be installed on your system." ], "description": [ "For NZBGet versions 18+, this script needs to be:", diff --git a/test_data/listgroups_resp.json b/test_data/listgroups_resp.json new file mode 100644 index 0000000..09c18af --- /dev/null +++ b/test_data/listgroups_resp.json @@ -0,0 +1,83 @@ +{ + "version": "1.1", + "result": [ + { + "FirstID": 39, + "LastID": 39, + "RemainingSizeLo": 1732652478, + "RemainingSizeHi": 0, + "RemainingSizeMB": 1652, + "PausedSizeLo": 1732652478, + "PausedSizeHi": 0, + "PausedSizeMB": 1652, + "RemainingFileCount": 41, + "RemainingParCount": 10, + "MinPriority": 0, + "MaxPriority": 0, + "ActiveDownloads": 0, + "Status": "PAUSED", + "NZBID": 39, + "NZBName": "bad", + "NZBNicename": "bad", + "Kind": "NZB", + "URL": "", + "NZBFilename": "bad.nzb", + "DestDir": "C:\\ProgramData\\NZBGet\\intermediate\\bad.#39", + "FinalDir": "", + "Category": "", + "ParStatus": "NONE", + "ExParStatus": "NONE", + "UnpackStatus": "NONE", + "MoveStatus": "NONE", + "ScriptStatus": "NONE", + "DeleteStatus": "NONE", + "MarkStatus": "NONE", + "UrlStatus": "NONE", + "FileSizeLo": 1732652478, + "FileSizeHi": 0, + "FileSizeMB": 1652, + "FileCount": 41, + "MinPostTime": 1294832829, + "MaxPostTime": 1294832829, + "TotalArticles": 4380, + "SuccessArticles": 0, + "FailedArticles": 0, + "Health": 1000, + "CriticalHealth": 894, + "DupeKey": "", + "DupeScore": 0, + "DupeMode": "SCORE", + "Deleted": false, + "DownloadedSizeLo": 0, + "DownloadedSizeHi": 0, + "DownloadedSizeMB": 0, + "DownloadTimeSec": 0, + "PostTotalTimeSec": 0, + "ParTimeSec": 0, + "RepairTimeSec": 0, + "UnpackTimeSec": 0, + "MessageCount": 3, + "ExtraParBlocks": 0, + "Parameters": [ + { + "Name": "*Unpack:", + "Value": "yes" + }, + { + "Name": "Completion:", + "Value": "yes" + }, + { + "Name": "CnpNZBFileName", + "Value": "C:\\ProgramData\\NZBGet\\nzb\\bad.nzb.queued" + } + ], + "ScriptStatuses": [], + "ServerStats": [], + "PostInfoText": "NONE", + "PostStageProgress": 71727040, + "PostStageTimeSec": 0, + "Log": [] + } + ] +} \ No newline at end of file diff --git a/test_data/nzb_filename.queued b/test_data/nzb_filename.queued new file mode 100644 index 0000000..e69de29 diff --git a/test_data/result_resp.xml b/test_data/result_resp.xml new file mode 100644 index 0000000..1620681 --- /dev/null +++ b/test_data/result_resp.xml @@ -0,0 +1,47 @@ + + +RemainingSizeLo0 +RemainingSizeHi0 +RemainingSizeMB0 +ForcedSizeLo0 +ForcedSizeHi0 +ForcedSizeMB0 +DownloadedSizeLo0 +DownloadedSizeHi0 +DownloadedSizeMB0 +MonthSizeLo294171462 +MonthSizeHi3 +MonthSizeMB12568 +DaySizeLo269393759 +DaySizeHi0 +DaySizeMB256 +ArticleCacheLo0 +ArticleCacheHi0 +ArticleCacheMB0 +DownloadRate0 +AverageDownloadRate0 +DownloadLimit0 +ThreadCount16 +ParJobCount0 +PostJobCount0 +UpTimeSec36 +DownloadTimeSec0 +ServerPausedfalse +DownloadPausedfalse +Download2Pausedfalse +ServerStandBytrue +PostPausedfalse +ScanPausedfalse +QuotaReached680816640 +FreeDiskSpaceHi36 +FreeDiskSpaceMB148105 +ServerTime1714653165 +ResumeTime0 +QueueScriptCount1 +NewsServers + +ID1 +Activetrue + + + \ No newline at end of file diff --git a/test_data/status_resp.xml b/test_data/status_resp.xml new file mode 100644 index 0000000..1620681 --- /dev/null +++ b/test_data/status_resp.xml @@ -0,0 +1,47 @@ + + +RemainingSizeLo0 +RemainingSizeHi0 +RemainingSizeMB0 +ForcedSizeLo0 +ForcedSizeHi0 +ForcedSizeMB0 +DownloadedSizeLo0 +DownloadedSizeHi0 +DownloadedSizeMB0 +MonthSizeLo294171462 +MonthSizeHi3 +MonthSizeMB12568 +DaySizeLo269393759 +DaySizeHi0 +DaySizeMB256 +ArticleCacheLo0 +ArticleCacheHi0 +ArticleCacheMB0 +DownloadRate0 +AverageDownloadRate0 +DownloadLimit0 +ThreadCount16 +ParJobCount0 +PostJobCount0 +UpTimeSec36 +DownloadTimeSec0 +ServerPausedfalse +DownloadPausedfalse +Download2Pausedfalse +ServerStandBytrue +PostPausedfalse +ScanPausedfalse +QuotaReached680816640 +FreeDiskSpaceHi36 +FreeDiskSpaceMB148105 +ServerTime1714653165 +ResumeTime0 +QueueScriptCount1 +NewsServers + +ID1 +Activetrue + + + \ No newline at end of file diff --git a/tests.py b/tests.py new file mode 100644 index 0000000..434dd80 --- /dev/null +++ b/tests.py @@ -0,0 +1,197 @@ +# +# Copyright (C) 2024 Denis +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with the program. If not, see . +# + +import http +import json +import threading +import unittest +import sys +from os.path import dirname +import os +import subprocess +import xmlrpc.server +import xml.etree.cElementTree as ET +import shutil + +SUCCESS = 93 +NONE = 95 +ERROR = 94 + +ROOT = dirname(__file__) +TMP_DIR = ROOT + os.sep + "tmp" +TEST_DATA_DIR = ROOT + os.sep + "test_data" +HOST = "127.0.0.1" +USERNAME = "TestUser" +PASSWORD = "TestPassword" +PORT = "6789" + + +def clean_up(): + if os.path.exists(TMP_DIR): + shutil.rmtree(TMP_DIR) + + +def parse_member(member): + name = member.find("name").text + value_elem = member.find("value") + + if value_elem.find("i4") is not None: + value = int(value_elem.find("i4").text) + elif value_elem.find("boolean") is not None: + value = value_elem.find("boolean").text == "true" + elif value_elem.find("array") is not None: + value = parse_array(value_elem.find("array")) + + return name, value + + +def parse_array(array_elem): + array_data = {} + for member in array_elem.find("data").findall("member"): + name = member.find("name").text + value_elem = member.find("value") + if value_elem.find("i4") is not None: + value = int(value_elem.find("i4").text) + elif value_elem.find("boolean") is not None: + value = value_elem.find("boolean").text == "true" + array_data[name] = value + return array_data + + +class NZBGetServer(http.server.BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + f = open(TEST_DATA_DIR + "/listgroups_resp.json") + data = json.load(f) + formatted = json.dumps(data, separators=(",\n", " : "), indent=0) + self.wfile.write(formatted.encode("utf-8")) + f.close() + + def do_POST(self): + self.log_request() + self.send_response(200) + self.send_header("Content-Type", "text/xml") + self.end_headers() + with open(TEST_DATA_DIR + "/status_resp.xml", "r") as f: + data = f.read().replace("\n", "").strip() + root = ET.fromstring(data) + response_dict = {} + for member in root.findall("member"): + name, value = parse_member(member) + response_dict[name] = value + + # Serialize Python object to XML-RPC response + response = xmlrpc.client.dumps( + (response_dict,), allow_none=False, encoding=None + ) + + # Send the serialized response to the client + self.wfile.write(response.encode("utf-8")) + + +def get_python(): + if os.name == "nt": + return "python" + return "python3" + + +def run_script(): + sys.stdout.flush() + proc = subprocess.Popen( + [get_python(), ROOT + "/main.py"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=os.environ.copy(), + ) + out, err = proc.communicate() + proc.pid + ret_code = proc.returncode + return (out.decode(), int(ret_code), err.decode()) + + +def set_defaults_env(): + # NZBGet global options + os.environ["NZBOP_CONTROLPORT"] = PORT + os.environ["NZBOP_CONTROLIP"] = HOST + os.environ["NZBOP_CONTROLUSERNAME"] = USERNAME + os.environ["NZBOP_CONTROLPASSWORD"] = PASSWORD + os.environ["NZBOP_TEMPDIR"] = TMP_DIR + os.environ["NZBNA_QUEUEDFILE"] = "nzb_filename" + + +class Tests(unittest.TestCase): + + def test_scheduler_mode(self): + set_defaults_env() + os.environ["NZBSP_TASKID"] = "ID" + server = http.server.HTTPServer((HOST, int(PORT)), NZBGetServer) + thread = threading.Thread(target=server.serve_forever) + thread.start() + [out, code, err] = run_script() + server.shutdown() + server.server_close() + thread.join() + del os.environ["NZBSP_TASKID"] + clean_up() + self.assertEqual(code, 0) + + def test_queue_mode(self): + set_defaults_env() + os.environ["NZBNA_NZBNAME"] = "nzb_filename" + os.environ["NZBNA_EVENT"] = "NZB_DOWNLOADED" + os.environ["NZBNA_QUEUEDFILE"] = "nzb_filename.queued" + server = http.server.HTTPServer((HOST, int(PORT)), NZBGetServer) + thread = threading.Thread(target=server.serve_forever) + thread.start() + [out, code, err] = run_script() + server.shutdown() + server.server_close() + thread.join() + del os.environ["NZBNA_NZBNAME"] + del os.environ["NZBNA_EVENT"] + del os.environ["NZBNA_QUEUEDFILE"] + self.assertEqual(code, 0) + + def test_scan_mode(self): + set_defaults_env() + os.environ["NZBNP_NZBNAME"] = "nzb_filename" + os.environ["NZBNP_CATEGORY"] = "Movies" + os.environ["NZBNP_FILENAME"] = "nzb_filename.queued" + os.environ["NZBOP_NZBDIR"] = TEST_DATA_DIR + server = http.server.HTTPServer((HOST, int(PORT)), NZBGetServer) + thread = threading.Thread(target=server.serve_forever) + thread.start() + [out, code, err] = run_script() + server.shutdown() + server.server_close() + thread.join() + del os.environ["NZBNP_NZBNAME"] + del os.environ["NZBNP_CATEGORY"] + self.assertEqual(code, 0) + + def test_manifest(self): + with open(ROOT + "/manifest.json", encoding="utf-8") as file: + try: + json.loads(file.read()) + except ValueError as e: + self.fail("manifest.json is not valid.") + + +if __name__ == "__main__": + unittest.main()