From 1f1134be23577ecaa0c06f0e9f0bd447ee3b3376 Mon Sep 17 00:00:00 2001 From: Phil Borman Date: Mon, 6 Aug 2018 20:49:57 +0200 Subject: [PATCH 1/7] Reorganised searchdelay, better messages --- lazylibrarian/providers.py | 18 +++- lazylibrarian/searchbook.py | 201 ++++++++++++++++++------------------ 2 files changed, 120 insertions(+), 99 deletions(-) diff --git a/lazylibrarian/providers.py b/lazylibrarian/providers.py index 0f7e477d7..7269a2b59 100644 --- a/lazylibrarian/providers.py +++ b/lazylibrarian/providers.py @@ -32,39 +32,46 @@ def test_provider(name, host=None, api=None): - logger.debug("Testing provider %s" % name) book = {'searchterm': 'Agatha+Christie', 'library': 'eBook'} if name == 'TPB': + logger.debug("Testing provider %s" % name) if host: lazylibrarian.CONFIG['TPB_HOST'] = host return TPB(book, test=True), "Pirate Bay" if name == 'WWT': + logger.debug("Testing provider %s" % name) if host: lazylibrarian.CONFIG['WWT_HOST'] = host return WWT(book, test=True), "WorldWideTorrents" if name == 'KAT': + logger.debug("Testing provider %s" % name) if host: lazylibrarian.CONFIG['KAT_HOST'] = host return KAT(book, test=True), "KickAss Torrents" if name == 'ZOO': + logger.debug("Testing provider %s" % name) if host: lazylibrarian.CONFIG['ZOO_HOST'] = host return ZOO(book, test=True), "Zooqle" if name == 'LIME': + logger.debug("Testing provider %s" % name) if host: lazylibrarian.CONFIG['LIME_HOST'] = host return LIME(book, test=True), "LimeTorrents" if name == 'TDL': + logger.debug("Testing provider %s" % name) if host: lazylibrarian.CONFIG['TDL_HOST'] = host return TDL(book, test=True), "TorrentDownloads" if name == 'GEN': + logger.debug("Testing provider %s" % name) if host: lazylibrarian.CONFIG['GEN_HOST'] = host if api: lazylibrarian.CONFIG['GEN_SEARCH'] = api return GEN(book, prov='GEN', test=True), "LibGen 1" if name == 'GEN2': + logger.debug("Testing provider %s" % name) if host: lazylibrarian.CONFIG['GEN2_HOST'] = host if api: @@ -75,6 +82,9 @@ def test_provider(name, host=None, api=None): prov = name.split('_')[1] for provider in lazylibrarian.RSS_PROV: if provider['NAME'] == 'RSS_%s' % prov: + if provider['DISPNAME']: + name = provider['DISPNAME'] + logger.debug("Testing provider %s" % name) if not host: host = provider['HOST'] if 'goodreads' in host and 'list_rss' in host: @@ -98,6 +108,9 @@ def test_provider(name, host=None, api=None): prov = name.split('_')[1] for provider in lazylibrarian.TORZNAB_PROV: if provider['NAME'] == 'Torznab%s' % prov: + if provider['DISPNAME']: + name = provider['DISPNAME'] + logger.debug("Testing provider %s" % name) if provider['MANUAL']: logger.debug("Capabilities are set to manual for %s" % provider['NAME']) else: @@ -121,6 +134,9 @@ def test_provider(name, host=None, api=None): prov = name.split('_')[1] for provider in lazylibrarian.NEWZNAB_PROV: if provider['NAME'] == 'Newznab%s' % prov: + if provider['DISPNAME']: + name = provider['DISPNAME'] + logger.debug("Testing provider %s" % name) if provider['MANUAL']: logger.debug("Capabilities are set to manual for %s" % provider['NAME']) else: diff --git a/lazylibrarian/searchbook.py b/lazylibrarian/searchbook.py index 580bfda5f..9552cf9c7 100644 --- a/lazylibrarian/searchbook.py +++ b/lazylibrarian/searchbook.py @@ -120,7 +120,7 @@ def search_book(books=None, library=None): if library is None or library == 'eBook': if searchbook['Status'] == "Wanted": # not just audiobook wanted cmd = 'SELECT BookID from wanted WHERE BookID=? and AuxInfo="eBook" and Status="Snatched"' - snatched = myDB.match(cmd, (searchbook["BookID"], )) + snatched = myDB.match(cmd, (searchbook["BookID"],)) if snatched: logger.warn('eBook %s %s already marked snatched in wanted table' % (searchbook['AuthorName'], searchbook['BookName'])) @@ -136,7 +136,7 @@ def search_book(books=None, library=None): if library is None or library == 'AudioBook': if searchbook['AudioStatus'] == "Wanted": # in case we just wanted eBook cmd = 'SELECT BookID from wanted WHERE BookID=? and AuxInfo="AudioBook" and Status="Snatched"' - snatched = myDB.match(cmd, (searchbook["BookID"], )) + snatched = myDB.match(cmd, (searchbook["BookID"],)) if snatched: logger.warn('AudioBook %s %s already marked snatched in wanted table' % (searchbook['AuthorName'], searchbook['BookName'])) @@ -161,74 +161,37 @@ def search_book(books=None, library=None): lazylibrarian.NO_RSS_MSG = timenow modelist.remove('rss') - if lazylibrarian.CONFIG['DELAYSEARCH']: - for book in searchlist: - # have we searched for the book before + book_count = 0 + for book in searchlist: + do_search = True + if lazylibrarian.CONFIG['DELAYSEARCH']: res = myDB.match('SELECT * FROM failedsearch WHERE BookID=? AND Library=?', (book['bookid'], book['library'])) - if res: + if not res: + logger.debug("SearchDelay: %s %s has not failed before" % (book['library'], book['bookid'])) + else: skipped = check_int(res['Count'], 0) interval = check_int(res['Interval'], 0) if skipped < interval: - logger.debug("Only searching for %s %s every %s" % - (book['bookid'], book['library'], res['Interval'])) - searchlist.remove(book) + logger.debug("SearchDelay: %s %s not due (%d/%d)" % + (book['library'], book['bookid'], skipped, interval)) myDB.action("UPDATE failedsearch SET Count=? WHERE BookID=? AND Library=?", (skipped + 1, book['bookid'], book['library'])) + do_search = False + else: + logger.debug("SearchDelay: %s %s due this time (%d/%d)" % + (book['library'], book['bookid'], skipped, interval)) - book_count = 0 - for book in searchlist: matches = [] - for mode in modelist: - # first attempt, try author/title in category "book" - if book['library'] == 'AudioBook': - searchtype = 'audio' - else: - searchtype = 'book' - - resultlist = None - if mode == 'nzb' and 'nzb' in modelist: - resultlist, nprov = IterateOverNewzNabSites(book, searchtype) - if not nprov: - # don't nag. Show warning message no more than every 20 mins - timenow = int(time.time()) - if check_int(lazylibrarian.NO_NZB_MSG, 0) + 1200 < timenow: - logger.warn('No nzb providers are available. Check config and blocklist') - lazylibrarian.NO_NZB_MSG = timenow - modelist.remove('nzb') - elif mode == 'tor' and 'tor' in modelist: - resultlist, nprov = IterateOverTorrentSites(book, searchtype) - if not nprov: - # don't nag. Show warning message no more than every 20 mins - timenow = int(time.time()) - if check_int(lazylibrarian.NO_TOR_MSG, 0) + 1200 < timenow: - logger.warn('No tor providers are available. Check config and blocklist') - lazylibrarian.NO_TOR_MSG = timenow - modelist.remove('tor') - elif mode == 'direct' and 'direct' in modelist: - resultlist, nprov = IterateOverDirectSites(book, searchtype) - if not nprov: - # don't nag. Show warning message no more than every 20 mins - timenow = int(time.time()) - if check_int(lazylibrarian.NO_DIRECT_MSG, 0) + 1200 < timenow: - logger.warn('No direct providers are available. Check config and blocklist') - lazylibrarian.NO_DIRECT_MSG = timenow - modelist.remove('direct') - elif mode == 'rss' and 'rss' in modelist: - if rss_resultlist: - resultlist = rss_resultlist + if do_search: + for mode in modelist: + # first attempt, try author/title in category "book" + if book['library'] == 'AudioBook': + searchtype = 'audio' else: - logger.debug("No active rss providers found") - modelist.remove('rss') + searchtype = 'book' - if resultlist: - match = findBestResult(resultlist, book, searchtype, mode) - else: - match = None - - # if you can't find the book, try author/title without any "(extended details, series etc)" - if not goodEnough(match) and '(' in book['bookName']: - searchtype = 'short' + searchtype + resultlist = None if mode == 'nzb' and 'nzb' in modelist: resultlist, nprov = IterateOverNewzNabSites(book, searchtype) if not nprov: @@ -257,55 +220,97 @@ def search_book(books=None, library=None): lazylibrarian.NO_DIRECT_MSG = timenow modelist.remove('direct') elif mode == 'rss' and 'rss' in modelist: - resultlist = rss_resultlist + if rss_resultlist: + resultlist = rss_resultlist + else: + logger.debug("No active rss providers found") + modelist.remove('rss') if resultlist: match = findBestResult(resultlist, book, searchtype, mode) else: match = None - # if you can't find the book under "books", you might find under general search - # general search is the same as booksearch for torrents and rss, no need to check again - if not goodEnough(match): - searchtype = 'general' - if mode == 'nzb' and 'nzb' in modelist: - resultlist, nprov = IterateOverNewzNabSites(book, searchtype) - if not nprov: - # don't nag. Show warning message no more than every 20 mins - timenow = int(time.time()) - if check_int(lazylibrarian.NO_NZB_MSG, 0) + 1200 < timenow: - logger.warn('No nzb providers are available. Check config and blocklist') - lazylibrarian.NO_NZB_MSG = timenow - modelist.remove('nzb') - if resultlist: - match = findBestResult(resultlist, book, searchtype, mode) - else: - match = None + # if you can't find the book, try author/title without any "(extended details, series etc)" + if not goodEnough(match) and '(' in book['bookName']: + searchtype = 'short' + searchtype + if mode == 'nzb' and 'nzb' in modelist: + resultlist, nprov = IterateOverNewzNabSites(book, searchtype) + if not nprov: + # don't nag. Show warning message no more than every 20 mins + timenow = int(time.time()) + if check_int(lazylibrarian.NO_NZB_MSG, 0) + 1200 < timenow: + logger.warn('No nzb providers are available. Check config and blocklist') + lazylibrarian.NO_NZB_MSG = timenow + modelist.remove('nzb') + elif mode == 'tor' and 'tor' in modelist: + resultlist, nprov = IterateOverTorrentSites(book, searchtype) + if not nprov: + # don't nag. Show warning message no more than every 20 mins + timenow = int(time.time()) + if check_int(lazylibrarian.NO_TOR_MSG, 0) + 1200 < timenow: + logger.warn('No tor providers are available. Check config and blocklist') + lazylibrarian.NO_TOR_MSG = timenow + modelist.remove('tor') + elif mode == 'direct' and 'direct' in modelist: + resultlist, nprov = IterateOverDirectSites(book, searchtype) + if not nprov: + # don't nag. Show warning message no more than every 20 mins + timenow = int(time.time()) + if check_int(lazylibrarian.NO_DIRECT_MSG, 0) + 1200 < timenow: + logger.warn('No direct providers are available. Check config and blocklist') + lazylibrarian.NO_DIRECT_MSG = timenow + modelist.remove('direct') + elif mode == 'rss' and 'rss' in modelist: + resultlist = rss_resultlist - # if still not found, try general search again without any "(extended details, series etc)" - if not goodEnough(match) and '(' in book['searchterm']: - searchtype = 'shortgeneral' - if mode == 'nzb' and 'nzb' in modelist: - resultlist, _ = IterateOverNewzNabSites(book, searchtype) - if not nprov: - # don't nag. Show warning message no more than every 20 mins - timenow = int(time.time()) - if check_int(lazylibrarian.NO_NZB_MSG, 0) + 1200 < timenow: - logger.warn('No nzb providers are available. Check config and blocklist') - lazylibrarian.NO_NZB_MSG = timenow - modelist.remove('nzb') if resultlist: match = findBestResult(resultlist, book, searchtype, mode) else: match = None - if not goodEnough(match): - logger.info("%s Searches for %s %s returned no results." % - (mode.upper(), book['library'], book['searchterm'])) - else: - logger.info("Found %s result: %s %s%%, %s priority %s" % - (mode.upper(), searchtype, match[0], match[1]['NZBprov'], match[3])) - matches.append(match) + # if you can't find the book under "books", you might find under general search + # general search is the same as booksearch for torrents and rss, no need to check again + if not goodEnough(match): + searchtype = 'general' + if mode == 'nzb' and 'nzb' in modelist: + resultlist, nprov = IterateOverNewzNabSites(book, searchtype) + if not nprov: + # don't nag. Show warning message no more than every 20 mins + timenow = int(time.time()) + if check_int(lazylibrarian.NO_NZB_MSG, 0) + 1200 < timenow: + logger.warn('No nzb providers are available. Check config and blocklist') + lazylibrarian.NO_NZB_MSG = timenow + modelist.remove('nzb') + if resultlist: + match = findBestResult(resultlist, book, searchtype, mode) + else: + match = None + + # if still not found, try general search again without any "(extended details, series etc)" + if not goodEnough(match) and '(' in book['searchterm']: + searchtype = 'shortgeneral' + if mode == 'nzb' and 'nzb' in modelist: + resultlist, _ = IterateOverNewzNabSites(book, searchtype) + if not nprov: + # don't nag. Show warning message no more than every 20 mins + timenow = int(time.time()) + if check_int(lazylibrarian.NO_NZB_MSG, 0) + 1200 < timenow: + logger.warn('No nzb providers are available. Check config and blocklist') + lazylibrarian.NO_NZB_MSG = timenow + modelist.remove('nzb') + if resultlist: + match = findBestResult(resultlist, book, searchtype, mode) + else: + match = None + + if not goodEnough(match): + logger.info("%s Searches for %s %s returned no results." % + (mode.upper(), book['library'], book['searchterm'])) + else: + logger.info("Found %s result: %s %s%%, %s priority %s" % + (mode.upper(), searchtype, match[0], match[1]['NZBprov'], match[3])) + matches.append(match) if matches: highest = max(matches, key=lambda s: (s[0], s[3])) # sort on percentage and priority @@ -314,7 +319,7 @@ def search_book(books=None, library=None): if downloadResult(highest, book) > 1: book_count += 1 # we found it myDB.action("DELETE from failedsearch WHERE BookID=? AND Library=?", (book['bookid'], book['library'])) - elif len(modelist) and lazylibrarian.CONFIG['DELAYSEARCH']: + elif lazylibrarian.CONFIG['DELAYSEARCH'] and do_search and len(modelist): res = myDB.match('SELECT * FROM failedsearch WHERE BookID=? AND Library=?', (book['bookid'], book['library'])) if res: @@ -331,4 +336,4 @@ def search_book(books=None, library=None): except Exception: logger.error('Unhandled exception in search_book: %s' % traceback.format_exc()) finally: - threading.currentThread().name = "WEBSERVER" + threading.currentThread().name = "WEBSERVER" From e11da4ba4ad0c3cfe670d12bfa2c747ac3f0e47f Mon Sep 17 00:00:00 2001 From: Phil Borman Date: Wed, 8 Aug 2018 19:34:44 +0200 Subject: [PATCH 2/7] Move system info to top bar --- data/interfaces/bookstrap/config.html | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/data/interfaces/bookstrap/config.html b/data/interfaces/bookstrap/config.html index f72a120dc..5d38f3e1a 100644 --- a/data/interfaces/bookstrap/config.html +++ b/data/interfaces/bookstrap/config.html @@ -7,6 +7,7 @@
Shutdown Restart + %if lazylibrarian.CONFIG['USER_ACCOUNTS'] == True: @@ -389,12 +390,6 @@

${title}

-
-
-
- -
-
From 5910a51a52156594efdfb88aa69d98f7f31ec67b Mon Sep 17 00:00:00 2001 From: Phil Borman Date: Wed, 8 Aug 2018 19:35:09 +0200 Subject: [PATCH 3/7] Fix filter reset button --- data/interfaces/bookstrap/base.html | 39 ++++++++++++++++++----------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/data/interfaces/bookstrap/base.html b/data/interfaces/bookstrap/base.html index 10856f8a8..e41c30af5 100644 --- a/data/interfaces/bookstrap/base.html +++ b/data/interfaces/bookstrap/base.html @@ -26,23 +26,29 @@ ${next.headIncludes()} - + ${next.javascriptIncludes()}