From 027005314d81885792ca6e80a092925cd89b8af1 Mon Sep 17 00:00:00 2001 From: Phil Borman Date: Fri, 23 Jun 2017 21:09:56 +0200 Subject: [PATCH 01/22] Separate direct downloader from tor --- lazylibrarian/__init__.py | 11 +++++++++-- lazylibrarian/api.py | 6 +++--- lazylibrarian/common.py | 9 +++++---- lazylibrarian/manualbook.py | 10 ++++++++-- lazylibrarian/providers.py | 34 +++++++++++++++++++++++++++++----- lazylibrarian/resultlist.py | 13 +++---------- lazylibrarian/searchbook.py | 26 ++++++++++++++++++-------- lazylibrarian/searchmag.py | 20 +++++++++++++++++++- lazylibrarian/torrentparser.py | 18 ++++++++++++------ lazylibrarian/webServe.py | 17 +++++++++++------ 10 files changed, 117 insertions(+), 47 deletions(-) diff --git a/lazylibrarian/__init__.py b/lazylibrarian/__init__.py index 02bf6ceed..4f3be28c0 100644 --- a/lazylibrarian/__init__.py +++ b/lazylibrarian/__init__.py @@ -996,8 +996,15 @@ def USE_RSS(): def USE_TOR(): count = 0 - for provider in [CONFIG['KAT'], CONFIG['TPB'], CONFIG['ZOO'], CONFIG['LIME'], - CONFIG['TDL'], CONFIG['GEN'], CONFIG['WWT']]: + for provider in [CONFIG['KAT'], CONFIG['TPB'], CONFIG['ZOO'], CONFIG['LIME'], CONFIG['TDL'], CONFIG['WWT']]: + if bool(provider): + count += 1 + return count + + +def USE_DIRECT(): + count = 0 + for provider in [CONFIG['GEN']]: if bool(provider): count += 1 return count diff --git a/lazylibrarian/api.py b/lazylibrarian/api.py index 134fd78f6..6b449344d 100644 --- a/lazylibrarian/api.py +++ b/lazylibrarian/api.py @@ -465,7 +465,7 @@ def _forceActiveAuthorsUpdate(self, **kwargs): threading.Thread(target=dbUpdate, name='API-DBUPDATE', args=[refresh]).start() def _forceMagSearch(self, **kwargs): - if lazylibrarian.USE_NZB() or lazylibrarian.USE_TOR() or lazylibrarian.USE_RSS(): + if lazylibrarian.USE_NZB() or lazylibrarian.USE_TOR() or lazylibrarian.USE_RSS() or lazylibrarian.USE_DIRECT(): if 'wait' in kwargs: search_magazines(None, True) else: @@ -478,7 +478,7 @@ def _forceBookSearch(self, **kwargs): library = kwargs['type'] else: library = None - if lazylibrarian.USE_NZB() or lazylibrarian.USE_TOR() or lazylibrarian.USE_RSS(): + if lazylibrarian.USE_NZB() or lazylibrarian.USE_TOR() or lazylibrarian.USE_RSS() or lazylibrarian.USE_DIRECT(): if 'wait' in kwargs: search_book(library=library) else: @@ -709,7 +709,7 @@ def _searchBook(self, **kwargs): else: library = None - if lazylibrarian.USE_NZB()or lazylibrarian.USE_TOR() or lazylibrarian.USE_RSS(): + if lazylibrarian.USE_NZB()or lazylibrarian.USE_TOR() or lazylibrarian.USE_RSS() or lazylibrarian.USE_DIRECT(): if 'wait' in kwargs: search_book(books=books, library=library) else: diff --git a/lazylibrarian/common.py b/lazylibrarian/common.py index 771f2e7b2..c0e5448a2 100644 --- a/lazylibrarian/common.py +++ b/lazylibrarian/common.py @@ -135,13 +135,14 @@ def scheduleJob(action='Start', target=None): minutes=int(lazylibrarian.CONFIG['SCAN_INTERVAL'])) logger.debug("%s %s job in %s minutes" % (action, target, lazylibrarian.CONFIG['SCAN_INTERVAL'])) elif 'search_magazines' in target and int(lazylibrarian.CONFIG['SEARCH_INTERVAL']): - if lazylibrarian.USE_TOR() or lazylibrarian.USE_NZB() or lazylibrarian.USE_RSS(): + if lazylibrarian.USE_TOR() or lazylibrarian.USE_NZB() \ + or lazylibrarian.USE_RSS() or lazylibrarian.USE_DIRECT(): lazylibrarian.SCHED.add_interval_job( lazylibrarian.searchmag.cron_search_magazines, minutes=int(lazylibrarian.CONFIG['SEARCH_INTERVAL'])) logger.debug("%s %s job in %s minutes" % (action, target, lazylibrarian.CONFIG['SEARCH_INTERVAL'])) elif 'search_book' in target and int(lazylibrarian.CONFIG['SEARCH_INTERVAL']): - if lazylibrarian.USE_NZB() or lazylibrarian.USE_TOR(): + if lazylibrarian.USE_NZB() or lazylibrarian.USE_TOR() or lazylibrarian.USE_DIRECT(): lazylibrarian.SCHED.add_interval_job( lazylibrarian.searchbook.cron_search_book, minutes=int(lazylibrarian.CONFIG['SEARCH_INTERVAL'])) @@ -264,7 +265,7 @@ def checkRunningJobs(): if snatched: ensureRunning('processDir') if wanted: - if lazylibrarian.USE_NZB() or lazylibrarian.USE_TOR(): + if lazylibrarian.USE_NZB() or lazylibrarian.USE_TOR() or lazylibrarian.USE_DIRECT(): ensureRunning('search_book') if lazylibrarian.USE_RSS(): ensureRunning('search_rss_book') @@ -272,7 +273,7 @@ def checkRunningJobs(): scheduleJob('Stop', 'search_book') scheduleJob('Stop', 'search_rss_book') - if lazylibrarian.USE_NZB() or lazylibrarian.USE_TOR() or lazylibrarian.USE_RSS(): + if lazylibrarian.USE_NZB() or lazylibrarian.USE_TOR() or lazylibrarian.USE_RSS() or lazylibrarian.USE_DIRECT(): ensureRunning('search_magazines') else: scheduleJob('Stop', 'search_magazines') diff --git a/lazylibrarian/manualbook.py b/lazylibrarian/manualbook.py index f14adf69d..5c97e540c 100644 --- a/lazylibrarian/manualbook.py +++ b/lazylibrarian/manualbook.py @@ -17,7 +17,8 @@ import urllib from lazylibrarian.formatter import getList, unaccented_str, plural from lazylibrarian import logger, database -from lazylibrarian.providers import IterateOverRSSSites, IterateOverTorrentSites, IterateOverNewzNabSites +from lazylibrarian.providers import IterateOverRSSSites, IterateOverTorrentSites, IterateOverNewzNabSites, \ + IterateOverDirectSites from lib.fuzzywuzzy import fuzz @@ -56,7 +57,8 @@ def searchItem(item=None, bookid=None, cat=None): logger.debug('Forcing general search') cat = 'general' - nproviders = lazylibrarian.USE_NZB() + lazylibrarian.USE_TOR() + lazylibrarian.USE_RSS() + nproviders = lazylibrarian.USE_NZB() + lazylibrarian.USE_TOR() + \ + lazylibrarian.USE_RSS() + lazylibrarian.USE_DIRECT() logger.debug('Searching %s provider%s (%s) for %s' % (nproviders, plural(nproviders), cat, searchterm)) if lazylibrarian.USE_NZB(): @@ -67,6 +69,10 @@ def searchItem(item=None, bookid=None, cat=None): resultlist, nproviders = IterateOverTorrentSites(book, cat) if nproviders: results += resultlist + if lazylibrarian.USE_DIRECT(): + resultlist, nproviders = IterateOverDirectSites(book, cat) + if nproviders: + results += resultlist if lazylibrarian.USE_RSS(): resultlist, nproviders = IterateOverRSSSites() if nproviders: diff --git a/lazylibrarian/providers.py b/lazylibrarian/providers.py index 46098a9b9..2d7959222 100644 --- a/lazylibrarian/providers.py +++ b/lazylibrarian/providers.py @@ -25,8 +25,8 @@ def get_searchterm(book, searchType): - authorname = cleanName(book['authorName']) - bookname = cleanName(book['bookName']) + authorname = cleanName(book['authorName'], "'") + bookname = cleanName(book['bookName'], "'") if searchType in ['book', 'audio'] or 'short' in searchType: if bookname == authorname and book['bookSub']: # books like "Spike Milligan: Man of Letters" @@ -240,7 +240,7 @@ def IterateOverTorrentSites(book=None, searchType=None): authorname, bookname = get_searchterm(book, searchType) book['searchterm'] = authorname + ' ' + bookname - for prov in ['KAT', 'WWT', 'TPB', 'ZOO', 'TDL', 'GEN', 'LIME']: + for prov in ['KAT', 'WWT', 'TPB', 'ZOO', 'TDL', 'LIME']: if lazylibrarian.CONFIG[prov] and not ProviderIsBlocked(prov): logger.debug('[IterateOverTorrentSites] - %s' % lazylibrarian.CONFIG[prov + '_HOST']) if prov == 'KAT': @@ -255,8 +255,6 @@ def IterateOverTorrentSites(book=None, searchType=None): # results, error = EXTRA(book) elif prov == 'TDL': results, error = TDL(book) - elif prov == 'GEN': - results, error = GEN(book) elif prov == 'LIME': results, error = LIME(book) else: @@ -273,6 +271,32 @@ def IterateOverTorrentSites(book=None, searchType=None): return resultslist, providers +def IterateOverDirectSites(book=None, searchType=None): + resultslist = [] + providers = 0 + if searchType != 'mag' and searchType != 'general': + authorname, bookname = get_searchterm(book, searchType) + book['searchterm'] = authorname + ' ' + bookname + + for prov in ['GEN']: + if lazylibrarian.CONFIG[prov] and not ProviderIsBlocked(prov): + logger.debug('[IterateOverTorrentSites] - %s' % lazylibrarian.CONFIG[prov + '_HOST']) + if prov == 'GEN': + results, error = GEN(book) + else: + results = '' + error = '' + logger.error('IterateOverDirectSites called with unknown provider [%s]' % prov) + + if error: + BlockProvider(prov, error) + else: + resultslist += results + providers += 1 + + return resultslist, providers + + def IterateOverRSSSites(): resultslist = [] providers = 0 diff --git a/lazylibrarian/resultlist.py b/lazylibrarian/resultlist.py index 386d6de84..169f5d56b 100644 --- a/lazylibrarian/resultlist.py +++ b/lazylibrarian/resultlist.py @@ -84,9 +84,7 @@ def findBestResult(resultlist, book, searchtype, source): if source == 'nzb': prefix = 'nzb' - elif source == 'tor': - prefix = 'tor_' - else: # rss returns torrents + else: # rss returns same names as torrents prefix = 'tor_' logger.debug('Searching %s %s results for best %s match' % (len(resultlist), source, auxinfo)) @@ -97,11 +95,8 @@ def findBestResult(resultlist, book, searchtype, source): resultTitle = re.sub(r"\s\s+", " ", resultTitle) # remove extra whitespace Author_match = fuzz.token_set_ratio(author, resultTitle) Book_match = fuzz.token_set_ratio(title, resultTitle) - stype = source.upper() - if res[prefix + 'prov'] == 'libgen': - stype = "DIR" logger.debug(u"%s author/book Match: %s/%s for %s at %s" % - (stype, Author_match, Book_match, resultTitle, res[prefix + 'prov'])) + (source.upper(), Author_match, Book_match, resultTitle, res[prefix + 'prov'])) rejected = False @@ -145,10 +140,8 @@ def findBestResult(resultlist, book, searchtype, source): if source == 'nzb': mode = res['nzbmode'] # nzb, torznab - elif source == 'tor': - mode = res['tor_type'] # torrent, magnet, direct else: - mode = res['tor_type'] # torrent, magnet, nzb + mode = res['tor_type'] # torrent, magnet, nzb(from rss), direct controlValueDict = {"NZBurl": url} newValueDict = { diff --git a/lazylibrarian/searchbook.py b/lazylibrarian/searchbook.py index 637844167..a3852d7d0 100644 --- a/lazylibrarian/searchbook.py +++ b/lazylibrarian/searchbook.py @@ -19,7 +19,8 @@ import lazylibrarian from lazylibrarian import logger, database from lazylibrarian.formatter import plural -from lazylibrarian.providers import IterateOverNewzNabSites, IterateOverTorrentSites, IterateOverRSSSites +from lazylibrarian.providers import IterateOverNewzNabSites, IterateOverTorrentSites, IterateOverRSSSites, \ + IterateOverDirectSites from lazylibrarian.resultlist import findBestResult, downloadResult @@ -40,7 +41,6 @@ def search_book(books=None, library=None): library is "eBook" or "AudioBook" or None to search all book types """ # noinspection PyBroadException - print "***",books,library try: threadname = threading.currentThread().name if "Thread-" in threadname: @@ -79,7 +79,8 @@ def search_book(books=None, library=None): logger.debug("SearchBooks - No books to search for") return - nproviders = lazylibrarian.USE_NZB() + lazylibrarian.USE_TOR() + lazylibrarian.USE_RSS() + nproviders = lazylibrarian.USE_NZB() + lazylibrarian.USE_TOR() + \ + lazylibrarian.USE_RSS() + lazylibrarian.USE_DIRECT() if nproviders == 0: logger.debug("SearchBooks - No providers to search") @@ -90,6 +91,8 @@ def search_book(books=None, library=None): modelist.append('nzb') if lazylibrarian.USE_TOR(): modelist.append('tor') + if lazylibrarian.USE_DIRECT(): + modelist.append('direct') if lazylibrarian.USE_RSS(): modelist.append('rss') @@ -150,6 +153,11 @@ def search_book(books=None, library=None): if not nproviders: logger.debug("No active tor providers found") modelist.remove('tor') + elif mode == 'direct': + resultlist, nproviders = IterateOverDirectSites(book, searchtype) + if not nproviders: + logger.debug("No active direct providers found") + modelist.remove('direct') elif mode == 'rss': if rss_resultlist: resultlist = rss_resultlist @@ -174,7 +182,12 @@ def search_book(books=None, library=None): resultlist, nproviders = IterateOverTorrentSites(book, searchtype) if not nproviders: logger.debug("No active tor providers found") - modelist.remove('nzb') + modelist.remove('tor') + elif mode == 'direct': + resultlist, nproviders = IterateOverDirectSites(book, searchtype) + if not nproviders: + logger.debug("No active direct providers found") + modelist.remove('direct') elif mode == 'rss': resultlist = rss_resultlist @@ -214,11 +227,8 @@ def search_book(books=None, library=None): logger.info("%s Searches for %s %s returned no results." % (mode.upper(), book['library'], book['searchterm'])) else: - smode = mode.upper() - if match[2]['NZBprov'] == 'libgen': - smode = 'DIR' logger.info("Found %s result: %s %s%%, %s priority %s" % - (smode, searchtype, match[0], match[2]['NZBprov'], match[4])) + (mode.upper(), searchtype, match[0], match[2]['NZBprov'], match[4])) matches.append(match) if matches: diff --git a/lazylibrarian/searchmag.py b/lazylibrarian/searchmag.py index 381167bfb..3af15f4eb 100644 --- a/lazylibrarian/searchmag.py +++ b/lazylibrarian/searchmag.py @@ -25,7 +25,8 @@ from lazylibrarian.formatter import plural, now, unaccented_str, replace_all, unaccented, \ nzbdate2format, getList, month2num, datecompare, check_int, check_year from lazylibrarian.notifiers import notify_snatch, custom_notify_snatch -from lazylibrarian.providers import IterateOverNewzNabSites, IterateOverTorrentSites, IterateOverRSSSites +from lazylibrarian.providers import IterateOverNewzNabSites, IterateOverTorrentSites, IterateOverRSSSites, \ + IterateOverDirectSites from lazylibrarian.downloadmethods import NZBDownloadMethod, TORDownloadMethod from lib.fuzzywuzzy import fuzz @@ -94,6 +95,23 @@ def search_magazines(mags=None, reset=False): if not nproviders: logger.warn('No nzb providers are set. Check config for NEWZNAB or TORZNAB providers') + if lazylibrarian.USE_DIRECT(): + dir_resultlist, nproviders = IterateOverDirectSites(book, 'mag') + if not nproviders: + logger.warn('No direct providers are set. Check config for DIRECT providers') + + if dir_resultlist: + for item in dir_resultlist: # reformat the results so they look like nzbs + resultlist.append({ + 'bookid': item['bookid'], + 'nzbprov': item['tor_prov'], + 'nzbtitle': item['tor_title'], + 'nzburl': item['tor_url'], + 'nzbdate': 'Fri, 01 Jan 1970 00:00:00 +0100', # fake date as none returned + 'nzbsize': item['tor_size'], + 'nzbmode': 'torrent' + }) + if lazylibrarian.USE_TOR(): tor_resultlist, nproviders = IterateOverTorrentSites(book, 'mag') if not nproviders: diff --git a/lazylibrarian/torrentparser.py b/lazylibrarian/torrentparser.py index b22a06d6d..27232ad41 100644 --- a/lazylibrarian/torrentparser.py +++ b/lazylibrarian/torrentparser.py @@ -22,7 +22,7 @@ from lazylibrarian import logger from lazylibrarian.cache import fetchURL from lazylibrarian.formatter import plural, unaccented, formatAuthorName -from lib.BeautifulSoup import BeautifulSoup +from lib.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup def url_fix(s, charset='utf-8'): @@ -731,8 +731,11 @@ def GEN(book=None): td = row.findAll('td') if 'index.php' in search and len(td) > 3: try: - author = formatAuthorName(unaccented(td[0].text)) - title = unaccented(td[2].text) + res = str(BeautifulStoneSoup(td[0].text, + convertEntities=BeautifulStoneSoup.HTML_ENTITIES)) + author = formatAuthorName(res) + title = str(BeautifulStoneSoup(td[2].text, + convertEntities=BeautifulStoneSoup.HTML_ENTITIES)) temp = str(td[4]) temp = temp.split('onmouseout')[1] extn = temp.split('">')[1].split('(')[0] @@ -744,8 +747,11 @@ def GEN(book=None): elif 'search.php' in search and len(td) > 8: try: - author = formatAuthorName(unaccented(td[1].text)) - title = unaccented(str(td[2]).split('>')[2].split('<')[0].strip()) + res = str(BeautifulStoneSoup(td[1].text, + convertEntities=BeautifulStoneSoup.HTML_ENTITIES)) + author = formatAuthorName(res) + title = str(td[2]).split('>')[2].split('<')[0].strip() + title = BeautifulStoneSoup(title, convertEntities=BeautifulStoneSoup.HTML_ENTITIES) link = str(td[2]).split('href="')[1].split('?')[1].split('"')[0] size = unaccented(td[7].text).upper() extn = td[8].text @@ -815,7 +821,7 @@ def GEN(book=None): 'tor_title': title, 'tor_url': url, 'tor_size': str(size), - 'tor_type': 'download', + 'tor_type': 'direct', 'priority': lazylibrarian.CONFIG['GEN_DLPRIORITY'] }) logger.debug('Found %s, Size %s' % (title, size)) diff --git a/lazylibrarian/webServe.py b/lazylibrarian/webServe.py index 8e6ca6e91..3ce8e46ae 100644 --- a/lazylibrarian/webServe.py +++ b/lazylibrarian/webServe.py @@ -656,7 +656,7 @@ def booksearch(self, author=None, title=None, bookid=None, action=None): @cherrypy.expose def countProviders(self): cherrypy.response.headers['Cache-Control'] = "max-age=0,no-cache,no-store" - count = lazylibrarian.USE_NZB() + lazylibrarian.USE_TOR() + lazylibrarian.USE_RSS() + count = lazylibrarian.USE_NZB() + lazylibrarian.USE_TOR() + lazylibrarian.USE_RSS() + lazylibrarian.USE_DIRECT() return "Searching %s providers, please wait..." % count @cherrypy.expose @@ -891,7 +891,8 @@ def addBook(self, bookid=None): @cherrypy.expose def startBookSearch(self, books=None): if books: - if lazylibrarian.USE_NZB() or lazylibrarian.USE_TOR() or lazylibrarian.USE_RSS(): + if lazylibrarian.USE_NZB() or lazylibrarian.USE_TOR() \ + or lazylibrarian.USE_RSS() or lazylibrarian.USE_DIRECT(): threading.Thread(target=search_book, name='SEARCHBOOK', args=[books]).start() logger.debug(u"Searching for book with id: " + books[0]["bookid"]) else: @@ -1258,7 +1259,8 @@ def markBooks(self, AuthorID=None, seriesid=None, action=None, redirect=None, ** if not bookid == 'book_table_length': books.append({"bookid": bookid}) - if lazylibrarian.USE_NZB() or lazylibrarian.USE_TOR() or lazylibrarian.USE_RSS(): + if lazylibrarian.USE_NZB() or lazylibrarian.USE_TOR() \ + or lazylibrarian.USE_RSS() or lazylibrarian.USE_DIRECT(): threading.Thread(target=search_book, name='SEARCHBOOK', args=[books]).start() if redirect == "author": @@ -1694,7 +1696,8 @@ def searchForMag(self, bookid=None): @cherrypy.expose def startMagazineSearch(self, mags=None): if mags: - if lazylibrarian.USE_NZB() or lazylibrarian.USE_TOR() or lazylibrarian.USE_RSS(): + if lazylibrarian.USE_NZB() or lazylibrarian.USE_TOR() \ + or lazylibrarian.USE_RSS() or lazylibrarian.USE_DIRECT(): threading.Thread(target=search_magazines, name='SEARCHMAG', args=[mags, False]).start() logger.debug(u"Searching for magazine with title: %s" % mags[0]["bookid"]) else: @@ -2244,11 +2247,13 @@ def forceProcess(self, source=None): @cherrypy.expose def forceSearch(self, source=None): if source == "magazines": - if lazylibrarian.USE_NZB() or lazylibrarian.USE_TOR() or lazylibrarian.USE_RSS(): + if lazylibrarian.USE_NZB() or lazylibrarian.USE_TOR() \ + or lazylibrarian.USE_RSS() or lazylibrarian.USE_DIRECT(): if 'SEARCHALLMAG' not in [n.name for n in [t for t in threading.enumerate()]]: threading.Thread(target=search_magazines, name='SEARCHALLMAG', args=[]).start() elif source in ["books", "audio"]: - if lazylibrarian.USE_NZB() or lazylibrarian.USE_TOR() or lazylibrarian.USE_RSS(): + if lazylibrarian.USE_NZB() or lazylibrarian.USE_TOR() \ + or lazylibrarian.USE_RSS() or lazylibrarian.USE_DIRECT(): if 'SEARCHALLBOOKS' not in [n.name for n in [t for t in threading.enumerate()]]: threading.Thread(target=search_book, name='SEARCHALLBOOKS', args=[]).start() else: From e238945693d8655c36b10d859e25b592b04fdbef Mon Sep 17 00:00:00 2001 From: Phil Borman Date: Fri, 23 Jun 2017 21:54:05 +0200 Subject: [PATCH 02/22] Moved direct provider into separate file --- lazylibrarian/directparser.py | 227 +++++++++++++++++++++++++++++++++ lazylibrarian/providers.py | 5 +- lazylibrarian/resultlist.py | 2 +- lazylibrarian/torrentparser.py | 198 +--------------------------- 4 files changed, 233 insertions(+), 199 deletions(-) create mode 100644 lazylibrarian/directparser.py diff --git a/lazylibrarian/directparser.py b/lazylibrarian/directparser.py new file mode 100644 index 000000000..a7d7b3ff7 --- /dev/null +++ b/lazylibrarian/directparser.py @@ -0,0 +1,227 @@ +# This file is part of Lazylibrarian. +# +# Lazylibrarian is free software':'you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Lazylibrarian is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Lazylibrarian. If not, see . + +import urllib +import urlparse +import traceback + +import lazylibrarian +from lazylibrarian import logger +from lazylibrarian.cache import fetchURL +from lazylibrarian.formatter import plural, unaccented, formatAuthorName +from lib.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup + + +def url_fix(s, charset='utf-8'): + if isinstance(s, unicode): + s = s.encode(charset, 'ignore') + scheme, netloc, path, qs, anchor = urlparse.urlsplit(s) + path = urllib.quote(path, '/%') + qs = urllib.quote_plus(qs, ':&=') + return urlparse.urlunsplit((scheme, netloc, path, qs, anchor)) + + +def GEN(book=None): + errmsg = '' + provider = "libgen" + host = lazylibrarian.CONFIG['GEN_HOST'] + search = lazylibrarian.CONFIG['GEN_SEARCH'] + if not str(host)[:4] == "http": + host = 'http://' + host + + page = 1 + results = [] + next_page = True + + while next_page: + if not search or not search.endswith('.php'): + search = 'search.php' + if not 'index.php' in search and not 'search.php' in search: + search = 'search.php' + if search[0] == '/': + search = search[1:] + + if 'index.php' in search: + params = { + "s": book['searchterm'] + } + if page > 1: + params['page'] = page + + providerurl = url_fix(host + "/%s" % search) + searchURL = providerurl + "?%s" % urllib.urlencode(params) + else: + params = { + "view": "simple", + "open": 0, + "phrase": 0, + "column": "def", + "res": 100, + "req": book['searchterm'] + } + if page > 1: + params['page'] = page + + providerurl = url_fix(host + "/%s" % search) + searchURL = providerurl + "?%s" % urllib.urlencode(params) + + next_page = False + result, success = fetchURL(searchURL) + if not success: + # may return 404 if no results, not really an error + if '404' in result: + logger.debug(u"No results found from %s for %s" % (provider, book['searchterm'])) + elif '111' in result: + # looks like libgen has ip based access limits + logger.error('Access forbidden. Please wait a while before trying %s again.' % provider) + errmsg = result + else: + logger.debug(searchURL) + logger.debug('Error fetching page data from %s: %s' % (provider, result)) + errmsg = result + + result = False + + if result: + logger.debug(u'Parsing results from %s' % (searchURL, provider)) + try: + soup = BeautifulSoup(result) + try: + table = soup.findAll('table')[2] # un-named table + if table: + rows = table.findAll('tr') + except IndexError: # no results table in result page + rows = [] + + if 'search.php' in search and len(rows) > 1: + rows = rows[1:] + + for row in rows: + author = '' + title = '' + size = '' + extn = '' + link = '' + td = row.findAll('td') + if 'index.php' in search and len(td) > 3: + try: + res = str(BeautifulStoneSoup(td[0].text, + convertEntities=BeautifulStoneSoup.HTML_ENTITIES)) + author = formatAuthorName(res) + title = str(BeautifulStoneSoup(td[2].text, + convertEntities=BeautifulStoneSoup.HTML_ENTITIES)) + temp = str(td[4]) + temp = temp.split('onmouseout')[1] + extn = temp.split('">')[1].split('(')[0] + size = temp.split('">')[1].split('(')[1].split(')')[0] + size = size.upper() + link = temp.split('href=')[1].split('"')[1] + except IndexError as e: + logger.debug('Error parsing libgen index.php results: %s' % str(e)) + + elif 'search.php' in search and len(td) > 8: + try: + res = str(BeautifulStoneSoup(td[1].text, + convertEntities=BeautifulStoneSoup.HTML_ENTITIES)) + author = formatAuthorName(res) + title = str(td[2]).split('>')[2].split('<')[0].strip() + title = BeautifulStoneSoup(title, convertEntities=BeautifulStoneSoup.HTML_ENTITIES) + link = str(td[2]).split('href="')[1].split('?')[1].split('"')[0] + size = unaccented(td[7].text).upper() + extn = td[8].text + except IndexError as e: + logger.debug('Error parsing libgen search.php results; %s' % str(e)) + + if not size: + size = 0 + else: + try: + mult = 1 + if 'K' in size: + size = size.split('K')[0] + mult = 1024 + elif 'M' in size: + size = size.split('M')[0] + mult = 1024 * 1024 + elif 'G' in size: + size = size.split('G')[0] + mult = 1024 * 1024 * 1024 + size = int(float(size) * mult) + except (ValueError, IndexError): + size = 0 + + if link and title: + if author: + title = author.strip() + ' ' + title.strip() + if extn: + title = title + '.' + extn + + if "/ads.php?" in link: + url = url_fix(host + link) + else: + url = url_fix(host + "/ads.php?" + link) + bookresult, success = fetchURL(url) + if not success: + # may return 404 if no results, not really an error + if '404' in bookresult: + logger.debug(u"No results found from %s for %s" % (provider, book['searchterm'])) + else: + logger.debug(url) + logger.debug('Error fetching link data from %s: %s' % (provider, bookresult)) + errmsg = bookresult + bookresult = False + + if bookresult: + url = None + new_soup = BeautifulSoup(bookresult) + for link in new_soup.findAll('a'): + output = link.get('href') + if output: + if output.startswith('http') and '/get.php' in output: + url = output + break + elif '/get.php' in output: + url = '/get.php' + output.split('/get.php')[1] + break + elif '/download/book' in output: + url = '/download/book' + output.split('/download/book')[1] + break + if url and not url.startswith('http'): + url = url_fix(host + url) + + results.append({ + 'bookid': book['bookid'], + 'tor_prov': provider, + 'tor_title': title, + 'tor_url': url, + 'tor_size': str(size), + 'tor_type': 'direct', + 'priority': lazylibrarian.CONFIG['GEN_DLPRIORITY'] + }) + logger.debug('Found %s, Size %s' % (title, size)) + next_page = True + + except Exception as e: + logger.error(u"An error occurred in the %s parser: %s" % (provider, str(e))) + logger.debug('%s: %s' % (provider, traceback.format_exc())) + + page += 1 + if 0 < lazylibrarian.CONFIG['MAX_PAGES'] < page: + logger.warn('Maximum results page search reached, still more results available') + next_page = False + + logger.debug(u"Found %i result%s from %s for %s" % + (len(results), plural(len(results)), provider, book['searchterm'])) + return results, errmsg diff --git a/lazylibrarian/providers.py b/lazylibrarian/providers.py index 2d7959222..d2de42c24 100644 --- a/lazylibrarian/providers.py +++ b/lazylibrarian/providers.py @@ -21,7 +21,8 @@ from lazylibrarian import logger from lazylibrarian.cache import fetchURL from lazylibrarian.formatter import age, today, plural, cleanName, unaccented, getList, check_int -from lazylibrarian.torrentparser import KAT, WWT, TPB, ZOO, TDL, GEN, LIME +from lazylibrarian.torrentparser import KAT, WWT, TPB, ZOO, TDL, LIME +from lazylibrarian.directparser import GEN def get_searchterm(book, searchType): @@ -280,7 +281,7 @@ def IterateOverDirectSites(book=None, searchType=None): for prov in ['GEN']: if lazylibrarian.CONFIG[prov] and not ProviderIsBlocked(prov): - logger.debug('[IterateOverTorrentSites] - %s' % lazylibrarian.CONFIG[prov + '_HOST']) + logger.debug('[IterateOverDirectSites] - %s' % lazylibrarian.CONFIG[prov + '_HOST']) if prov == 'GEN': results, error = GEN(book) else: diff --git a/lazylibrarian/resultlist.py b/lazylibrarian/resultlist.py index 169f5d56b..82ca0068d 100644 --- a/lazylibrarian/resultlist.py +++ b/lazylibrarian/resultlist.py @@ -95,7 +95,7 @@ def findBestResult(resultlist, book, searchtype, source): resultTitle = re.sub(r"\s\s+", " ", resultTitle) # remove extra whitespace Author_match = fuzz.token_set_ratio(author, resultTitle) Book_match = fuzz.token_set_ratio(title, resultTitle) - logger.debug(u"%s author/book Match: %s/%s for %s at %s" % + logger.debug(u"%s author/book Match: %s/%s %s at %s" % (source.upper(), Author_match, Book_match, resultTitle, res[prefix + 'prov'])) rejected = False diff --git a/lazylibrarian/torrentparser.py b/lazylibrarian/torrentparser.py index 27232ad41..290a703cd 100644 --- a/lazylibrarian/torrentparser.py +++ b/lazylibrarian/torrentparser.py @@ -21,8 +21,8 @@ import lib.feedparser as feedparser from lazylibrarian import logger from lazylibrarian.cache import fetchURL -from lazylibrarian.formatter import plural, unaccented, formatAuthorName -from lib.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup +from lazylibrarian.formatter import plural, unaccented +from lib.BeautifulSoup import BeautifulSoup def url_fix(s, charset='utf-8'): @@ -647,200 +647,6 @@ def LIME(book=None): return results, errmsg -def GEN(book=None): - errmsg = '' - provider = "libgen" - host = lazylibrarian.CONFIG['GEN_HOST'] - search = lazylibrarian.CONFIG['GEN_SEARCH'] - if not str(host)[:4] == "http": - host = 'http://' + host - - page = 1 - results = [] - next_page = True - - while next_page: - if not search or not search.endswith('.php'): - search = 'search.php' - if not 'index.php' in search and not 'search.php' in search: - search = 'search.php' - if search[0] == '/': - search = search[1:] - - if 'index.php' in search: - params = { - "s": book['searchterm'] - } - if page > 1: - params['page'] = page - - providerurl = url_fix(host + "/%s" % search) - searchURL = providerurl + "?%s" % urllib.urlencode(params) - else: - params = { - "view": "simple", - "open": 0, - "phrase": 0, - "column": "def", - "res": 100, - "req": book['searchterm'] - } - if page > 1: - params['page'] = page - - providerurl = url_fix(host + "/%s" % search) - searchURL = providerurl + "?%s" % urllib.urlencode(params) - - next_page = False - result, success = fetchURL(searchURL) - if not success: - # may return 404 if no results, not really an error - if '404' in result: - logger.debug(u"No results found from %s for %s" % (provider, book['searchterm'])) - elif '111' in result: - # looks like libgen has ip based access limits - logger.error('Access forbidden. Please wait a while before trying %s again.' % provider) - errmsg = result - else: - logger.debug(searchURL) - logger.debug('Error fetching page data from %s: %s' % (provider, result)) - errmsg = result - - result = False - - if result: - logger.debug(u'Parsing results from %s' % (searchURL, provider)) - try: - soup = BeautifulSoup(result) - try: - table = soup.findAll('table')[2] # un-named table - if table: - rows = table.findAll('tr') - except IndexError: # no results table in result page - rows = [] - - if 'search.php' in search and len(rows) > 1: - rows = rows[1:] - - for row in rows: - author = '' - title = '' - size = '' - extn = '' - link = '' - td = row.findAll('td') - if 'index.php' in search and len(td) > 3: - try: - res = str(BeautifulStoneSoup(td[0].text, - convertEntities=BeautifulStoneSoup.HTML_ENTITIES)) - author = formatAuthorName(res) - title = str(BeautifulStoneSoup(td[2].text, - convertEntities=BeautifulStoneSoup.HTML_ENTITIES)) - temp = str(td[4]) - temp = temp.split('onmouseout')[1] - extn = temp.split('">')[1].split('(')[0] - size = temp.split('">')[1].split('(')[1].split(')')[0] - size = size.upper() - link = temp.split('href=')[1].split('"')[1] - except IndexError as e: - logger.debug('Error parsing libgen index.php results: %s' % str(e)) - - elif 'search.php' in search and len(td) > 8: - try: - res = str(BeautifulStoneSoup(td[1].text, - convertEntities=BeautifulStoneSoup.HTML_ENTITIES)) - author = formatAuthorName(res) - title = str(td[2]).split('>')[2].split('<')[0].strip() - title = BeautifulStoneSoup(title, convertEntities=BeautifulStoneSoup.HTML_ENTITIES) - link = str(td[2]).split('href="')[1].split('?')[1].split('"')[0] - size = unaccented(td[7].text).upper() - extn = td[8].text - except IndexError as e: - logger.debug('Error parsing libgen search.php results; %s' % str(e)) - - if not size: - size = 0 - else: - try: - mult = 1 - if 'K' in size: - size = size.split('K')[0] - mult = 1024 - elif 'M' in size: - size = size.split('M')[0] - mult = 1024 * 1024 - elif 'G' in size: - size = size.split('G')[0] - mult = 1024 * 1024 * 1024 - size = int(float(size) * mult) - except (ValueError, IndexError): - size = 0 - - if link and title: - if author: - title = author.strip() + ' ' + title.strip() - if extn: - title = title + '.' + extn - - if "/ads.php?" in link: - url = url_fix(host + link) - else: - url = url_fix(host + "/ads.php?" + link) - bookresult, success = fetchURL(url) - if not success: - # may return 404 if no results, not really an error - if '404' in bookresult: - logger.debug(u"No results found from %s for %s" % (provider, book['searchterm'])) - else: - logger.debug(url) - logger.debug('Error fetching link data from %s: %s' % (provider, bookresult)) - errmsg = bookresult - bookresult = False - - if bookresult: - url = None - new_soup = BeautifulSoup(bookresult) - for link in new_soup.findAll('a'): - output = link.get('href') - if output: - if output.startswith('http') and '/get.php' in output: - url = output - break - elif '/get.php' in output: - url = '/get.php' + output.split('/get.php')[1] - break - elif '/download/book' in output: - url = '/download/book' + output.split('/download/book')[1] - break - if url and not url.startswith('http'): - url = url_fix(host + url) - - results.append({ - 'bookid': book['bookid'], - 'tor_prov': provider, - 'tor_title': title, - 'tor_url': url, - 'tor_size': str(size), - 'tor_type': 'direct', - 'priority': lazylibrarian.CONFIG['GEN_DLPRIORITY'] - }) - logger.debug('Found %s, Size %s' % (title, size)) - next_page = True - - except Exception as e: - logger.error(u"An error occurred in the %s parser: %s" % (provider, str(e))) - logger.debug('%s: %s' % (provider, traceback.format_exc())) - - page += 1 - if 0 < lazylibrarian.CONFIG['MAX_PAGES'] < page: - logger.warn('Maximum results page search reached, still more results available') - next_page = False - - logger.debug(u"Found %i result%s from %s for %s" % - (len(results), plural(len(results)), provider, book['searchterm'])) - return results, errmsg - - def TDL(book=None): errmsg = '' provider = "torrentdownloads" From 4cd9c7f1b1efdf3ee1d8c6eac838a60a33fa3b64 Mon Sep 17 00:00:00 2001 From: Phil Borman Date: Fri, 23 Jun 2017 22:07:13 +0200 Subject: [PATCH 03/22] Code tidying --- lazylibrarian/directparser.py | 8 ++++---- lazylibrarian/librarysync.py | 3 ++- lazylibrarian/manualbook.py | 6 +++--- lazylibrarian/nzbget.py | 3 ++- lazylibrarian/searchbook.py | 3 +-- lazylibrarian/webServe.py | 14 +++++++------- 6 files changed, 19 insertions(+), 18 deletions(-) diff --git a/lazylibrarian/directparser.py b/lazylibrarian/directparser.py index a7d7b3ff7..1e7225c54 100644 --- a/lazylibrarian/directparser.py +++ b/lazylibrarian/directparser.py @@ -48,7 +48,7 @@ def GEN(book=None): while next_page: if not search or not search.endswith('.php'): search = 'search.php' - if not 'index.php' in search and not 'search.php' in search: + if 'index.php' not in search and 'search.php' not in search: search = 'search.php' if search[0] == '/': search = search[1:] @@ -118,10 +118,10 @@ def GEN(book=None): if 'index.php' in search and len(td) > 3: try: res = str(BeautifulStoneSoup(td[0].text, - convertEntities=BeautifulStoneSoup.HTML_ENTITIES)) + convertEntities=BeautifulStoneSoup.HTML_ENTITIES)) author = formatAuthorName(res) title = str(BeautifulStoneSoup(td[2].text, - convertEntities=BeautifulStoneSoup.HTML_ENTITIES)) + convertEntities=BeautifulStoneSoup.HTML_ENTITIES)) temp = str(td[4]) temp = temp.split('onmouseout')[1] extn = temp.split('">')[1].split('(')[0] @@ -134,7 +134,7 @@ def GEN(book=None): elif 'search.php' in search and len(td) > 8: try: res = str(BeautifulStoneSoup(td[1].text, - convertEntities=BeautifulStoneSoup.HTML_ENTITIES)) + convertEntities=BeautifulStoneSoup.HTML_ENTITIES)) author = formatAuthorName(res) title = str(td[2]).split('>')[2].split('<')[0].strip() title = BeautifulStoneSoup(title, convertEntities=BeautifulStoneSoup.HTML_ENTITIES) diff --git a/lazylibrarian/librarysync.py b/lazylibrarian/librarysync.py index 32f444f5e..44a73079d 100644 --- a/lazylibrarian/librarysync.py +++ b/lazylibrarian/librarysync.py @@ -683,7 +683,8 @@ def LibraryScan(startdir=None, library='eBook'): myDB.action( 'UPDATE books set Status="Open" where BookID="%s"' % bookid) myDB.action( - 'UPDATE books set BookLibrary="%s" where BookID="%s"' % (now(), bookid)) + 'UPDATE books set BookLibrary="%s" where BookID="%s"' % + (now(), bookid)) # check and store book location so we can check if it gets (re)moved book_filename = os.path.join(r, files) diff --git a/lazylibrarian/manualbook.py b/lazylibrarian/manualbook.py index 5c97e540c..25f0233a1 100644 --- a/lazylibrarian/manualbook.py +++ b/lazylibrarian/manualbook.py @@ -58,7 +58,7 @@ def searchItem(item=None, bookid=None, cat=None): cat = 'general' nproviders = lazylibrarian.USE_NZB() + lazylibrarian.USE_TOR() + \ - lazylibrarian.USE_RSS() + lazylibrarian.USE_DIRECT() + lazylibrarian.USE_RSS() + lazylibrarian.USE_DIRECT() logger.debug('Searching %s provider%s (%s) for %s' % (nproviders, plural(nproviders), cat, searchterm)) if lazylibrarian.USE_NZB(): @@ -133,8 +133,8 @@ def searchItem(item=None, bookid=None, cat=None): searchresults.append(result) - # from operator import itemgetter - # searchresults = sorted(searchresults, key=itemgetter('score'), reverse=True) + # from operator import itemgetter + # searchresults = sorted(searchresults, key=itemgetter('score'), reverse=True) logger.debug('Found %s %s results for %s' % (len(searchresults), cat, searchterm)) return searchresults diff --git a/lazylibrarian/nzbget.py b/lazylibrarian/nzbget.py index a87235a55..41781ca8e 100644 --- a/lazylibrarian/nzbget.py +++ b/lazylibrarian/nzbget.py @@ -166,7 +166,8 @@ def sendNZB(nzb, cmd=None, nzbID=None): # (Positive number representing NZBID of the queue item. 0 and negative numbers represent error codes.) elif nzbget_version >= 13: nzbget_result = nzbGetRPC.append(nzb.name + ".nzb", nzbcontent64 if nzbcontent64 is not None - else nzb.url, lazylibrarian.CONFIG['NZBGET_CATEGORY'], lazylibrarian.CONFIG['NZBGET_PRIORITY'], False, + else nzb.url, lazylibrarian.CONFIG['NZBGET_CATEGORY'], + lazylibrarian.CONFIG['NZBGET_PRIORITY'], False, False, dupekey, dupescore, "score") if nzbget_result <= 0: nzbget_result = False diff --git a/lazylibrarian/searchbook.py b/lazylibrarian/searchbook.py index a3852d7d0..84c7ea64d 100644 --- a/lazylibrarian/searchbook.py +++ b/lazylibrarian/searchbook.py @@ -74,13 +74,12 @@ def search_book(books=None, library=None): else: logger.debug("SearchBooks - BookID %s is not in the database" % book['bookid']) - if len(searchbooks) == 0: logger.debug("SearchBooks - No books to search for") return nproviders = lazylibrarian.USE_NZB() + lazylibrarian.USE_TOR() + \ - lazylibrarian.USE_RSS() + lazylibrarian.USE_DIRECT() + lazylibrarian.USE_RSS() + lazylibrarian.USE_DIRECT() if nproviders == 0: logger.debug("SearchBooks - No providers to search") diff --git a/lazylibrarian/webServe.py b/lazylibrarian/webServe.py index 3ce8e46ae..bd4fd0e6d 100644 --- a/lazylibrarian/webServe.py +++ b/lazylibrarian/webServe.py @@ -651,7 +651,7 @@ def booksearch(self, author=None, title=None, bookid=None, action=None): if action.startswith('a_'): library = 'AudioBook' return serve_template(templatename="manualsearch.html", title=library + ' Search Results: "' + - searchterm + '"', bookid=bookid, results=results, library=library) + searchterm + '"', bookid=bookid, results=results, library=library) @cherrypy.expose def countProviders(self): @@ -676,7 +676,7 @@ def snatchBook(self, bookid=None, mode=None, provider=None, url=None, size=None, "NZBmode": mode, "AuxInfo": library, "Status": "Skipped" - } + } myDB.upsert("wanted", newValueDict, controlValueDict) AuthorID = bookdata["AuthorID"] url = urllib.unquote_plus(url) @@ -892,7 +892,7 @@ def addBook(self, bookid=None): def startBookSearch(self, books=None): if books: if lazylibrarian.USE_NZB() or lazylibrarian.USE_TOR() \ - or lazylibrarian.USE_RSS() or lazylibrarian.USE_DIRECT(): + or lazylibrarian.USE_RSS() or lazylibrarian.USE_DIRECT(): threading.Thread(target=search_book, name='SEARCHBOOK', args=[books]).start() logger.debug(u"Searching for book with id: " + books[0]["bookid"]) else: @@ -1260,7 +1260,7 @@ def markBooks(self, AuthorID=None, seriesid=None, action=None, redirect=None, ** books.append({"bookid": bookid}) if lazylibrarian.USE_NZB() or lazylibrarian.USE_TOR() \ - or lazylibrarian.USE_RSS() or lazylibrarian.USE_DIRECT(): + or lazylibrarian.USE_RSS() or lazylibrarian.USE_DIRECT(): threading.Thread(target=search_book, name='SEARCHBOOK', args=[books]).start() if redirect == "author": @@ -1697,7 +1697,7 @@ def searchForMag(self, bookid=None): def startMagazineSearch(self, mags=None): if mags: if lazylibrarian.USE_NZB() or lazylibrarian.USE_TOR() \ - or lazylibrarian.USE_RSS() or lazylibrarian.USE_DIRECT(): + or lazylibrarian.USE_RSS() or lazylibrarian.USE_DIRECT(): threading.Thread(target=search_magazines, name='SEARCHMAG', args=[mags, False]).start() logger.debug(u"Searching for magazine with title: %s" % mags[0]["bookid"]) else: @@ -2248,12 +2248,12 @@ def forceProcess(self, source=None): def forceSearch(self, source=None): if source == "magazines": if lazylibrarian.USE_NZB() or lazylibrarian.USE_TOR() \ - or lazylibrarian.USE_RSS() or lazylibrarian.USE_DIRECT(): + or lazylibrarian.USE_RSS() or lazylibrarian.USE_DIRECT(): if 'SEARCHALLMAG' not in [n.name for n in [t for t in threading.enumerate()]]: threading.Thread(target=search_magazines, name='SEARCHALLMAG', args=[]).start() elif source in ["books", "audio"]: if lazylibrarian.USE_NZB() or lazylibrarian.USE_TOR() \ - or lazylibrarian.USE_RSS() or lazylibrarian.USE_DIRECT(): + or lazylibrarian.USE_RSS() or lazylibrarian.USE_DIRECT(): if 'SEARCHALLBOOKS' not in [n.name for n in [t for t in threading.enumerate()]]: threading.Thread(target=search_book, name='SEARCHALLBOOKS', args=[]).start() else: From 669d648c31c95a6ffb4c909d1042091553a7cb32 Mon Sep 17 00:00:00 2001 From: Phil Borman Date: Fri, 23 Jun 2017 23:04:53 +0200 Subject: [PATCH 04/22] Added second libgen slot in config --- data/interfaces/bookstrap/config.html | 21 +++++++++++++++++++++ data/interfaces/default/config.html | 9 +++++++++ lazylibrarian/__init__.py | 6 +++++- lazylibrarian/directparser.py | 10 ++++++---- lazylibrarian/providers.py | 13 ++++--------- 5 files changed, 45 insertions(+), 14 deletions(-) diff --git a/data/interfaces/bookstrap/config.html b/data/interfaces/bookstrap/config.html index c5a828dfc..09f75ca6a 100644 --- a/data/interfaces/bookstrap/config.html +++ b/data/interfaces/bookstrap/config.html @@ -1087,6 +1087,27 @@

${title}

Search Examples: search.php foreignfiction/index.php +
+ +
+ + + + +
+ +
+ Search + +
+
+ Priority + +
+ + Search Examples: search.php foreignfiction/index.php + +


diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index 526403686..15e0270ed 100644 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -820,6 +820,15 @@

Direct Download Providers

libgen + <% + if lazylibrarian.CONFIG['GEN2'] == True: + checked = 'checked="checked"' + else: + checked = '' + %> + libgen + + diff --git a/lazylibrarian/__init__.py b/lazylibrarian/__init__.py index 4f3be28c0..69a073a32 100644 --- a/lazylibrarian/__init__.py +++ b/lazylibrarian/__init__.py @@ -266,6 +266,10 @@ 'GEN_SEARCH': ('str', 'GEN', 'search.php'), 'GEN': ('bool', 'GEN', 0), 'GEN_DLPRIORITY': ('int', 'GEN', 0), + 'GEN2_HOST': ('str', 'GEN', 'libgen.io'), + 'GEN2_SEARCH': ('str', 'GEN', 'foreignfiction/index.php'), + 'GEN2': ('bool', 'GEN', 0), + 'GEN2_DLPRIORITY': ('int', 'GEN', 0), 'LIME_HOST': ('str', 'LIME', 'https://www.limetorrents.cc'), 'LIME': ('bool', 'LIME', 0), 'LIME_DLPRIORITY': ('int', 'LIME', 0), @@ -1004,7 +1008,7 @@ def USE_TOR(): def USE_DIRECT(): count = 0 - for provider in [CONFIG['GEN']]: + for provider in [CONFIG['GEN'], CONFIG['GEN2']]: if bool(provider): count += 1 return count diff --git a/lazylibrarian/directparser.py b/lazylibrarian/directparser.py index 1e7225c54..04f82630a 100644 --- a/lazylibrarian/directparser.py +++ b/lazylibrarian/directparser.py @@ -33,14 +33,16 @@ def url_fix(s, charset='utf-8'): return urlparse.urlunsplit((scheme, netloc, path, qs, anchor)) -def GEN(book=None): +def GEN(book=None, prov=None): errmsg = '' provider = "libgen" - host = lazylibrarian.CONFIG['GEN_HOST'] - search = lazylibrarian.CONFIG['GEN_SEARCH'] + if prov is None: + prov = 'GEN' + host = lazylibrarian.CONFIG[prov + '_HOST'] if not str(host)[:4] == "http": host = 'http://' + host + search = lazylibrarian.CONFIG[prov + '_SEARCH'] page = 1 results = [] next_page = True @@ -208,7 +210,7 @@ def GEN(book=None): 'tor_url': url, 'tor_size': str(size), 'tor_type': 'direct', - 'priority': lazylibrarian.CONFIG['GEN_DLPRIORITY'] + 'priority': lazylibrarian.CONFIG[prov + '_DLPRIORITY'] }) logger.debug('Found %s, Size %s' % (title, size)) next_page = True diff --git a/lazylibrarian/providers.py b/lazylibrarian/providers.py index d2de42c24..0ee789465 100644 --- a/lazylibrarian/providers.py +++ b/lazylibrarian/providers.py @@ -279,16 +279,11 @@ def IterateOverDirectSites(book=None, searchType=None): authorname, bookname = get_searchterm(book, searchType) book['searchterm'] = authorname + ' ' + bookname - for prov in ['GEN']: + for prov in ['GEN', 'GEN2']: if lazylibrarian.CONFIG[prov] and not ProviderIsBlocked(prov): - logger.debug('[IterateOverDirectSites] - %s' % lazylibrarian.CONFIG[prov + '_HOST']) - if prov == 'GEN': - results, error = GEN(book) - else: - results = '' - error = '' - logger.error('IterateOverDirectSites called with unknown provider [%s]' % prov) - + logger.debug('[IterateOverDirectSites] - %s %s' % (lazylibrarian.CONFIG[prov + '_HOST'], + lazylibrarian.CONFIG[prov + '_SEARCH'])) + results, error = GEN(book, prov) if error: BlockProvider(prov, error) else: From 13fe64d152768f74a62da3aebf2351c14d22325e Mon Sep 17 00:00:00 2001 From: Phil Borman Date: Fri, 23 Jun 2017 23:46:03 +0200 Subject: [PATCH 05/22] Extra libgen params --- lazylibrarian/directparser.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lazylibrarian/directparser.py b/lazylibrarian/directparser.py index 04f82630a..b504686cc 100644 --- a/lazylibrarian/directparser.py +++ b/lazylibrarian/directparser.py @@ -57,7 +57,10 @@ def GEN(book=None, prov=None): if 'index.php' in search: params = { - "s": book['searchterm'] + "s": book['searchterm'], + "f_lang": "All", + "f_columns": 0, + "f_ext": "All" } if page > 1: params['page'] = page @@ -139,7 +142,7 @@ def GEN(book=None, prov=None): convertEntities=BeautifulStoneSoup.HTML_ENTITIES)) author = formatAuthorName(res) title = str(td[2]).split('>')[2].split('<')[0].strip() - title = BeautifulStoneSoup(title, convertEntities=BeautifulStoneSoup.HTML_ENTITIES) + title = str(BeautifulStoneSoup(title, convertEntities=BeautifulStoneSoup.HTML_ENTITIES)) link = str(td[2]).split('href="')[1].split('?')[1].split('"')[0] size = unaccented(td[7].text).upper() extn = td[8].text From ab0ae1d16f932d13d39fc434ca00d32aa01df59f Mon Sep 17 00:00:00 2001 From: Phil Borman Date: Fri, 23 Jun 2017 23:53:30 +0200 Subject: [PATCH 06/22] optimised imports --- lazylibrarian/__init__.py | 2 +- lazylibrarian/api.py | 4 ++-- lazylibrarian/bookwork.py | 2 +- lazylibrarian/dbupgrade.py | 4 ++-- lazylibrarian/directparser.py | 2 +- lazylibrarian/importer.py | 10 +++++----- lazylibrarian/magazinescan.py | 4 ++-- lazylibrarian/manualbook.py | 5 +++-- lazylibrarian/notifiers/__init__.py | 4 ++-- lazylibrarian/notifiers/custom_notify.py | 3 ++- lazylibrarian/notifiers/prowl.py | 5 +++-- lazylibrarian/providers.py | 5 +++-- lazylibrarian/searchmag.py | 2 +- lazylibrarian/torrentparser.py | 2 +- lazylibrarian/versioncheck.py | 2 +- 15 files changed, 30 insertions(+), 26 deletions(-) diff --git a/lazylibrarian/__init__.py b/lazylibrarian/__init__.py index 69a073a32..c16139626 100644 --- a/lazylibrarian/__init__.py +++ b/lazylibrarian/__init__.py @@ -15,6 +15,7 @@ from __future__ import with_statement +import ConfigParser import calendar import json import locale @@ -24,7 +25,6 @@ import threading import time import webbrowser -import ConfigParser import cherrypy from lazylibrarian import logger, postprocess, searchbook, searchrss, librarysync, versioncheck, database, \ diff --git a/lazylibrarian/api.py b/lazylibrarian/api.py index 6b449344d..1a7e60a6e 100644 --- a/lazylibrarian/api.py +++ b/lazylibrarian/api.py @@ -13,12 +13,12 @@ # You should have received a copy of the GNU General Public License # along with Lazylibrarian. If not, see . +import ConfigParser import Queue import json import os import shutil import threading -import ConfigParser import lazylibrarian from lazylibrarian import logger, database @@ -36,8 +36,8 @@ from lazylibrarian.magazinescan import magazineScan, create_covers from lazylibrarian.manualbook import searchItem from lazylibrarian.postprocess import processDir, processAlternate -from lazylibrarian.searchmag import search_magazines from lazylibrarian.searchbook import search_book +from lazylibrarian.searchmag import search_magazines cmd_dict = {'help': 'list available commands. ' + 'Time consuming commands take an optional &wait parameter if you want to wait for completion, ' + diff --git a/lazylibrarian/bookwork.py b/lazylibrarian/bookwork.py index d62c50e59..201d20be2 100644 --- a/lazylibrarian/bookwork.py +++ b/lazylibrarian/bookwork.py @@ -17,12 +17,12 @@ import shutil import time import urllib -from lib.fuzzywuzzy import fuzz import lazylibrarian from lazylibrarian import logger, database from lazylibrarian.cache import cache_img, fetchURL, get_xml_request from lazylibrarian.formatter import safe_unicode, plural, cleanName, unaccented, formatAuthorName +from lib.fuzzywuzzy import fuzz def setAllBookAuthors(): diff --git a/lazylibrarian/dbupgrade.py b/lazylibrarian/dbupgrade.py index 8ce03e07b..5244ab099 100644 --- a/lazylibrarian/dbupgrade.py +++ b/lazylibrarian/dbupgrade.py @@ -15,11 +15,11 @@ from __future__ import with_statement -import os -import time import datetime +import os import shutil import threading +import time import traceback import lazylibrarian diff --git a/lazylibrarian/directparser.py b/lazylibrarian/directparser.py index b504686cc..aa610b8e2 100644 --- a/lazylibrarian/directparser.py +++ b/lazylibrarian/directparser.py @@ -13,9 +13,9 @@ # You should have received a copy of the GNU General Public License # along with Lazylibrarian. If not, see . +import traceback import urllib import urlparse -import traceback import lazylibrarian from lazylibrarian import logger diff --git a/lazylibrarian/importer.py b/lazylibrarian/importer.py index 31b67e0ac..4705610b9 100644 --- a/lazylibrarian/importer.py +++ b/lazylibrarian/importer.py @@ -13,19 +13,19 @@ # You should have received a copy of the GNU General Public License # along with Lazylibrarian. If not, see . -import traceback -import threading -import lazylibrarian import Queue -from lib.fuzzywuzzy import fuzz - +import threading +import traceback from operator import itemgetter + +import lazylibrarian from lazylibrarian import logger, database from lazylibrarian.bookwork import getAuthorImage from lazylibrarian.cache import cache_img from lazylibrarian.formatter import today, unaccented, formatAuthorName from lazylibrarian.gb import GoogleBooks from lazylibrarian.gr import GoodReads +from lib.fuzzywuzzy import fuzz def addAuthorNameToDB(author=None, refresh=False, addbooks=True): diff --git a/lazylibrarian/magazinescan.py b/lazylibrarian/magazinescan.py index 1460b8c8f..743c5fa0f 100644 --- a/lazylibrarian/magazinescan.py +++ b/lazylibrarian/magazinescan.py @@ -17,13 +17,13 @@ import os import platform import re +import shutil import subprocess import traceback from hashlib import sha1 -import shutil -import lib.zipfile as zipfile import lazylibrarian +import lib.zipfile as zipfile from lazylibrarian import database, logger from lazylibrarian.common import setperm from lazylibrarian.formatter import getList, is_valid_booktype, plural diff --git a/lazylibrarian/manualbook.py b/lazylibrarian/manualbook.py index 25f0233a1..76c24adc3 100644 --- a/lazylibrarian/manualbook.py +++ b/lazylibrarian/manualbook.py @@ -13,10 +13,11 @@ # You should have received a copy of the GNU General Public License # along with Lazylibrarian. If not, see . -import lazylibrarian import urllib -from lazylibrarian.formatter import getList, unaccented_str, plural + +import lazylibrarian from lazylibrarian import logger, database +from lazylibrarian.formatter import getList, unaccented_str, plural from lazylibrarian.providers import IterateOverRSSSites, IterateOverTorrentSites, IterateOverNewzNabSites, \ IterateOverDirectSites from lib.fuzzywuzzy import fuzz diff --git a/lazylibrarian/notifiers/__init__.py b/lazylibrarian/notifiers/__init__.py index f56c9dedb..b0516f3be 100644 --- a/lazylibrarian/notifiers/__init__.py +++ b/lazylibrarian/notifiers/__init__.py @@ -18,14 +18,14 @@ import androidpn import boxcar +import custom_notify import email_notify -import prowl import nma +import prowl import pushbullet import pushover import slack import tweet -import custom_notify from lazylibrarian import logger # online diff --git a/lazylibrarian/notifiers/custom_notify.py b/lazylibrarian/notifiers/custom_notify.py index 1d49347ad..d3f72e8f7 100644 --- a/lazylibrarian/notifiers/custom_notify.py +++ b/lazylibrarian/notifiers/custom_notify.py @@ -13,8 +13,9 @@ # You should have received a copy of the GNU General Public License # along with LazyLibrarian. If not, see . -import lazylibrarian import subprocess + +import lazylibrarian from lazylibrarian import logger, database from lazylibrarian.common import notifyStrings, NOTIFY_SNATCH, NOTIFY_DOWNLOAD diff --git a/lazylibrarian/notifiers/prowl.py b/lazylibrarian/notifiers/prowl.py index 3d6ca3c89..12f85eb81 100644 --- a/lazylibrarian/notifiers/prowl.py +++ b/lazylibrarian/notifiers/prowl.py @@ -1,8 +1,9 @@ +from httplib import HTTPSConnection +from urllib import urlencode + import lazylibrarian from lazylibrarian import logger from lazylibrarian.common import notifyStrings, NOTIFY_SNATCH, NOTIFY_DOWNLOAD -from httplib import HTTPSConnection -from urllib import urlencode class Prowl_Notifier: diff --git a/lazylibrarian/providers.py b/lazylibrarian/providers.py index 0ee789465..a20450a37 100644 --- a/lazylibrarian/providers.py +++ b/lazylibrarian/providers.py @@ -13,16 +13,17 @@ # You should have received a copy of the GNU General Public License # along with Lazylibrarian. If not, see . +import time import urllib from xml.etree import ElementTree -import time + import lazylibrarian import lib.feedparser as feedparser from lazylibrarian import logger from lazylibrarian.cache import fetchURL +from lazylibrarian.directparser import GEN from lazylibrarian.formatter import age, today, plural, cleanName, unaccented, getList, check_int from lazylibrarian.torrentparser import KAT, WWT, TPB, ZOO, TDL, LIME -from lazylibrarian.directparser import GEN def get_searchterm(book, searchType): diff --git a/lazylibrarian/searchmag.py b/lazylibrarian/searchmag.py index 3af15f4eb..d784cbb82 100644 --- a/lazylibrarian/searchmag.py +++ b/lazylibrarian/searchmag.py @@ -22,12 +22,12 @@ import lazylibrarian from lazylibrarian import logger, database from lazylibrarian.common import scheduleJob +from lazylibrarian.downloadmethods import NZBDownloadMethod, TORDownloadMethod from lazylibrarian.formatter import plural, now, unaccented_str, replace_all, unaccented, \ nzbdate2format, getList, month2num, datecompare, check_int, check_year from lazylibrarian.notifiers import notify_snatch, custom_notify_snatch from lazylibrarian.providers import IterateOverNewzNabSites, IterateOverTorrentSites, IterateOverRSSSites, \ IterateOverDirectSites -from lazylibrarian.downloadmethods import NZBDownloadMethod, TORDownloadMethod from lib.fuzzywuzzy import fuzz diff --git a/lazylibrarian/torrentparser.py b/lazylibrarian/torrentparser.py index 290a703cd..5a5453591 100644 --- a/lazylibrarian/torrentparser.py +++ b/lazylibrarian/torrentparser.py @@ -13,9 +13,9 @@ # You should have received a copy of the GNU General Public License # along with Lazylibrarian. If not, see . +import traceback import urllib import urlparse -import traceback import lazylibrarian import lib.feedparser as feedparser diff --git a/lazylibrarian/versioncheck.py b/lazylibrarian/versioncheck.py index 23e0655de..7de84e0f6 100644 --- a/lazylibrarian/versioncheck.py +++ b/lazylibrarian/versioncheck.py @@ -16,11 +16,11 @@ import os import platform import re -import time import socket import subprocess import tarfile import threading +import time import urllib2 import lazylibrarian From 19a386aa537836b5f9e0b1700b58fb646538adeb Mon Sep 17 00:00:00 2001 From: Phil Borman Date: Sat, 24 Jun 2017 14:10:21 +0200 Subject: [PATCH 07/22] Code tidying --- data/interfaces/bookstrap/config.html | 10 +- data/interfaces/default/config.html | 390 +++++++++++++------------- lazylibrarian/torrentparser.py | 2 +- 3 files changed, 204 insertions(+), 198 deletions(-) diff --git a/data/interfaces/bookstrap/config.html b/data/interfaces/bookstrap/config.html index 09f75ca6a..8c1592624 100644 --- a/data/interfaces/bookstrap/config.html +++ b/data/interfaces/bookstrap/config.html @@ -1084,9 +1084,15 @@

${title}

- Search Examples: search.php foreignfiction/index.php + Search Examples: search.php or foreignfiction/index.php + <% + if lazylibrarian.CONFIG['GEN2'] == True: + checked = 'checked="checked"' + else: + checked = '' + %>
@@ -1105,7 +1111,7 @@

${title}

- Search Examples: search.php foreignfiction/index.php + Search Examples: search.php or foreignfiction/index.php
diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index 15e0270ed..bc429536a 100644 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -52,11 +52,11 @@

  ${title}

<% - if lazylibrarian.CONFIG['HTTPS_ENABLED'] == True: - checked = 'checked="checked"' - else: - checked = '' - %> + if lazylibrarian.CONFIG['HTTPS_ENABLED'] == True: + checked = 'checked="checked"' + else: + checked = '' +%>
@@ -77,11 +77,11 @@

  ${title}


@@ -105,19 +105,19 @@

  ${title}

<% - if lazylibrarian.CONFIG['LAUNCH_BROWSER'] == True: - checked = 'checked="checked"' - else: - checked = '' - %> + if lazylibrarian.CONFIG['LAUNCH_BROWSER'] == True: + checked = 'checked="checked"' + else: + checked = '' + %> <% - if lazylibrarian.CONFIG['API_ENABLED'] == True: - checked = 'checked="checked"' - else: - checked = '' - %> + if lazylibrarian.CONFIG['API_ENABLED'] == True: + checked = 'checked="checked"' + else: + checked = '' + %>
<% - if lazylibrarian.CONFIG['TOR_CONVERT_MAGNET'] == True: - checked = 'checked="checked"' - else: - checked = '' - %> + if lazylibrarian.CONFIG['TOR_CONVERT_MAGNET'] == True: + checked = 'checked="checked"' + else: + checked = '' + %>
<% - if lazylibrarian.CONFIG['PREFER_MAGNET'] == True: - checked = 'checked="checked"' - else: - checked = '' - %> + if lazylibrarian.CONFIG['PREFER_MAGNET'] == True: + checked = 'checked="checked"' + else: + checked = '' + %>
@@ -671,11 +671,11 @@

Torrents

<% - if lazylibrarian.CONFIG['KEEP_SEEDING'] == True: - checked = 'checked="checked"' - else: - checked = '' - %> + if lazylibrarian.CONFIG['KEEP_SEEDING'] == True: + checked = 'checked="checked"' + else: + checked = '' + %>
@@ -703,11 +703,11 @@

Newznab Providers


%for prov in lazylibrarian.NEWZNAB_PROV: <% - if prov['ENABLED'] == True: - checked = 'checked="checked"' - else: - checked = '' - %> + if prov['ENABLED'] == True: + checked = 'checked="checked"' + else: + checked = '' + %> @@ -721,11 +721,11 @@

Torznab Providers


%for prov in lazylibrarian.TORZNAB_PROV: <% - if prov['ENABLED'] == True: - checked = 'checked="checked"' - else: - checked = '' - %> + if prov['ENABLED'] == True: + checked = 'checked="checked"' + else: + checked = '' + %> @@ -740,11 +740,11 @@

RSS Providers


%for prov in lazylibrarian.RSS_PROV: <% - if prov['ENABLED'] == True: - checked = 'checked="checked"' - else: - checked = '' - %> + if prov['ENABLED'] == True: + checked = 'checked="checked"' + else: + checked = '' + %>
@@ -755,56 +755,56 @@

RSS Providers

Torrent Providers


<% - if lazylibrarian.CONFIG['KAT'] == True: - checked = 'checked="checked"' - else: - checked = '' - %> + if lazylibrarian.CONFIG['KAT'] == True: + checked = 'checked="checked"' + else: + checked = '' + %> kickass
<% - if lazylibrarian.CONFIG['WWT'] == True: - checked = 'checked="checked"' - else: - checked = '' - %> + if lazylibrarian.CONFIG['WWT'] == True: + checked = 'checked="checked"' + else: + checked = '' + %> WorldWideTorrent
<% - if lazylibrarian.CONFIG['TPB'] == True: - checked = 'checked="checked"' - else: - checked = '' - %> + if lazylibrarian.CONFIG['TPB'] == True: + checked = 'checked="checked"' + else: + checked = '' + %> piratebay
<% - if lazylibrarian.CONFIG['ZOO'] == True: - checked = 'checked="checked"' - else: - checked = '' - %> + if lazylibrarian.CONFIG['ZOO'] == True: + checked = 'checked="checked"' + else: + checked = '' + %> zooqle
<% - if lazylibrarian.CONFIG['TDL'] == True: - checked = 'checked="checked"' - else: - checked = '' - %> + if lazylibrarian.CONFIG['TDL'] == True: + checked = 'checked="checked"' + else: + checked = '' + %> torrentdownload
<% - if lazylibrarian.CONFIG['LIME'] == True: - checked = 'checked="checked"' - else: - checked = '' - %> + if lazylibrarian.CONFIG['LIME'] == True: + checked = 'checked="checked"' + else: + checked = '' + %> limetorrent
@@ -812,20 +812,20 @@

Torrent Providers

Direct Download Providers


<% - if lazylibrarian.CONFIG['GEN'] == True: - checked = 'checked="checked"' - else: - checked = '' - %> + if lazylibrarian.CONFIG['GEN'] == True: + checked = 'checked="checked"' + else: + checked = '' + %> libgen <% - if lazylibrarian.CONFIG['GEN2'] == True: - checked = 'checked="checked"' - else: - checked = '' - %> + if lazylibrarian.CONFIG['GEN2'] == True: + checked = 'checked="checked"' + else: + checked = '' + %> libgen diff --git a/lazylibrarian/torrentparser.py b/lazylibrarian/torrentparser.py index 5a5453591..7ed269c2c 100644 --- a/lazylibrarian/torrentparser.py +++ b/lazylibrarian/torrentparser.py @@ -310,7 +310,7 @@ def WWT(book=None): next_page = False result, success = fetchURL(searchURL) if not success: - # seems KAT returns 404 if no results, not really an error + # might return 404 if no results, not really an error if '404' in result: logger.debug(u"No results found from %s for %s" % (provider, book['searchterm'])) else: From b75e65bec5dc4e09eaf2cb152fa1d771eba59d8e Mon Sep 17 00:00:00 2001 From: Phil Borman Date: Sun, 25 Jun 2017 18:54:20 +0200 Subject: [PATCH 08/22] html tidying --- data/interfaces/bookstrap/config.html | 400 +++++++++++++------------- 1 file changed, 202 insertions(+), 198 deletions(-) diff --git a/data/interfaces/bookstrap/config.html b/data/interfaces/bookstrap/config.html index 8c1592624..10457bbcb 100644 --- a/data/interfaces/bookstrap/config.html +++ b/data/interfaces/bookstrap/config.html @@ -861,266 +861,270 @@

${title}

-
+
Newznab Providers %for provider in lazylibrarian.NEWZNAB_PROV: - <% - checked = '' - if provider['ENABLED'] == True: - checked = 'checked="checked"' - %> -
-
- - - - - -
- - -
- Priority - + <% + checked = '' + if provider['ENABLED'] == True: + checked = 'checked="checked"' + %> +
+
+ + + + + +
+ + +
+ Priority + +
-
%endfor
-
+
Torznab Providers %for provider in lazylibrarian.TORZNAB_PROV: - <% - checked = '' - if provider['ENABLED'] == True: + <% + checked = '' + if provider['ENABLED'] == True: checked = 'checked="checked"' - %> -
-
- - - - - -
- - -
- Priority - + %> +
+
+ + + + + +
+ + +
+ Priority + +
-
%endfor
-
+
RSS Providers
%for provider in lazylibrarian.RSS_PROV: - <% + <% checked = '' if provider['ENABLED'] == True: checked = 'checked="checked"' - %> -
- -
- - - - + %> +
+ +
+ + + + +
+
+ Priority + +
-
- Priority - -
-
%endfor
-
- Torrent Providers - <% - if lazylibrarian.CONFIG['KAT'] == True: +
+
+ Torrent Providers + <% + if lazylibrarian.CONFIG['KAT'] == True: checked = 'checked="checked"' - else: + else: checked = '' %> -
- -
- - - - -
-
- Priority - +
+ +
+ + + + +
+
+ Priority + +
-
- <% - if lazylibrarian.CONFIG['WWT'] == True: + <% + if lazylibrarian.CONFIG['WWT'] == True: checked = 'checked="checked"' - else: + else: checked = '' %> -
- -
- - - - -
-
- Priority - +
+ +
+ + + + +
+
+ Priority + +
-
- <% - if lazylibrarian.CONFIG['TPB'] == True: + <% + if lazylibrarian.CONFIG['TPB'] == True: checked = 'checked="checked"' - else: + else: checked = '' - %> -
- -
- - - - -
-
- Priority - + %> +
+ +
+ + + + +
+
+ Priority + +
-
- <% - if lazylibrarian.CONFIG['ZOO'] == True: + <% + if lazylibrarian.CONFIG['ZOO'] == True: checked = 'checked="checked"' - else: + else: checked = '' %> -
- -
- - - - -
-
- Priority - +
+ +
+ + + + +
+
+ Priority + +
-
- <% - if lazylibrarian.CONFIG['TDL'] == True: + <% + if lazylibrarian.CONFIG['TDL'] == True: checked = 'checked="checked"' - else: + else: checked = '' %> -
- -
- - - - -
-
- Priority - +
+ +
+ + + + +
+
+ Priority + +
-
- <% - if lazylibrarian.CONFIG['LIME'] == True: + <% + if lazylibrarian.CONFIG['LIME'] == True: checked = 'checked="checked"' - else: + else: checked = '' %> -
- -
- - - - -
-
- Priority - +
+ +
+ + + + +
+
+ Priority + +
-
-
-
- Direct Download Providers - <% - if lazylibrarian.CONFIG['GEN'] == True: +
+
+
+
+ Direct Download Providers + <% + if lazylibrarian.CONFIG['GEN'] == True: checked = 'checked="checked"' - else: + else: checked = '' %> -
- -
- - +
+ +
+ + + + +
+ +
+ Search + +
+
+ Priority + +
+ + Search Examples: search.php or foreignfiction/index.php - -
- -
- Search -
-
- Priority - -
- - Search Examples: search.php or foreignfiction/index.php - -
- <% - if lazylibrarian.CONFIG['GEN2'] == True: + <% + if lazylibrarian.CONFIG['GEN2'] == True: checked = 'checked="checked"' - else: + else: checked = '' %> -
- -
- - +
+ +
+ + + + +
+ +
+ Search + +
+
+ Priority + +
+ + Search Examples: search.php or foreignfiction/index.php - -
- -
- Search - -
-
- Priority -
- - Search Examples: search.php or foreignfiction/index.php - -
-
- -

+
+
+ +

+
-
From 44f0da06892de8935e04da71f2031234d38eb906 Mon Sep 17 00:00:00 2001 From: Phil Borman Date: Sun, 25 Jun 2017 18:54:30 +0200 Subject: [PATCH 09/22] sqlite commands now use tuples instead of string substitution --- lazylibrarian/api.py | 28 ++-- lazylibrarian/bookwork.py | 51 ++++---- lazylibrarian/common.py | 12 +- lazylibrarian/csvfile.py | 22 ++-- lazylibrarian/dbupgrade.py | 50 ++++---- lazylibrarian/downloadmethods.py | 26 ++-- lazylibrarian/gb.py | 31 +++-- lazylibrarian/gr.py | 28 ++-- lazylibrarian/importer.py | 40 +++--- lazylibrarian/librarysync.py | 61 +++++---- lazylibrarian/magazinescan.py | 12 +- lazylibrarian/manualbook.py | 6 +- lazylibrarian/notifiers/custom_notify.py | 4 +- lazylibrarian/postprocess.py | 53 ++++---- lazylibrarian/resultlist.py | 10 +- lazylibrarian/searchbook.py | 5 +- lazylibrarian/searchmag.py | 15 +-- lazylibrarian/searchrss.py | 7 +- lazylibrarian/webServe.py | 156 +++++++++++------------ 19 files changed, 301 insertions(+), 316 deletions(-) diff --git a/lazylibrarian/api.py b/lazylibrarian/api.py index 1a7e60a6e..fb3eed66a 100644 --- a/lazylibrarian/api.py +++ b/lazylibrarian/api.py @@ -388,9 +388,9 @@ def _removeMagazine(self, **kwargs): self.id = kwargs['name'] myDB = database.DBConnection() - myDB.action('DELETE from magazines WHERE Title="%s"' % self.id) - myDB.action('DELETE from wanted WHERE BookID="%s"' % self.id) - myDB.action('DELETE from issues WHERE Title="%s"' % self.id) + myDB.action('DELETE from magazines WHERE Title=?', (self.id,)) + myDB.action('DELETE from wanted WHERE BookID=?', (self.id,)) + myDB.action('DELETE from issues WHERE Title=?', (self.id,)) def _pauseAuthor(self, **kwargs): if 'id' not in kwargs: @@ -618,13 +618,11 @@ def _moveBook(self, **kwargs): return try: myDB = database.DBConnection() - authordata = myDB.match( - 'SELECT AuthorName from authors WHERE AuthorID="%s"' % kwargs['toid']) + authordata = myDB.match('SELECT AuthorName from authors WHERE AuthorID=?', (kwargs['toid'],)) if not authordata: self.data = "No destination author [%s] in the database" % kwargs['toid'] else: - bookdata = myDB.match( - 'SELECT AuthorID, BookName from books where BookID="%s"' % kwargs['id']) + bookdata = myDB.match('SELECT AuthorID, BookName from books where BookID=?', (kwargs['id'],)) if not bookdata: self.data = "No bookid [%s] in the database" % kwargs['id'] else: @@ -648,19 +646,17 @@ def _moveBooks(self, **kwargs): try: myDB = database.DBConnection() q = 'SELECT bookid,books.authorid from books,authors where books.AuthorID = authors.AuthorID' - q += ' and authorname="%s"' % kwargs['fromname'] - fromhere = myDB.select(q) + q += ' and authorname=?' + fromhere = myDB.select(q, (kwargs['fromname'],)) - tohere = myDB.match( - 'SELECT authorid from authors where authorname="%s"' % kwargs['toname']) + tohere = myDB.match('SELECT authorid from authors where authorname=?', (kwargs['toname'],)) if not len(fromhere): self.data = "No books by [%s] in the database" % kwargs['fromname'] else: if not tohere: self.data = "No destination author [%s] in the database" % kwargs['toname'] else: - myDB.action('UPDATE books SET authorid="%s", where authorname="%s"' % - (tohere[0], kwargs['fromname'])) + myDB.action('UPDATE books SET authorid=?, where authorname=?', (tohere[0], kwargs['fromname'])) self.data = "Moved %s books from %s to %s" % (len(fromhere), kwargs['fromname'], kwargs['toname']) update_totals(fromhere[0][1]) # we moved from here update_totals(tohere[0]) # to here @@ -725,12 +721,12 @@ def _removeAuthor(self, **kwargs): self.id = kwargs['id'] myDB = database.DBConnection() - authorsearch = myDB.select('SELECT AuthorName from authors WHERE AuthorID="%s"' % id) + authorsearch = myDB.select('SELECT AuthorName from authors WHERE AuthorID=?', (kwargs['id'],)) if len(authorsearch): # to stop error if try to remove an author while they are still loading AuthorName = authorsearch[0]['AuthorName'] logger.info(u"Removing all references to author: %s" % AuthorName) - myDB.action('DELETE from authors WHERE AuthorID="%s"' % id) - myDB.action('DELETE from books WHERE AuthorID="%s"' % id) + myDB.action('DELETE from authors WHERE AuthorID=?', (kwargs['id'],)) + myDB.action('DELETE from books WHERE AuthorID=?', (kwargs['id'],)) def _writeCFG(self, **kwargs): if 'name' not in kwargs: diff --git a/lazylibrarian/bookwork.py b/lazylibrarian/bookwork.py index 201d20be2..ca6dab307 100644 --- a/lazylibrarian/bookwork.py +++ b/lazylibrarian/bookwork.py @@ -31,7 +31,7 @@ def setAllBookAuthors(): myDB.action('create table bookauthors (AuthorID TEXT, BookID TEXT, UNIQUE (AuthorID, BookID))') books = myDB.select('SELECT AuthorID,BookID from books') for item in books: - myDB.action('insert into bookauthors (AuthorID, BookID) values (%s, %s)' % + myDB.action('insert into bookauthors (AuthorID, BookID) values (?, ?)', (item['AuthorID'], item['BookID']), suppress='UNIQUE') # also need to drop authorid from books table once it all works properly totalauthors = 0 @@ -57,7 +57,7 @@ def setBookAuthors(book): if authtype in ['primary author', 'main author', 'secondary author']: if author['role'] in ['Author', '—'] and author['work'] == 'all editions': name = formatAuthorName(unaccented(author['name'])) - exists = myDB.match('select authorid from authors where authorname = "%s"' % name) + exists = myDB.match('select authorid from authors where authorname=?', (name,)) if exists: authorid = exists['authorid'] else: @@ -67,7 +67,7 @@ def setBookAuthors(book): newauthors += 1 if authorid: # suppress duplicates in bookauthors - myDB.action('INSERT into bookauthors (AuthorID, BookID) VALUES ("%s", "%s")' % + myDB.action('INSERT into bookauthors (AuthorID, BookID) VALUES (?, ?)', (authorid, book['bookid']), suppress='UNIQUE') newrefs += 1 except Exception as e: @@ -157,16 +157,16 @@ def setSeries(seriesdict=None, bookid=None, seriesauthors=True, seriesdisplay=Tr myDB = database.DBConnection() if bookid: # delete any old series-member entries - myDB.action('DELETE from member WHERE BookID="%s"' % bookid) + myDB.action('DELETE from member WHERE BookID=?', (bookid,)) for item in seriesdict: - match = myDB.match('SELECT SeriesID from series where SeriesName="%s" COLLATE NOCASE' % item) + match = myDB.match('SELECT SeriesID from series where SeriesName=? COLLATE NOCASE', (item,)) if not match: # new series, need to set status and get SeriesID - myDB.action('INSERT into series (SeriesName, Status) VALUES ("%s", "Active")' % item) - match = myDB.match('SELECT SeriesID from series where SeriesName="%s"' % item) + myDB.action('INSERT into series (SeriesName, Status) VALUES (?, ?)', (item, "Active")) + match = myDB.match('SELECT SeriesID from series where SeriesName=?', (item,)) # don't ask librarything what other books are in the series - leave for user to query if series wanted # _ = getSeriesMembers(match['SeriesID']) - book = myDB.match('SELECT AuthorID from books where BookID="%s"' % bookid) + book = myDB.match('SELECT AuthorID from books where BookID=?', (bookid,)) if match and book: controlValueDict = {"BookID": bookid, "SeriesID": match['SeriesID']} newValueDict = {"SeriesNum": seriesdict[item]} @@ -174,7 +174,7 @@ def setSeries(seriesdict=None, bookid=None, seriesauthors=True, seriesdisplay=Tr # database versions earlier than 17 don't have seriesauthors table # but this function is used in dbupgrade v15 if seriesauthors: - myDB.action('INSERT INTO seriesauthors ("SeriesID", "AuthorID") VALUES ("%s", "%s")' % + myDB.action('INSERT INTO seriesauthors ("SeriesID", "AuthorID") VALUES (?, ?)', (match['SeriesID'], book['AuthorID']), suppress='UNIQUE') else: logger.debug('Unable to set series for book %s, %s' % (bookid, repr(seriesdict))) @@ -189,7 +189,7 @@ def setSeries(seriesdict=None, bookid=None, seriesauthors=True, seriesdisplay=Tr if series and newseries: series += '
' series += newseries - myDB.action('UPDATE books SET SeriesDisplay="%s" WHERE BookID="%s"' % (series, bookid)) + myDB.action('UPDATE books SET SeriesDisplay=? WHERE BookID=?', (series, bookid)) # removed deleteEmptySeries as setSeries slows down drastically if run in a loop # eg dbupgrade or setAllBookSeries. Better to tidy up all empties when loop finished @@ -203,7 +203,7 @@ def setStatus(bookid=None, seriesdict=None, default=None): if not bookid: return default - match = myDB.match('SELECT Status,AuthorID,BookName from books WHERE BookID="%s"' % bookid) + match = myDB.match('SELECT Status,AuthorID,BookName from books WHERE BookID=?', (bookid,)) if not match: return default @@ -218,7 +218,7 @@ def setStatus(bookid=None, seriesdict=None, default=None): bookname = match['BookName'] # Is the book part of any series we want? for item in seriesdict: - match = myDB.match('SELECT Status from series where SeriesName="%s" COLLATE NOCASE' % item) + match = myDB.match('SELECT Status from series where SeriesName=? COLLATE NOCASE', (item,)) if match['Status'] == 'Wanted': new_status = 'Wanted' logger.debug('Marking %s as %s, series %s' % (bookname, new_status, item)) @@ -227,7 +227,7 @@ def setStatus(bookid=None, seriesdict=None, default=None): if not new_status: # Is it part of any series we don't want? for item in seriesdict: - match = myDB.match('SELECT Status from series where SeriesName="%s" COLLATE NOCASE' % item) + match = myDB.match('SELECT Status from series where SeriesName=? COLLATE NOCASE', (item,)) if match['Status'] == 'Skipped': new_status = 'Skipped' logger.debug('Marking %s as %s, series %s' % (bookname, new_status, item)) @@ -235,14 +235,14 @@ def setStatus(bookid=None, seriesdict=None, default=None): if not new_status: # Author we don't want? - match = myDB.match('SELECT Status from authors where AuthorID="%s"' % authorid) + match = myDB.match('SELECT Status from authors where AuthorID=?', (authorid,)) if match['Status'] in ['Paused', 'Ignored']: new_status = 'Skipped' logger.debug('Marking %s as %s, author %s' % (bookname, new_status, match['Status'])) # If none of these, leave default "newbook" or "newauthor" status if new_status: - myDB.action('UPDATE books SET Status="%s" WHERE BookID="%s"' % (new_status, bookid)) + myDB.action('UPDATE books SET Status=? WHERE BookID=?', (new_status, bookid)) return new_status return default @@ -254,11 +254,11 @@ def deleteEmptySeries(): series = myDB.select('SELECT SeriesID,SeriesName from series') count = 0 for item in series: - match = myDB.match('SELECT BookID from member where SeriesID="%s"' % item['SeriesID']) + match = myDB.match('SELECT BookID from member where SeriesID=?', (item['SeriesID'],)) if not match: logger.debug('Deleting empty series %s' % item['SeriesName']) count += 1 - myDB.action('DELETE from series where SeriesID="%s"' % item['SeriesID']) + myDB.action('DELETE from series where SeriesID=?', (item['SeriesID'],)) return count @@ -313,13 +313,14 @@ def getBookWork(bookID=None, reason=None, seriesID=None): if bookID: # need to specify authors.AuthorName here as function is called during dbupgrade v15 to v16 # while books.authorname column is still present - cmd = 'select BookName,authors.AuthorName,BookISBN from books,authors where bookID="%s"' % bookID + cmd = 'select BookName,authors.AuthorName,BookISBN from books,authors where bookID=?' cmd += ' and books.AuthorID = authors.AuthorID' cacheLocation = "WorkCache" + item = myDB.match(cmd, (bookID,)) else: - cmd = 'select SeriesName from series where SeriesID="%s"' % seriesID + cmd = 'select SeriesName from series where SeriesID=?' cacheLocation = "SeriesCache" - item = myDB.match(cmd) + item = myDB.match(cmd, (seriesID,)) if item: cacheLocation = os.path.join(lazylibrarian.CACHEDIR, cacheLocation) if bookID: @@ -502,7 +503,7 @@ def getSeriesAuthors(seriesid): myDB = database.DBConnection() result = myDB.match("select count('AuthorID') as counter from authors") start = int(result['counter']) - result = myDB.match('select SeriesName from series where SeriesID="%s"' % seriesid) + result = myDB.match('select SeriesName from series where SeriesID=?', (seriesid,)) seriesname = result['SeriesName'] members = getSeriesMembers(seriesid) if members: @@ -648,7 +649,7 @@ def getBookCover(bookID=None): lazylibrarian.CACHE_MISS = int(lazylibrarian.CACHE_MISS) + 1 myDB = database.DBConnection() - item = myDB.match('select BookFile from books where bookID="%s"' % bookID) + item = myDB.match('select BookFile from books where bookID=?', (bookID,)) if item: bookfile = item['BookFile'] if bookfile: # we may have a cover.jpg in the same folder @@ -678,9 +679,9 @@ def getBookCover(bookID=None): logger.debug('getBookCover: Image not found in work page for %s' % bookID) # not found in librarything work page, try to get a cover from goodreads or google instead - cmd = 'select BookName,AuthorName,BookLink from books,authors where bookID="%s"' % bookID + cmd = 'select BookName,AuthorName,BookLink from books,authors where bookID=?' cmd += ' and books.AuthorID = authors.AuthorID' - item = myDB.match(cmd) + item = myDB.match(cmd, (bookID,)) if item: title = safe_unicode(item['BookName']).encode(lazylibrarian.SYS_ENCODING) author = safe_unicode(item['AuthorName']).encode(lazylibrarian.SYS_ENCODING) @@ -764,7 +765,7 @@ def getAuthorImage(authorid=None): lazylibrarian.CACHE_MISS = int(lazylibrarian.CACHE_MISS) + 1 myDB = database.DBConnection() - authors = myDB.select('select AuthorName from authors where AuthorID="%s"' % authorid) + authors = myDB.select('select AuthorName from authors where AuthorID=?', (authorid,)) if authors: authorname = safe_unicode(authors[0][0]).encode(lazylibrarian.SYS_ENCODING) safeparams = urllib.quote_plus("author %s" % authorname) diff --git a/lazylibrarian/common.py b/lazylibrarian/common.py index c0e5448a2..8fe8f3a81 100644 --- a/lazylibrarian/common.py +++ b/lazylibrarian/common.py @@ -406,7 +406,7 @@ def cleanCache(): except IndexError: logger.error('Clean Cache: Error splitting %s' % cached_file) continue - item = myDB.match('select BookID from books where BookID="%s"' % bookid) + item = myDB.match('select BookID from books where BookID=?', (bookid,)) if not item: # WorkPage no longer referenced in database, delete cached_file os.remove(target) @@ -431,7 +431,7 @@ def cleanCache(): except IndexError: logger.error('Clean Cache: Error splitting %s' % cached_file) continue - item = myDB.match('select SeriesID from series where SeriesID="%s"' % seriesid) + item = myDB.match('select SeriesID from series where SeriesID=?', (seriesid,)) if not item: # SeriesPage no longer referenced in database, delete cached_file os.remove(target) @@ -455,7 +455,7 @@ def cleanCache(): except IndexError: logger.error('Clean Cache: Error splitting %s' % cached_file) continue - item = myDB.match('select AuthorID from authors where AuthorID="%s"' % imgid) + item = myDB.match('select AuthorID from authors where AuthorID=?', (imgid,)) if not item: # Author Image no longer referenced in database, delete cached_file os.remove(target) @@ -472,7 +472,7 @@ def cleanCache(): except IndexError: logger.error('Clean Cache: Error splitting %s' % cached_file) continue - item = myDB.match('select BookID from books where BookID="%s"' % imgid) + item = myDB.match('select BookID from books where BookID=?', (imgid,)) if not item: # Book Image no longer referenced in database, delete cached_file os.remove(target) @@ -511,7 +511,7 @@ def cleanCache(): else: cleaned += 1 logger.debug('Cover missing for %s %s' % (item['BookName'], imgfile)) - myDB.action('update books set BookImg="images/nocover.png" where Bookid="%s"' % item['BookID']) + myDB.action('update books set BookImg="images/nocover.png" where Bookid=?', (item['BookID'],)) msg = "Cleaned %i missing cover file%s, kept %i" % (cleaned, plural(cleaned), kept) result.append(msg) @@ -538,7 +538,7 @@ def cleanCache(): else: cleaned += 1 logger.debug('Image missing for %s %s' % (item['AuthorName'], imgfile)) - myDB.action('update authors set AuthorImg="images/nophoto.png" where AuthorID="%s"' % item['AuthorID']) + myDB.action('update authors set AuthorImg="images/nophoto.png" where AuthorID=?', (item['AuthorID'],)) msg = "Cleaned %i missing author image%s, kept %i" % (cleaned, plural(cleaned), kept) result.append(msg) diff --git a/lazylibrarian/csvfile.py b/lazylibrarian/csvfile.py index 49ef17636..e1f00a8c2 100644 --- a/lazylibrarian/csvfile.py +++ b/lazylibrarian/csvfile.py @@ -43,8 +43,8 @@ def export_CSV(search_dir=None, status="Wanted"): myDB = database.DBConnection() cmd = 'SELECT BookID,AuthorName,BookName,BookIsbn,books.AuthorID FROM books,authors ' - cmd += 'WHERE books.Status = "%s" and books.AuthorID = authors.AuthorID' % status - find_status = myDB.select(cmd) + cmd += 'WHERE books.Status=? and books.AuthorID = authors.AuthorID' + find_status = myDB.select(cmd, (status,)) if not find_status: logger.warn(u"No books marked as %s" % status) @@ -96,21 +96,21 @@ def finditem(item, authorname, headers): # try to find book in our database using bookid or isbn, or if that fails, name matching cmd = 'SELECT AuthorName,BookName,BookID,books.Status FROM books,authors where books.AuthorID = authors.AuthorID ' if bookid: - fullcmd = cmd + 'and BookID=%s' % bookid - bookmatch = myDB.match(fullcmd) + fullcmd = cmd + 'and BookID=?' + bookmatch = myDB.match(fullcmd, (bookid,)) if not bookmatch: if is_valid_isbn(isbn10): - fullcmd = cmd + 'and BookIsbn="%s"' % isbn10 - bookmatch = myDB.match(fullcmd) + fullcmd = cmd + 'and BookIsbn=?' + bookmatch = myDB.match(fullcmd, (isbn10,)) if not bookmatch: if is_valid_isbn(isbn13): - fullcmd = cmd + 'and BookIsbn="%s"' % isbn13 - bookmatch = myDB.match(fullcmd) + fullcmd = cmd + 'and BookIsbn=?' + bookmatch = myDB.match(fullcmd, (isbn13,)) if not bookmatch: bookid = find_book_in_db(myDB, authorname, bookname) if bookid: - fullcmd = cmd + 'and BookID="%s"' % bookid - bookmatch = myDB.match(fullcmd) + fullcmd = cmd + 'and BookID=?' + bookmatch = myDB.match(fullcmd, (bookid,)) return bookmatch @@ -171,7 +171,7 @@ def import_CSV(search_dir=None): if isinstance(title, str): title = title.decode(lazylibrarian.SYS_ENCODING) - authmatch = myDB.match('SELECT * FROM authors where AuthorName="%s"' % authorname) + authmatch = myDB.match('SELECT * FROM authors where AuthorName=?', (authorname,)) if authmatch: logger.debug(u"CSV: Author %s found in database" % authorname) diff --git a/lazylibrarian/dbupgrade.py b/lazylibrarian/dbupgrade.py index 5244ab099..6629b6cf8 100644 --- a/lazylibrarian/dbupgrade.py +++ b/lazylibrarian/dbupgrade.py @@ -324,7 +324,7 @@ def dbupgrade(db_current_version): nzbsize = units["NZBsize"] nzbsize = nzbsize.split(' ')[0] myDB.action( - 'UPDATE wanted SET NZBsize = "%s" WHERE BookID = "%s"' % (nzbsize, units["BookID"])) + 'UPDATE wanted SET NZBsize=? WHERE BookID=?', (nzbsize, units["BookID"])) upgradelog.write("%s v2: %s\n" % (time.ctime(), lazylibrarian.UPDATE_MSG)) except Exception as e: msg = 'Error removing units from wanted table: ' + str(e) @@ -380,7 +380,7 @@ def dbupgrade(db_current_version): issueid = issue['IssueID'] issuedate = str(issue['IssueDate']) issuedate = issuedate.zfill(4) - myDB.action('UPDATE issues SET IssueDate="%s" WHERE IssueID="%s"' % (issuedate, issueid)) + myDB.action('UPDATE issues SET IssueDate=? WHERE IssueID=?', (issuedate, issueid)) upgradelog.write("%s v5: %s\n" % (time.ctime(), lazylibrarian.UPDATE_MSG)) mags = myDB.select( @@ -397,7 +397,7 @@ def dbupgrade(db_current_version): title = mag['Title'] issuedate = str(mag['IssueDate']) issuedate = issuedate.zfill(4) - myDB.action('UPDATE magazines SET IssueDate="%s" WHERE Title="%s"' % (issuedate, title)) + myDB.action('UPDATE magazines SET IssueDate=? WHERE Title=?', (issuedate, title)) upgradelog.write("%s v5: %s\n" % (time.ctime(), lazylibrarian.UPDATE_MSG)) upgradelog.write("%s v5: complete\n" % time.ctime()) @@ -435,7 +435,7 @@ def dbupgrade(db_current_version): img = image['AuthorImg'] img = img[7:] myDB.action( - 'UPDATE authors SET AuthorImg="%s" WHERE AuthorID="%s"' % (img, image['AuthorID'])) + 'UPDATE authors SET AuthorImg=? WHERE AuthorID=?', (img, image['AuthorID'])) img = img[6:] srcfile = os.path.join(src, img) if os.path.isfile(srcfile): @@ -460,7 +460,7 @@ def dbupgrade(db_current_version): lazylibrarian.UPDATE_MSG = "Moving book images to new location: %s of %s" % (cnt, tot) img = image['BookImg'] img = img[7:] - myDB.action('UPDATE books SET BookImg="%s" WHERE BookID="%s"' % (img, image['BookID'])) + myDB.action('UPDATE books SET BookImg=? WHERE BookID=?', (img, image['BookID'])) img = img[6:] srcfile = os.path.join(src, img) if os.path.isfile(srcfile): @@ -512,11 +512,11 @@ def dbupgrade(db_current_version): lazylibrarian.UPDATE_MSG = 'Updating last book image for %s' % book['AuthorName'] if book['LastBook']: match = myDB.match( - 'SELECT BookImg from books WHERE AuthorID="%s" AND BookName="%s"' % + 'SELECT BookImg from books WHERE AuthorID=? AND BookName=?', (book['AuthorID'], book['LastBook'])) if match: - myDB.action('UPDATE authors SET LastBookImg="%s" WHERE AuthorID=%s' % ( - match['BookImg'], book['AuthorID'])) + myDB.action('UPDATE authors SET LastBookImg=? WHERE AuthorID=?', + (match['BookImg'], book['AuthorID'])) upgradelog.write("%s v11: %s\n" % (time.ctime(), lazylibrarian.UPDATE_MSG)) upgradelog.write("%s v11: complete\n" % time.ctime()) @@ -533,13 +533,13 @@ def dbupgrade(db_current_version): for mag in mags: lazylibrarian.UPDATE_MSG = 'Updating last issue image for %s' % mag['Title'] match = myDB.match( - 'SELECT IssueFile from issues WHERE IssueAcquired="%s" AND Title="%s"' % + 'SELECT IssueFile from issues WHERE IssueAcquired=? AND Title=?', (mag['LastAcquired'], mag['Title'])) if match: coverfile = os.path.splitext(match['IssueFile'])[0] + '.jpg' if os.path.exists(coverfile): - myDB.action('UPDATE magazines SET LatestCover="%s" WHERE Title="%s"' % ( - coverfile, mag['Title'])) + myDB.action('UPDATE magazines SET LatestCover=? WHERE Title=?', + (coverfile, mag['Title'])) upgradelog.write("%s v12: %s\n" % (time.ctime(), lazylibrarian.UPDATE_MSG)) upgradelog.write("%s v12: complete\n" % time.ctime()) @@ -584,7 +584,7 @@ def dbupgrade(db_current_version): try: shutil.move(srcfile, os.path.join(src, "author", img)) myDB.action( - 'UPDATE authors SET AuthorImg="cache/author/%s" WHERE AuthorID="%s"' % + 'UPDATE authors SET AuthorImg="cache/author/?" WHERE AuthorID=?', (img, image['AuthorID'])) except Exception as e: logger.warn("dbupgrade: %s" % str(e)) @@ -624,7 +624,7 @@ def dbupgrade(db_current_version): if os.path.isfile(srcfile): try: shutil.move(srcfile, os.path.join(src, "book", img)) - myDB.action('UPDATE books SET BookImg="cache/book/%s" WHERE BookID="%s"' % + myDB.action('UPDATE books SET BookImg="cache/book/?" WHERE BookID=?', (img, image['BookID'])) except Exception as e: logger.warn("dbupgrade: %s" % str(e)) @@ -714,7 +714,7 @@ def dbupgrade(db_current_version): cnt += 1 lazylibrarian.UPDATE_MSG = "Updating seriesauthors: %s of %s" % (cnt, tot) if item['AuthorID']: - myDB.action('insert into seriesauthors (SeriesID, AuthorID) values ("%s", "%s")' % + myDB.action('insert into seriesauthors (SeriesID, AuthorID) values (?, ?)', (item['SeriesID'], item['AuthorID']), suppress='UNIQUE') myDB.action('DROP TABLE IF EXISTS temp_table') @@ -743,7 +743,7 @@ def dbupgrade(db_current_version): for item in series: cnt += 1 lazylibrarian.UPDATE_MSG = "Updating seriesauthors: %s of %s" % (cnt, tot) - myDB.action('insert into seriesauthors (SeriesID, AuthorID) values ("%s", "%s")' % + myDB.action('insert into seriesauthors (SeriesID, AuthorID) values (?, ?)', (item['SeriesID'], item['AuthorID']), suppress='UNIQUE') myDB.action('DROP TABLE temp_table') lazylibrarian.UPDATE_MSG = 'Reorganisation of seriesauthors complete' @@ -763,9 +763,9 @@ def dbupgrade(db_current_version): cnt += 1 lazylibrarian.UPDATE_MSG = "Updating series display: %s of %s" % (cnt, tot) cmd = 'SELECT SeriesName,SeriesNum from series,member WHERE ' - cmd += 'series.SeriesID = member.SeriesID and member.BookID="%s"' % book['BookID'] + cmd += 'series.SeriesID = member.SeriesID and member.BookID=?' - whichseries = myDB.select(cmd) + whichseries = myDB.select(cmd, (book['BookID'],)) series = '' for item in whichseries: @@ -775,7 +775,7 @@ def dbupgrade(db_current_version): series += '
' series += newseries - myDB.action('UPDATE books SET SeriesDisplay="%s" WHERE BookID="%s"' % + myDB.action('UPDATE books SET SeriesDisplay=? WHERE BookID=?', (series, book['BookID'])) lazylibrarian.UPDATE_MSG = 'Reorganisation of series display complete' @@ -802,7 +802,7 @@ def dbupgrade(db_current_version): mod += 1 t = os.path.getctime(book['BookFile']) filedate = datetime.datetime.utcfromtimestamp(int(t)).strftime("%Y-%m-%d %H:%M:%S") - myDB.action('UPDATE books SET BookLibrary="%s" WHERE BookID="%s"' % + myDB.action('UPDATE books SET BookLibrary=? WHERE BookID=?', (filedate, book['BookID'])) lazylibrarian.UPDATE_MSG = 'Adding BookLibrary date complete, %s/%s books' % (mod, cnt) @@ -822,13 +822,13 @@ def dbupgrade(db_current_version): myDB.action('CREATE TABLE IF NOT EXISTS downloads (Count INTEGER, Provider TEXT)') downloads = myDB.select('SELECT NZBprov from wanted WHERE Status="Processed"') for download in downloads: - entry = myDB.match('SELECT Count FROM downloads where Provider="%s"' % download['NZBprov']) + entry = myDB.match('SELECT Count FROM downloads where Provider=?', (download['NZBprov'],)) if entry: counter = int(entry['Count']) - myDB.action('UPDATE downloads SET Count=%s WHERE Provider="%s"' % + myDB.action('UPDATE downloads SET Count=? WHERE Provider=?', (counter + 1, download['NZBprov'])) else: - myDB.action('INSERT into downloads (Count, Provider) VALUES (%s, "%s")' % + myDB.action('INSERT into downloads (Count, Provider) VALUES (?, ?)', (1, download['NZBprov'])) upgradelog.write("%s v21: complete\n" % time.ctime()) @@ -842,14 +842,14 @@ def dbupgrade(db_current_version): upgradelog.write("%s: %s\n" % (time.ctime(), msg)) for author in authors: authorid = author["AuthorID"] - myDB.action('DELETE from authors WHERE AuthorID="%s"' % authorid) - myDB.action('DELETE from books WHERE AuthorID="%s"' % authorid) + myDB.action('DELETE from authors WHERE AuthorID=?', (authorid,)) + myDB.action('DELETE from books WHERE AuthorID=?', (authorid,)) except Exception as e: msg = 'Delete unnamed author error: ' + str(e) logger.error(msg) upgradelog.write("%s: %s\n" % (time.ctime(), msg)) - myDB.action('PRAGMA user_version = %s' % db_current_version) + myDB.action('PRAGMA user_version=%s' % db_current_version) lazylibrarian.UPDATE_MSG = 'Cleaning Database' upgradelog.write("%s: %s\n" % (time.ctime(), lazylibrarian.UPDATE_MSG)) myDB.action('vacuum') diff --git a/lazylibrarian/downloadmethods.py b/lazylibrarian/downloadmethods.py index 0731d0998..79d200108 100644 --- a/lazylibrarian/downloadmethods.py +++ b/lazylibrarian/downloadmethods.py @@ -89,15 +89,15 @@ def NZBDownloadMethod(bookid=None, nzbtitle=None, nzburl=None, library='eBook'): if downloadID: logger.debug('Nzbfile has been downloaded from ' + str(nzburl)) if library == 'eBook': - myDB.action('UPDATE books SET status = "Snatched" WHERE BookID="%s"' % bookid) + myDB.action('UPDATE books SET status="Snatched" WHERE BookID=?', (bookid,)) elif library == 'AudioBook': - myDB.action('UPDATE books SET audiostatus = "Snatched" WHERE BookID="%s"' % bookid) - myDB.action('UPDATE wanted SET status = "Snatched", Source = "%s", DownloadID = "%s" WHERE NZBurl="%s"' % + myDB.action('UPDATE books SET audiostatus = "Snatched" WHERE BookID=?', (bookid,)) + myDB.action('UPDATE wanted SET status="Snatched", Source=?, DownloadID=? WHERE NZBurl=?', (Source, downloadID, nzburl)) return True else: logger.error(u'Failed to download nzb @ %s' % (nzburl, Source)) - myDB.action('UPDATE wanted SET status = "Failed" WHERE NZBurl="%s"' % nzburl) + myDB.action('UPDATE wanted SET status="Failed" WHERE NZBurl=?', (nzburl,)) return False @@ -155,15 +155,15 @@ def DirectDownloadMethod(bookid=None, tor_title=None, tor_url=None, bookname=Non if downloadID: logger.debug(u'File %s has been downloaded from %s' % (tor_title, tor_url)) if library == 'eBook': - myDB.action('UPDATE books SET status = "Snatched" WHERE BookID="%s"' % bookid) + myDB.action('UPDATE books SET status="Snatched" WHERE BookID=?', (bookid,)) elif library == 'AudioBook': - myDB.action('UPDATE books SET audiostatus = "Snatched" WHERE BookID="%s"' % bookid) - myDB.action('UPDATE wanted SET status = "Snatched", Source = "%s", DownloadID = "%s" WHERE NZBurl="%s"' % + myDB.action('UPDATE books SET audiostatus="Snatched" WHERE BookID=?', (bookid,)) + myDB.action('UPDATE wanted SET status="Snatched", Source=?, DownloadID=? WHERE NZBurl=?', (Source, downloadID, full_url)) return True else: logger.error(u'Failed to download file @ %s' % (full_url, tor_url)) - myDB.action('UPDATE wanted SET status = "Failed" WHERE NZBurl="%s"' % full_url) + myDB.action('UPDATE wanted SET status="Failed" WHERE NZBurl=?', (full_url,)) return False @@ -344,10 +344,10 @@ def TORDownloadMethod(bookid=None, tor_title=None, tor_url=None, library='eBook' if downloadID: if library == 'eBook': - myDB.action('UPDATE books SET status = "Snatched" WHERE BookID="%s"' % bookid) + myDB.action('UPDATE books SET status="Snatched" WHERE BookID=?', (bookid,)) elif library == 'AudioBook': - myDB.action('UPDATE books SET audiostatus = "Snatched" WHERE BookID="%s"' % bookid) - myDB.action('UPDATE wanted SET status = "Snatched", Source = "%s", DownloadID = "%s" WHERE NZBurl="%s"' % + myDB.action('UPDATE books SET audiostatus="Snatched" WHERE BookID=?', (bookid,)) + myDB.action('UPDATE wanted SET status="Snatched", Source=?, DownloadID=? WHERE NZBurl=?', (Source, downloadID, full_url)) if tor_title: if downloadID.upper() in tor_title.upper(): @@ -355,11 +355,11 @@ def TORDownloadMethod(bookid=None, tor_title=None, tor_url=None, library='eBook' else: tor_title = unaccented_str(tor_title) logger.debug('%s setting torrent name to [%s]' % (Source, tor_title)) - myDB.action('UPDATE wanted SET NZBtitle = "%s" WHERE NZBurl="%s"' % (tor_title, full_url)) + myDB.action('UPDATE wanted SET NZBtitle=? WHERE NZBurl=?', (tor_title, full_url)) return True else: logger.error(u'Failed to download torrent from %s, %s' % (Source, tor_url)) - myDB.action('UPDATE wanted SET status = "Failed" WHERE NZBurl="%s"' % full_url) + myDB.action('UPDATE wanted SET status="Failed" WHERE NZBurl=?', (full_url,)) return False diff --git a/lazylibrarian/gb.py b/lazylibrarian/gb.py index 799d5593f..36cc2c120 100644 --- a/lazylibrarian/gb.py +++ b/lazylibrarian/gb.py @@ -249,7 +249,7 @@ def find_results(self, searchterm=None, queue=None): bookid = item['id'] author = myDB.select( - 'SELECT AuthorID FROM authors WHERE AuthorName = "%s"' % Author.replace('"', '""')) + 'SELECT AuthorID FROM authors WHERE AuthorName=?', (Author.replace('"', '""'),)) if author: AuthorID = author[0]['authorid'] else: @@ -391,7 +391,7 @@ def get_author_books(self, authorid=None, authorname=None, bookstatus="Skipped", if booklang == "Unknown" or booklang == "en": googlelang = booklang match = False - lang = myDB.match('SELECT lang FROM languages where isbn = "%s"' % isbnhead) + lang = myDB.match('SELECT lang FROM languages where isbn=?', (isbnhead,)) if lang: booklang = lang['lang'] cache_hits += 1 @@ -417,7 +417,7 @@ def get_author_books(self, authorid=None, authorname=None, bookstatus="Skipped", match = True break if match: - myDB.action('insert into languages values ("%s", "%s")' % + myDB.action('insert into languages values (?, ?)', (isbnhead, booklang)) logger.debug(u"GB language: " + booklang) @@ -435,7 +435,7 @@ def get_author_books(self, authorid=None, authorname=None, bookstatus="Skipped", if resp != 'invalid' and resp != 'unknown': booklang = resp # found a language code match = True - myDB.action('insert into languages values ("%s", "%s")' % + myDB.action('insert into languages values (?, ?)', (isbnhead, booklang)) logger.debug(u"LT language: " + booklang) except Exception as e: @@ -540,7 +540,7 @@ def get_author_books(self, authorid=None, authorname=None, bookstatus="Skipped", # GoodReads sometimes has multiple bookids for the same book (same author/title, different # editions) and sometimes uses the same bookid if the book is the same but the title is # slightly different. Not sure if googlebooks does too, but we only want one... - existing_book = myDB.match('SELECT Status,Manual FROM books WHERE BookID = "%s"' % bookid) + existing_book = myDB.match('SELECT Status,Manual FROM books WHERE BookID=?', (bookid,)) if existing_book: book_status = existing_book['Status'] locked = existing_book['Manual'] @@ -574,9 +574,8 @@ def get_author_books(self, authorid=None, authorname=None, bookstatus="Skipped", if not rejected: cmd = 'SELECT BookID FROM books,authors WHERE books.AuthorID = authors.AuthorID' - cmd += ' and BookName = "%s" COLLATE NOCASE and AuthorName = "%s" COLLATE NOCASE' % \ - (bookname.replace('"', '""'), authorname.replace('"', '""')) - match = myDB.match(cmd) + cmd += ' and BookName=? COLLATE NOCASE and AuthorName=? COLLATE NOCASE', + match = myDB.match(cmd, (bookname.replace('"', '""'), authorname.replace('"', '""'))) if match: if match['BookID'] != bookid: # we have a different book with this author/title already logger.debug('Rejecting bookid %s for [%s][%s] already got %s' % @@ -586,8 +585,8 @@ def get_author_books(self, authorid=None, authorname=None, bookstatus="Skipped", if not rejected: cmd = 'SELECT AuthorName,BookName FROM books,authors' - cmd += ' WHERE authors.AuthorID = books.AuthorID AND BookID="%s"' % bookid - match = myDB.match(cmd) + cmd += ' WHERE authors.AuthorID = books.AuthorID AND BookID=?' + match = myDB.match(cmd, (bookid,)) if match: # we have a book with this bookid already if bookname != match['BookName'] or authorname != match['AuthorName']: logger.debug('Rejecting bookid %s for [%s][%s] already got bookid for [%s][%s]' % @@ -684,9 +683,9 @@ def get_author_books(self, authorid=None, authorname=None, bookstatus="Skipped", deleteEmptySeries() logger.debug('[%s] The Google Books API was hit %s time%s to populate book list' % (authorname, api_hits, plural(api_hits))) - - lastbook = myDB.match('SELECT BookName, BookLink, BookDate, BookImg from books WHERE AuthorID="%s" \ - AND Status != "Ignored" order by BookDate DESC' % authorid) + cmd = 'SELECT BookName, BookLink, BookDate, BookImg from books WHERE AuthorID=?' + cmd += ' AND Status != "Ignored" order by BookDate DESC' + lastbook = myDB.match(cmd, (authorid,)) if lastbook: # maybe there are no books [remaining] for this author lastbookname = lastbook['BookName'] @@ -718,7 +717,7 @@ def get_author_books(self, authorid=None, authorname=None, bookstatus="Skipped", logger.debug("Found %s book%s by author marked as Ignored" % (book_ignore_count, plural(book_ignore_count))) logger.debug("Imported/Updated %s book%s for author" % (resultcount, plural(resultcount))) - myDB.action('insert into stats values ("%s", %i, %i, %i, %i, %i, %i, %i, %i, %i)' % + myDB.action('insert into stats values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', (authorname.replace('"', '""'), api_hits, gr_lang_hits, lt_lang_hits, gb_lang_change, cache_hits, ignored, removedResults, not_cached, duplicates)) @@ -835,9 +834,9 @@ def find_book(bookid=None, queue=None): author = GR.find_author_id() if author: AuthorID = author['authorid'] - match = myDB.match('SELECT AuthorID from authors WHERE AuthorID="%s"' % AuthorID) + match = myDB.match('SELECT AuthorID from authors WHERE AuthorID=?', (AuthorID,)) if not match: - match = myDB.match('SELECT AuthorID from authors WHERE AuthorName="%s"' % author['authorname']) + match = myDB.match('SELECT AuthorID from authors WHERE AuthorName=?', (author['authorname'],)) if match: logger.debug('%s: Changing authorid from %s to %s' % (author['authorname'], AuthorID, match['AuthorID'])) diff --git a/lazylibrarian/gr.py b/lazylibrarian/gr.py index 110eeb9eb..294011652 100644 --- a/lazylibrarian/gr.py +++ b/lazylibrarian/gr.py @@ -378,7 +378,7 @@ def get_author_books(self, authorid=None, authorname=None, bookstatus="Skipped", if bookLanguage == "Unknown" and isbnhead: # Nothing in the isbn dictionary, try any cached results - match = myDB.match('SELECT lang FROM languages where isbn = "%s"' % isbnhead) + match = myDB.match('SELECT lang FROM languages where isbn=?', (isbnhead,)) if match: bookLanguage = match['lang'] cache_hits += 1 @@ -399,7 +399,7 @@ def get_author_books(self, authorid=None, authorname=None, bookstatus="Skipped", bookLanguage = "Unknown" else: bookLanguage = resp # found a language code - myDB.action('insert into languages values ("%s", "%s")' % + myDB.action('insert into languages values (?, ?)', (isbnhead, bookLanguage)) logger.debug(u"LT language %s: %s" % (isbnhead, bookLanguage)) except Exception as e: @@ -452,7 +452,7 @@ def get_author_books(self, authorid=None, authorname=None, bookstatus="Skipped", if isbnhead != "": # if GR didn't give an isbn we can't cache it # just use language for this book - myDB.action('insert into languages values ("%s", "%s")' % + myDB.action('insert into languages values (?, ?)', (isbnhead, bookLanguage)) logger.debug("GoodReads reports language [%s] for %s" % (bookLanguage, isbnhead)) @@ -533,9 +533,8 @@ def get_author_books(self, authorid=None, authorname=None, bookstatus="Skipped", if not rejected: cmd = 'SELECT BookID FROM books,authors WHERE books.AuthorID = authors.AuthorID' - cmd += ' and BookName = "%s" COLLATE NOCASE and AuthorName = "%s" COLLATE NOCASE' % \ - (bookname, authorNameResult.replace('"', '""')) - match = myDB.match(cmd) + cmd += ' and BookName=? COLLATE NOCASE and AuthorName=? COLLATE NOCASE' + match = myDB.match(cmd, (bookname, authorNameResult.replace('"', '""'))) if match: if match['BookID'] != bookid: # we have a different book with this author/title already @@ -546,8 +545,8 @@ def get_author_books(self, authorid=None, authorname=None, bookstatus="Skipped", if not rejected: cmd = 'SELECT AuthorName,BookName FROM books,authors' - cmd += ' WHERE authors.AuthorID = books.AuthorID AND BookID=%s' % bookid - match = myDB.match(cmd) + cmd += ' WHERE authors.AuthorID = books.AuthorID AND BookID=?' + match = myDB.match(cmd, (bookid,)) if match: # we have a book with this bookid already if bookname != match['BookName'] or authorNameResult != match['AuthorName']: @@ -562,7 +561,7 @@ def get_author_books(self, authorid=None, authorname=None, bookstatus="Skipped", rejected = True if check_status or not rejected: - existing_book = myDB.match('SELECT Status,Manual FROM books WHERE BookID = "%s"' % bookid) + existing_book = myDB.match('SELECT Status,Manual FROM books WHERE BookID=?', (bookid,)) if existing_book: book_status = existing_book['Status'] locked = existing_book['Manual'] @@ -683,8 +682,9 @@ def get_author_books(self, authorid=None, authorname=None, bookstatus="Skipped", resultxml = None deleteEmptySeries() - lastbook = myDB.match('SELECT BookName, BookLink, BookDate, BookImg from books WHERE AuthorID="%s" \ - AND Status != "Ignored" order by BookDate DESC' % authorid) + cmd = 'SELECT BookName, BookLink, BookDate, BookImg from books WHERE AuthorID=?' + cmd += ' AND Status != "Ignored" order by BookDate DESC' + lastbook = myDB.match(cmd, (authorid,)) if lastbook: lastbookname = lastbook['BookName'] lastbooklink = lastbook['BookLink'] @@ -718,7 +718,7 @@ def get_author_books(self, authorid=None, authorname=None, bookstatus="Skipped", logger.debug("Found %s book%s by author marked as Ignored" % (book_ignore_count, plural(book_ignore_count))) logger.debug("Imported/Updated %s book%s" % (modified_count, plural(modified_count))) - myDB.action('insert into stats values ("%s", %i, %i, %i, %i, %i, %i, %i, %i, %i)' % + myDB.action('insert into stats values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', (authorname.replace('"', '""'), api_hits, gr_lang_hits, lt_lang_hits, gb_lang_change, cache_hits, ignored, removedResults, not_cached, duplicates)) @@ -784,9 +784,9 @@ def find_book(self, bookid=None, queue=None): author = GR.find_author_id() if author: AuthorID = author['authorid'] - match = myDB.match('SELECT AuthorID from authors WHERE AuthorID="%s"' % AuthorID) + match = myDB.match('SELECT AuthorID from authors WHERE AuthorID=?', (AuthorID,)) if not match: - match = myDB.match('SELECT AuthorID from authors WHERE AuthorName="%s"' % author['authorname']) + match = myDB.match('SELECT AuthorID from authors WHERE AuthorName=?', (author['authorname'],)) if match: logger.debug('%s: Changing authorid from %s to %s' % (author['authorname'], AuthorID, match['AuthorID'])) diff --git a/lazylibrarian/importer.py b/lazylibrarian/importer.py index 4705610b9..abdfc678e 100644 --- a/lazylibrarian/importer.py +++ b/lazylibrarian/importer.py @@ -41,7 +41,7 @@ def addAuthorNameToDB(author=None, refresh=False, addbooks=True): author = formatAuthorName(author) # Check if the author exists, and import the author if not, - check_exist_author = myDB.match('SELECT AuthorID FROM authors where AuthorName="%s"' % author.replace('"', '""')) + check_exist_author = myDB.match('SELECT AuthorID FROM authors where AuthorName=?', (author.replace('"', '""'),)) if not check_exist_author and lazylibrarian.CONFIG['ADD_AUTHOR']: logger.debug('Author %s not found in database, trying to add' % author) @@ -88,14 +88,14 @@ def addAuthorNameToDB(author=None, refresh=False, addbooks=True): authorid = author_gr['authorid'] # this new authorname may already be in the # database, so check again - check_exist_author = myDB.match('SELECT AuthorID FROM authors where AuthorID="%s"' % authorid) + check_exist_author = myDB.match('SELECT AuthorID FROM authors where AuthorID=?', (authorid,)) if check_exist_author: logger.debug('Found goodreads authorname %s in database' % author) else: logger.info("Adding new author [%s]" % author) try: addAuthorToDB(authorname=author, refresh=refresh, authorid=authorid, addbooks=addbooks) - check_exist_author = myDB.match('SELECT AuthorID FROM authors where AuthorID="%s"' % authorid) + check_exist_author = myDB.match('SELECT AuthorID FROM authors where AuthorID=?', (authorid,)) if check_exist_author: new = True except Exception as e: @@ -126,7 +126,7 @@ def addAuthorToDB(authorname=None, refresh=False, authorid=None, addbooks=True): controlValueDict = {"AuthorID": authorid} newValueDict = {"Status": "Loading"} - dbauthor = myDB.match("SELECT * from authors WHERE AuthorID='%s'" % authorid) + dbauthor = myDB.match("SELECT * from authors WHERE AuthorID=?", (authorid,)) if not dbauthor: authorname = 'unknown author' logger.debug("Adding new author id %s to database" % authorid) @@ -159,18 +159,18 @@ def addAuthorToDB(authorname=None, refresh=False, authorid=None, addbooks=True): else: logger.warn(u"Nothing found for %s" % authorid) if not dbauthor: - myDB.action('DELETE from authors WHERE AuthorID="%s"' % authorid) + myDB.action('DELETE from authors WHERE AuthorID=?', (authorid,)) if authorname and not match: authorname = ' '.join(authorname.split()) # ensure no extra whitespace GR = GoodReads(authorname) author = GR.find_author_id(refresh=refresh) - query = "SELECT * from authors WHERE AuthorName='%s'" % authorname.replace("'", "''") - dbauthor = myDB.match(query) + query = "SELECT * from authors WHERE AuthorName=?" + dbauthor = myDB.match(query, (authorname.replace("'", "''"),)) if author and not dbauthor: # may have different name for same authorid (spelling?) - query = "SELECT * from authors WHERE AuthorID='%s'" % author['authorid'] - dbauthor = myDB.match(query) + query = "SELECT * from authors WHERE AuthorID=?" + dbauthor = myDB.match(query, (author['authorid'],)) authorname = dbauthor['AuthorName'] controlValueDict = {"AuthorName": authorname} @@ -208,7 +208,7 @@ def addAuthorToDB(authorname=None, refresh=False, authorid=None, addbooks=True): else: logger.warn(u"Nothing found for %s" % authorname) if not dbauthor: - myDB.action('DELETE from authors WHERE AuthorName="%s"' % authorname) + myDB.action('DELETE from authors WHERE AuthorName=?', (authorname,)) return if not match: logger.error("AddAuthorToDB: No matching result for authorname or authorid") @@ -216,7 +216,7 @@ def addAuthorToDB(authorname=None, refresh=False, authorid=None, addbooks=True): # if author is set to manual, should we allow replacing 'nophoto' ? new_img = False - match = myDB.match("SELECT Manual from authors WHERE AuthorID='%s'" % authorid) + match = myDB.match("SELECT Manual from authors WHERE AuthorID=?", (authorid,)) if not match or not match['Manual']: if authorimg and 'nophoto' in authorimg: newimg = getAuthorImage(authorid) @@ -275,21 +275,19 @@ def addAuthorToDB(authorname=None, refresh=False, authorid=None, addbooks=True): def update_totals(AuthorID): myDB = database.DBConnection() # author totals needs to be updated every time a book is marked differently - match = myDB.select('SELECT AuthorID from authors WHERE AuthorID="%s"' % AuthorID) + match = myDB.select('SELECT AuthorID from authors WHERE AuthorID=?', (AuthorID,)) if not match: logger.debug('Update_totals - authorid [%s] not found' % AuthorID) return - cmd = 'SELECT BookName, BookLink, BookDate from books WHERE AuthorID="%s"' % AuthorID + cmd = 'SELECT BookName, BookLink, BookDate from books WHERE AuthorID=?' cmd += ' AND Status != "Ignored" order by BookDate DESC' - lastbook = myDB.match(cmd) - cmd = 'SELECT count("BookID") as counter FROM books WHERE AuthorID="%s"' % AuthorID - cmd += ' AND Status != "Ignored"' - unignoredbooks = myDB.match(cmd) - totalbooks = myDB.match( - 'SELECT count("BookID") as counter FROM books WHERE AuthorID="%s"' % AuthorID) - cmd = 'SELECT count("BookID") as counter FROM books WHERE AuthorID="%s"' % AuthorID + lastbook = myDB.match(cmd, (AuthorID,)) + cmd = 'SELECT count("BookID") as counter FROM books WHERE AuthorID=? AND Status != "Ignored"' + unignoredbooks = myDB.match(cmd, (AuthorID,)) + totalbooks = myDB.match('SELECT count("BookID") as counter FROM books WHERE AuthorID=?', (AuthorID,)) + cmd = 'SELECT count("BookID") as counter FROM books WHERE AuthorID=?' cmd += ' AND (Status="Have" OR Status="Open" OR AudioStatus="Have" OR AudioStatus="Open")' - havebooks = myDB.match(cmd) + havebooks = myDB.match(cmd, (AuthorID,)) controlValueDict = {"AuthorID": AuthorID} newValueDict = { diff --git a/lazylibrarian/librarysync.py b/lazylibrarian/librarysync.py index 44a73079d..717581ca7 100644 --- a/lazylibrarian/librarysync.py +++ b/lazylibrarian/librarysync.py @@ -160,9 +160,8 @@ def find_book_in_db(myDB, author, book): logger.debug('Searching database for [%s] by [%s]' % (book, author)) cmd = 'SELECT BookID FROM books,authors where books.AuthorID = authors.AuthorID ' - cmd += 'and AuthorName="%s" COLLATE NOCASE and BookName="%s" COLLATE NOCASE' % \ - (author.replace('"', '""'), book.replace('"', '""')) - match = myDB.match(cmd) + cmd += 'and AuthorName=? COLLATE NOCASE and BookName=? COLLATE NOCASE' + match = myDB.match(cmd, (author.replace('"', '""'), book.replace('"', '""'))) if match: logger.debug('Exact match [%s]' % book) return match['BookID'] @@ -173,8 +172,8 @@ def find_book_in_db(myDB, author, book): # on books that should be matched # Maybe make ratios configurable in config.ini later cmd = 'SELECT BookID,BookName,BookISBN FROM books,authors where books.AuthorID = authors.AuthorID ' - cmd += 'and AuthorName="%s" COLLATE NOCASE' % author.replace('"', '""') - books = myDB.select(cmd) + cmd += 'and AuthorName=? COLLATE NOCASE' + books = myDB.select(cmd, (author.replace('"', '""'),)) best_ratio = 0 best_partial = 0 best_partname = 0 @@ -300,15 +299,14 @@ def LibraryScan(startdir=None, library='eBook'): authorname = ' '.join(author['AuthorName'].split()) # Have we got author name both with-and-without extra spaces? If so, merge them duplicate = myDB.match( - 'Select AuthorID,AuthorName FROM authors WHERE AuthorName="%s"' % authorname) + 'Select AuthorID,AuthorName FROM authors WHERE AuthorName=?', (authorname,)) if duplicate: - myDB.action('DELETE from authors where authorname="%s"' % author['AuthorName']) + myDB.action('DELETE from authors where authorname=?', (author['AuthorName'],)) if author['AuthorID'] != duplicate['AuthorID']: - myDB.action('UPDATE books set AuthorID="%s" WHERE AuthorID="%s"' % + myDB.action('UPDATE books set AuthorID=? WHERE AuthorID=?', (duplicate['AuthorID'], author['AuthorID'])) else: - myDB.action( - 'UPDATE authors set AuthorName="%s" WHERE AuthorID="%s"' % (authorname, authorid)) + myDB.action('UPDATE authors set AuthorName=? WHERE AuthorID=?', (authorname, authorid)) except Exception as e: logger.info('Error: ' + str(e)) @@ -334,7 +332,7 @@ def LibraryScan(startdir=None, library='eBook'): bookfile = book['BookFile'] if bookfile and not os.path.isfile(bookfile): - myDB.action('update books set Status="%s",BookFile="",BookLibrary="" where BookID="%s"' % + myDB.action('update books set Status=?,BookFile="",BookLibrary="" where BookID=?', (status, book['BookID'])) logger.warn('eBook %s - %s updated as not found on disk' % (book['AuthorName'], book['BookName'])) @@ -351,7 +349,7 @@ def LibraryScan(startdir=None, library='eBook'): bookfile = book['AudioFile'] if bookfile and not os.path.isfile(bookfile): - myDB.action('update books set AudioStatus="%s",AudioFile="",AudioLibrary="" where BookID="%s"' % + myDB.action('update books set AudioStatus=?,AudioFile="",AudioLibrary="" where BookID=?', (status, book['BookID'])) logger.warn('Audiobook %s - %s updated as not found on disk' % (book['AuthorName'], book['BookName'])) @@ -532,9 +530,9 @@ def LibraryScan(startdir=None, library='eBook'): isbnhead = isbn[0:3] else: isbnhead = isbn[3:6] - match = myDB.match('SELECT lang FROM languages where isbn = "%s"' % isbnhead) + match = myDB.match('SELECT lang FROM languages where isbn=?', (isbnhead,)) if not match: - myDB.action('insert into languages values ("%s", "%s")' % (isbnhead, language)) + myDB.action('insert into languages values (?, ?)', (isbnhead, language)) logger.debug("Cached Lang [%s] ISBN [%s]" % (language, isbnhead)) else: logger.debug("Already cached Lang [%s] ISBN [%s]" % (language, isbnhead)) @@ -568,7 +566,7 @@ def LibraryScan(startdir=None, library='eBook'): bookid = gb_id if bookid: - match = myDB.match('SELECT BookID FROM books where BookID = "%s"' % bookid) + match = myDB.match('SELECT BookID FROM books where BookID=?', (bookid,)) if not match: msg = 'Unable to find book %s by %s in database, trying to add it using ' if bookid == gr_id: @@ -583,14 +581,14 @@ def LibraryScan(startdir=None, library='eBook'): GB_ID = GoogleBooks(gb_id) GB_ID.find_book(gb_id, None) # see if it's there now... - match = myDB.match('SELECT BookID from books where BookID="%s"' % bookid) + match = myDB.match('SELECT BookID from books where BookID=?', (bookid,)) if not match: logger.debug("Unable to add bookid %s to database" % bookid) bookid = "" if not bookid and isbn: # See if the isbn is in our database - match = myDB.match('SELECT BookID FROM books where BookIsbn = "%s"' % isbn) + match = myDB.match('SELECT BookID FROM books where BookIsbn=?', (isbn,)) if match: bookid = match['BookID'] @@ -653,7 +651,7 @@ def LibraryScan(startdir=None, library='eBook'): logger.debug("Setting language from metadata %s : %s" % ( booktitle, language)) myDB.action( - 'UPDATE books SET BookLang="%s" WHERE BookID="%s"' % + 'UPDATE books SET BookLang=? WHERE BookID=?', (language, bookid)) break if not bookid: @@ -669,8 +667,8 @@ def LibraryScan(startdir=None, library='eBook'): if bookid: cmd = 'SELECT books.Status, AudioStatus, BookFile, AudioFile, AuthorName, BookName' cmd += ' from books,authors where books.AuthorID = authors.AuthorID' - cmd += ' and BookID="%s"' % bookid - check_status = myDB.match(cmd) + cmd += ' and BookID=?' + check_status = myDB.match(cmd, (bookid,)) if not check_status: logger.debug('Unable to find bookid %s in database' % bookid) @@ -681,9 +679,9 @@ def LibraryScan(startdir=None, library='eBook'): # we found a new book new_book_count += 1 myDB.action( - 'UPDATE books set Status="Open" where BookID="%s"' % bookid) + 'UPDATE books set Status="Open" where BookID=?', (bookid,)) myDB.action( - 'UPDATE books set BookLibrary="%s" where BookID="%s"' % + 'UPDATE books set BookLibrary=? where BookID=?', (now(), bookid)) # check and store book location so we can check if it gets (re)moved @@ -698,7 +696,7 @@ def LibraryScan(startdir=None, library='eBook'): break if not check_status['BookFile']: # no previous location - myDB.action('UPDATE books set BookFile="%s" where BookID="%s"' % + myDB.action('UPDATE books set BookFile=? where BookID=?', (book_filename, bookid)) # location may have changed since last scan elif book_filename != check_status['BookFile']: @@ -708,7 +706,7 @@ def LibraryScan(startdir=None, library='eBook'): logger.debug("%s %s matched %s BookID %s, [%s][%s]" % (author, book, check_status['Status'], bookid, check_status['AuthorName'], check_status['BookName'])) - myDB.action('UPDATE books set BookFile="%s" where BookID="%s"' % + myDB.action('UPDATE books set BookFile=? where BookID=?', (book_filename, bookid)) elif library == 'Audio': @@ -716,10 +714,9 @@ def LibraryScan(startdir=None, library='eBook'): # we found a new audiobook new_book_count += 1 myDB.action( - 'UPDATE books set AudioStatus="Open" where BookID="%s"' % bookid) + 'UPDATE books set AudioStatus="Open" where BookID=?', (bookid,)) myDB.action( - 'UPDATE books set AudioLibrary="%s" where BookID="%s"' % ( - now(), bookid)) + 'UPDATE books set AudioLibrary=? where BookID=?', (now(), bookid)) # store audiobook location so we can check if it gets (re)moved book_filename = os.path.join(r, files) # link to the first part of multi-part audiobooks @@ -729,7 +726,7 @@ def LibraryScan(startdir=None, library='eBook'): break if not check_status['AudioFile']: # no previous location - myDB.action('UPDATE books set AudioFile="%s" where BookID="%s"' % + myDB.action('UPDATE books set AudioFile=? where BookID=?', (book_filename, bookid)) # location may have changed since last scan elif book_filename != check_status['AudioFile']: @@ -739,7 +736,7 @@ def LibraryScan(startdir=None, library='eBook'): logger.debug("%s %s matched %s BookID %s, [%s][%s]" % (author, book, check_status['AudioStatus'], bookid, check_status['AuthorName'], check_status['BookName'])) - myDB.action('UPDATE books set AudioFile="%s" where BookID="%s"' % + myDB.action('UPDATE books set AudioFile=? where BookID=?', (book_filename, bookid)) # update cover file to cover.jpg in book folder (if exists) @@ -829,7 +826,7 @@ def LibraryScan(startdir=None, library='eBook'): # bookname = item['bookname'] newimg, success = cache_img("book", bookid, bookimg) if success: - myDB.action('update books set BookImg="%s" where BookID="%s"' % (newimg, bookid)) + myDB.action('update books set BookImg=? where BookID=?', (newimg, bookid)) images = myDB.select('select AuthorID, AuthorImg, AuthorName from authors where AuthorImg like "http%"') if len(images): @@ -840,14 +837,14 @@ def LibraryScan(startdir=None, library='eBook'): # authorname = item['authorname'] newimg, success = cache_img("author", authorid, authorimg) if success: - myDB.action('update authors set AuthorImg="%s" where AuthorID="%s"' % (newimg, authorid)) + myDB.action('update authors set AuthorImg=? where AuthorID=?', (newimg, authorid)) # On full scan, update bookcounts for all authors, not just new ones - refresh may have located # new books for existing authors especially if switched provider gb/gr or changed wanted languages authors = myDB.select('select AuthorID from authors') else: # On single author/book import, just update bookcount for that author - authors = myDB.select('select AuthorID from authors where AuthorName = "%s"' % author.replace('"', '""')) + authors = myDB.select('select AuthorID from authors where AuthorName=?', (author.replace('"', '""'),)) logger.debug('Updating bookcounts for %i author%s' % (len(authors), plural(len(authors)))) for author in authors: diff --git a/lazylibrarian/magazinescan.py b/lazylibrarian/magazinescan.py index 743c5fa0f..cadfb9b9d 100644 --- a/lazylibrarian/magazinescan.py +++ b/lazylibrarian/magazinescan.py @@ -269,7 +269,7 @@ def magazineScan(): issuefile = mag['IssueFile'] if issuefile and not os.path.isfile(issuefile): - myDB.action('DELETE from Issues where issuefile="%s"' % issuefile) + myDB.action('DELETE from Issues where issuefile=?', (issuefile,)) logger.info('Issue %s - %s deleted as not found on disk' % (title, issuedate)) controlValueDict = {"Title": title} newValueDict = { @@ -285,11 +285,11 @@ def magazineScan(): # now check the magazine titles and delete any with no issues for mag in mags: title = mag['Title'] - count = myDB.select('SELECT COUNT(Title) as counter FROM issues WHERE Title="%s"' % title) + count = myDB.select('SELECT COUNT(Title) as counter FROM issues WHERE Title=?', (title,)) issues = count[0]['counter'] if not issues: logger.debug('Magazine %s deleted as no issues found' % title) - myDB.action('DELETE from magazines WHERE Title="%s"' % title) + myDB.action('DELETE from magazines WHERE Title=?', (title,)) logger.info(' Checking [%s] for magazines' % mag_path) @@ -351,7 +351,7 @@ def magazineScan(): # is this magazine already in the database? mag_entry = myDB.match( - 'SELECT LastAcquired, IssueDate, MagazineAdded from magazines WHERE Title="%s"' % title) + 'SELECT LastAcquired, IssueDate, MagazineAdded from magazines WHERE Title=?', (title,)) if not mag_entry: # need to add a new magazine to the database newValueDict = { @@ -379,8 +379,8 @@ def magazineScan(): # is this issue already in the database? controlValueDict = {"Title": title, "IssueDate": issuedate} issue_id = create_id("%s %s" % (title, issuedate)) - iss_entry = myDB.match('SELECT Title from issues WHERE Title="%s" and IssueDate="%s"' % ( - title, issuedate)) + iss_entry = myDB.match('SELECT Title from issues WHERE Title=? and IssueDate=?', + (title, issuedate)) if not iss_entry: newValueDict = { "IssueAcquired": iss_acquired, diff --git a/lazylibrarian/manualbook.py b/lazylibrarian/manualbook.py index 76c24adc3..72bf80d32 100644 --- a/lazylibrarian/manualbook.py +++ b/lazylibrarian/manualbook.py @@ -47,9 +47,9 @@ def searchItem(item=None, bookid=None, cat=None): if cat in ['book', 'audio']: myDB = database.DBConnection() - cmd = 'SELECT authorName,bookName,bookSub from books,authors WHERE bookID="%s"' % bookid - cmd += ' and books.AuthorID=authors.AuthorID' - match = myDB.match(cmd) + cmd = 'SELECT authorName,bookName,bookSub from books,authors WHERE books.AuthorID=authors.AuthorID' + cmd += ' and bookID=?' + match = myDB.match(cmd, (bookid,)) if match: book['authorName'] = match['authorName'] book['bookName'] = match['bookName'] diff --git a/lazylibrarian/notifiers/custom_notify.py b/lazylibrarian/notifiers/custom_notify.py index d3f72e8f7..79c2095e3 100644 --- a/lazylibrarian/notifiers/custom_notify.py +++ b/lazylibrarian/notifiers/custom_notify.py @@ -41,9 +41,9 @@ def _notify(message, event, force=False): data = myDB.match('SELECT * from books') else: # message is a bookid or a magazineid - data = myDB.match('SELECT * from books where BookID="%s"' % message) + data = myDB.match('SELECT * from books where BookID=?', (message,)) if not data: - data = myDB.match('SELECT * from magazines where BookID="%s"' % message) + data = myDB.match('SELECT * from magazines where BookID=?', (message,)) dictionary = dict(zip(data.keys(), data)) dictionary['Event'] = event diff --git a/lazylibrarian/postprocess.py b/lazylibrarian/postprocess.py index 32ed66a3e..acc8cbbf6 100644 --- a/lazylibrarian/postprocess.py +++ b/lazylibrarian/postprocess.py @@ -44,12 +44,12 @@ def update_downloads(provider): myDB = database.DBConnection() - entry = myDB.match('SELECT Count FROM downloads where Provider="%s"' % provider) + entry = myDB.match('SELECT Count FROM downloads where Provider=?', (provider,)) if entry: counter = int(entry['Count']) - myDB.action('UPDATE downloads SET Count=%s WHERE Provider="%s"' % (counter + 1, provider)) + myDB.action('UPDATE downloads SET Count=? WHERE Provider=?', (counter + 1, provider)) else: - myDB.action('INSERT into downloads (Count, Provider) VALUES (%s, "%s")' % (1, provider)) + myDB.action('INSERT into downloads (Count, Provider) VALUES (?, ?)', (1, provider)) def processAlternate(source_dir=None): @@ -102,7 +102,7 @@ def processAlternate(source_dir=None): bookname = metadata['title'] myDB = database.DBConnection() authorid = '' - authmatch = myDB.match('SELECT * FROM authors where AuthorName="%s"' % authorname) + authmatch = myDB.match('SELECT * FROM authors where AuthorName=?', (authorname,)) if not authmatch: # try goodreads preferred authorname @@ -118,7 +118,7 @@ def processAlternate(source_dir=None): authorid = author_gr['authorid'] logger.debug("GoodReads reports [%s] for [%s]" % (grauthorname, authorname)) authorname = grauthorname - authmatch = myDB.match('SELECT * FROM authors where AuthorID="%s"' % authorid) + authmatch = myDB.match('SELECT * FROM authors where AuthorID=?', (authorid,)) if authmatch: logger.debug("ALT: Author %s found in database" % authorname) @@ -269,8 +269,7 @@ def processDir(reset=False): matchtitle = unaccented_str(book['NZBtitle']) if torrentname and torrentname != matchtitle: logger.debug("%s Changing [%s] to [%s]" % (book['Source'], matchtitle, torrentname)) - myDB.action('UPDATE wanted SET NZBtitle = "%s" WHERE NZBurl = "%s"' % - (torrentname, book['NZBurl'])) + myDB.action('UPDATE wanted SET NZBtitle=? WHERE NZBurl=?', (torrentname, book['NZBurl'])) matchtitle = torrentname # here we could also check percentage downloaded or eta or status? @@ -394,9 +393,9 @@ def processDir(reset=False): mostrecentissue = '' logger.debug(u'Found match (%s%%): %s for %s %s' % (match, pp_path, book_type, book['NZBtitle'])) - cmd = 'SELECT AuthorName,BookName from books,authors WHERE BookID="%s"' % book['BookID'] + cmd = 'SELECT AuthorName,BookName from books,authors WHERE BookID=?' cmd += ' and books.AuthorID = authors.AuthorID' - data = myDB.match(cmd) + data = myDB.match(cmd, (book['BookID'],)) if data: # it's ebook/audiobook logger.debug(u'Processing %s %s' % (book_type, book['BookID'])) authorname = data['AuthorName'] @@ -418,7 +417,7 @@ def processDir(reset=False): dest_dir = lazylibrarian.DIRECTORY('Audio') dest_path = os.path.join(dest_dir, dest_path).encode(lazylibrarian.SYS_ENCODING) else: - data = myDB.match('SELECT IssueDate from magazines WHERE Title="%s"' % book['BookID']) + data = myDB.match('SELECT IssueDate from magazines WHERE Title=?', (book['BookID'],)) if data: # it's a magazine logger.debug(u'Processing magazine %s' % book['BookID']) # AuxInfo was added for magazine release date, normally housed in 'magazines' @@ -543,9 +542,9 @@ def processDir(reset=False): # if it's a book, reset status so we try for a different version # if it's a magazine, user can select a different one from pastissues table if book_type == 'eBook': - myDB.action('UPDATE books SET status = "Wanted" WHERE BookID="%s"' % book['BookID']) + myDB.action('UPDATE books SET status="Wanted" WHERE BookID=?', (book['BookID'],)) elif book_type == 'AudioBook': - myDB.action('UPDATE books SET audiostatus = "Wanted" WHERE BookID="%s"' % book['BookID']) + myDB.action('UPDATE books SET audiostatus="Wanted" WHERE BookID=?', (book['BookID'],)) # at this point, as it failed we should move it or it will get postprocessed # again (and fail again) @@ -581,7 +580,7 @@ def processDir(reset=False): if not extn or extn not in skipped_extensions: bookID = entry.split("LL.(")[1].split(")")[0] logger.debug("Book with id: %s found in download directory" % bookID) - data = myDB.match('SELECT BookFile from books WHERE BookID="%s"' % bookID) + data = myDB.match('SELECT BookFile from books WHERE BookID=?', (bookID,)) if data and data['BookFile'] and os.path.isfile(data['BookFile']): logger.debug('Skipping BookID %s, already exists' % bookID) else: @@ -636,10 +635,10 @@ def processDir(reset=False): # change status to "Failed", and ask downloader to delete task and files if book['BookID'] != 'unknown': if book_type == 'eBook': - myDB.action('UPDATE books SET status = "Wanted" WHERE BookID="%s"' % book['BookID']) + myDB.action('UPDATE books SET status="Wanted" WHERE BookID=?', (book['BookID'],)) elif book_type == 'AudioBook': - myDB.action('UPDATE books SET audiostatus = "Wanted" WHERE BookID="%s"' % book['BookID']) - myDB.action('UPDATE wanted SET Status="Failed" WHERE BookID="%s"' % book['BookID']) + myDB.action('UPDATE books SET audiostatus="Wanted" WHERE BookID=?', (book['BookID'],)) + myDB.action('UPDATE wanted SET Status="Failed" WHERE BookID=?', (book['BookID'],)) delete_task(book['Source'], book['DownloadID'], True) # Check if postprocessor needs to run again @@ -716,13 +715,12 @@ def import_book(pp_path=None, bookID=None): return False myDB = database.DBConnection() - cmd = 'SELECT AuthorName,BookName from books,authors WHERE BookID="%s"' % bookID - cmd += ' and books.AuthorID = authors.AuthorID' - data = myDB.match(cmd) + cmd = 'SELECT AuthorName,BookName from books,authors WHERE BookID=? and books.AuthorID = authors.AuthorID' + data = myDB.match(cmd, (bookID,)) if data: - cmd = 'SELECT BookID, NZBprov, AuxInfo FROM wanted WHERE BookID="%s" and Status="Snatched"' % bookID + cmd = 'SELECT BookID, NZBprov, AuxInfo FROM wanted WHERE BookID=? and Status="Snatched"' # we may have snatched an ebook and audiobook of the same title/id - was_snatched = myDB.select(cmd) + was_snatched = myDB.select(cmd, (bookID,)) want_audio = False want_ebook = False for item in was_snatched: @@ -801,16 +799,16 @@ def import_book(pp_path=None, bookID=None): else: logger.error('Postprocessing for %s has failed: %s' % (global_name, dest_file)) logger.error('Warning - Residual files remain in %s.fail' % pp_path) - was_snatched = myDB.match('SELECT BookID FROM wanted WHERE BookID="%s" and Status="Snatched"' % bookID) + was_snatched = myDB.match('SELECT BookID FROM wanted WHERE BookID=? and Status="Snatched"', (bookID,)) if was_snatched: controlValueDict = {"BookID": bookID} newValueDict = {"Status": "Failed", "NZBDate": now()} myDB.upsert("wanted", newValueDict, controlValueDict) # reset status so we try for a different version if book_type == 'AudioBook': - myDB.action('UPDATE books SET audiostatus = "Wanted" WHERE BookID="%s"' % bookID) + myDB.action('UPDATE books SET audiostatus="Wanted" WHERE BookID=?', (bookID,)) else: - myDB.action('UPDATE books SET status = "Wanted" WHERE BookID="%s"' % bookID) + myDB.action('UPDATE books SET status="Wanted" WHERE BookID=?', (bookID,)) try: os.rename(pp_path, pp_path + '.fail') logger.error('Warning - Residual files remain in %s.fail' % pp_path) @@ -848,7 +846,7 @@ def processExtras(dest_file=None, global_name=None, bookid=None, book_type="eBoo myDB.upsert("books", newValueDict, controlValueDict) # update authors book counts - match = myDB.match('SELECT AuthorID FROM books WHERE BookID="%s"' % bookid) + match = myDB.match('SELECT AuthorID FROM books WHERE BookID=?', (bookid,)) if match: update_totals(match['AuthorID']) @@ -866,9 +864,8 @@ def processExtras(dest_file=None, global_name=None, bookid=None, book_type="eBoo processAutoAdd(dest_path) cmd = 'SELECT AuthorName,BookID,BookName,BookDesc,BookIsbn,BookImg,BookDate,' - cmd += 'BookLang,BookPub from books,authors WHERE BookID="%s"' % bookid - cmd += ' and books.AuthorID = authors.AuthorID' - data = myDB.match(cmd) + cmd += 'BookLang,BookPub from books,authors WHERE BookID=? and books.AuthorID = authors.AuthorID' + data = myDB.match(cmd, (bookid,)) if not data: logger.error('processExtras: No data found for bookid %s' % bookid) return diff --git a/lazylibrarian/resultlist.py b/lazylibrarian/resultlist.py index 82ca0068d..ebb90e1ca 100644 --- a/lazylibrarian/resultlist.py +++ b/lazylibrarian/resultlist.py @@ -106,7 +106,7 @@ def findBestResult(resultlist, book, searchtype, source): logger.debug("Rejecting %s, no URL found" % resultTitle) if not rejected: - already_failed = myDB.match('SELECT * from wanted WHERE NZBurl="%s" and Status="Failed"' % url) + already_failed = myDB.match('SELECT * from wanted WHERE NZBurl=? and Status="Failed"', (url,)) if already_failed: logger.debug("Rejecting %s, blacklisted at %s" % (resultTitle, already_failed['NZBprov'])) rejected = True @@ -216,11 +216,11 @@ def downloadResult(match, book): auxinfo = 'eBook' if auxinfo == 'eBook': - snatchedbooks = myDB.match('SELECT BookID from books WHERE BookID="%s" and Status="Snatched"' % - newValueDict["BookID"]) + snatchedbooks = myDB.match('SELECT BookID from books WHERE BookID=? and Status="Snatched"', + (newValueDict["BookID"],)) else: - snatchedbooks = myDB.match('SELECT BookID from books WHERE BookID="%s" and AudioStatus="Snatched"' % - newValueDict["BookID"]) + snatchedbooks = myDB.match('SELECT BookID from books WHERE BookID=? and AudioStatus="Snatched"', + (newValueDict["BookID"],)) if snatchedbooks: logger.debug('%s %s already marked snatched' % (book['authorName'], book['bookName'])) diff --git a/lazylibrarian/searchbook.py b/lazylibrarian/searchbook.py index 84c7ea64d..98d9295d1 100644 --- a/lazylibrarian/searchbook.py +++ b/lazylibrarian/searchbook.py @@ -65,9 +65,8 @@ def search_book(books=None, library=None): # The user has added a new book for book in books: cmd = 'SELECT BookID, AuthorName, BookName, BookSub, books.Status, AudioStatus ' - cmd += 'from books,authors WHERE BookID="%s" ' % book['bookid'] - cmd += 'AND books.AuthorID = authors.AuthorID' - results = myDB.select(cmd) + cmd += 'from books,authors WHERE BookID=? AND books.AuthorID = authors.AuthorID' + results = myDB.select(cmd, (book['bookid'],)) if results: for terms in results: searchbooks.append(terms) diff --git a/lazylibrarian/searchmag.py b/lazylibrarian/searchmag.py index d784cbb82..a50afaaff 100644 --- a/lazylibrarian/searchmag.py +++ b/lazylibrarian/searchmag.py @@ -56,7 +56,7 @@ def search_magazines(mags=None, reset=False): searchmags = [] for magazine in mags: searchmags_temp = myDB.select('SELECT Title, Regex, LastAcquired, IssueDate from magazines \ - WHERE Title="%s" AND Status="Active"' % (magazine['bookid'])) + WHERE Title=? AND Status="Active"', (magazine['bookid'],)) for terms in searchmags_temp: searchmags.append(terms) @@ -173,7 +173,7 @@ def search_magazines(mags=None, reset=False): nzbdate = nzbdate2format(nzbdate_temp) nzbmode = nzb['nzbmode'] - results = myDB.match('SELECT * from magazines WHERE Title="%s"' % bookid) + results = myDB.match('SELECT * from magazines WHERE Title=?', (bookid,)) if not results: logger.debug('Magazine [%s] does not match search term [%s].' % (nzbtitle, bookid)) bad_name += 1 @@ -232,11 +232,10 @@ def search_magazines(mags=None, reset=False): rejected = True if not rejected: - already_failed = myDB.match('SELECT * from wanted WHERE NZBurl="%s" and Status="Failed"' % - nzburl) - if already_failed: + blocked = myDB.match('SELECT * from wanted WHERE NZBurl=? and Status="Failed"', (nzburl,)) + if blocked: logger.debug("Rejecting %s, blacklisted at %s" % - (nzbtitle_formatted, already_failed['NZBprov'])) + (nzbtitle_formatted, blocked['NZBprov'])) rejected = True if not rejected: @@ -451,8 +450,8 @@ def search_magazines(mags=None, reset=False): # and status has been user-set ( we only delete the "Skipped" ones ) # In "wanted" table it might be already snatched/downloading/processing - mag_entry = myDB.match('SELECT * from %s WHERE NZBtitle="%s" and NZBprov="%s"' % ( - insert_table, nzbtitle, nzbprov)) + mag_entry = myDB.match('SELECT * from ? WHERE NZBtitle=? and NZBprov=?', + (insert_table, nzbtitle, nzbprov)) if not mag_entry: controlValueDict = { "NZBtitle": nzbtitle, diff --git a/lazylibrarian/searchrss.py b/lazylibrarian/searchrss.py index 3d4e708b0..4bc4677d5 100644 --- a/lazylibrarian/searchrss.py +++ b/lazylibrarian/searchrss.py @@ -62,7 +62,7 @@ def search_rss_book(books=None, library=None): # not sure if anyone would use a goodreads wishlist if not using goodreads interface... logger.debug('Processing %s item%s in wishlists' % (len(resultlist), plural(len(resultlist)))) if book['rss_bookid'] and lazylibrarian.CONFIG['BOOK_API'] == "GoodReads": - bookmatch = myDB.match('select Status,BookName from books where bookid="%s"' % book['rss_bookid']) + bookmatch = myDB.match('select Status,BookName from books where bookid=?', (book['rss_bookid'],)) if bookmatch: bookstatus = bookmatch['Status'] bookname = bookmatch['BookName'] @@ -148,9 +148,8 @@ def search_rss_book(books=None, library=None): # The user has added a new book for book in books: cmd = 'SELECT BookID, AuthorName, BookName, BookSub, books.Status, AudioStatus ' - cmd += 'from books,authors WHERE BookID="%s" ' % book['bookid'] - cmd += 'AND books.AuthorID = authors.AuthorID' - results = myDB.select(cmd) + cmd += 'from books,authors WHERE BookID=? AND books.AuthorID = authors.AuthorID' + results = myDB.select(cmd, (book['bookid'],)) for terms in results: searchbooks.append(terms) diff --git a/lazylibrarian/webServe.py b/lazylibrarian/webServe.py index bd4fd0e6d..4a073d856 100644 --- a/lazylibrarian/webServe.py +++ b/lazylibrarian/webServe.py @@ -122,18 +122,22 @@ def getSeries(self, iDisplayStart=0, iDisplayLength=100, iSortCol_0=0, sSortDir_ cmd = 'SELECT series.SeriesID,AuthorName,SeriesName,series.Status,seriesauthors.AuthorID,series.SeriesID' cmd += ' from series,authors,seriesauthors' cmd += ' where authors.AuthorID=seriesauthors.AuthorID and series.SeriesID=seriesauthors.SeriesID' + args = [] if whichStatus not in ['All', 'None']: - cmd += ' and series.Status="%s"' % whichStatus - + cmd += ' and series.Status=?' + args.append(whichStatus) if AuthorID and not AuthorID == 'None': - match = myDB.match('SELECT AuthorName from authors WHERE AuthorID="%s"' % AuthorID) + match = myDB.match('SELECT AuthorName from authors WHERE AuthorID=?', (AuthorID,)) if match: title = "%s Series" % match['AuthorName'] - cmd += ' and seriesauthors.AuthorID="%s"' % AuthorID + cmd += ' and seriesauthors.AuthorID=?' + args.append(AuthorID) cmd += ' GROUP BY series.seriesID' cmd += ' order by AuthorName,SeriesName' - - rowlist = myDB.select(cmd) + if args: + rowlist = myDB.select(cmd, tuple(args)) + else: + rowlist = myDB.select(cmd) # turn the sqlite rowlist into a list of lists filtered = [] @@ -168,7 +172,7 @@ def series(self, AuthorID=None, whichStatus=None): myDB = database.DBConnection() title = "Series" if AuthorID: - match = myDB.match('SELECT AuthorName from authors WHERE AuthorID="%s"' % AuthorID) + match = myDB.match('SELECT AuthorName from authors WHERE AuthorID=?', (AuthorID,)) if match: title = "%s Series" % match['AuthorName'] return serve_template(templatename="series.html", title=title, authorid=AuthorID, series=[], @@ -180,14 +184,14 @@ def seriesMembers(self, seriesid): cmd = 'SELECT SeriesName,series.SeriesID,AuthorName,seriesauthors.AuthorID' cmd += ' from series,authors,seriesauthors' cmd += ' where authors.AuthorID=seriesauthors.AuthorID and series.SeriesID=seriesauthors.SeriesID' - cmd += ' and series.SeriesID="%s"' % seriesid - series = myDB.match(cmd) + cmd += ' and series.SeriesID=?' + series = myDB.match(cmd, (seriesid,)) cmd = 'SELECT member.BookID,BookName,SeriesNum,BookImg,books.Status,AuthorName,authors.AuthorID' cmd += ' from member,series,books,authors' cmd += ' where series.SeriesID=member.SeriesID and books.BookID=member.BookID' cmd += ' and books.AuthorID=authors.AuthorID and books.Status != "Ignored"' - cmd += ' and series.SeriesID="%s" order by SeriesName' % seriesid - members = myDB.select(cmd) + cmd += ' and series.SeriesID=? order by SeriesName' + members = myDB.select(cmd, (seriesid,)) # is it a multi-author series? multi = "False" authorid = '' @@ -210,7 +214,7 @@ def markSeries(self, action=None, **args): # ouch dirty workaround... if not seriesid == 'book_table_length': if action in ["Wanted", "Active", "Skipped", "Ignored"]: - match = myDB.match('SELECT SeriesName from series WHERE SeriesID = "%s"' % seriesid) + match = myDB.match('SELECT SeriesName from series WHERE SeriesID=?', (seriesid,)) if match: myDB.upsert("series", {'Status': action}, {'SeriesID': seriesid}) logger.debug(u'Status set to "%s" for "%s"' % (action, match['SeriesName'])) @@ -453,14 +457,12 @@ def authorPage(self, AuthorID, BookLang=None, library='eBook', Ignored=False): myDB = database.DBConnection() if Ignored: languages = myDB.select( - "SELECT DISTINCT BookLang from books WHERE AuthorID = '%s' AND Status ='Ignored'" % AuthorID) + "SELECT DISTINCT BookLang from books WHERE AuthorID=? AND Status ='Ignored'", (AuthorID,)) else: languages = myDB.select( - "SELECT DISTINCT BookLang from books WHERE AuthorID = '%s' AND Status !='Ignored'" % AuthorID) - - queryauthors = "SELECT * from authors WHERE AuthorID = '%s'" % AuthorID + "SELECT DISTINCT BookLang from books WHERE AuthorID=? AND Status !='Ignored'", (AuthorID,)) - author = myDB.match(queryauthors) + author = myDB.match("SELECT * from authors WHERE AuthorID=?", (AuthorID,)) types = ['eBook'] if lazylibrarian.SHOW_AUDIO: @@ -479,8 +481,7 @@ def pauseAuthor(self, AuthorID): self.label_thread() myDB = database.DBConnection() - authorsearch = myDB.match( - 'SELECT AuthorName from authors WHERE AuthorID="%s"' % AuthorID) + authorsearch = myDB.match('SELECT AuthorName from authors WHERE AuthorID=?', (AuthorID,)) if authorsearch: AuthorName = authorsearch['AuthorName'] logger.info(u"Pausing author: %s" % AuthorName) @@ -500,8 +501,7 @@ def resumeAuthor(self, AuthorID): self.label_thread() myDB = database.DBConnection() - authorsearch = myDB.match( - 'SELECT AuthorName from authors WHERE AuthorID="%s"' % AuthorID) + authorsearch = myDB.match('SELECT AuthorName from authors WHERE AuthorID=?', (AuthorID,)) if authorsearch: AuthorName = authorsearch['AuthorName'] logger.info(u"Resuming author: %s" % AuthorName) @@ -521,8 +521,7 @@ def ignoreAuthor(self, AuthorID): self.label_thread() myDB = database.DBConnection() - authorsearch = myDB.match( - 'SELECT AuthorName from authors WHERE AuthorID="%s"' % AuthorID) + authorsearch = myDB.match('SELECT AuthorName from authors WHERE AuthorID=?', (AuthorID,)) if authorsearch: AuthorName = authorsearch['AuthorName'] logger.info(u"Ignoring author: %s" % AuthorName) @@ -541,14 +540,13 @@ def removeAuthor(self, AuthorID): self.label_thread() myDB = database.DBConnection() - authorsearch = myDB.match( - 'SELECT AuthorName from authors WHERE AuthorID="%s"' % AuthorID) + authorsearch = myDB.match('SELECT AuthorName from authors WHERE AuthorID=?', (AuthorID,)) if authorsearch: # to stop error if try to remove an author while they are still loading AuthorName = authorsearch['AuthorName'] logger.info(u"Removing all references to author: %s" % AuthorName) - myDB.action('DELETE from authors WHERE AuthorID="%s"' % AuthorID) - myDB.action('DELETE from seriesauthors WHERE AuthorID="%s"' % AuthorID) - myDB.action('DELETE from books WHERE AuthorID="%s"' % AuthorID) + myDB.action('DELETE from authors WHERE AuthorID=', (AuthorID,)) + myDB.action('DELETE from seriesauthors WHERE AuthorID=?', (AuthorID,)) + myDB.action('DELETE from books WHERE AuthorID=?', (AuthorID,)) raise cherrypy.HTTPRedirect("home") @cherrypy.expose @@ -556,7 +554,7 @@ def refreshAuthor(self, AuthorID): self.label_thread() myDB = database.DBConnection() - authorsearch = myDB.match('SELECT AuthorName from authors WHERE AuthorID="%s"' % AuthorID) + authorsearch = myDB.match('SELECT AuthorName from authors WHERE AuthorID=?', (AuthorID,)) if authorsearch: # to stop error if try to refresh an author while they are still loading threading.Thread(target=addAuthorToDB, name='REFRESHAUTHOR', args=[None, True, AuthorID]).start() raise cherrypy.HTTPRedirect("authorPage?AuthorID=%s" % AuthorID) @@ -569,7 +567,7 @@ def libraryScanAuthor(self, AuthorID, **kwargs): self.label_thread() myDB = database.DBConnection() - authorsearch = myDB.match('SELECT AuthorName from authors WHERE AuthorID="%s"' % AuthorID) + authorsearch = myDB.match('SELECT AuthorName from authors WHERE AuthorID=?', (AuthorID,)) if authorsearch: # to stop error if try to refresh an author while they are still loading AuthorName = authorsearch['AuthorName'] @@ -590,8 +588,8 @@ def libraryScanAuthor(self, AuthorID, **kwargs): else: sourcefile = 'BookFile' cmd = 'SELECT %s from books,authors where books.AuthorID = authors.AuthorID' % sourcefile - cmd += ' and AuthorName="%s" and %s <> ""' % (AuthorName, sourcefile) - anybook = myDB.match(cmd) + cmd += ' and AuthorName=? and %s <> ""' % sourcefile + anybook = myDB.match(cmd, (AuthorName,)) if anybook: authordir = safe_unicode(os.path.dirname(os.path.dirname(anybook[sourcefile]))) if os.path.isdir(authordir): @@ -664,7 +662,7 @@ def snatchBook(self, bookid=None, mode=None, provider=None, url=None, size=None, self.label_thread() logger.debug("snatch bookid %s mode=%s from %s url=[%s]" % (bookid, mode, provider, url)) myDB = database.DBConnection() - bookdata = myDB.match('SELECT AuthorID, BookName from books WHERE BookID="%s"' % bookid) + bookdata = myDB.match('SELECT AuthorID, BookName from books WHERE BookID=?', (bookid,)) if bookdata: controlValueDict = {"NZBurl": url} newValueDict = { @@ -734,8 +732,10 @@ def getBooks(self, iDisplayStart=0, iDisplayLength=100, iSortCol_0=0, sSortDir_0 cmd += ' from books,authors where books.AuthorID = authors.AuthorID' status_type = 'books.status' + args = [] if kwargs['source'] == "Manage": - cmd += ' and books.STATUS="%s"' % kwargs['whichStatus'] + cmd += ' and books.STATUS=?' + args.append(kwargs['whichStatus']) elif kwargs['source'] == "Books": cmd += ' and books.STATUS !="Skipped" AND books.STATUS !="Ignored"' elif kwargs['source'] == "Audio": @@ -751,7 +751,8 @@ def getBooks(self, iDisplayStart=0, iDisplayLength=100, iSortCol_0=0, sSortDir_0 else: status_type = 'books.status' - cmd += ' and books.AuthorID="%s"' % kwargs['AuthorID'] + cmd += ' and books.AuthorID=?' + args.append(kwargs['AuthorID'] if 'ignored' in kwargs and kwargs['ignored'] == "True": cmd += ' and %s="Ignored"' % status_type else: @@ -760,9 +761,10 @@ def getBooks(self, iDisplayStart=0, iDisplayLength=100, iSortCol_0=0, sSortDir_0 if kwargs['source'] in ["Books", "Author", "Audio"]: # for these we need to check and filter on BookLang if set if 'booklang' in kwargs and kwargs['booklang'] != '' and kwargs['booklang'] != 'None': - cmd += ' and BOOKLANG="%s"' % kwargs['booklang'] + cmd += ' and BOOKLANG=?' + args.append(kwargs['booklang']) - rowlist = myDB.select(cmd) + rowlist = myDB.select(cmd, tuple(args)) # At his point we want to sort and filter _before_ adding the html as it's much quicker # turn the sqlite rowlist into a list of lists d = [] @@ -866,7 +868,7 @@ def get_alphanum_key_func(key): def addBook(self, bookid=None): myDB = database.DBConnection() AuthorID = "" - match = myDB.match('SELECT AuthorID from books WHERE BookID="%s"' % bookid) + match = myDB.match('SELECT AuthorID from books WHERE BookID=?', (bookid,)) if match: myDB.upsert("books", {'Status': 'Wanted'}, {'BookID': bookid}) AuthorID = match['AuthorID'] @@ -904,7 +906,7 @@ def startBookSearch(self, books=None): def searchForBook(self, bookid=None): myDB = database.DBConnection() AuthorID = '' - bookdata = myDB.match('SELECT AuthorID from books WHERE BookID="%s"' % bookid) + bookdata = myDB.match('SELECT AuthorID from books WHERE BookID=?', (bookid,)) if bookdata: AuthorID = bookdata["AuthorID"] @@ -934,9 +936,9 @@ def mimetype(filename): def openBook(self, bookid=None, source=None): self.label_thread() myDB = database.DBConnection() - cmd = 'SELECT BookFile,AudioFile,AuthorName,BookName from books,authors WHERE BookID="%s"' % bookid + cmd = 'SELECT BookFile,AudioFile,AuthorName,BookName from books,authors WHERE BookID=?' cmd += ' and books.AuthorID = authors.AuthorID' - bookdata = myDB.match(cmd) + bookdata = myDB.match(cmd, (bookid,)) if bookdata: if source == 'audio': source = 'AudioBook' @@ -962,7 +964,7 @@ def editAuthor(self, authorid=None): myDB = database.DBConnection() - data = myDB.match('SELECT * from authors WHERE AuthorID="%s"' % authorid) + data = myDB.match('SELECT * from authors WHERE AuthorID=?', (authorid,)) if data: return serve_template(templatename="editauthor.html", title="Edit Author", config=data) else: @@ -974,7 +976,7 @@ def authorUpdate(self, authorid='', authorname='', authorborn='', authordeath='' myDB = database.DBConnection() if authorid: - authdata = myDB.match('SELECT * from authors WHERE AuthorID="%s"' % authorid) + authdata = myDB.match('SELECT * from authors WHERE AuthorID=?', (authorid,)) if authdata: edited = "" if authorborn == 'None': @@ -995,7 +997,7 @@ def authorUpdate(self, authorid='', authorname='', authorborn='', authordeath='' edited += "Manual " if not (authdata["AuthorName"] == authorname): - match = myDB.match('SELECT AuthorName from authors where AuthorName="%s"' % authorname) + match = myDB.match('SELECT AuthorName from authors where AuthorName=?', (authorname,)) if match: logger.debug("Unable to rename, new author name %s already exists" % authorname) authorname = authdata["AuthorName"] @@ -1089,11 +1091,11 @@ def editBook(self, bookid=None): authors = myDB.select( "SELECT AuthorName from authors WHERE Status !='Ignored' ORDER by AuthorName COLLATE NOCASE") cmd = 'SELECT BookName,BookID,BookSub,BookGenre,BookLang,books.Manual,AuthorName,books.AuthorID ' - cmd += 'from books,authors WHERE books.AuthorID = authors.AuthorID and BookID="%s"' % bookid - bookdata = myDB.match(cmd) + cmd += 'from books,authors WHERE books.AuthorID = authors.AuthorID and BookID=?' + bookdata = myDB.match(cmd, (bookid,)) cmd = 'SELECT SeriesName, SeriesNum from member,series ' - cmd += 'where series.SeriesID=member.SeriesID and BookID="%s"' % bookid - seriesdict = myDB.select(cmd) + cmd += 'where series.SeriesID=member.SeriesID and BookID=?' + seriesdict = myDB.select(cmd, (bookid,)) if bookdata: return serve_template(templatename="editbook.html", title="Edit Book", config=bookdata, seriesdict=seriesdict, authors=authors) @@ -1106,8 +1108,8 @@ def bookUpdate(self, bookname='', bookid='', booksub='', bookgenre='', booklang= myDB = database.DBConnection() if bookid: cmd = 'SELECT BookName,BookSub,BookGenre,BookLang,books.Manual,AuthorName,books.AuthorID ' - cmd += 'from books,authors WHERE books.AuthorID = authors.AuthorID and BookID="%s"' % bookid - bookdata = myDB.match(cmd) + cmd += 'from books,authors WHERE books.AuthorID = authors.AuthorID and BookID=?' + bookdata = myDB.match(cmd, (bookid,)) if bookdata: edited = '' moved = False @@ -1139,8 +1141,8 @@ def bookUpdate(self, bookname='', bookid='', booksub='', bookgenre='', booklang= myDB.upsert("books", newValueDict, controlValueDict) cmd = 'SELECT SeriesName, SeriesNum from member,series ' - cmd += 'where series.SeriesID=member.SeriesID and BookID="%s"' % bookid - old_series = myDB.select(cmd) + cmd += 'where series.SeriesID=member.SeriesID and BookID=?' + old_series = myDB.select(cmd, (bookid,)) old_dict = {} new_dict = {} dict_counter = 0 @@ -1178,8 +1180,7 @@ def bookUpdate(self, bookname='', bookid='', booksub='', bookgenre='', booklang= logger.debug('Book [%s] has not been changed' % bookname) if moved: - authordata = myDB.match( - 'SELECT AuthorID from authors WHERE AuthorName="%s"' % authorname) + authordata = myDB.match('SELECT AuthorID from authors WHERE AuthorName=?', (authorname,)) if authordata: controlValueDict = {'BookID': bookid} newValueDict = {'AuthorID': authordata['AuthorID']} @@ -1213,7 +1214,7 @@ def markBooks(self, AuthorID=None, seriesid=None, action=None, redirect=None, ** # ouch dirty workaround... if not bookid == 'book_table_length': if action in ["Wanted", "Have", "Ignored", "Skipped"]: - title = myDB.match('SELECT BookName from books WHERE BookID = "%s"' % bookid) + title = myDB.match('SELECT BookName from books WHERE BookID=?', (bookid,)) if title: bookname = title['BookName'] if library == 'eBook': @@ -1224,7 +1225,7 @@ def markBooks(self, AuthorID=None, seriesid=None, action=None, redirect=None, ** logger.debug(u'AudioStatus set to "%s" for "%s"' % (action, bookname)) if action in ["Remove", "Delete"]: bookdata = myDB.match( - 'SELECT AuthorID,Bookname,BookFile,AudioFile from books WHERE BookID = "%s"' % bookid) + 'SELECT AuthorID,Bookname,BookFile,AudioFile from books WHERE BookID=?', (bookid,)) if bookdata: AuthorID = bookdata['AuthorID'] bookname = bookdata['BookName'] @@ -1240,12 +1241,12 @@ def markBooks(self, AuthorID=None, seriesid=None, action=None, redirect=None, ** except Exception as e: logger.debug('rmtree failed on %s, %s' % (bookfile, str(e))) - authorcheck = myDB.match('SELECT AuthorID from authors WHERE AuthorID = "%s"' % AuthorID) + authorcheck = myDB.match('SELECT AuthorID from authors WHERE AuthorID=?', (AuthorID,)) if authorcheck: myDB.upsert("books", {"Status": "Ignored"}, {"BookID": bookid}) logger.debug(u'Status set to Ignored for "%s"' % bookname) else: - myDB.action('delete from books where bookid="%s"' % bookid) + myDB.action('delete from books where bookid=?', (bookid,)) logger.info(u'Removed "%s" from database' % bookname) if redirect == "author" or len(authorcheck): @@ -1358,8 +1359,7 @@ def magazines(self): if magazines: for mag in magazines: title = mag['Title'] - count = myDB.match( - 'SELECT COUNT(Title) as counter FROM issues WHERE Title="%s"' % title) + count = myDB.match('SELECT COUNT(Title) as counter FROM issues WHERE Title=?', (title,)) if count: issues = count['counter'] else: @@ -1390,7 +1390,7 @@ def magazines(self): def issuePage(self, title): myDB = database.DBConnection() - issues = myDB.select('SELECT * from issues WHERE Title="%s" order by IssueDate DESC' % title) + issues = myDB.select('SELECT * from issues WHERE Title=? order by IssueDate DESC', (title,)) if not len(issues): raise cherrypy.HTTPRedirect("magazines") @@ -1443,8 +1443,8 @@ def getPastIssues(self, iDisplayStart=0, iDisplayLength=100, iSortCol_0=0, sSort iDisplayLength = int(iDisplayLength) lazylibrarian.CONFIG['DISPLAYLENGTH'] = iDisplayLength # need to filter on whichStatus - rowlist = myDB.select( - 'SELECT NZBurl, NZBtitle, NZBdate, Auxinfo, NZBprov from pastissues WHERE Status=' + kwargs['whichStatus']) + rowlist = myDB.select('SELECT NZBurl, NZBtitle, NZBdate, Auxinfo, NZBprov from pastissues WHERE Status=?', + (kwargs['whichStatus'],)) rows = [] filtered = [] if len(rowlist): @@ -1479,7 +1479,7 @@ def openMag(self, bookid=None): bookid = urllib.unquote_plus(bookid) myDB = database.DBConnection() # we may want to open an issue with a hashed bookid - mag_data = myDB.match('SELECT * from issues WHERE IssueID="%s"' % bookid) + mag_data = myDB.match('SELECT * from issues WHERE IssueID=?', (bookid,)) if mag_data: IssueFile = mag_data["IssueFile"] if IssueFile and os.path.isfile(IssueFile): @@ -1487,7 +1487,7 @@ def openMag(self, bookid=None): return serve_file(IssueFile, self.mimetype(IssueFile), "attachment") # or we may just have a title to find magazine in issues table - mag_data = myDB.select('SELECT * from issues WHERE Title="%s"' % bookid) + mag_data = myDB.select('SELECT * from issues WHERE Title=?', (bookid,)) if len(mag_data) <= 0: # no issues! raise cherrypy.HTTPRedirect("magazines") elif len(mag_data) == 1 and lazylibrarian.CONFIG['MAG_SINGLE']: # we only have one issue, get it @@ -1521,18 +1521,18 @@ def markPastIssues(self, action=None, **args): nzburl2 = '' if not nzburl2: - title = myDB.select('SELECT * from pastissues WHERE NZBurl="%s"' % nzburl) + title = myDB.select('SELECT * from pastissues WHERE NZBurl=?', (nzburl,)) else: - title = myDB.select('SELECT * from pastissues WHERE NZBurl="%s" OR NZBurl="%s"' % (nzburl, nzburl2)) + title = myDB.select('SELECT * from pastissues WHERE NZBurl=? OR NZBurl=?', (nzburl, nzburl2)) for item in title: nzburl = item['NZBurl'] if action == 'Remove': - myDB.action('DELETE from pastissues WHERE NZBurl="%s"' % nzburl) + myDB.action('DELETE from pastissues WHERE NZBurl=?', (nzburl,)) logger.debug(u'Item %s removed from past issues' % nzburl) maglist.append({'nzburl': nzburl}) elif action in ['Have', 'Ignored', 'Skipped']: - myDB.action('UPDATE pastissues set status="%s" WHERE NZBurl="%s"' % (action, nzburl)) + myDB.action('UPDATE pastissues set status=? WHERE NZBurl=?', (action, nzburl)) logger.debug(u'Item %s removed from past issues' % nzburl) maglist.append({'nzburl': nzburl}) elif action == 'Wanted': @@ -1598,14 +1598,14 @@ def markIssues(self, action=None, **args): for item in args: # ouch dirty workaround... if not item == 'book_table_length': - issue = myDB.match('SELECT IssueFile,Title,IssueDate from issues WHERE IssueID="%s"' % item) + issue = myDB.match('SELECT IssueFile,Title,IssueDate from issues WHERE IssueID=?', (item,)) if issue: if action == "Delete": result = self.deleteIssue(issue['IssueFile']) if result: logger.info(u'Issue %s of %s deleted from disc' % (issue['IssueDate'], issue['Title'])) if action == "Remove" or action == "Delete": - myDB.action('DELETE from issues WHERE IssueID="%s"' % item) + myDB.action('DELETE from issues WHERE IssueID=?', (item,)) logger.info(u'Issue %s of %s removed from database' % (issue['IssueDate'], issue['Title'])) raise cherrypy.HTTPRedirect("magazines") @@ -1645,7 +1645,7 @@ def markMagazines(self, action=None, **args): myDB.upsert("magazines", newValueDict, controlValueDict) logger.info(u'Status of magazine %s changed to %s' % (item, action)) if action == "Delete": - issues = myDB.select('SELECT IssueFile from issues WHERE Title="%s"' % item) + issues = myDB.select('SELECT IssueFile from issues WHERE Title=?', (item,)) logger.debug(u'Deleting magazine %s from disc' % item) issuedir = '' for issue in issues: # delete all issues of this magazine @@ -1665,9 +1665,9 @@ def markMagazines(self, action=None, **args): logger.debug(u'Magazine directory %s is not empty' % magdir) logger.info(u'Magazine %s deleted from disc' % item) if action == "Remove" or action == "Delete": - myDB.action('DELETE from magazines WHERE Title="%s"' % item) - myDB.action('DELETE from pastissues WHERE BookID="%s"' % item) - myDB.action('DELETE from issues WHERE Title="%s"' % item) + myDB.action('DELETE from magazines WHERE Title=?', (item,)) + myDB.action('DELETE from pastissues WHERE BookID=?', (item,)) + myDB.action('DELETE from issues WHERE Title=?', (item,)) logger.info(u'Magazine %s removed from database' % item) if action == "Reset": controlValueDict = {"Title": item} @@ -1686,7 +1686,7 @@ def markMagazines(self, action=None, **args): def searchForMag(self, bookid=None): myDB = database.DBConnection() bookid = urllib.unquote_plus(bookid) - bookdata = myDB.match('SELECT * from magazines WHERE Title="%s"' % bookid) + bookdata = myDB.match('SELECT * from magazines WHERE Title=?', (bookid,)) if bookdata: # start searchthreads mags = [{"bookid": bookid}] @@ -1720,7 +1720,7 @@ def addMagazine(self, title=None): # replace any non-ascii quotes/apostrophes with ascii ones eg "Collector's" dic = {u'\u2018': u"'", u'\u2019': u"'", u'\u201c': u'"', u'\u201d': u'"'} title = replace_all(title, dic) - exists = myDB.match('SELECT Title from magazines WHERE Title="%s"' % title) + exists = myDB.match('SELECT Title from magazines WHERE Title=?', (title,)) if exists: logger.debug("Magazine %s already exists (%s)" % (title, exists['Title'])) else: @@ -1986,7 +1986,7 @@ def clearhistory(self, status=None): myDB.action("DELETE from wanted WHERE Status != 'Skipped' and Status != 'Ignored'") else: logger.info(u"Clearing history where status is %s" % status) - myDB.action('DELETE from wanted WHERE Status="%s"' % status) + myDB.action('DELETE from wanted WHERE Status=?', (status,)) raise cherrypy.HTTPRedirect("history") @cherrypy.expose From 6b1ef57a2e9628d4d3926191da6a7bad70a0b764 Mon Sep 17 00:00:00 2001 From: Phil Borman Date: Sun, 25 Jun 2017 18:56:37 +0200 Subject: [PATCH 10/22] Fix typo --- lazylibrarian/webServe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lazylibrarian/webServe.py b/lazylibrarian/webServe.py index 4a073d856..b3ad84d85 100644 --- a/lazylibrarian/webServe.py +++ b/lazylibrarian/webServe.py @@ -752,7 +752,7 @@ def getBooks(self, iDisplayStart=0, iDisplayLength=100, iSortCol_0=0, sSortDir_0 status_type = 'books.status' cmd += ' and books.AuthorID=?' - args.append(kwargs['AuthorID'] + args.append(kwargs['AuthorID']) if 'ignored' in kwargs and kwargs['ignored'] == "True": cmd += ' and %s="Ignored"' % status_type else: From 54ce4f5bedc48cab0cd806a7b85875ad3cb074e8 Mon Sep 17 00:00:00 2001 From: Phil Borman Date: Sun, 25 Jun 2017 23:50:25 +0200 Subject: [PATCH 11/22] Added button to clear download counters --- data/interfaces/bookstrap/history.html | 5 +++++ lazylibrarian/webServe.py | 16 ++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/data/interfaces/bookstrap/history.html b/data/interfaces/bookstrap/history.html index 3e0fd3ffd..dc391bad2 100644 --- a/data/interfaces/bookstrap/history.html +++ b/data/interfaces/bookstrap/history.html @@ -77,6 +77,11 @@

${title}

primary: { label: "Close", className: 'btn-primary' + }, + prompt: { + label: "Clear Counters", + className: 'btn-primary', + callback: function(result){ $.get("cleardownloads", function(e) {}); } } } }); diff --git a/lazylibrarian/webServe.py b/lazylibrarian/webServe.py index b3ad84d85..5e88d4d0b 100644 --- a/lazylibrarian/webServe.py +++ b/lazylibrarian/webServe.py @@ -2018,6 +2018,22 @@ def showblocked(self): logger.debug(result) return result + @cherrypy.expose + def cleardownloads(self): + cherrypy.response.headers[ + 'Cache-Control'] = "max-age=0,no-cache,no-store" + # clear download counters + myDB = database.DBConnection() + count = myDB.match('SELECT COUNT(Provider) as counter FROM downloads') + if count: + num = count['counter'] + else: + num = 0 + result = 'Deleted download counter for %s provider%s' % (num, plural(num)) + myDB.action('DELETE from downloads') + logger.debug(result) + return result + @cherrypy.expose def showdownloads(self): cherrypy.response.headers[ From 7ef39e35f035343bb7dda7dde474ae7186e06657 Mon Sep 17 00:00:00 2001 From: Phil Borman Date: Mon, 26 Jun 2017 13:03:11 +0200 Subject: [PATCH 12/22] Message changes --- lazylibrarian/gb.py | 3 ++- lazylibrarian/gr.py | 6 +++--- lazylibrarian/librarysync.py | 3 ++- lazylibrarian/magazinescan.py | 2 +- lazylibrarian/nzbget.py | 10 +++++----- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/lazylibrarian/gb.py b/lazylibrarian/gb.py index 36cc2c120..510d7c9ad 100644 --- a/lazylibrarian/gb.py +++ b/lazylibrarian/gb.py @@ -853,6 +853,7 @@ def find_book(bookid=None, queue=None): "DateAdded": today(), "Status": "Ignored" } + authorname = author['authorname'] myDB.upsert("authors", newValueDict, controlValueDict) else: logger.warn("No AuthorID for %s, unable to add book %s" % (authorname, bookname)) @@ -879,7 +880,7 @@ def find_book(bookid=None, queue=None): } myDB.upsert("books", newValueDict, controlValueDict) - logger.info("%s added to the books database" % bookname) + logger.info("%s by %s added to the books database" % (bookname, authorname)) if 'nocover' in bookimg or 'nophoto' in bookimg: # try to get a cover from librarything diff --git a/lazylibrarian/gr.py b/lazylibrarian/gr.py index 294011652..e9a381eb5 100644 --- a/lazylibrarian/gr.py +++ b/lazylibrarian/gr.py @@ -779,8 +779,7 @@ def find_book(self, bookid=None, queue=None): bookrate = float(rootxml.find('./book/average_rating').text) bookpages = rootxml.find('.book/num_pages').text - name = authorname - GR = GoodReads(name) + GR = GoodReads(authorname) author = GR.find_author_id() if author: AuthorID = author['authorid'] @@ -803,6 +802,7 @@ def find_book(self, bookid=None, queue=None): "DateAdded": today(), "Status": "Ignored" } + authorname = author['authorname'] myDB.upsert("authors", newValueDict, controlValueDict) else: logger.warn("No AuthorID for %s, unable to add book %s" % (authorname, bookname)) @@ -839,7 +839,7 @@ def find_book(self, bookid=None, queue=None): } myDB.upsert("books", newValueDict, controlValueDict) - logger.info("%s added to the books database" % bookname) + logger.info("%s by %s added to the books database" % (bookname, authorname)) if 'nocover' in bookimg or 'nophoto' in bookimg: # try to get a cover from librarything diff --git a/lazylibrarian/librarysync.py b/lazylibrarian/librarysync.py index 717581ca7..9a0fd5a12 100644 --- a/lazylibrarian/librarysync.py +++ b/lazylibrarian/librarysync.py @@ -190,7 +190,8 @@ def find_book_in_db(myDB, author, book): if book_partname == book_lower: book_partname = '' - logger.debug('Found %s book%s by [%s] in database' % (len(books), plural(len(books)), author)) + logger.debug('Searching %s book%s by [%s] in database for [%s]' % + (len(books), plural(len(books)), author, book)) for a_book in books: # tidy up everything to raise fuzziness scores diff --git a/lazylibrarian/magazinescan.py b/lazylibrarian/magazinescan.py index cadfb9b9d..5a435d2f3 100644 --- a/lazylibrarian/magazinescan.py +++ b/lazylibrarian/magazinescan.py @@ -380,7 +380,7 @@ def magazineScan(): controlValueDict = {"Title": title, "IssueDate": issuedate} issue_id = create_id("%s %s" % (title, issuedate)) iss_entry = myDB.match('SELECT Title from issues WHERE Title=? and IssueDate=?', - (title, issuedate)) + (title, issuedate)) if not iss_entry: newValueDict = { "IssueAcquired": iss_acquired, diff --git a/lazylibrarian/nzbget.py b/lazylibrarian/nzbget.py index 41781ca8e..6409dddc8 100644 --- a/lazylibrarian/nzbget.py +++ b/lazylibrarian/nzbget.py @@ -31,7 +31,7 @@ def checkLink(): # socket.setdefaulttimeout(2) - test = sendNZB('', cmd="test") + test = sendNZB(None, cmd="test") # socket.setdefaulttimeout(None) if test: return "NZBget connection successful" @@ -40,11 +40,11 @@ def checkLink(): def deleteNZB(nzbID, remove_data=False): if remove_data: - sendNZB('', 'GroupFinalDelete', nzbID) - return sendNZB('', 'HistoryFinalDelete', nzbID) + sendNZB(None, 'GroupFinalDelete', nzbID) + return sendNZB(None, 'HistoryFinalDelete', nzbID) else: - sendNZB('', 'GroupDelete', nzbID) - return sendNZB('', 'HistoryDelete', nzbID) + sendNZB(None, 'GroupDelete', nzbID) + return sendNZB(None, 'HistoryDelete', nzbID) def sendNZB(nzb, cmd=None, nzbID=None): From 0c0cfcf1a1f5b0a45c63be589e71edecf5177e21 Mon Sep 17 00:00:00 2001 From: Phil Borman Date: Mon, 26 Jun 2017 17:33:20 +0200 Subject: [PATCH 13/22] Message changes --- lazylibrarian/gr.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lazylibrarian/gr.py b/lazylibrarian/gr.py index e9a381eb5..d4a9889ee 100644 --- a/lazylibrarian/gr.py +++ b/lazylibrarian/gr.py @@ -526,9 +526,11 @@ def get_author_books(self, authorid=None, authorname=None, bookstatus="Skipped", else: logger.debug('Ignoring %s for %s, role is %s' % (bookname, authorNameResult, role)) - else: - logger.debug('Ignoring %s for %s, authorid %s' % - (bookname, authorNameResult, aid)) + # else: # multiple authors or wrong author + # logger.debug('Ignoring %s for %s, authorid %s' % + # (bookname, authorNameResult, aid)) + if not amatch: + logger.debug('Ignoring %s for %s, wrong author?' % (bookname, authorNameResult)) rejected = not amatch if not rejected: From cd99ed3b5bb0805ed6269eb8908e82d98c1295cc Mon Sep 17 00:00:00 2001 From: Phil Borman Date: Mon, 26 Jun 2017 22:24:53 +0200 Subject: [PATCH 14/22] Show libraryscan in progress with refresh on relevant pages --- data/interfaces/bookstrap/audio.html | 6 ++++ data/interfaces/bookstrap/books.html | 6 ++++ data/interfaces/bookstrap/magazines.html | 6 ++++ lazylibrarian/__init__.py | 5 +++- lazylibrarian/librarysync.py | 36 ++++++++++++++++++++++-- lazylibrarian/magazinescan.py | 3 ++ lazylibrarian/webServe.py | 4 +-- 7 files changed, 61 insertions(+), 5 deletions(-) diff --git a/data/interfaces/bookstrap/audio.html b/data/interfaces/bookstrap/audio.html index e6718301f..9d58e47ed 100644 --- a/data/interfaces/bookstrap/audio.html +++ b/data/interfaces/bookstrap/audio.html @@ -29,6 +29,9 @@ <%def name="body()">

${title}

+ %if lazylibrarian.AUDIO_UPDATE == True: +

Library scan in progress ...

+ %endif
@@ -71,6 +74,9 @@

${title}

<%def name="headIncludes()"> + %if lazylibrarian.AUDIO_UPDATE == True: + + %endif <%def name="javascriptIncludes()">