From 002b863cd8240c2b2fe042fa2a922697c8f2d710 Mon Sep 17 00:00:00 2001 From: Blake Warner Date: Wed, 13 Feb 2019 12:09:21 -0500 Subject: [PATCH] --- CDNSP.py | 8 +- Fs/BaseFs.py | 2 +- Fs/File.py | 6 +- Fs/Hfs0.py | 16 +- Fs/Ivfc.py | 16 +- Fs/Nacp.py | 4 +- Fs/Nca.py | 16 +- Fs/Nsp.py | 18 +-- Fs/Pfs0.py | 16 +- Fs/Rom.py | 8 +- Fs/Ticket.py | 6 +- Fs/Xci.py | 2 +- Server/Controller/Api.py | 21 ++- Server/__init__.py | 6 +- cdn/Atum.py | 8 +- cdn/Shogun.py | 10 +- cdn/Superfly.py | 10 +- nut.py | 291 +++++++++++++------------------------ {lib => nut}/Config.py | 45 +++--- {lib => nut}/Hex.py | 2 +- {lib => nut}/Keys.py | 4 +- {lib => nut}/Nsps.py | 8 +- {lib => nut}/Print.py | 2 +- {lib => nut}/Status.py | 2 +- {lib => nut}/Title.py | 8 +- {lib => nut}/Titles.py | 62 ++++---- {lib => nut}/Usb.py | 11 +- {lib => nut}/Users.py | 2 +- nut/__init__.py | 103 +++++++++++++ {lib => nut}/aes128.py | 0 {lib => nut}/blockchain.py | 24 +-- server.py | 239 ++++++++++++++++++++++++++++++ 32 files changed, 623 insertions(+), 353 deletions(-) rename {lib => nut}/Config.py (89%) mode change 100755 => 100644 rename {lib => nut}/Hex.py (97%) mode change 100755 => 100644 rename {lib => nut}/Keys.py (98%) mode change 100755 => 100644 rename {lib => nut}/Nsps.py (96%) mode change 100755 => 100644 rename {lib => nut}/Print.py (88%) rename {lib => nut}/Status.py (93%) rename {lib => nut}/Title.py (99%) mode change 100755 => 100644 rename {lib => nut}/Titles.py (99%) mode change 100755 => 100644 rename {lib => nut}/Usb.py (97%) rename {lib => nut}/Users.py (99%) create mode 100644 nut/__init__.py rename {lib => nut}/aes128.py (100%) mode change 100755 => 100644 rename {lib => nut}/blockchain.py (97%) create mode 100644 server.py diff --git a/CDNSP.py b/CDNSP.py index 6bd3c23b4..c5bfc58cd 100644 --- a/CDNSP.py +++ b/CDNSP.py @@ -17,13 +17,13 @@ from hashlib import sha256 from struct import pack as pk, unpack as upk from io import TextIOWrapper -import Titles +from nut import Titles import requests import unidecode import urllib3 -import Print -import Status -import Config +from nut import Print +from nut import Status +from nut import Config #Global Vars titlekey_list = [] diff --git a/Fs/BaseFs.py b/Fs/BaseFs.py index e257a6e17..4019ff5cb 100644 --- a/Fs/BaseFs.py +++ b/Fs/BaseFs.py @@ -1,5 +1,5 @@ from Fs.File import File -import Print +from nut import Print from binascii import hexlify as hx, unhexlify as uhx class BaseFs(File): diff --git a/Fs/File.py b/Fs/File.py index 5ae386a2d..467c86cf5 100755 --- a/Fs/File.py +++ b/Fs/File.py @@ -1,8 +1,8 @@ from enum import IntEnum import Fs.Type -import aes128 -import Print -import Hex +from nut import aes128 +from nut import Print +from nut import Hex from binascii import hexlify as hx, unhexlify as uhx class BaseFile: diff --git a/Fs/Hfs0.py b/Fs/Hfs0.py index af4f26c9d..2d015f814 100644 --- a/Fs/Hfs0.py +++ b/Fs/Hfs0.py @@ -1,7 +1,7 @@ -import aes128 -import Title -import Titles -import Hex +from nut import aes128 +from nut import Title +from nut import Titles +from nut import Hex from binascii import hexlify as hx, unhexlify as uhx from struct import pack as pk, unpack as upk from Fs.File import File @@ -12,10 +12,10 @@ import os import re import pathlib -import Keys -import Config -import Print -import Nsps +from nut import Keys +from nut import Config +from nut import Print +from nut import Nsps import Fs from tqdm import tqdm diff --git a/Fs/Ivfc.py b/Fs/Ivfc.py index ccc9578dd..bf34738f4 100644 --- a/Fs/Ivfc.py +++ b/Fs/Ivfc.py @@ -1,7 +1,7 @@ -import aes128 -import Title -import Titles -import Hex +from nut import aes128 +from nut import Title +from nut import Titles +from nut import Hex from binascii import hexlify as hx, unhexlify as uhx from struct import pack as pk, unpack as upk from Fs.File import File @@ -10,10 +10,10 @@ import os import re import pathlib -import Keys -import Config -import Print -import Nsps +from nut import Keys +from nut import Config +from nut import Print +from nut import Nsps from tqdm import tqdm MEDIA_SIZE = 0x200 diff --git a/Fs/Nacp.py b/Fs/Nacp.py index 79cf7a88b..e58a267c8 100644 --- a/Fs/Nacp.py +++ b/Fs/Nacp.py @@ -2,8 +2,8 @@ import Fs.Type from binascii import hexlify as hx, unhexlify as uhx from enum import IntEnum -import Print -import Keys +from nut import Print +from nut import Keys class NacpLanguageType(IntEnum): AmericanEnglish = 0 diff --git a/Fs/Nca.py b/Fs/Nca.py index a295f3584..b757576b7 100644 --- a/Fs/Nca.py +++ b/Fs/Nca.py @@ -1,7 +1,7 @@ -import aes128 -import Title -import Titles -import Hex +from nut import aes128 +from nut import Title +from nut import Titles +from nut import Hex from binascii import hexlify as hx, unhexlify as uhx from struct import pack as pk, unpack as upk from hashlib import sha256 @@ -9,10 +9,10 @@ import os import re import pathlib -import Keys -import Config -import Print -import Nsps +from nut import Keys +from nut import Config +from nut import Print +from nut import Nsps from tqdm import tqdm import Fs from Fs.File import File diff --git a/Fs/Nsp.py b/Fs/Nsp.py index e5e9b8dcc..7fe6929c2 100644 --- a/Fs/Nsp.py +++ b/Fs/Nsp.py @@ -1,7 +1,7 @@ -import aes128 -import Title -import Titles -import Hex +from nut import aes128 +from nut import Title +from nut import Titles +from nut import Hex from binascii import hexlify as hx, unhexlify as uhx from struct import pack as pk, unpack as upk from Fs.File import File @@ -10,16 +10,16 @@ import os import re import pathlib -import Keys -import Config -import Print -import Nsps +from nut import Keys +from nut import Config +from nut import Print +from nut import Nsps from tqdm import tqdm from Fs.Pfs0 import Pfs0 from Fs.Ticket import Ticket from Fs.Nca import Nca import shutil -import blockchain +from nut import blockchain MEDIA_SIZE = 0x200 diff --git a/Fs/Pfs0.py b/Fs/Pfs0.py index 852f6c8a4..7a5681356 100644 --- a/Fs/Pfs0.py +++ b/Fs/Pfs0.py @@ -1,7 +1,7 @@ -import aes128 -import Title -import Titles -import Hex +from nut import aes128 +from nut import Title +from nut import Titles +from nut import Hex from binascii import hexlify as hx, unhexlify as uhx from struct import pack as pk, unpack as upk from Fs.File import File @@ -10,10 +10,10 @@ import os import re import pathlib -import Keys -import Config -import Print -import Nsps +from nut import Keys +from nut import Config +from nut import Print +from nut import Nsps from tqdm import tqdm from Fs.BaseFs import BaseFs diff --git a/Fs/Rom.py b/Fs/Rom.py index e0e4ea5d8..5868e075e 100644 --- a/Fs/Rom.py +++ b/Fs/Rom.py @@ -5,10 +5,10 @@ import os import re import pathlib -import Keys -import Config -import Print -import Nsps +from nut import Keys +from nut import Config +from nut import Print +from nut import Nsps from tqdm import tqdm from Fs.BaseFs import BaseFs from Fs.Ivfc import Ivfc diff --git a/Fs/Ticket.py b/Fs/Ticket.py index abf093cf6..adcaf2116 100644 --- a/Fs/Ticket.py +++ b/Fs/Ticket.py @@ -1,9 +1,9 @@ from Fs.File import File import Fs.Type from binascii import hexlify as hx, unhexlify as uhx -import Print -import Keys -import blockchain +from nut import Print +from nut import Keys +from nut import blockchain class Ticket(File): diff --git a/Fs/Xci.py b/Fs/Xci.py index 6c52f3a95..d58f7d3a4 100644 --- a/Fs/Xci.py +++ b/Fs/Xci.py @@ -2,7 +2,7 @@ from Fs.File import File from Fs.Hfs0 import Hfs0 import os -import Print +from nut import Print MEDIA_SIZE = 0x200 diff --git a/Server/Controller/Api.py b/Server/Controller/Api.py index 460912a94..07c1b7568 100644 --- a/Server/Controller/Api.py +++ b/Server/Controller/Api.py @@ -1,18 +1,18 @@ -import Titles +from nut import Titles import json -import Titles -import Status -import Nsps -import Print +from nut import Titles +from nut import Status +from nut import Nsps +from nut import Print import Server -import Config -import Hex +from nut import Config +from nut import Hex import socket import struct import time import nut import cdn -import blockchain +from nut import blockchain import urllib.parse import requests @@ -216,6 +216,7 @@ def getDownload(request, response, start = None, end = None): start = int(request.bits[-2]) end = int(request.bits[-1]) + #chunkSize = 0x1000000 chunkSize = 0x400000 with open(nsp.path, "rb") as f: @@ -266,16 +267,20 @@ def getDownload(request, response, start = None, end = None): size = end - start i = 0 + status = Status.create(size, 'Downloading ' + os.path.basename(nsp.path)) while i < size: chunk = f.read(min(size-i, chunkSize)) i += len(chunk) + status.add(len(chunk)) + if chunk: pass response.write(chunk) else: break + status.close() except BaseException as e: Print.error('NSP download exception: ' + str(e)) if response.bytesSent == 0: diff --git a/Server/__init__.py b/Server/__init__.py index 22ad64729..9e818447d 100644 --- a/Server/__init__.py +++ b/Server/__init__.py @@ -3,13 +3,13 @@ import socket import socketserver import time -import Config +from nut import Config import sys import os import re -import Print +from nut import Print import urllib -import Users +from nut import Users import base64 from urllib.parse import urlparse from urllib.parse import parse_qs diff --git a/cdn/Atum.py b/cdn/Atum.py index 71b166ba5..997673e24 100644 --- a/cdn/Atum.py +++ b/cdn/Atum.py @@ -17,13 +17,13 @@ from hashlib import sha256 from struct import pack as pk, unpack as upk from io import TextIOWrapper -import Titles +from nut import Titles import requests import unidecode import urllib3 -import Print -import Status -import Config +from nut import Print +from nut import Status +from nut import Config quiet = False truncateName = False diff --git a/cdn/Shogun.py b/cdn/Shogun.py index 0b349dd11..075079a38 100644 --- a/cdn/Shogun.py +++ b/cdn/Shogun.py @@ -17,16 +17,16 @@ from hashlib import sha256 from struct import pack as pk, unpack as upk from io import TextIOWrapper -import Titles +from nut import Titles import requests import unidecode import urllib3 -import Print -import Status -import Config +from nut import Print +from nut import Status +from nut import Config import os import hashlib -import Title +from nut import Title import cdn.Superfly import cdn diff --git a/cdn/Superfly.py b/cdn/Superfly.py index d92c2349b..b986fc0d1 100644 --- a/cdn/Superfly.py +++ b/cdn/Superfly.py @@ -17,16 +17,16 @@ from hashlib import sha256 from struct import pack as pk, unpack as upk from io import TextIOWrapper -import Titles +from nut import Titles import requests import unidecode import urllib3 -import Print -import Status -import Config +from nut import Print +from nut import Status +from nut import Config import os import hashlib -import Title +from nut import Title import cdn diff --git a/nut.py b/nut.py index d7b650005..d49bbd381 100755 --- a/nut.py +++ b/nut.py @@ -12,22 +12,22 @@ import urllib3 import json -os.chdir(os.path.dirname(os.path.abspath(__file__))) +#os.chdir(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, 'lib') +#sys.path.insert(0, 'nut') -from Title import Title -import Titles -import Nsps +from nut import Title +from nut import Titles +from nut import Nsps import CDNSP import Fs -import Config +from nut import Config import requests -import Hex -import Print +from nut import Hex +from nut import Print import threading import signal -import Status +from nut import Status import time import colorama import Server @@ -38,14 +38,14 @@ import queue try: - import blockchain + import nut.blockchain except: raise def logMissingTitles(file): - initTitles() - initFiles() + nut.initTitles() + nut.initFiles() f = open(file,"w", encoding="utf-8-sig") @@ -58,8 +58,8 @@ def logMissingTitles(file): f.close() def logNcaDeltas(file): - initTitles() - initFiles() + nut.initTitles() + nut.initFiles() x = open(file,"w", encoding="utf-8-sig") @@ -80,7 +80,7 @@ def logNcaDeltas(file): x.close() def updateDb(url, c=0): - initTitles() + nut.initTitles() c += 1 @@ -176,8 +176,8 @@ def startDownloadThreads(): downloadThreadsStarted = True - initTitles() - initFiles() + nut.initTitles() + nut.initFiles() threads = [] for i in range(Config.threads): @@ -188,8 +188,8 @@ def startDownloadThreads(): threads.append(t) def downloadAll(wait = True): - initTitles() - initFiles() + nut.initTitles() + nut.initFiles() global activeDownloads global status @@ -306,32 +306,12 @@ def startBaseScan(): def export(file, cols = ['id', 'rightsId', 'key', 'isUpdate', 'isDLC', 'isDemo', 'baseName', 'name', 'version', 'region']): - initTitles() + nut.initTitles() Titles.export(file, cols) - -global hasScanned -hasScanned = False - -def scan(): - global hasScanned - - #if hasScanned: - # return - hasScanned = True - initTitles() - initFiles() - - - refreshRegions() - importRegion(Config.region, Config.language) - - r = Nsps.scan(Config.paths.scan) - Titles.save() - return r def organize(): - initTitles() - initFiles() + nut.initTitles() + nut.initFiles() #scan() Print.info('organizing') @@ -358,8 +338,8 @@ def organize(): Nsps.save() def refresh(titleRightsOnly = False): - initTitles() - initFiles() + nut.initTitles() + nut.initFiles() i = 0 for k, f in Nsps.files.items(): try: @@ -382,8 +362,8 @@ def refresh(titleRightsOnly = False): Titles.save() def scanLatestTitleUpdates(): - initTitles() - initFiles() + nut.initTitles() + nut.initFiles() for k,i in CDNSP.get_versionUpdates().items(): id = str(k).upper() @@ -407,8 +387,8 @@ def scanLatestTitleUpdates(): Titles.save() def updateVersions(force = True): - initTitles() - initFiles() + nut.initTitles() + nut.initFiles() i = 0 for k,t in Titles.items(): @@ -437,33 +417,10 @@ def updateVersions(force = True): Titles.save() -isInitTitles = False - -def initTitles(): - global isInitTitles - if isInitTitles: - return - - isInitTitles = True - - Titles.load() - - Nsps.load() - Titles.queue.load() - -isInitFiles = False -def initFiles(): - global isInitFiles - if isInitFiles: - return - - isInitFiles = True - - Nsps.load() def unlockAll(): - initTitles() - initFiles() + nut.initTitles() + nut.initFiles() for k,f in Nsps.files.items(): if f.isUnlockable(): @@ -479,7 +436,7 @@ def unlockAll(): Print.info('error unlocking: ' + str(e)) def exportVerifiedKeys(fileName): - initTitles() + nut.initTitles() with open(fileName, 'w') as f: f.write('id|key|version\n') for tid,key in blockchain.blockchain.export().items(): @@ -488,7 +445,7 @@ def exportVerifiedKeys(fileName): f.write(str(title.rightsId) + '|' + str(key) + '|' + str(title.version) + '\n') def exportKeys(fileName): - initTitles() + nut.initTitles() with open(fileName, 'w') as f: f.write('id|key|version\n') for tid,title in Titles.items(): @@ -513,65 +470,11 @@ def submitKeys(): Print.info(str(e)) raise -def refreshRegions(): - for region in Config.regionLanguages(): - for language in Config.regionLanguages()[region]: - for i in Titles.data(region, language): - regionTitle = Titles.data(region, language)[i] - - if regionTitle.id: - title = Titles.get(regionTitle.id, None, None) - - if not hasattr(title, 'regions') or not title.regions: - title.regions = [] - if not hasattr(title, 'languages') or not title.languages: - title.languages = [] - - if not region in title.regions: - title.regions.append(region) - - if not language in title.languages: - title.languages.append(language) - Titles.save() - -def importRegion(region = 'US', language = 'en'): - if not region in Config.regionLanguages() or language not in Config.regionLanguages()[region]: - Print.error('Could not locate %s/%s !' % (region, language)) - return False - - for region2 in Config.regionLanguages(): - for language2 in Config.regionLanguages()[region2]: - for nsuId, regionTitle in Titles.data(region2, language2).items(): - if not regionTitle.id: - continue - title = Titles.get(regionTitle.id, None, None) - title.importFrom(regionTitle, region2, language2) - - for region2 in Config.regionLanguages(): - for language2 in Config.regionLanguages()[region2]: - if language2 != language: - continue - for nsuId, regionTitle in Titles.data(region2, language2).items(): - if not regionTitle.id: - continue - title = Titles.get(regionTitle.id, None, None) - title.importFrom(regionTitle, region2, language2) - - - for nsuId, regionTitle in Titles.data(region, language).items(): - if not regionTitle.id: - continue - - title = Titles.get(regionTitle.id, None, None) - title.importFrom(regionTitle, region, language) - - Titles.loadTxtDatabases() - Titles.save() def scrapeShogun(): - initTitles() - initFiles() + nut.initTitles() + nut.initFiles() for region in cdn.regions(): cdn.Shogun.scrapeTitles(region) @@ -590,8 +493,8 @@ def scrapeShogunWorker(q): q.task_done() def scrapeShogunThreaded(): - initTitles() - initFiles() + nut.initTitles() + nut.initFiles() scrapeThreads = [] numThreads = 4 @@ -602,9 +505,9 @@ def scrapeShogunThreaded(): q.put(region) for i in range(numThreads): - t = threading.Thread(target=scrapeShogunWorker, args=[q]) - t.daemon = True - t.start() + t = threading.Thread(target=scrapeShogunWorker, args=[q]) + t.daemon = True + t.start() scrapeThreads.append(t) q.join() @@ -617,12 +520,12 @@ def scrapeShogunThreaded(): Titles.saveAll() def genTinfoilTitles(): - initTitles() - initFiles() + nut.initTitles() + nut.initFiles() for region, languages in Config.regionLanguages().items(): for language in languages: - importRegion(region, language) + nut.importRegion(region, language) Titles.save('titledb/titles.%s.%s.json' % (region, language), False) #Print.info('%s - %s' % (region, language)) scanLatestTitleUpdates() @@ -819,7 +722,7 @@ def organizeNcas(dir): Print.info(' `"\'') if args.extract: - initTitles() + nut.initTitles() for filePath in args.extract: #f = Fs.Nsp(filePath, 'rb') f = Fs.factory(filePath) @@ -838,20 +741,20 @@ def organizeNcas(dir): if args.update_titles: - initTitles() + nut.initTitles() for url in Config.titleUrls: updateDb(url) Titles.loadTxtDatabases() Titles.save() if args.submit_keys: - initTitles() - initFiles() + nut.initTitles() + nut.initFiles() submitKeys() if args.seteshop: - #initTitles() - #initFiles() + #nut.initTitles() + #nut.initFiles() f = Fs.factory(args.seteshop) f.open(args.seteshop, 'r+b') f.setGameCard(False) @@ -862,7 +765,7 @@ def organizeNcas(dir): exit(0) if args.refresh_regions: - refreshRegions() + nut.refreshRegions() exit(0) if args.import_region: @@ -872,7 +775,7 @@ def organizeNcas(dir): args.language = args.language.lower() - importRegion(region, args.language) + nut.importRegion(region, args.language) exit(0) if args.usb: @@ -880,38 +783,38 @@ def organizeNcas(dir): import Usb except BaseException as e: Print.error('pip3 install pyusb, required for USB coms: ' + str(e)) - scan() + nut.scan() Usb.daemon() if args.download: - initTitles() - initFiles() + nut.initTitles() + nut.initFiles() for d in args.download: download(d) if args.scan: - initTitles() - initFiles() - scan() + nut.initTitles() + nut.initFiles() + nut.scan() if args.refresh: - initTitles() - initFiles() + nut.initTitles() + nut.initFiles() refresh(False) if args.read_rightsids: - initTitles() - initFiles() + nut.initTitles() + nut.initFiles() refresh(True) if args.organize: - initTitles() - initFiles() + nut.initTitles() + nut.initFiles() organize() if args.set_masterkey1: - initTitles() - initFiles() + nut.initTitles() + nut.initFiles() f = Fs.Nsp(args.set_masterkey1, 'r+b') f.setMasterKeyRev(0) f.flush() @@ -919,8 +822,8 @@ def organizeNcas(dir): pass if args.set_masterkey2: - initTitles() - initFiles() + nut.initTitles() + nut.initFiles() f = Fs.Nsp(args.set_masterkey2, 'r+b') f.setMasterKeyRev(2) f.flush() @@ -928,8 +831,8 @@ def organizeNcas(dir): pass if args.set_masterkey3: - initTitles() - initFiles() + nut.initTitles() + nut.initFiles() f = Fs.Nsp(args.set_masterkey3, 'r+b') f.setMasterKeyRev(3) f.flush() @@ -937,8 +840,8 @@ def organizeNcas(dir): pass if args.set_masterkey4: - initTitles() - initFiles() + nut.initTitles() + nut.initFiles() f = Fs.Nsp(args.set_masterkey4, 'r+b') f.setMasterKeyRev(4) f.flush() @@ -946,8 +849,8 @@ def organizeNcas(dir): pass if args.set_masterkey5: - initTitles() - initFiles() + nut.initTitles() + nut.initFiles() f = Fs.Nsp(args.set_masterkey5, 'r+b') f.setMasterKeyRev(5) f.flush() @@ -955,8 +858,8 @@ def organizeNcas(dir): pass if args.remove_title_rights: - initTitles() - initFiles() + nut.initTitles() + nut.initFiles() for fileName in args.remove_title_rights: try: f = Fs.Nsp(fileName, 'r+b') @@ -976,8 +879,8 @@ def organizeNcas(dir): Print.info('Title key is INVALID %s - %s' % (args.verify[0], args.verify[1])) if args.info: - initTitles() - initFiles() + nut.initTitles() + nut.initFiles() if re.search(r'^[A-Fa-f0-9]+$', args.info.strip(), re.I | re.M | re.S): Print.info('%s version = %s' % (args.info.upper(), CDNSP.get_version(args.info.lower()))) else: @@ -1003,8 +906,8 @@ def organizeNcas(dir): if len(args.scrape_shogun) == 0: scrapeShogunThreaded() else: - initTitles() - initFiles() + nut.initTitles() + nut.initFiles() for i in args.scrape_shogun: if len(i) == 16: l = cdn.Shogun.ids(i) @@ -1023,8 +926,8 @@ def organizeNcas(dir): genTinfoilTitles() if args.scrape_title: - initTitles() - initFiles() + nut.initTitles() + nut.initFiles() if not Titles.contains(args.scrape_title): Print.error('Could not find title ' + args.scrape_title) @@ -1035,8 +938,8 @@ def organizeNcas(dir): pprint.pprint(Titles.get(args.scrape_title).__dict__) if args.scrape or args.scrape_delta: - initTitles() - initFiles() + nut.initTitles() + nut.initFiles() threads = [] for i in range(scrapeThreads): @@ -1064,8 +967,8 @@ def organizeNcas(dir): pass if args.unlock: - initTitles() - initFiles() + nut.initTitles() + nut.initFiles() Print.info('opening ' + args.unlock) f = Fs.Nsp(args.unlock, 'r+b') f.unlock() @@ -1077,13 +980,13 @@ def organizeNcas(dir): Titles.save() if args.export: - initTitles() - initFiles() + nut.initTitles() + nut.initFiles() export(args.export) if args.export_versions: - initTitles() - initFiles() + nut.initTitles() + nut.initFiles() export(args.export_versions, ['rightsId', 'version']) if args.missing: @@ -1091,13 +994,13 @@ def organizeNcas(dir): if args.server: startDownloadThreads() - initTitles() - initFiles() + nut.initTitles() + nut.initFiles() Server.run() if args.blockchain: - initTitles() - initFiles() + nut.initTitles() + nut.initFiles() try: import blockchain except: @@ -1105,15 +1008,15 @@ def organizeNcas(dir): blockchain.run() if len(sys.argv)==1: - scan() + nut.scan() organize() downloadAll() scanLatestTitleUpdates() export('titledb/versions.txt', ['id', 'version']) if args.scan_dlc != None: - initTitles() - initFiles() + nut.initTitles() + nut.initFiles() queue = Titles.Queue() if len(args.scan_dlc) > 0: for id in args.scan_dlc: @@ -1125,8 +1028,8 @@ def organizeNcas(dir): startDlcScan(queue) if args.scan_base != None: - initTitles() - initFiles() + nut.initTitles() + nut.initFiles() startBaseScan() if args.export_verified_keys: diff --git a/lib/Config.py b/nut/Config.py old mode 100755 new mode 100644 similarity index 89% rename from lib/Config.py rename to nut/Config.py index 4417d9553..77e6f496c --- a/lib/Config.py +++ b/nut/Config.py @@ -6,7 +6,7 @@ class Server: def __init__(self): - self.hostname = 'localhost' + self.hostname = '0.0.0.0' self.port = 9000 class Cdn: @@ -321,32 +321,43 @@ def regionLanguages(fileName = 'titledb/languages.json'): if g_regionLanguages: return g_regionLanguages - with open(fileName, encoding="utf-8-sig") as f: - g_regionLanguages = json.loads(f.read()) + g_regionLanguages = [] + + try: + with open(fileName, encoding="utf-8-sig") as f: + g_regionLanguages = json.loads(f.read()) + except: + pass return g_regionLanguages def loadTitleWhitelist(): global titleWhitelist titleWhitelist = [] - with open('conf/whitelist.txt', encoding="utf8") as f: - for line in f.readlines(): - titleWhitelist.append(line.strip().upper()) + try: + with open('conf/whitelist.txt', encoding="utf8") as f: + for line in f.readlines(): + titleWhitelist.append(line.strip().upper()) + except: + pass def loadTitleBlacklist(): global titleBlacklist titleBlacklist = [] - with open('conf/blacklist.txt', encoding="utf8") as f: - for line in f.readlines(): - id = line.split('|')[0].strip().upper() - if id: - titleBlacklist.append(id) - - with open('conf/retailOnly.blacklist', encoding="utf8") as f: - for line in f.readlines(): - id = line.split('|')[0].strip().upper() - if id: - titleBlacklist.append(id) + try: + with open('conf/blacklist.txt', encoding="utf8") as f: + for line in f.readlines(): + id = line.split('|')[0].strip().upper() + if id: + titleBlacklist.append(id) + + with open('conf/retailOnly.blacklist', encoding="utf8") as f: + for line in f.readlines(): + id = line.split('|')[0].strip().upper() + if id: + titleBlacklist.append(id) + except: + pass loadTitleWhitelist() loadTitleBlacklist() \ No newline at end of file diff --git a/lib/Hex.py b/nut/Hex.py old mode 100755 new mode 100644 similarity index 97% rename from lib/Hex.py rename to nut/Hex.py index 483512306..798771f79 --- a/lib/Hex.py +++ b/nut/Hex.py @@ -1,5 +1,5 @@ from string import ascii_letters, digits, punctuation -import Print +from nut import Print def bufferToHex(buffer, start, count): accumulator = '' diff --git a/lib/Keys.py b/nut/Keys.py old mode 100755 new mode 100644 similarity index 98% rename from lib/Keys.py rename to nut/Keys.py index 67ceb1884..11a8ec90a --- a/lib/Keys.py +++ b/nut/Keys.py @@ -1,7 +1,7 @@ import re -import aes128 +from nut import aes128 from binascii import hexlify as hx, unhexlify as uhx -import Print +from nut import Print keys = {} titleKeks = [] diff --git a/lib/Nsps.py b/nut/Nsps.py old mode 100755 new mode 100644 similarity index 96% rename from lib/Nsps.py rename to nut/Nsps.py index 1113f264d..5680a5f6a --- a/lib/Nsps.py +++ b/nut/Nsps.py @@ -4,9 +4,9 @@ import Fs import pathlib import re -import Status +from nut import Status import time -import Print +from nut import Print import threading import json @@ -30,9 +30,9 @@ def getByTitleId(id): return f return None -def scan(base): +def scan(base, force = False): global hasScanned - if hasScanned: + if hasScanned and not force: return hasScanned = True diff --git a/lib/Print.py b/nut/Print.py similarity index 88% rename from lib/Print.py rename to nut/Print.py index 443f0b04f..96b8340bd 100644 --- a/lib/Print.py +++ b/nut/Print.py @@ -1,4 +1,4 @@ -import Status +from nut import Status global silent enableInfo = True diff --git a/lib/Status.py b/nut/Status.py similarity index 93% rename from lib/Status.py rename to nut/Status.py index 5476710ea..6ed17403c 100644 --- a/lib/Status.py +++ b/nut/Status.py @@ -1,7 +1,7 @@ import tqdm import time import threading -import Config +from nut import Config import json import sys diff --git a/lib/Title.py b/nut/Title.py old mode 100755 new mode 100644 similarity index 99% rename from lib/Title.py rename to nut/Title.py index 6ad91f9b1..6cf736960 --- a/lib/Title.py +++ b/nut/Title.py @@ -4,8 +4,8 @@ import re import json import CDNSP -import Titles -import Print +from nut import Titles +from nut import Print from bs4 import BeautifulSoup import requests @@ -13,9 +13,9 @@ import datetime import calendar import threading -import Nsps +from nut import Nsps import urllib.request -import Config +from nut import Config import cdn.Shogun try: diff --git a/lib/Titles.py b/nut/Titles.py old mode 100755 new mode 100644 similarity index 99% rename from lib/Titles.py rename to nut/Titles.py index d5badb8db..8fc1bde63 --- a/lib/Titles.py +++ b/nut/Titles.py @@ -4,10 +4,10 @@ import re import time import json -import Title +from nut import Title import operator -import Config -import Print +from nut import Config +from nut import Print import threading import cdn @@ -192,34 +192,34 @@ def load(): confLock.release() #loadTxtDatabases() -def parsePersonalKeys(path): - Print.info('loading personal keys ' + path) - parsed_keys = {} - with open(path, encoding='utf8', errors='ignore') as f: - lines = f.readlines() - - for line in lines: - if 'Ticket' in line: - pass - elif 'Rights ID' in line: - rid = line.split(': ')[1].strip() - elif 'Title ID' in line: - tid = line.split(': ')[1].strip() - elif 'Titlekey' in line: - tkey = line.split(': ')[1].strip() - - if not tid.endswith('800'): - parsed_keys[rid] = tkey - - for rightsId, key in parsed_keys.items(): - rightsId = rightsId.upper() - key = key.upper() - titleId = rightsId[0:16] - title = get(titleId) - if title.key != key: - title.setId(rightsId) - title.setKey(key) - Print.info('Added new title key for %s[%s]' % (title.name, titleId)) +def parsePersonalKeys(path): + Print.info('loading personal keys ' + path) + parsed_keys = {} + with open(path, encoding='utf8', errors='ignore') as f: + lines = f.readlines() + + for line in lines: + if 'Ticket' in line: + pass + elif 'Rights ID' in line: + rid = line.split(': ')[1].strip() + elif 'Title ID' in line: + tid = line.split(': ')[1].strip() + elif 'Titlekey' in line: + tkey = line.split(': ')[1].strip() + + if not tid.endswith('800'): + parsed_keys[rid] = tkey + + for rightsId, key in parsed_keys.items(): + rightsId = rightsId.upper() + key = key.upper() + titleId = rightsId[0:16] + title = get(titleId) + if title.key != key: + title.setId(rightsId) + title.setKey(key) + Print.info('Added new title key for %s[%s]' % (title.name, titleId)) #print("{}|{}".format(k, v)) def loadTxtDatabases(): diff --git a/lib/Usb.py b/nut/Usb.py similarity index 97% rename from lib/Usb.py rename to nut/Usb.py index 44c5b6f24..7febace79 100644 --- a/lib/Usb.py +++ b/nut/Usb.py @@ -50,15 +50,17 @@ import sys from binascii import hexlify as hx, unhexlify as uhx from pathlib import Path -import Titles +from nut import Titles import Server import Server.Controller.Api -import Print +from nut import Print import time from urllib.parse import urlparse from urllib.parse import parse_qs import Server.Controller.Api +global status +status = 'initializing' def getFiles(): for k, t in Titles.items(): @@ -164,8 +166,10 @@ def poll_commands(in_ep, out_ep): print('failed to read!') def daemon(): + global status while True: try: + status = 'disconnected' while True: dev = usb.core.find(idVendor=0x057E, idProduct=0x3000) @@ -174,6 +178,7 @@ def daemon(): time.sleep(1) Print.info('USB Connected') + status = 'connected' dev.reset() dev.set_configuration() @@ -189,5 +194,5 @@ def daemon(): poll_commands(in_ep, out_ep) except BaseException as e: - print(str(e)) + print('usb exception: ' + str(e)) time.sleep(1) \ No newline at end of file diff --git a/lib/Users.py b/nut/Users.py similarity index 99% rename from lib/Users.py rename to nut/Users.py index 3b96b6508..452c58ff4 100644 --- a/lib/Users.py +++ b/nut/Users.py @@ -1,6 +1,6 @@ import os import re -import Print +from nut import Print global users users = {} diff --git a/nut/__init__.py b/nut/__init__.py new file mode 100644 index 000000000..76cd42589 --- /dev/null +++ b/nut/__init__.py @@ -0,0 +1,103 @@ +from nut import Titles +from nut import Nsps +from nut import Print + +def refreshRegions(): + for region in Config.regionLanguages(): + for language in Config.regionLanguages()[region]: + for i in Titles.data(region, language): + regionTitle = Titles.data(region, language)[i] + + if regionTitle.id: + title = Titles.get(regionTitle.id, None, None) + + if not hasattr(title, 'regions') or not title.regions: + title.regions = [] + + if not hasattr(title, 'languages') or not title.languages: + title.languages = [] + + if not region in title.regions: + title.regions.append(region) + + if not language in title.languages: + title.languages.append(language) + Titles.save() + +def importRegion(region = 'US', language = 'en'): + if not region in Config.regionLanguages() or language not in Config.regionLanguages()[region]: + Print.error('Could not locate %s/%s !' % (region, language)) + return False + + for region2 in Config.regionLanguages(): + for language2 in Config.regionLanguages()[region2]: + for nsuId, regionTitle in Titles.data(region2, language2).items(): + if not regionTitle.id: + continue + title = Titles.get(regionTitle.id, None, None) + title.importFrom(regionTitle, region2, language2) + + for region2 in Config.regionLanguages(): + for language2 in Config.regionLanguages()[region2]: + if language2 != language: + continue + for nsuId, regionTitle in Titles.data(region2, language2).items(): + if not regionTitle.id: + continue + title = Titles.get(regionTitle.id, None, None) + title.importFrom(regionTitle, region2, language2) + + + for nsuId, regionTitle in Titles.data(region, language).items(): + if not regionTitle.id: + continue + + title = Titles.get(regionTitle.id, None, None) + title.importFrom(regionTitle, region, language) + + Titles.loadTxtDatabases() + Titles.save() + +isInitTitles = False + +def initTitles(): + global isInitTitles + if isInitTitles: + return + + isInitTitles = True + + Titles.load() + + Nsps.load() + Titles.queue.load() + +isInitFiles = False +def initFiles(): + global isInitFiles + if isInitFiles: + return + + isInitFiles = True + + Nsps.load() + +global hasScanned +hasScanned = False + +def scan(): + global hasScanned + + #if hasScanned: + # return + hasScanned = True + initTitles() + initFiles() + + + refreshRegions() + importRegion(Config.region, Config.language) + + r = Nsps.scan(Config.paths.scan) + Titles.save() + return r diff --git a/lib/aes128.py b/nut/aes128.py old mode 100755 new mode 100644 similarity index 100% rename from lib/aes128.py rename to nut/aes128.py diff --git a/lib/blockchain.py b/nut/blockchain.py similarity index 97% rename from lib/blockchain.py rename to nut/blockchain.py index 315428cde..3e7e3c36b 100644 --- a/lib/blockchain.py +++ b/nut/blockchain.py @@ -5,8 +5,8 @@ from urllib.parse import urlparse from uuid import uuid4 import os -import Config -import Nsps +from nut import Config +from nut import Nsps import Fs import Fs.File from Fs import File @@ -16,9 +16,9 @@ from Fs import Pfs0 from Fs.Nca import NcaHeader from Fs import Type -import Keys -import Print -import Hex +from nut import Keys +from nut import Print +from nut import Hex from binascii import hexlify as hx, unhexlify as uhx import requests @@ -150,11 +150,15 @@ def __init__(self): self.new_block(previous_hash='1') def save(self): - with open('titledb/blockchain.json', 'w') as outfile: - obj = [] - for i in self.chain: - obj.append(i.serialize()) - json.dump(obj, outfile, indent=4) + try: + os.mkdir('titledb') + with open('titledb/blockchain.json', 'w') as outfile: + obj = [] + for i in self.chain: + obj.append(i.serialize()) + json.dump(obj, outfile, indent=4) + except: + pass def load(self): try: diff --git a/server.py b/server.py new file mode 100644 index 000000000..c28b02a07 --- /dev/null +++ b/server.py @@ -0,0 +1,239 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- + +import sys +import os +import re +import pathlib +import urllib3 +import json +import Server + +import nut +from nut import Title +from nut import Titles +from nut import Nsps +import CDNSP +from nut import Config +import time + +import Server +import pprint +import random +from nut import Usb +import threading +from nut import Status +import time +import socket + +import sys +from PyQt5.QtWidgets import QMainWindow, QApplication, QWidget, QAction, QTableWidget,QTableWidgetItem,QVBoxLayout,QDesktopWidget, QTabWidget, QProgressBar, QLabel,QHBoxLayout, QLineEdit, QPushButton +from PyQt5.QtGui import QIcon +from PyQt5.QtCore import pyqtSlot,Qt,QTimer +from PyQt5 import QtWidgets + +def getIpAddress(): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.connect(("8.8.8.8", 80)) + ip = s.getsockname()[0] + s.close() + return ip + +def formatSpeed(n): + return str(round(n / 1000 / 1000, 1)) + 'MB/s' + +class Header: + def __init__(self, app): + self.app = app + self.layout = QHBoxLayout() + + self.textbox = QLineEdit(app) + self.textbox.setText(os.path.abspath(Config.paths.scan)) + self.textbox.textChanged.connect(self.updatePath) + self.layout.addWidget(self.textbox) + + self.scan = QPushButton('Scan', app) + self.scan.clicked.connect(app.on_scan) + self.layout.addWidget(self.scan) + + self.serverInfo = QLabel("IP: " + getIpAddress() + " Port: " + str(Config.server.port)) + self.serverInfo.setFixedWidth(300) + self.serverInfo.setAlignment(Qt.AlignCenter) + + self.layout.addWidget(self.serverInfo) + self.usbStatus = QLabel("USB Status: " + str(Usb.status)) + + self.usbStatus.setFixedWidth(200) + + self.layout.addWidget(self.usbStatus) + + self.timer = QTimer() + self.timer.setInterval(1000) + self.timer.timeout.connect(self.tick) + self.timer.start() + + def updatePath(self): + Config.paths.scan = self.textbox.text() + + def tick(self): + self.usbStatus.setText("USB Status: " + str(Usb.status)) + +class Progress: + def __init__(self, app): + self.app = app + self.progress = QProgressBar(app) + self.text = QLabel() + self.speed = QLabel() + self.text.resize(100, 40) + self.speed.resize(100, 40) + + self.layout = QHBoxLayout() + self.layout.addWidget(self.text) + self.layout.addWidget(self.progress) + self.layout.addWidget(self.speed) + + self.timer = QTimer() + self.timer.setInterval(250) + self.timer.timeout.connect(self.tick) + self.timer.start() + + def resetStatus(self): + self.progress.setValue(0) + self.text.setText('') + self.speed.setText('') + + def tick(self): + for i in Status.lst: + if i.isOpen(): + try: + self.progress.setValue(i.i / i.size * 100) + self.text.setText(i.desc) + self.speed.setText(formatSpeed(i.a / (time.clock() - i.ats))) + except: + self.resetStatus() + break + else: + self.resetStatus() + if len(Status.lst) == 0: + self.resetStatus() + +class App(QWidget): + + def __init__(self): + super().__init__() + screen = QDesktopWidget().screenGeometry() + self.title = 'NUT USB / Web Server' + self.left = screen.width() / 4 + self.top = screen.height() / 4 + self.width = screen.width() / 2 + self.height = screen.height() / 2 + #self.setWindowState(Qt.WindowMaximized) + self.initUI() + + + def initUI(self): + self.setWindowTitle(self.title) + self.setGeometry(self.left, self.top, self.width, self.height) + + self.createTable() + + self.layout = QVBoxLayout() + + self.header = Header(self) + self.layout.addLayout(self.header.layout) + + self.layout.addWidget(self.tableWidget) + + self.progress = Progress(self) + self.layout.addLayout(self.progress.layout) + + self.setLayout(self.layout) + + self.show() + + def createTable(self): + self.tableWidget = QTableWidget() + self.tableWidget.setColumnCount(4) + + headers = [QTableWidgetItem("File"), QTableWidgetItem("Title ID"), QTableWidgetItem("Type"), QTableWidgetItem("Size")] + + i = 0 + for h in headers: + self.tableWidget.setHorizontalHeaderItem(i, h) + i = i + 1 + + header = self.tableWidget.horizontalHeader() + i = 0 + for h in headers: + header.setSectionResizeMode(i, QtWidgets.QHeaderView.Stretch if i == 0 else QtWidgets.QHeaderView.ResizeToContents) + i = i + 1 + + self.tableWidget.setSortingEnabled(True) + + self.refreshTable() + + @pyqtSlot() + def on_scan(self): + self.tableWidget.setRowCount(0) + Nsps.scan(Config.paths.scan, True) + self.refreshTable() + + def refreshTable(self): + self.tableWidget.setRowCount(len(Nsps.files)) + i = 0 + for k, f in Nsps.files.items(): + title = f.title() + if f.path.endswith('.nsx'): + continue + + self.tableWidget.setItem(i,0, QTableWidgetItem(os.path.basename(f.path))) + self.tableWidget.setItem(i,1, QTableWidgetItem(str(f.titleId))) + self.tableWidget.setItem(i,2, QTableWidgetItem("UPD" if title.isUpdate else ("DLC" if title.isDLC else "BASE"))) + self.tableWidget.setItem(i,3, QTableWidgetItem(str(f.fileSize or title.size))) + i = i + 1 + + self.tableWidget.setRowCount(i) + +threadRun = True + +def usbThread(): + Usb.daemon() + +def nutThread(): + Server.run() + +def initThread(app): + nut.scan() + app.refreshTable() + +if __name__ == '__main__': + urllib3.disable_warnings() + + + print(' ,;:;;,') + print(' ;;;;;') + print(' .=\', ;:;;:,') + print(' /_\', "=. \';:;:;') + print(' @=:__, \,;:;:\'') + print(' _(\.= ;:;;\'') + print(' `"_( _/="`') + print(' `"\'') + + nut.initTitles() + nut.initFiles() + + app = QApplication(sys.argv) + ex = App() + + threads = [] + threads.append(threading.Thread(target=initThread, args=[ex])) + threads.append(threading.Thread(target=usbThread, args=[])) + threads.append(threading.Thread(target=nutThread, args=[])) + + for t in threads: + t.start() + + sys.exit(app.exec_()) + + print('fin') +