diff --git a/plugins/DupFileManager/DupFileManager.dev.py b/plugins/DupFileManager/DupFileManager.dev.py deleted file mode 100644 index 630e16e2..00000000 --- a/plugins/DupFileManager/DupFileManager.dev.py +++ /dev/null @@ -1,1440 +0,0 @@ -# Description: This is a Stash plugin which manages duplicate files. -# By David Maisonave (aka Axter) Jul-2024 (https://www.axter.com/) -# Get the latest developers version from following link: https://github.com/David-Maisonave/Axter-Stash/tree/main/plugins/DupFileManager -# Note: To call this script outside of Stash, pass argument --url -# Example: python DupFileManager.py --url http://localhost:9999 -a -try: - import ModulesValidate - ModulesValidate.modulesInstalled(["send2trash", "requests"], silent=True) -except Exception as e: - import traceback, sys - tb = traceback.format_exc() - print(f"ModulesValidate Exception. Error: {e}\nTraceBack={tb}", file=sys.stderr) -import os, sys, time, pathlib, argparse, platform, shutil, traceback, logging, requests -from datetime import datetime -from StashPluginHelper import StashPluginHelper -from stashapi.stash_types import PhashDistance -from DupFileManager_config import config # Import config from DupFileManager_config.py -from DupFileManager_report_config import report_config - -# ToDo: make sure the following line of code works -config += report_config - -parser = argparse.ArgumentParser() -parser.add_argument('--url', '-u', dest='stash_url', type=str, help='Add Stash URL') -parser.add_argument('--trace', '-t', dest='trace', action='store_true', help='Enables debug trace mode.') -parser.add_argument('--add_dup_tag', '-a', dest='dup_tag', action='store_true', help='Set a tag to duplicate files.') -parser.add_argument('--clear_dup_tag', '-c', dest='clear_tag', action='store_true', help='Clear duplicates of duplicate tags.') -parser.add_argument('--del_tag_dup', '-d', dest='del_tag', action='store_true', help='Only delete scenes having DuplicateMarkForDeletion tag.') -parser.add_argument('--remove_dup', '-r', dest='remove', action='store_true', help='Remove (delete) duplicate files.') -parse_args = parser.parse_args() - -settings = { - "matchDupDistance": 0, - "mergeDupFilename": False, - "whitelistDelDupInSameFolder": False, - "zvWhitelist": "", - "zwGraylist": "", - "zxBlacklist": "", - "zyMaxDupToProcess": 0, - "zySwapHighRes": False, - "zySwapLongLength": False, - "zySwapBetterBitRate": False, - "zySwapCodec": False, - "zySwapBetterFrameRate": False, - "zzDebug": False, - "zzTracing": False, - - "zzObsoleteSettingsCheckVer2": False, # This is a hidden variable that is NOT displayed in the UI - - # Obsolete setting names - "zWhitelist": "", - "zxGraylist": "", - "zyBlacklist": "", - "zyMatchDupDistance": 0, - "zSwapHighRes": False, - "zSwapLongLength": False, - "zSwapBetterBitRate": False, - "zSwapCodec": False, - "zSwapBetterFrameRate": False, -} -stash = StashPluginHelper( - stash_url=parse_args.stash_url, - debugTracing=parse_args.trace, - settings=settings, - config=config, - maxbytes=10*1024*1024, - DebugTraceFieldName="zzTracing", - DebugFieldName="zzDebug", - ) -stash.convertToAscii = True - -advanceMenuOptions = [ "applyCombo", "applyComboBlacklist", "pathToDelete", "pathToDeleteBlacklist", "sizeToDeleteLess", "sizeToDeleteGreater", "sizeToDeleteBlacklistLess", "sizeToDeleteBlacklistGreater", "durationToDeleteLess", "durationToDeleteGreater", "durationToDeleteBlacklistLess", "durationToDeleteBlacklistGreater", - "commonResToDeleteLess", "commonResToDeleteEq", "commonResToDeleteGreater", "commonResToDeleteBlacklistLess", "commonResToDeleteBlacklistEq", "commonResToDeleteBlacklistGreater", "resolutionToDeleteLess", "resolutionToDeleteEq", "resolutionToDeleteGreater", - "resolutionToDeleteBlacklistLess", "resolutionToDeleteBlacklistEq", "resolutionToDeleteBlacklistGreater", "ratingToDeleteLess", "ratingToDeleteEq", "ratingToDeleteGreater", "ratingToDeleteBlacklistLess", "ratingToDeleteBlacklistEq", "ratingToDeleteBlacklistGreater", - "tagToDelete", "tagToDeleteBlacklist", "titleToDelete", "titleToDeleteBlacklist", "pathStrToDelete", "pathStrToDeleteBlacklist"] - -doJsonReturnModeTypes = ["tag_duplicates_task", "removeDupTag", "addExcludeTag", "removeExcludeTag", "mergeTags", "getLocalDupReportPath", - "createDuplicateReportWithoutTagging", "deleteLocalDupReportHtmlFiles", "clear_duplicate_tags_task", - "deleteAllDupFileManagerTags", "deleteBlackListTaggedDuplicatesTask", "deleteTaggedDuplicatesLwrResOrLwrDuration", - "deleteBlackListTaggedDuplicatesLwrResOrLwrDuration"] -doJsonReturnModeTypes += [advanceMenuOptions] -doJsonReturn = False -if len(sys.argv) < 2 and stash.PLUGIN_TASK_NAME in doJsonReturnModeTypes: - doJsonReturn = True - stash.log_to_norm = stash.LogTo.FILE -elif stash.PLUGIN_TASK_NAME == "doEarlyExit": - time.sleep(3) - stash.Log("Doing early exit because of task name") - time.sleep(3) - exit(0) - -stash.Log("******************* Starting *******************") -if len(sys.argv) > 1: - stash.Log(f"argv = {sys.argv}") -else: - stash.Debug(f"No command line arguments. JSON_INPUT['args'] = {stash.JSON_INPUT['args']}; PLUGIN_TASK_NAME = {stash.PLUGIN_TASK_NAME}; argv = {sys.argv}") -stash.status(logLevel=logging.DEBUG) - -obsoleteSettingsToConvert = {"zWhitelist" : "zvWhitelist", "zxGraylist" : "zwGraylist", "zyBlacklist" : "zxBlacklist", "zyMatchDupDistance" : "matchDupDistance", "zSwapHighRes" : "zySwapHighRes", "zSwapLongLength" : "zySwapLongLength", "zSwapBetterBitRate" : "zySwapBetterBitRate", "zSwapCodec" : "zySwapCodec", "zSwapBetterFrameRate" : "zySwapBetterFrameRate"} -stash.replaceObsoleteSettings(obsoleteSettingsToConvert, "zzObsoleteSettingsCheckVer2") - - -LOG_STASH_N_PLUGIN = stash.LogTo.STASH if stash.CALLED_AS_STASH_PLUGIN else stash.LogTo.CONSOLE + stash.LogTo.FILE -listSeparator = stash.Setting('listSeparator', ',', notEmpty=True) -addPrimaryDupPathToDetails = stash.Setting('addPrimaryDupPathToDetails') -clearAllDupfileManagerTags = stash.Setting('clearAllDupfileManagerTags') -doGeneratePhash = stash.Setting('doGeneratePhash') -mergeDupFilename = stash.Setting('mergeDupFilename') -moveToTrashCan = False if stash.Setting('permanentlyDelete') else True -alternateTrashCanPath = stash.Setting('dup_path') -whitelistDelDupInSameFolder = stash.Setting('whitelistDelDupInSameFolder') -graylistTagging = stash.Setting('graylistTagging') -maxDupToProcess = int(stash.Setting('zyMaxDupToProcess')) -significantTimeDiff = float(stash.Setting('significantTimeDiff')) -toRecycleBeforeSwap = stash.Setting('toRecycleBeforeSwap') -cleanAfterDel = stash.Setting('cleanAfterDel') - -swapHighRes = stash.Setting('zySwapHighRes') -swapLongLength = stash.Setting('zySwapLongLength') -swapBetterBitRate = stash.Setting('zySwapBetterBitRate') -swapCodec = stash.Setting('zySwapCodec') -swapBetterFrameRate = stash.Setting('zySwapBetterFrameRate') -favorLongerFileName = stash.Setting('favorLongerFileName') -favorLargerFileSize = stash.Setting('favorLargerFileSize') -favorBitRateChange = stash.Setting('favorBitRateChange') -favorHighBitRate = stash.Setting('favorHighBitRate') -favorFrameRateChange = stash.Setting('favorFrameRateChange') -favorHigherFrameRate = stash.Setting('favorHigherFrameRate') -favorCodecRanking = stash.Setting('favorCodecRanking') -codecRankingSetToUse = stash.Setting('codecRankingSetToUse') -if codecRankingSetToUse == 4: - codecRanking = stash.Setting('codecRankingSet4') -elif codecRankingSetToUse == 3: - codecRanking = stash.Setting('codecRankingSet3') -elif codecRankingSetToUse == 2: - codecRanking = stash.Setting('codecRankingSet2') -else: - codecRanking = stash.Setting('codecRankingSet1') -skipIfTagged = stash.Setting('skipIfTagged') -killScanningPostProcess = stash.Setting('killScanningPostProcess') -tagLongDurationLowRes = stash.Setting('tagLongDurationLowRes') -bitRateIsImporantComp = stash.Setting('bitRateIsImporantComp') -codecIsImporantComp = stash.Setting('codecIsImporantComp') - -excludeFromReportIfSignificantTimeDiff = False - -matchDupDistance = int(stash.Setting('matchDupDistance')) -matchPhaseDistance = PhashDistance.EXACT -matchPhaseDistanceText = "Exact Match" -if stash.PLUGIN_TASK_NAME == "tag_duplicates_task" and 'Target' in stash.JSON_INPUT['args']: - if stash.JSON_INPUT['args']['Target'].startswith("0"): - matchDupDistance = 0 - elif stash.JSON_INPUT['args']['Target'].startswith("1"): - matchDupDistance = 1 - elif stash.JSON_INPUT['args']['Target'].startswith("2"): - matchDupDistance = 2 - elif stash.JSON_INPUT['args']['Target'].startswith("3"): - matchDupDistance = 3 - - if stash.JSON_INPUT['args']['Target'].find(":") == 1: - significantTimeDiff = float(stash.JSON_INPUT['args']['Target'][2:]) - excludeFromReportIfSignificantTimeDiff = True - -if matchDupDistance == 1: - matchPhaseDistance = PhashDistance.HIGH - matchPhaseDistanceText = "High Match" -elif matchDupDistance == 2: - matchPhaseDistance = PhashDistance.MEDIUM - matchPhaseDistanceText = "Medium Match" -elif matchDupDistance == 3: - matchPhaseDistance = PhashDistance.LOW - matchPhaseDistanceText = "Low Match" - -# significantTimeDiff can not be higher than 1 and shouldn't be lower than .5 -if significantTimeDiff > 1: - significantTimeDiff = float(1.00) -if significantTimeDiff < .25: - significantTimeDiff = float(0.25) - - -duplicateMarkForDeletion = stash.Setting('DupFileTag') -if duplicateMarkForDeletion == "": - duplicateMarkForDeletion = 'DuplicateMarkForDeletion' - -base1_duplicateMarkForDeletion = duplicateMarkForDeletion - -duplicateWhitelistTag = stash.Setting('DupWhiteListTag') -if duplicateWhitelistTag == "": - duplicateWhitelistTag = '_DuplicateWhitelistFile' - -excludeDupFileDeleteTag = stash.Setting('excludeDupFileDeleteTag') -if excludeDupFileDeleteTag == "": - excludeDupFileDeleteTag = '_ExcludeDuplicateMarkForDeletion' - -graylistMarkForDeletion = stash.Setting('graylistMarkForDeletion') -if graylistMarkForDeletion == "": - graylistMarkForDeletion = '_GraylistMarkForDeletion' - -longerDurationLowerResolution = stash.Setting('longerDurationLowerResolution') -if longerDurationLowerResolution == "": - longerDurationLowerResolution = '_LongerDurationLowerResolution' - -excludeMergeTags = [duplicateMarkForDeletion, duplicateWhitelistTag, excludeDupFileDeleteTag] - -if stash.Setting('underscoreDupFileTag') and not duplicateMarkForDeletion.startswith('_'): - duplicateMarkForDeletionWithOutUnderscore = duplicateMarkForDeletion - duplicateMarkForDeletion = "_" + duplicateMarkForDeletion - if stash.renameTag(duplicateMarkForDeletionWithOutUnderscore, duplicateMarkForDeletion): - stash.Log(f"Renamed tag {duplicateMarkForDeletionWithOutUnderscore} to {duplicateMarkForDeletion}") - stash.Trace(f"Added underscore to {duplicateMarkForDeletionWithOutUnderscore} = {duplicateMarkForDeletion}") - excludeMergeTags += [duplicateMarkForDeletion] -else: - stash.Trace(f"duplicateMarkForDeletion = {duplicateMarkForDeletion}") - -base2_duplicateMarkForDeletion = duplicateMarkForDeletion - -if stash.Setting('appendMatchDupDistance'): - duplicateMarkForDeletion += f"_{matchDupDistance}" - excludeMergeTags += [duplicateMarkForDeletion] - -stash.initMergeMetadata(excludeMergeTags) - -graylist = stash.Setting('zwGraylist').split(listSeparator) -graylist = [item.lower() for item in graylist] -if graylist == [""] : graylist = [] -stash.Trace(f"graylist = {graylist}") -whitelist = stash.Setting('zvWhitelist').split(listSeparator) -whitelist = [item.lower() for item in whitelist] -if whitelist == [""] : whitelist = [] -stash.Trace(f"whitelist = {whitelist}") -blacklist = stash.Setting('zxBlacklist').split(listSeparator) -blacklist = [item.lower() for item in blacklist] -if blacklist == [""] : blacklist = [] -stash.Trace(f"blacklist = {blacklist}") - -def realpath(path): - """ - get_symbolic_target for win - """ - try: - import win32file - f = win32file.CreateFile(path, win32file.GENERIC_READ, - win32file.FILE_SHARE_READ, None, - win32file.OPEN_EXISTING, - win32file.FILE_FLAG_BACKUP_SEMANTICS, None) - target = win32file.GetFinalPathNameByHandle(f, 0) - # an above gives us something like u'\\\\?\\C:\\tmp\\scalarizr\\3.3.0.7978' - return target.strip('\\\\?\\') - except ImportError: - handle = open_dir(path) - target = get_symbolic_target(handle) - check_closed(handle) - return target - -def isReparsePoint(path): - import win32api - import win32con - from parse_reparsepoint import Navigator - FinalPathname = realpath(path) - stash.Log(f"(path='{path}') (FinalPathname='{FinalPathname}')") - if FinalPathname != path: - stash.Log(f"Symbolic link '{path}'") - return True - if not os.path.isdir(path): - path = os.path.dirname(path) - return win32api.GetFileAttributes(path) & win32con.FILE_ATTRIBUTE_REPARSE_POINT - -def testReparsePointAndSymLink(merge=False, deleteDup=False): - stash.Trace(f"Debug Tracing (platform.system()={platform.system()})") - myTestPath1 = r"B:\V\V\Tip\POV - Holly Molly petite ginger anal slut - RedTube.mp4" # not a reparse point or symbolic link - myTestPath2 = r"B:\_\SpecialSet\Amateur Anal Attempts\BRCC test studio name.m2ts" # reparse point - myTestPath3 = r"B:\_\SpecialSet\Amateur Anal Attempts\Amateur Anal Attempts 4.mp4" #symbolic link - myTestPath4 = r"E:\Stash\plugins\RenameFile\README.md" #symbolic link - myTestPath5 = r"E:\_\David-Maisonave\Axter-Stash\plugins\RenameFile\README.md" #symbolic link - myTestPath6 = r"E:\_\David-Maisonave\Axter-Stash\plugins\DeleteMe\Renamer\README.md" # not reparse point - stash.Log(f"Testing '{myTestPath1}'") - if isReparsePoint(myTestPath1): - stash.Log(f"isSymLink '{myTestPath1}'") - else: - stash.Log(f"Not isSymLink '{myTestPath1}'") - - if isReparsePoint(myTestPath2): - stash.Log(f"isSymLink '{myTestPath2}'") - else: - stash.Log(f"Not isSymLink '{myTestPath2}'") - - if isReparsePoint(myTestPath3): - stash.Log(f"isSymLink '{myTestPath3}'") - else: - stash.Log(f"Not isSymLink '{myTestPath3}'") - - if isReparsePoint(myTestPath4): - stash.Log(f"isSymLink '{myTestPath4}'") - else: - stash.Log(f"Not isSymLink '{myTestPath4}'") - - if isReparsePoint(myTestPath5): - stash.Log(f"isSymLink '{myTestPath5}'") - else: - stash.Log(f"Not isSymLink '{myTestPath5}'") - - if isReparsePoint(myTestPath6): - stash.Log(f"isSymLink '{myTestPath6}'") - else: - stash.Log(f"Not isSymLink '{myTestPath6}'") - return - -detailPrefix = "BaseDup=" -detailPostfix = "\n" - -def setTagId(tagName, sceneDetails, DupFileToKeep, TagReason="", ignoreAutoTag=False): - details = "" - ORG_DATA_DICT = {'id' : sceneDetails['id']} - dataDict = ORG_DATA_DICT.copy() - doAddTag = True - if addPrimaryDupPathToDetails: - BaseDupStr = f"{detailPrefix}{DupFileToKeep['files'][0]['path']}\n{stash.STASH_URL}/scenes/{DupFileToKeep['id']}\n{TagReason}(matchDupDistance={matchPhaseDistanceText})\n{detailPostfix}" - if sceneDetails['details'] == "": - details = BaseDupStr - elif not sceneDetails['details'].startswith(detailPrefix): - details = f"{BaseDupStr};\n{sceneDetails['details']}" - for tag in sceneDetails['tags']: - if tag['name'] == tagName: - doAddTag = False - break - if doAddTag: - stash.addTag(sceneDetails, tagName, ignoreAutoTag=ignoreAutoTag) - if details != "": - dataDict.update({'details' : details}) - if dataDict != ORG_DATA_DICT: - stash.updateScene(dataDict) - stash.Trace(f"[setTagId] Updated {sceneDetails['files'][0]['path']} with metadata {dataDict} and tag {tagName}", toAscii=True) - else: - stash.Trace(f"[setTagId] Nothing to update {sceneDetails['files'][0]['path']} already has tag {tagName}.", toAscii=True) - return doAddTag - -def setTagId_withRetry(tagName, sceneDetails, DupFileToKeep, TagReason="", ignoreAutoTag=False, retryCount = 12, sleepSecondsBetweenRetry = 5): - errMsg = None - for i in range(0, retryCount): - try: - if errMsg != None: - stash.Warn(errMsg) - return setTagId(tagName, sceneDetails, DupFileToKeep, TagReason, ignoreAutoTag) - except (requests.exceptions.ConnectionError, ConnectionResetError): - tb = traceback.format_exc() - errMsg = f"[setTagId] Exception calling setTagId. Will retry; count({i}); Error: {e}\nTraceBack={tb}" - except Exception as e: - tb = traceback.format_exc() - errMsg = f"[setTagId] Unknown exception calling setTagId. Will retry; count({i}); Error: {e}\nTraceBack={tb}" - time.sleep(sleepSecondsBetweenRetry) - -def hasSameDir(path1, path2): - if pathlib.Path(path1).resolve().parent == pathlib.Path(path2).resolve().parent: - return True - return False - -def sendToTrash(path): - if not os.path.isfile(path): - stash.Warn(f"File does not exist: {path}.", toAscii=True) - return False - try: - from send2trash import send2trash # Requirement: pip install Send2Trash - send2trash(path) - return True - except Exception as e: - stash.Error(f"Failed to send file {path} to recycle bin. Error: {e}", toAscii=True) - try: - if os.path.isfile(path): - os.remove(path) - return True - except Exception as e: - stash.Error(f"Failed to delete file {path}. Error: {e}", toAscii=True) - return False -# If ckTimeDiff=False: Does durration2 have significant more time than durration1 -def significantTimeDiffCheck(durration1, durration2, ckTimeDiff = False): # If ckTimeDiff=True: is time different significant in either direction. - if not isinstance(durration1, int) and 'files' in durration1: - durration1 = int(durration1['files'][0]['duration']) - durration2 = int(durration2['files'][0]['duration']) - timeDiff = getTimeDif(durration1, durration2) - if ckTimeDiff and timeDiff > 1: - timeDiff = getTimeDif(durration2, durration1) - if timeDiff < significantTimeDiff: - return True - return False - -def getTimeDif(durration1, durration2): # Where durration1 is ecpected to be smaller than durration2 IE(45/60=.75) - return durration1 / durration2 - -def isBetterVideo(scene1, scene2, swapCandidateCk = False): # is scene2 better than scene1 - # Prioritize higher reslution over codec, bit rate, and frame rate - if int(scene1['files'][0]['width']) * int(scene1['files'][0]['height']) > int(scene2['files'][0]['width']) * int(scene2['files'][0]['height']): - return False - if (favorBitRateChange and swapCandidateCk == False) or (swapCandidateCk and swapBetterBitRate): - if (favorHighBitRate and int(scene2['files'][0]['bit_rate']) > int(scene1['files'][0]['bit_rate'])) or (not favorHighBitRate and int(scene2['files'][0]['bit_rate']) < int(scene1['files'][0]['bit_rate'])): - stash.Trace(f"[isBetterVideo]:[favorHighBitRate={favorHighBitRate}] Better bit rate. {scene1['files'][0]['path']}={scene1['files'][0]['bit_rate']} v.s. {scene2['files'][0]['path']}={scene2['files'][0]['bit_rate']}") - return True - if (favorCodecRanking and swapCandidateCk == False) or (swapCandidateCk and swapCodec): - scene1CodecRank = stash.indexStartsWithInList(codecRanking, scene1['files'][0]['video_codec']) - scene2CodecRank = stash.indexStartsWithInList(codecRanking, scene2['files'][0]['video_codec']) - if scene2CodecRank < scene1CodecRank: - stash.Trace(f"[isBetterVideo] Better codec. {scene1['files'][0]['path']}={scene1['files'][0]['video_codec']}:Rank={scene1CodecRank} v.s. {scene2['files'][0]['path']}={scene2['files'][0]['video_codec']}:Rank={scene2CodecRank}") - return True - if (favorFrameRateChange and swapCandidateCk == False) or (swapCandidateCk and swapBetterFrameRate): - if (favorHigherFrameRate and int(scene2['files'][0]['frame_rate']) > int(scene1['files'][0]['frame_rate'])) or (not favorHigherFrameRate and int(scene2['files'][0]['frame_rate']) < int(scene1['files'][0]['frame_rate'])): - stash.Trace(f"[isBetterVideo]:[favorHigherFrameRate={favorHigherFrameRate}] Better frame rate. {scene1['files'][0]['path']}={scene1['files'][0]['frame_rate']} v.s. {scene2['files'][0]['path']}={scene2['files'][0]['frame_rate']}") - return True - return False - -def significantMoreTimeCompareToBetterVideo(scene1, scene2): # is scene2 better than scene1 - if isinstance(scene1, int): - scene1 = stash.find_scene(scene1) - scene2 = stash.find_scene(scene2) - if int(scene1['files'][0]['duration']) >= int(scene2['files'][0]['duration']): - return False - if int(scene1['files'][0]['width']) * int(scene1['files'][0]['height']) > int(scene2['files'][0]['width']) * int(scene2['files'][0]['height']): - if significantTimeDiffCheck(scene1, scene2): - if tagLongDurationLowRes: - didAddTag = setTagId_withRetry(longerDurationLowerResolution, scene2, scene1, ignoreAutoTag=True) - stash.Log(f"Tagged sene2 with tag {longerDurationLowerResolution}, because scene1 is better video, but it has significant less time ({getTimeDif(int(scene1['files'][0]['duration']), int(scene2['files'][0]['duration']))}%) compare to scene2; scene1={scene1['files'][0]['path']} (ID={scene1['id']})(duration={scene1['files'][0]['duration']}); scene2={scene2['files'][0]['path']} (ID={scene2['id']}) (duration={scene1['files'][0]['duration']}); didAddTag={didAddTag}") - else: - stash.Warn(f"Scene1 is better video, but it has significant less time ({getTimeDif(int(scene1['files'][0]['duration']), int(scene2['files'][0]['duration']))}%) compare to scene2; Scene1={scene1['files'][0]['path']} (ID={scene1['id']})(duration={scene1['files'][0]['duration']}); Scene2={scene2['files'][0]['path']} (ID={scene2['id']}) (duration={scene1['files'][0]['duration']})") - return False - return True - -def allThingsEqual(scene1, scene2): # If all important things are equal, return true - if int(scene1['files'][0]['duration']) != int(scene2['files'][0]['duration']): - return False - if scene1['files'][0]['width'] != scene2['files'][0]['width']: - return False - if scene1['files'][0]['height'] != scene2['files'][0]['height']: - return False - if bitRateIsImporantComp and scene1['files'][0]['bit_rate'] != scene2['files'][0]['bit_rate']: - return False - if codecIsImporantComp and scene1['files'][0]['video_codec'] != scene2['files'][0]['video_codec']: - return False - return True - -def isSwapCandidate(DupFileToKeep, DupFile): - # Don't move if both are in whitelist - if stash.startsWithInList(whitelist, DupFileToKeep['files'][0]['path']) and stash.startsWithInList(whitelist, DupFile['files'][0]['path']): - return False - if swapHighRes and int(DupFileToKeep['files'][0]['width']) * int(DupFileToKeep['files'][0]['height']) > int(DupFile['files'][0]['width']) * int(DupFile['files'][0]['height']): - if not significantTimeDiffCheck(DupFileToKeep, DupFile): - return True - else: - stash.Warn(f"File '{DupFileToKeep['files'][0]['path']}' has a higher resolution than '{DupFile['files'][0]['path']}', but the duration is significantly shorter.", toAscii=True) - if swapLongLength and int(DupFileToKeep['files'][0]['duration']) > int(DupFile['files'][0]['duration']): - if int(DupFileToKeep['files'][0]['width']) >= int(DupFile['files'][0]['width']) or int(DupFileToKeep['files'][0]['height']) >= int(DupFile['files'][0]['height']): - return True - if isBetterVideo(DupFile, DupFileToKeep, swapCandidateCk=True): - if not significantTimeDiffCheck(DupFileToKeep, DupFile): - return True - else: - stash.Warn(f"File '{DupFileToKeep['files'][0]['path']}' has better codec/bit-rate than '{DupFile['files'][0]['path']}', but the duration is significantly shorter; DupFileToKeep-ID={DupFileToKeep['id']};DupFile-ID={DupFile['id']};BitRate {DupFileToKeep['files'][0]['bit_rate']} vs {DupFile['files'][0]['bit_rate']};Codec {DupFileToKeep['files'][0]['video_codec']} vs {DupFile['files'][0]['video_codec']};FrameRate {DupFileToKeep['files'][0]['frame_rate']} vs {DupFile['files'][0]['frame_rate']};", toAscii=True) - return False - -dupWhitelistTagId = None -def addDupWhitelistTag(): - global dupWhitelistTagId - stash.Trace(f"Adding tag duplicateWhitelistTag = {duplicateWhitelistTag}") - descp = 'Tag added to duplicate scenes which are in the whitelist. This means there are two or more duplicates in the whitelist.' - dupWhitelistTagId = stash.createTagId(duplicateWhitelistTag, descp, ignoreAutoTag=True) - stash.Trace(f"dupWhitelistTagId={dupWhitelistTagId} name={duplicateWhitelistTag}") - -excludeDupFileDeleteTagId = None -def addExcludeDupTag(): - global excludeDupFileDeleteTagId - stash.Trace(f"Adding tag excludeDupFileDeleteTag = {excludeDupFileDeleteTag}") - descp = 'Excludes duplicate scene from DupFileManager tagging and deletion process. A scene having this tag will not get deleted by DupFileManager' - excludeDupFileDeleteTagId = stash.createTagId(excludeDupFileDeleteTag, descp, ignoreAutoTag=True) - stash.Trace(f"dupWhitelistTagId={excludeDupFileDeleteTagId} name={excludeDupFileDeleteTag}") - -def isTaggedExcluded(Scene): - for tag in Scene['tags']: - if tag['name'] == excludeDupFileDeleteTag: - return True - return False - -def isWorseKeepCandidate(DupFileToKeep, Scene): - if not stash.startsWithInList(whitelist, Scene['files'][0]['path']) and stash.startsWithInList(whitelist, DupFileToKeep['files'][0]['path']): - return True - if not stash.startsWithInList(graylist, Scene['files'][0]['path']) and stash.startsWithInList(graylist, DupFileToKeep['files'][0]['path']): - return True - if not stash.startsWithInList(blacklist, DupFileToKeep['files'][0]['path']) and stash.startsWithInList(blacklist, Scene['files'][0]['path']): - return True - - if stash.startsWithInList(graylist, Scene['files'][0]['path']) and stash.startsWithInList(graylist, DupFileToKeep['files'][0]['path']) and stash.indexStartsWithInList(graylist, DupFileToKeep['files'][0]['path']) < stash.indexStartsWithInList(graylist, Scene['files'][0]['path']): - return True - if stash.startsWithInList(blacklist, DupFileToKeep['files'][0]['path']) and stash.startsWithInList(blacklist, Scene['files'][0]['path']) and stash.indexStartsWithInList(blacklist, DupFileToKeep['files'][0]['path']) < stash.indexStartsWithInList(blacklist, Scene['files'][0]['path']): - return True - return False - -def killScanningJobs(): - try: - if killScanningPostProcess: - stash.stopJobs(1, "Scanning...") - except Exception as e: - tb = traceback.format_exc() - stash.Error(f"Exception while trying to kill scan jobs; Error: {e}\nTraceBack={tb}") - -def getPath(Scene, getParent = False): - path = stash.asc2(Scene['files'][0]['path']) - path = path.replace("'", "") - path = path.replace("\\\\", "\\") - if getParent: - return pathlib.Path(path).resolve().parent - return path - -def getHtmlReportTableRow(qtyResults, tagDuplicates): - htmlReportPrefix = stash.Setting('htmlReportPrefix') - htmlReportPrefix = htmlReportPrefix.replace('http://127.0.0.1:9999/graphql', stash.url) - htmlReportPrefix = htmlReportPrefix.replace('http://localhost:9999/graphql', stash.url) - if tagDuplicates == False: - htmlReportPrefix = htmlReportPrefix.replace('") - fileHtmlReport.write(f"") - fileHtmlReport.write(f"") - fileHtmlReport.write(f"") - fileHtmlReport.write(f"") - # ToDo: Add following buttons: - # rename file - if dupFileExist and tagDuplicates: - fileHtmlReport.write(f"") - fileHtmlReport.write(f"") - fileHtmlReport.write(f"") - if dupFileExist: - fileHtmlReport.write(f"[Folder]") - fileHtmlReport.write(f"[Play]") - else: - fileHtmlReport.write("[File NOT Exist]") - fileHtmlReport.write("

") - - videoPreview = f"" - if htmlIncludeImagePreview: - imagePreview = f"
  • \"\"\"\"
" - fileHtmlReport.write(f"{getSceneID(DupFileToKeep['id'])}
{videoPreview}{imagePreview}
") - else: - fileHtmlReport.write(f"{getSceneID(DupFileToKeep['id'])}{videoPreview}") - fileHtmlReport.write(f"{getSceneID(DupFileToKeep['id'])}{getPath(DupFileToKeep)}") - fileHtmlReport.write(f"

") - fileHtmlReport.write(f"
ResDurrationBitRateCodecFrameRatesizeID
{DupFileToKeep['files'][0]['width']}x{DupFileToKeep['files'][0]['height']}{DupFileToKeep['files'][0]['duration']}{DupFileToKeep['files'][0]['bit_rate']}{DupFileToKeep['files'][0]['video_codec']}{DupFileToKeep['files'][0]['frame_rate']}{DupFileToKeep['files'][0]['size']}{DupFileToKeep['id']}
") - fileHtmlReport.write(f"") - fileHtmlReport.write(f"") - fileHtmlReport.write(f"") - if isTaggedExcluded(DupFileToKeep): - fileHtmlReport.write(f"") - fileHtmlReport.write(f"[Folder]") - if toKeepFileExist: - fileHtmlReport.write(f"[Play]") - else: - fileHtmlReport.write("[File NOT Exist]") - # ToDo: Add following buttons: - # rename file - fileHtmlReport.write(f"

") - - fileHtmlReport.write("\n") - - if QtyTagForDelPaginate >= htmlReportPaginate: - QtyTagForDelPaginate = 0 - fileHtmlReport.write("\n") - homeHtmReportLink = f"[Home]" - prevHtmReportLink = "" - if PaginateId > 0: - if PaginateId > 1: - prevHtmReport = htmlReportNameHomePage.replace(".html", f"_{PaginateId-1}.html") - else: - prevHtmReport = htmlReportNameHomePage - prevHtmReportLink = f"[Prev]" - nextHtmReport = htmlReportNameHomePage.replace(".html", f"_{PaginateId+1}.html") - nextHtmReportLink = f"[Next]" - fileHtmlReport.write(f"
{homeHtmReportLink}{prevHtmReportLink}{nextHtmReportLink}
") - fileHtmlReport.write(f"{stash.Setting('htmlReportPostfix')}") - fileHtmlReport.close() - PaginateId+=1 - fileHtmlReport = open(nextHtmReport, "w") - fileHtmlReport.write(f"{getHtmlReportTableRow(qtyResults, tagDuplicates)}\n") - if PaginateId > 1: - prevHtmReport = htmlReportNameHomePage.replace(".html", f"_{PaginateId-1}.html") - else: - prevHtmReport = htmlReportNameHomePage - prevHtmReportLink = f"[Prev]" - if len(DupFileSets) > (QtyTagForDel + htmlReportPaginate): - nextHtmReport = htmlReportNameHomePage.replace(".html", f"_{PaginateId+1}.html") - nextHtmReportLink = f"[Next]" - fileHtmlReport.write(f"
{homeHtmReportLink}{prevHtmReportLink}{nextHtmReportLink}
") - else: - stash.Debug(f"DupFileSets Qty = {len(DupFileSets)}; DupFileDetailList Qty = {len(DupFileDetailList)}; QtyTagForDel = {QtyTagForDel}; htmlReportPaginate = {htmlReportPaginate}; QtyTagForDel + htmlReportPaginate = {QtyTagForDel+htmlReportPaginate}") - fileHtmlReport.write(f"
{homeHtmReportLink}{prevHtmReportLink}
") - fileHtmlReport.write(f"{stash.Setting('htmlReportTable')}\n") - fileHtmlReport.write(f"{htmlReportTableRow}{htmlReportTableHeader}Scene{htmlReportTableHeader}Duplicate to Delete{htmlReportTableHeader}Scene-ToKeep{htmlReportTableHeader}Duplicate to Keep\n") - - if tagDuplicates and graylistTagging and stash.startsWithInList(graylist, DupFile['files'][0]['path']): - stash.addTag(DupFile, graylistMarkForDeletion, ignoreAutoTag=True) - if didAddTag: - QtyNewlyTag+=1 - if QtyTagForDel == 1: - stash.Log(f"Tagging duplicate {DupFile['files'][0]['path']} for deletion with tag {duplicateMarkForDeletion}", toAscii=True, printTo=LOG_STASH_N_PLUGIN) - else: - didAddTag = 1 if didAddTag else 0 - stash.Log(f"Tagging duplicate {DupFile['files'][0]['path']} for deletion;AddTag={didAddTag};Qty={QtyDup};Set={QtyDupSet} of {qtyResults};NewlyTag={QtyNewlyTag};isTag={QtyTagForDel}", toAscii=True, printTo=LOG_STASH_N_PLUGIN) - stash.Trace(SepLine) - if maxDupToProcess > 0 and ((QtyTagForDel > maxDupToProcess) or (QtyTagForDel == 0 and QtyDup > maxDupToProcess)): - break - - if fileHtmlReport != None: - fileHtmlReport.write("\n") - if PaginateId > 0: - homeHtmReportLink = f"[Home]" - if PaginateId > 1: - prevHtmReport = htmlReportNameHomePage.replace(".html", f"_{PaginateId-1}.html") - else: - prevHtmReport = htmlReportNameHomePage - prevHtmReportLink = f"[Prev]" - fileHtmlReport.write(f"
{homeHtmReportLink}{prevHtmReportLink}
") - fileHtmlReport.write(f"

Total Tagged for Deletion {QtyTagForDel}

\n") - fileHtmlReport.write(f"{stash.Setting('htmlReportPostfix')}") - fileHtmlReport.close() - stash.Log(f"************************************************************", printTo = stash.LogTo.STASH) - stash.Log(f"************************************************************", printTo = stash.LogTo.STASH) - stash.Log(f"View Stash duplicate report using Stash->Settings->Tools->[Duplicate File Report]", printTo = stash.LogTo.STASH) - stash.Log(f"************************************************************", printTo = stash.LogTo.STASH) - stash.Log(f"************************************************************", printTo = stash.LogTo.STASH) - - - stash.Debug("#####################################################") - stash.Log(f"QtyDupSet={QtyDupSet}, QtyDup={QtyDup}, QtyDeleted={QtyDeleted}, QtySwap={QtySwap}, QtyTagForDel={QtyTagForDel}, QtySkipForDel={QtySkipForDel}, QtyExcludeForDel={QtyExcludeForDel}, QtyExactDup={QtyExactDup}, QtyAlmostDup={QtyAlmostDup}, QtyMerge={QtyMerge}, QtyRealTimeDiff={QtyRealTimeDiff}", printTo=LOG_STASH_N_PLUGIN) - killScanningJobs() - if cleanAfterDel and deleteDup: - stash.Log("Adding clean jobs to the Task Queue", printTo=LOG_STASH_N_PLUGIN) - stash.metadata_clean() - stash.metadata_clean_generated() - stash.optimise_database() - if doGeneratePhash: - stash.metadata_generate({"phashes": True}) - sys.stdout.write("Report complete") - -def findCurrentTagId(tagNames): - # tagNames = [i for n, i in enumerate(tagNames) if i not in tagNames[:n]] - for tagName in tagNames: - tagId = stash.find_tags(q=tagName) - if len(tagId) > 0 and 'id' in tagId[0]: - stash.Debug(f"Using tag name {tagName} with Tag ID {tagId[0]['id']}") - return tagId[0]['id'] - return "-1" - -def toJson(data): - import json - # data = data.replace("'", '"') - data = data.replace("\\", "\\\\") - data = data.replace("\\\\\\\\", "\\\\") - return json.loads(data) - -def getAnAdvanceMenuOptionSelected(taskName, target, isBlackList, pathToDelete, sizeToDelete, durationToDelete, resolutionToDelete, ratingToDelete, tagToDelete, titleToDelete, pathStrToDelete, fileNotExistToDelete, compareToLess, compareToGreater): - stash.Log(f"Processing taskName = {taskName}, target = {target}") - if "Blacklist" in taskName: - isBlackList = True - if "Less" in taskName: - compareToLess = True - if "Greater" in taskName: - compareToGreater = True - - if "pathToDelete" in taskName: - pathToDelete = target.lower() - elif "sizeToDelete" in taskName: - sizeToDelete = int(target) - elif "durationToDelete" in taskName: - durationToDelete = int(target) - elif "commonResToDelete" in taskName: - resolutionToDelete = int(target) - elif "resolutionToDelete" in taskName: - resolutionToDelete = int(target) - elif "ratingToDelete" in taskName: - ratingToDelete = int(target) * 20 - elif "tagToDelete" in taskName: - tagToDelete = target.lower() - elif "titleToDelete" in taskName: - titleToDelete = target.lower() - elif "pathStrToDelete" in taskName: - pathStrToDelete = target.lower() - elif "fileNotExistToDelete" in taskName: - fileNotExistToDelete = True - return isBlackList, pathToDelete, sizeToDelete, durationToDelete, resolutionToDelete, ratingToDelete, tagToDelete, titleToDelete, pathStrToDelete, fileNotExistToDelete, compareToLess, compareToGreater - -def getAdvanceMenuOptionSelected(advanceMenuOptionSelected): - isBlackList = False - pathToDelete = "" - sizeToDelete = -1 - durationToDelete = -1 - resolutionToDelete = -1 - ratingToDelete = -1 - tagToDelete = "" - titleToDelete = "" - pathStrToDelete = "" - fileNotExistToDelete = False - compareToLess = False - compareToGreater = False - if advanceMenuOptionSelected: - if 'Target' in stash.JSON_INPUT['args']: - if "applyCombo" in stash.PLUGIN_TASK_NAME: - jsonObject = toJson(stash.JSON_INPUT['args']['Target']) - for taskName in jsonObject: - isBlackList, pathToDelete, sizeToDelete, durationToDelete, resolutionToDelete, ratingToDelete, tagToDelete, titleToDelete, pathStrToDelete, fileNotExistToDelete, compareToLess, compareToGreater = getAnAdvanceMenuOptionSelected(taskName, jsonObject[taskName], isBlackList, pathToDelete, sizeToDelete, durationToDelete, resolutionToDelete, ratingToDelete, tagToDelete, titleToDelete, pathStrToDelete, compareToLess, compareToGreater) - else: - return getAnAdvanceMenuOptionSelected(stash.PLUGIN_TASK_NAME, stash.JSON_INPUT['args']['Target'], isBlackList, pathToDelete, sizeToDelete, durationToDelete, resolutionToDelete, ratingToDelete, tagToDelete, titleToDelete, pathStrToDelete, compareToLess, compareToGreater) - return isBlackList, pathToDelete, sizeToDelete, durationToDelete, resolutionToDelete, ratingToDelete, tagToDelete, titleToDelete, pathStrToDelete, fileNotExistToDelete, compareToLess, compareToGreater - -# ////////////////////////////////////////////////////////////////////////////// -# ////////////////////////////////////////////////////////////////////////////// -def manageTagggedDuplicates(deleteScenes=False, clearTag=False, setGrayListTag=False, tagId=-1, advanceMenuOptionSelected=False): - if tagId == -1: - tagId = findCurrentTagId([duplicateMarkForDeletion, base1_duplicateMarkForDeletion, base2_duplicateMarkForDeletion, 'DuplicateMarkForDeletion', '_DuplicateMarkForDeletion']) - if int(tagId) < 0: - stash.Warn(f"Could not find tag ID for tag '{duplicateMarkForDeletion}'.") - return - - excludedTags = [duplicateMarkForDeletion] - if clearAllDupfileManagerTags: - excludedTags = [duplicateMarkForDeletion, duplicateWhitelistTag, excludeDupFileDeleteTag, graylistMarkForDeletion, longerDurationLowerResolution] - - isBlackList, pathToDelete, sizeToDelete, durationToDelete, resolutionToDelete, ratingToDelete, tagToDelete, titleToDelete, pathStrToDelete, fileNotExistToDelete, compareToLess, compareToGreater = getAdvanceMenuOptionSelected(advanceMenuOptionSelected) - if advanceMenuOptionSelected and deleteScenes and pathToDelete == "" and tagToDelete == "" and titleToDelete == "" and pathStrToDelete == "" and sizeToDelete == -1 and durationToDelete == -1 and resolutionToDelete == -1 and ratingToDelete == -1 and fileNotExistToDelete == False: - stash.Error("Running advance menu option with no options enabled.") - return - - QtyDup = 0 - QtyDeleted = 0 - QtyClearedTags = 0 - QtySetGraylistTag = 0 - QtyFailedQuery = 0 - stash.Debug("#########################################################################") - stash.startSpinningProcessBar() - scenes = stash.find_scenes(f={"tags": {"value":tagId, "modifier":"INCLUDES"}}, fragment='id tags {id name} files {path width height duration size video_codec bit_rate frame_rate} details title rating100') - stash.stopSpinningProcessBar() - qtyResults = len(scenes) - stash.Log(f"Found {qtyResults} scenes with tag ({duplicateMarkForDeletion})") - stash.setProgressBarIter(qtyResults) - for scene in scenes: - QtyDup += 1 - stash.progressBar(QtyDup, qtyResults) - # scene = stash.find_scene(sceneID['id']) - # if scene == None or len(scene) == 0: - # stash.Warn(f"Could not get scene data for scene ID {scene['id']}.") - # QtyFailedQuery += 1 - # continue - # stash.Trace(f"scene={scene}") - if clearTag: - QtyClearedTags += 1 - # ToDo: Add logic to exclude graylistMarkForDeletion - tags = [int(item['id']) for item in scene["tags"] if item['name'] not in excludedTags] - # if clearAllDupfileManagerTags: - # tags = [] - # for tag in scene["tags"]: - # if tag['name'] in excludedTags: - # continue - # tags += [int(tag['id'])] - stash.TraceOnce(f"tagId={tagId}, len={len(tags)}, tags = {tags}") - dataDict = {'id' : scene['id']} - if addPrimaryDupPathToDetails: - sceneDetails = scene['details'] - if sceneDetails.find(detailPrefix) == 0 and sceneDetails.find(detailPostfix) > 1: - Pos1 = sceneDetails.find(detailPrefix) - Pos2 = sceneDetails.find(detailPostfix) - sceneDetails = sceneDetails[0:Pos1] + sceneDetails[Pos2 + len(detailPostfix):] - dataDict.update({'details' : sceneDetails}) - dataDict.update({'tag_ids' : tags}) - stash.Log(f"Updating scene with {dataDict};QtyClearedTags={QtyClearedTags};Count={QtyDup} of {qtyResults}") - stash.updateScene(dataDict) - # stash.removeTag(scene, duplicateMarkForDeletion) - elif setGrayListTag: - if stash.startsWithInList(graylist, scene['files'][0]['path']): - QtySetGraylistTag+=1 - if stash.addTag(scene, graylistMarkForDeletion, ignoreAutoTag=True): - stash.Log(f"Added tag {graylistMarkForDeletion} to scene {scene['files'][0]['path']};QtySetGraylistTag={QtySetGraylistTag};Count={QtyDup} of {qtyResults}") - else: - stash.Trace(f"Scene already had tag {graylistMarkForDeletion}; {scene['files'][0]['path']}") - elif deleteScenes: - DupFileName = scene['files'][0]['path'] - DupFileNameOnly = pathlib.Path(DupFileName).stem - if advanceMenuOptionSelected: - if isBlackList: - if not stash.startsWithInList(blacklist, scene['files'][0]['path']): - continue - if pathToDelete != "": - if not DupFileName.lower().startswith(pathToDelete): - stash.Debug(f"Skipping file {DupFileName} because it does not start with {pathToDelete}.") - continue - if pathStrToDelete != "": - if not pathStrToDelete in DupFileName.lower(): - stash.Debug(f"Skipping file {DupFileName} because it does not contain value {pathStrToDelete}.") - continue - if sizeToDelete != -1: - compareTo = int(scene['files'][0]['size']) - if compareToLess: - if not (compareTo < sizeToDelete): - continue - elif compareToGreater: - if not (compareTo > sizeToDelete): - continue - else: - if not compareTo == sizeToDelete: - continue - if durationToDelete != -1: - compareTo = int(scene['files'][0]['duration']) - if compareToLess: - if not (compareTo < durationToDelete): - continue - elif compareToGreater: - if not (compareTo > durationToDelete): - continue - else: - if not compareTo == durationToDelete: - continue - if resolutionToDelete != -1: - compareTo = int(scene['files'][0]['width']) * int(scene['files'][0]['height']) - if compareToLess: - if not (compareTo < resolutionToDelete): - continue - elif compareToGreater: - if not (compareTo > resolutionToDelete): - continue - else: - if not compareTo == resolutionToDelete: - continue - if ratingToDelete != -1: - if scene['rating100'] == "None": - compareTo = 0 - else: - compareTo = int(scene['rating100']) - if compareToLess: - if not (compareTo < resolutionToDelete): - continue - elif compareToGreater: - if not (compareTo > resolutionToDelete): - continue - else: - if not compareTo == resolutionToDelete: - continue - if titleToDelete != "": - if not titleToDelete in scene['title'].lower(): - stash.Debug(f"Skipping file {DupFileName} because it does not contain value {titleToDelete} in title ({scene['title']}).") - continue - if tagToDelete != "": - doProcessThis = False - for tag in scene['tags']: - if tag['name'].lower() == tagToDelete: - doProcessThis = True - break - if doProcessThis == False: - continue - if fileNotExistToDelete: - if os.path.isfile(scene['files'][0]['path']): - continue - stash.Warn(f"Deleting duplicate '{DupFileName}'", toAscii=True, printTo=LOG_STASH_N_PLUGIN) - if alternateTrashCanPath != "": - destPath = f"{alternateTrashCanPath }{os.sep}{DupFileNameOnly}" - if os.path.isfile(destPath): - destPath = f"{alternateTrashCanPath }{os.sep}_{time.time()}_{DupFileNameOnly}" - shutil.move(DupFileName, destPath) - elif moveToTrashCan: - sendToTrash(DupFileName) - result = stash.destroyScene(scene['id'], delete_file=True) - QtyDeleted += 1 - stash.Debug(f"destroyScene result={result} for file {DupFileName};QtyDeleted={QtyDeleted};Count={QtyDup} of {qtyResults}", toAscii=True) - else: - stash.Error("manageTagggedDuplicates called with invlaid input arguments. Doing early exit.") - return - stash.Debug("#####################################################") - stash.Log(f"QtyDup={QtyDup}, QtyClearedTags={QtyClearedTags}, QtySetGraylistTag={QtySetGraylistTag}, QtyDeleted={QtyDeleted}, QtyFailedQuery={QtyFailedQuery}", printTo=LOG_STASH_N_PLUGIN) - killScanningJobs() - if deleteScenes and not advanceMenuOptionSelected: - if cleanAfterDel: - stash.Log("Adding clean jobs to the Task Queue", printTo=LOG_STASH_N_PLUGIN) - stash.metadata_clean() - stash.metadata_clean_generated() - stash.optimise_database() - - -def removeDupTag(): - if 'Target' not in stash.JSON_INPUT['args']: - stash.Error(f"Could not find Target in JSON_INPUT ({stash.JSON_INPUT['args']})") - return - scene = stash.JSON_INPUT['args']['Target'] - stash.Log(f"Processing scene ID# {scene}") - stash.removeTag(scene, duplicateMarkForDeletion) - stash.Log(f"Done removing tag from scene {scene}.") - jsonReturn = "{'removeDupTag' : 'complete', 'id': '" + f"{scene}" + "'}" - stash.Log(f"Sending json value {jsonReturn}") - sys.stdout.write(jsonReturn) - -def addExcludeTag(): - if 'Target' not in stash.JSON_INPUT['args']: - stash.Error(f"Could not find Target in JSON_INPUT ({stash.JSON_INPUT['args']})") - return - scene = stash.JSON_INPUT['args']['Target'] - stash.Log(f"Processing scene ID# {scene}") - stash.addTag(scene, excludeDupFileDeleteTag) - stash.Log(f"Done adding exclude tag to scene {scene}.") - sys.stdout.write("{" + f"addExcludeTag : 'complete', id: '{scene}'" + "}") - -def removeExcludeTag(): - if 'Target' not in stash.JSON_INPUT['args']: - stash.Error(f"Could not find Target in JSON_INPUT ({stash.JSON_INPUT['args']})") - return - scene = stash.JSON_INPUT['args']['Target'] - stash.Log(f"Processing scene ID# {scene}") - stash.removeTag(scene, excludeDupFileDeleteTag) - stash.Log(f"Done removing exclude tag from scene {scene}.") - sys.stdout.write("{" + f"removeExcludeTag : 'complete', id: '{scene}'" + "}") - -def getParseData(getSceneDetails1=True, getSceneDetails2=True): - if 'Target' not in stash.JSON_INPUT['args']: - stash.Error(f"Could not find Target in JSON_INPUT ({stash.JSON_INPUT['args']})") - return None, None - targetsSrc = stash.JSON_INPUT['args']['Target'] - targets = targetsSrc.split(":") - if len(targets) < 2: - stash.Error(f"Could not get both targets from string {targetsSrc}") - return None, None - stash.Log(f"Parsed targets {targets[0]} and {targets[1]}") - target1 = targets[0] - target2 = targets[1] - if getSceneDetails1: - target1 = stash.find_scene(int(target1)) - if getSceneDetails2: - target2 = stash.find_scene(int(target2)) - elif len(targets) > 2: - target2 = target2 + targets[2] - return target1, target2 - - -def mergeTags(): - scene1, scene2 = getParseData() - if scene1 == None or scene2 == None: - sys.stdout.write("{" + f"mergeTags : 'failed', id1: '{scene1}', id2: '{scene2}'" + "}") - return - stash.mergeMetadata(scene1, scene2) - stash.Log(f"Done merging scenes for scene {scene1['id']} and scene {scene2['id']}") - sys.stdout.write("{" + f"mergeTags : 'complete', id1: '{scene1['id']}', id2: '{scene2['id']}'" + "}") - -def getLocalDupReportPath(): - htmlReportExist = "true" if os.path.isfile(htmlReportName) else "false" - localPath = htmlReportName.replace("\\", "\\\\") - jsonReturn = "{'LocalDupReportExist' : " + f"{htmlReportExist}" + ", 'Path': '" + f"{localPath}" + "'}" - stash.Log(f"Sending json value {jsonReturn}") - sys.stdout.write(jsonReturn) - -def deleteLocalDupReportHtmlFiles(doJsonOutput = True): - htmlReportExist = "true" if os.path.isfile(htmlReportName) else "false" - if os.path.isfile(htmlReportName): - stash.Log(f"Deleting file {htmlReportName}") - os.remove(htmlReportName) - for x in range(2, 9999): - fileName = htmlReportName.replace(".html", f"_{x-1}.html") - stash.Debug(f"Checking if file '{fileName}' exist.") - if not os.path.isfile(fileName): - break - stash.Log(f"Deleting file {fileName}") - os.remove(fileName) - else: - stash.Log(f"Report file does not exist: {htmlReportName}") - if doJsonOutput: - jsonReturn = "{'LocalDupReportExist' : " + f"{htmlReportExist}" + ", 'Path': '" + f"{htmlReportName}" + "', 'qty': '" + f"{x}" + "'}" - stash.Log(f"Sending json value {jsonReturn}") - sys.stdout.write(jsonReturn) - -def removeTagFromAllScenes(tagName, deleteTags): - # ToDo: Replace code with SQL code if DB version 68 - tagId = stash.find_tags(q=tagName) - if len(tagId) > 0 and 'id' in tagId[0]: - if deleteTags: - stash.Debug(f"Deleting tag name {tagName} with Tag ID {tagId[0]['id']} from stash.") - stash.destroy_tag(int(tagId[0]['id'])) - else: - stash.Debug(f"Removing tag name {tagName} with Tag ID {tagId[0]['id']} from all scenes.") - manageTagggedDuplicates(clearTag=True, tagId=int(tagId[0]['id'])) - return True - return False - -def removeAllDupTagsFromAllScenes(deleteTags=False): - tagsToClear = [duplicateMarkForDeletion, base1_duplicateMarkForDeletion, base2_duplicateMarkForDeletion, graylistMarkForDeletion, longerDurationLowerResolution, duplicateWhitelistTag] - for x in range(0, 3): - tagsToClear += [base1_duplicateMarkForDeletion + f"_{x}"] - for x in range(0, 3): - tagsToClear += [base2_duplicateMarkForDeletion + f"_{x}"] - tagsToClear = list(set(tagsToClear)) # Remove duplicates - validTags = [] - for tagToClear in tagsToClear: - if removeTagFromAllScenes(tagToClear, deleteTags): - validTags +=[tagToClear] - if doJsonReturn: - jsonReturn = "{'removeAllDupTagFromAllScenes' : " + f"{duplicateMarkForDeletion}" + ", 'OtherTags': '" + f"{validTags}" + "'}" - stash.Log(f"Sending json value {jsonReturn}") - sys.stdout.write(jsonReturn) - else: - stash.Log(f"Clear tags {tagsToClear}") - -def insertDisplayNone(htmlReportName, scene): - stash.Log(f"Inserting display none for scene {scene} in file {htmlReportName}") - import in_place - doStyleEndTagCheck = True - with in_place.InPlace(htmlReportName) as file: - for line in file: - if doStyleEndTagCheck and line.startsWith(""): - file.write(f".ID_{scene}" + "{display:none;}") - doStyleEndTagCheck = False - file.write(line) - file.close() - -def hideScene(scene): - if os.path.isfile(htmlReportName): - insertDisplayNone(htmlReportName, scene) - for x in range(2, 9999): - fileName = htmlReportName.replace(".html", f"_{x-1}.html") - stash.Debug(f"Checking if file '{fileName}' exist.") - if not os.path.isfile(fileName): - break - insertDisplayNone(fileName, scene) - else: - stash.Log(f"Report file does not exist: {htmlReportName}") - -def deleteScene(hideInReport=True, deleteFile=True): - if 'Target' not in stash.JSON_INPUT['args']: - stash.Error(f"Could not find Target in JSON_INPUT ({stash.JSON_INPUT['args']})") - return - scene = stash.JSON_INPUT['args']['Target'] - stash.Log(f"Processing scene ID# {scene}") - result = stash.destroyScene(scene, delete_file=deleteFile) - if hideInReport: - hideScene(scene) - stash.Log(f"{stash.PLUGIN_TASK_NAME} complete for scene {scene} with results = {result}") - sys.stdout.write("{" + f"{stash.PLUGIN_TASK_NAME} : 'complete', id: '{scene}', result: '{result}'" + "}") - -def copyScene(moveScene=False): - scene1, scene2 = getParseData() - if scene1 == None or scene2 == None: - sys.stdout.write("{" + f"{stash.PLUGIN_TASK_NAME} : 'failed', id1: '{scene1}', id2: '{scene2}'" + "}") - return - if moveScene: - stash.mergeMetadata(scene1, scene2) - result = shutil.copy(scene1['file']['path'], scene2['file']['path']) - if moveScene: - result = stash.destroyScene(scene1['id'], delete_file=True) - stash.Log(f"destroyScene for scene {scene1['id']} results = {result}") - stash.Log(f"{stash.PLUGIN_TASK_NAME} complete for scene {scene1['id']} and {scene2['id]}") - sys.stdout.write("{" + f"{stash.PLUGIN_TASK_NAME} : 'complete', id1: '{scene1['id']}', id2: '{scene2['id']}', result: '{result}'" + "}") - -def renameFile(): - scene, newName = getParseData(getSceneDetails2=False) - if scene == None or newName == None: - sys.stdout.write("{" + f"{stash.PLUGIN_TASK_NAME} : 'failed', scene: '{scene}', newName: '{newName}'" + "}") - return - newName = scene['file']['path'].replace(pathlib.Path(scene['file']['path']).stem, newName) - result = shutil.rename(scene['file']['path'], newName) - stash.Log(f"{stash.PLUGIN_TASK_NAME} complete for scene {scene['id']} ;renamed to {newName}; result={resul}") - sys.stdout.write("{" + f"{stash.PLUGIN_TASK_NAME} : 'complete', scene: '{scene['id']}', newName: '{newName}', result: '{result}'" + "}") - - -# ToDo: Add to UI menu -# Remove all Dup tagged files (Just remove from stash, and leave file) -# Clear GraylistMarkForDel tag -# Delete GraylistMarkForDel tag -# Remove from stash all files no longer part of stash library -# Remove from stash all files in the Exclusion list (Not supporting regexps) -# ToDo: Add to advance menu -# Remove only graylist dup -# Exclude graylist from delete -# Include graylist in delete - -try: - if stash.PLUGIN_TASK_NAME == "tag_duplicates_task": - mangeDupFiles(tagDuplicates=True, merge=mergeDupFilename) - stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT") - elif stash.PLUGIN_TASK_NAME == "delete_tagged_duplicates_task": - manageTagggedDuplicates(deleteScenes=True) - stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT") - elif stash.PLUGIN_TASK_NAME == "delete_duplicates_task": - mangeDupFiles(deleteDup=True, merge=mergeDupFilename) - stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT") - elif stash.PLUGIN_TASK_NAME == "clear_duplicate_tags_task": - removeAllDupTagsFromAllScenes() - stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT") - elif stash.PLUGIN_TASK_NAME == "graylist_tag_task": - manageTagggedDuplicates(setGrayListTag=True) - stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT") - elif stash.PLUGIN_TASK_NAME == "generate_phash_task": - stash.metadata_generate({"phashes": True}) - stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT") - elif stash.PLUGIN_TASK_NAME == "deleteScene": - deleteScene() - stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT") - elif stash.PLUGIN_TASK_NAME == "removeScene": - deleteScene(deleteFile=False) - stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT") - elif stash.PLUGIN_TASK_NAME == "renameFile": - renameFile() - stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT") - elif stash.PLUGIN_TASK_NAME == "copyScene": - copyScene() - stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT") - elif stash.PLUGIN_TASK_NAME == "moveScene": - copyScene(moveScene=True) - stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT") - elif stash.PLUGIN_TASK_NAME == "removeDupTag": - removeDupTag() - stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT") - elif stash.PLUGIN_TASK_NAME == "addExcludeTag": - addExcludeTag() - stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT") - elif stash.PLUGIN_TASK_NAME == "removeExcludeTag": - removeExcludeTag() - stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT") - elif stash.PLUGIN_TASK_NAME == "mergeTags": - mergeTags() - stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT") - elif stash.PLUGIN_TASK_NAME == "getLocalDupReportPath": - getLocalDupReportPath() - stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT") - elif stash.PLUGIN_TASK_NAME == "deleteLocalDupReportHtmlFiles": - deleteLocalDupReportHtmlFiles() - stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT") - elif stash.PLUGIN_TASK_NAME == "createDuplicateReportWithoutTagging": - mangeDupFiles(tagDuplicates=False, merge=mergeDupFilename) - stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT") - elif stash.PLUGIN_TASK_NAME == "deleteAllDupFileManagerTags": - removeAllDupTagsFromAllScenes(deleteTags=True) - stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT") - elif stash.PLUGIN_TASK_NAME == "deleteBlackListTaggedDuplicatesTask": - mangeDupFiles(deleteDup=True, merge=mergeDupFilename, deleteBlacklistOnly=True) - stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT") - elif stash.PLUGIN_TASK_NAME == "deleteTaggedDuplicatesLwrResOrLwrDuration": - mangeDupFiles(deleteDup=True, merge=mergeDupFilename, deleteLowerResAndDuration=True) - stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT") - elif stash.PLUGIN_TASK_NAME == "deleteBlackListTaggedDuplicatesLwrResOrLwrDuration": - mangeDupFiles(deleteDup=True, merge=mergeDupFilename, deleteBlacklistOnly=True, deleteLowerResAndDuration=True) - stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT") - elif parse_args.dup_tag: - stash.PLUGIN_TASK_NAME = "dup_tag" - mangeDupFiles(tagDuplicates=True, merge=mergeDupFilename) - stash.Debug(f"Tag duplicate EXIT") - elif parse_args.del_tag: - stash.PLUGIN_TASK_NAME = "del_tag" - manageTagggedDuplicates(deleteScenes=True) - stash.Debug(f"Delete Tagged duplicates EXIT") - elif parse_args.clear_tag: - stash.PLUGIN_TASK_NAME = "clear_tag" - removeAllDupTagsFromAllScenes() - stash.Debug(f"Clear duplicate tags EXIT") - elif parse_args.remove: - stash.PLUGIN_TASK_NAME = "remove" - mangeDupFiles(deleteDup=True, merge=mergeDupFilename) - stash.Debug(f"Delete duplicate EXIT") - elif len(sys.argv) < 2 and stash.PLUGIN_TASK_NAME in advanceMenuOptions: - manageTagggedDuplicates(deleteScenes=True, advanceMenuOptionSelected=True) - stash.Debug(f"{stash.PLUGIN_TASK_NAME} EXIT") - else: - stash.Log(f"Nothing to do!!! (PLUGIN_ARGS_MODE={stash.PLUGIN_TASK_NAME})") -except Exception as e: - tb = traceback.format_exc() - - stash.Error(f"Exception while running DupFileManager Task({stash.PLUGIN_TASK_NAME}); Error: {e}\nTraceBack={tb}") - killScanningJobs() - stash.convertToAscii = False - stash.Error(f"Error: {e}\nTraceBack={tb}") - if doJsonReturn: - sys.stdout.write("{" + f"Exception : '{e}; See log file for TraceBack' " + "}") - -stash.Log("\n*********************************\nEXITING ***********************\n*********************************")