Skip to content

Commit

Permalink
Add now playing view and index cache
Browse files Browse the repository at this point in the history 
Adds support for the --nowplaying option which caches the last MSU pack that was generated in live reshuffle mode, queries the currently playing MSU track over qusb2snes every second, and prints out which MSU pack the currently playing track is from if it's changed.

Also greatly speeds up runtime by adding a cache for the track index, and adds the --reindex argument to allow manually rebuilding this index after adding new MSU packs.
  • Loading branch information
@krelbel
krelbel committed May 17, 2021
1 parent cdea9b7 commit bf12bb6
Show file tree
Hide file tree
Showing 2 changed files with 197 additions and 30 deletions.
214 changes: 185 additions & 29 deletions Main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,13 @@
import pprint
import sched, time
import datetime
import websockets
import json
import asyncio
import pickle
from tempfile import TemporaryDirectory

__version__ = '0.7.3'
__version__ = '0.8'

# Creates a shuffled MSU-1 pack for ALttP Randomizer from one or more source
# MSU-1 packs.
Expand Down Expand Up @@ -46,7 +50,7 @@
# directory and generate a new one. Track names picked will be saved in
# "shuffled-msushuffleroutput.log" (cleared on reruns)
#
# 3) LIVE RESHUFFLE METHOD (EXPERIMENTAL): Instead of simply running
# 3) LIVE RESHUFFLE METHOD: Instead of simply running
# **Main.py**, run **LiveReshuffle.py** or run in the command line as
# "python Main.py --live 10" (or any other positive integer) to
# generate a new shuffled MSU pack every few seconds. Will skip
Expand All @@ -56,6 +60,12 @@
# Edit **LiveReshuffle.py** to set a different reshuffle interval than
# the 10 second default.
#
# 4) LIVE RESHUFFLE + NOW PLAYING VIEW (EXPERIMENTAL): Run the command
# line as "python Main.py --live 10 --nowplaying" to run in live
# reshuffle mode (as described above) while polling qusb2snes for
# the currently playing MSU pack, printed to console and nowplaying.txt
# for use as an OBS streaming text source.
#
# 4) Load the ROM in an MSU-compatible emulator (works well with Snes9x 1.60)
#
# Additional options/usage notes:
Expand Down Expand Up @@ -97,6 +107,11 @@
# file name; useful for keeping tracks hidden from the shuffler without
# needing to move them out of the collection entirely.
#
# - Caches the track list in ./trackindex.pkl to avoid reindexing the entire
# collection every time the script is run. If run in the command line as
# "python Main.py --reindex", it will regenerate the track index. Use this
# to pick up any new MSU packs for the shuffler.
#
# Debugging options (not necessary for normal use):
#
# - This script uses hardlinks instead of copies by default to reduce disk
Expand Down Expand Up @@ -346,9 +361,11 @@ def copy_track(logger, srcpath, dst, rompath, dry_run, higan, forcerealcopy, liv
os.link(srcpath, tmpname)

os.replace(tmpname, dstpath)
return True
except PermissionError:
if not live:
logger.info(f"Failed to copy {srcpath} to {dstpath} during non-live update")
return False

# Build a dictionary mapping each possible track number to all matching tracks
# in the search directory; do this once, to avoid excess searching later.
Expand All @@ -360,11 +377,22 @@ def copy_track(logger, srcpath, dst, rompath, dry_run, higan, forcerealcopy, liv
# Index format:
# index[2] = ['../msu1/track-2.pcm', '../msu2/track-2.pcm']
def build_index(args):
global trackindex

if os.path.exists('trackindex.pkl') and not args.reindex:
with open('trackindex.pkl', 'rb') as f:
try:
trackindex = pickle.load(f)
except Exception as e:
print("Failed to load track index")

if trackindex:
print("Reusing track index, run with --reindex to pick up any new packs.")
return

print("Building index, this should take a few seconds.")
buildstarttime = datetime.datetime.now()

global trackindex

if (args.singleshuffle):
searchdir = args.singleshuffle
else:
Expand Down Expand Up @@ -409,7 +437,12 @@ def build_index(args):
buildtime = datetime.datetime.now() - buildstarttime
print(f"Index build took {buildtime.seconds}.{buildtime.microseconds} seconds")

def shuffle_all_tracks(rompath, fullshuffle, singleshuffle, dry_run, higan, forcerealcopy, live):
with open('trackindex.pkl', 'wb') as f:
# Saving track index as plaintext instead of HIGHEST_PROTOCOL since
# this is only loaded once, and plaintext may be useful for debugging.
pickle.dump(trackindex, f, 0)

def shuffle_all_tracks(rompath, fullshuffle, singleshuffle, dry_run, higan, forcerealcopy, live, nowplaying, cooldown, prevtrack):
logger = logging.getLogger('')
#For all found non-looping tracks, pick a random track with a matching
#track number from a random pack in the target directory.
Expand All @@ -418,30 +451,150 @@ def shuffle_all_tracks(rompath, fullshuffle, singleshuffle, dry_run, higan, forc
if not live:
logger.info("Non-looping tracks:")

with TemporaryDirectory(dir='.') as tmpdir:
for i in nonloopingfoundtracks:
winner = random.choice(trackindex[i])
copy_track(logger, winner, i, rompath, dry_run, higan, forcerealcopy, live, tmpdir)
if cooldown == 0:
cooldown = int(live)
with TemporaryDirectory(dir='.') as tmpdir:
oldwinnerlist = list()
if os.path.exists('winnerlist.pkl'):
with open('winnerlist.pkl', 'rb') as f:
try:
oldwinnerlist = pickle.load(f)
except Exception as e:
print("Failed to load tracklist")
winnerlist = list()
for i in nonloopingfoundtracks:
winner = random.choice(trackindex[i])
winnerlist.insert(i, winner)
copy_track(logger, winner, i, rompath, dry_run, higan, forcerealcopy, live, tmpdir)

#For all found looping tracks, pick a random track from a random pack
#in the target directory, with a matching track number by default, or
#a shuffled different looping track number if fullshuffle or
#singleshuffle are enabled.
if not live:
logger.info("Looping tracks:")
for i in loopingfoundtracks:
if (args.fullshuffle or args.singleshuffle):
dst = i
src = shuffledloopingfoundtracks[loopingfoundtracks.index(i)]
else:
dst = i
src = i
winner = random.choice(trackindex[src])
copied = copy_track(logger, winner, dst, rompath, dry_run, higan, forcerealcopy, live, tmpdir)
# if copy failed, use OLD winner...
if copied:
winnerlist.insert(i, winner)
else:
winnerlist.insert(i, oldwinnerlist[i])

with open('winnerlist.pkl', 'wb') as f:
pickle.dump(winnerlist, f, pickle.HIGHEST_PROTOCOL)
if live and not nowplaying:
shuffletime = datetime.datetime.now() - shufflestarttime
print("Reshuffling MSU pack every%s second%s, press ctrl+c or close the window to stop reshuffling. (shuffled in %d.%ds)" %(" " + str(int(live)) if int(live) != 1 else "", "s" if int(live) != 1 else "", shuffletime.seconds, shuffletime.microseconds))

#For all found looping tracks, pick a random track from a random pack
#in the target directory, with a matching track number by default, or
#a shuffled different looping track number if fullshuffle or
#singleshuffle are enabled.
if not live:
logger.info("Looping tracks:")
for i in loopingfoundtracks:
if (args.fullshuffle or args.singleshuffle):
dst = i
src = shuffledloopingfoundtracks[loopingfoundtracks.index(i)]
else:
dst = i
src = i
winner = random.choice(trackindex[src])
copy_track(logger, winner, dst, rompath, dry_run, higan, forcerealcopy, live, tmpdir)
if live:
shuffletime = datetime.datetime.now() - shufflestarttime
print("Reshuffling MSU pack every%s second%s, press ctrl+c or close the window to stop reshuffling. (shuffled in %d.%ds)" %(" " + str(int(live)) if int(live) != 1 else "", "s" if int(live) != 1 else "", shuffletime.seconds, shuffletime.microseconds))
s.enter(int(live), 1, shuffle_all_tracks, argument=(rompath, fullshuffle, singleshuffle, dry_run, higan, forcerealcopy, live))
if nowplaying:
newtrack = read_track(prevtrack)
prevtrack = newtrack
s.enter(1, 1, shuffle_all_tracks, argument=(rompath, fullshuffle, singleshuffle, dry_run, higan, forcerealcopy, live, nowplaying, cooldown - 1, prevtrack))

async def recv_loop(ws, recv_queue):
try:
async for msg in ws:
recv_queue.put_nowait(msg)
finally:
await ws.close()

# Print the track that's currently playing, and print its pack to
# nowplaying.txt which can be used as a streaming text file source.
def print_pack(path):
print("Now playing: " + path)
path_parts = list()
while True:
parts = os.path.split(path)
if parts[0] == path:
path_parts.insert(0, parts[0])
break
elif parts[1] == path:
path_parts.insert(0, parts[1])
break
else:
path = parts[0]
path_parts.insert(0, parts[1])
with open('nowplaying.txt', 'w') as f:
f.truncate(0)
print("MSU pack now playing:", file=f)
print(path_parts[1], file=f)

async def query(prevtrack):
addr = "ws://localhost:8080"
ws = await websockets.connect(addr, ping_timeout=None, ping_interval=None)
devlist = {
"Opcode": "DeviceList",
"Space": "SNES"
}
await ws.send(json.dumps(devlist))
reply = json.loads(await ws.recv())
devices = reply['Results'] if 'Results' in reply and len(reply['Results']) > 0 else None
if not devices:
print("Failed to connect to qusb2snes")
device = devices[0]
attachreq = {
"Opcode": "Attach",
"Space": "SNES",
"Operands": [device]
}
await ws.send(json.dumps(attachreq))

recv_queue = asyncio.Queue()
recv_task = asyncio.create_task(recv_loop(ws, recv_queue))
WRAM_START = 0xF50000

# Current MSU is $010B, per https://github.com/KatDevsGames/z3randomizer/blob/master/msu.asm#L126
REG_CURRENT_MSU_TRACK = 0x010B

address = WRAM_START + REG_CURRENT_MSU_TRACK
size = 1
readreq = {
"Opcode": "GetAddress",
"Space": "SNES",
"Operands": [hex(address)[2:], hex(size)[2:]]
}
await ws.send(json.dumps(readreq))
data = bytes()
while len(data) < 1:
try:
data += await asyncio.wait_for(recv_queue.get(), 1)
except asyncio.TimeoutError:
break

track = 0
if len(data) != 1:
print("Failed to query REG_CURRENT_MSU_TRACK")
else:
track = int(data[0])

if track != 0 and track != prevtrack:
if os.path.exists('winnerlist.pkl'):
winnerlist = list()
with open('winnerlist.pkl', 'rb') as f:
try:
winnerlist = pickle.load(f)
print_pack(str(winnerlist[track]))
except Exception as e:
print("Failed to load tracklist")

await ws.close()
return track

# Read the currently playing track over qusb2snes.
# TODO: This currently opens up a new qusb2snes connection every second.
# Eventually this should be smarter by keeping one connection alive instead.
def read_track(prevtrack):
track = asyncio.get_event_loop().run_until_complete(query(prevtrack))
return track

def generate_shuffled_msu(args, rompath):
logger = logging.getLogger('')
Expand Down Expand Up @@ -471,10 +624,10 @@ def generate_shuffled_msu(args, rompath):
nonloopingfoundtracks = [i for i in foundtracks if i in nonloopingtracks]

if args.live:
s.enter(1, 1, shuffle_all_tracks, argument=(rompath, args.fullshuffle, args.singleshuffle, args.dry_run, args.higan, args.forcerealcopy, args.live))
s.enter(1, 1, shuffle_all_tracks, argument=(rompath, args.fullshuffle, args.singleshuffle, args.dry_run, args.higan, args.forcerealcopy, args.live, args.nowplaying, int(args.live), 0))
s.run()
else:
shuffle_all_tracks(rompath, args.fullshuffle, args.singleshuffle, args.dry_run, args.higan, args.forcerealcopy, args.live)
shuffle_all_tracks(rompath, args.fullshuffle, args.singleshuffle, args.dry_run, args.higan, args.forcerealcopy, args.live, args.nowplaying, 0, 0)
logger.info('Done.')

def main(args):
Expand All @@ -491,6 +644,7 @@ def main(args):
# determine if the supplied rom is ON the same drive as the script. If not, realcopy is mandatory.
os.path.commonpath([os.path.abspath(rom), os.path.abspath(__file__)])
except:
print(f"Failed to find common path between {os.path.abspath(rom)} and {os.path.abspath(__file__)}, forcing real copies.")
args.forcerealcopy = True

if args.live and args.forcerealcopy:
Expand All @@ -509,6 +663,8 @@ def main(args):
parser.add_argument('--realcopy', help='Creates real copies of the source tracks instead of hardlinks', action='store_true', default=False)
parser.add_argument('--dry-run', help='Makes script print all filesystem commands that would be executed instead of actually executing them.', action='store_true', default=False)
parser.add_argument('--live', help='The interval at which to re-shuffle the entire pack, in seconds; will skip tracks currently in use.')
parser.add_argument('--nowplaying', help='EXPERIMENTAL: During live reshuffling, connect to qusb2snes to print the currently playing MSU pack to console and nowplaying.txt', action='store_true', default=False)
parser.add_argument('--reindex', help='Rebuild the index of MSU packs, this must be run to pick up any new packs or moved/deleted files in existing packs!', action='store_true', default=False)
parser.add_argument('--version', help='Print version number and exit.', action='store_true', default=False)

romlist = list()
Expand Down
13 changes: 12 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ Usage:
directory and generate a new one. Track names picked will be saved in
"shuffled-msushuffleroutput.log" (cleared on reruns)

3) LIVE RESHUFFLE METHOD (EXPERIMENTAL): Instead of simply running
3) LIVE RESHUFFLE METHOD: Instead of simply running
**Main.py**, run **LiveReshuffle.py** or run in the command line as
"python Main.py --live 10" (or any other positive integer) to
generate a new shuffled MSU pack every few seconds. Will skip
Expand All @@ -42,6 +42,12 @@ Usage:
Edit **LiveReshuffle.py** to set a different reshuffle interval than
the 10 second default.

4) LIVE RESHUFFLE + NOW PLAYING VIEW (EXPERIMENTAL): Run the command
line as "python Main.py --live 10 --nowplaying" to run in live
reshuffle mode (as described above) while polling qusb2snes for
the currently playing MSU pack, printed to console and nowplaying.txt
for use as an OBS streaming text source.

4) Load the ROM in an MSU-compatible emulator (works well with Snes9x 1.60)

Additional options/usage notes:
Expand Down Expand Up @@ -83,6 +89,11 @@ Additional options/usage notes:
file name; useful for keeping tracks hidden from the shuffler without
needing to move them out of the collection entirely.

- Caches the track list in ./trackindex.pkl to avoid reindexing the entire
collection every time the script is run. If run in the command line as
"python Main.py --reindex", it will regenerate the track index. Use this
to pick up any new MSU packs for the shuffler.

Debugging options (not necessary for normal use):

- This script uses hardlinks instead of copies by default to reduce disk
Expand Down

0 comments on commit bf12bb6

Please sign in to comment.