From 1bc5077779286ed51af9d01d7315863d6326298a Mon Sep 17 00:00:00 2001 From: ilonachan Date: Thu, 3 Oct 2024 21:01:04 +0200 Subject: [PATCH 01/17] Upgrade to GDA syntax, command documentation --- data/commands.json | 253 +++++++++++++++++++++++++++++++++++++++++++-- formats/gds.py | 150 +++++++++++++++++++-------- 2 files changed, 351 insertions(+), 52 deletions(-) diff --git a/data/commands.json b/data/commands.json index c88bdc1..857e390 100644 --- a/data/commands.json +++ b/data/commands.json @@ -1,10 +1,245 @@ { - "engine": 27, - "img_win": 31, - "puzzle_title": 186, - "puzzle_picarats": 195, - "circle_answer": 212, - "puzzle_info": 220, - "input_answer": 251, - "input_bg": 252 -} \ No newline at end of file + "engine": { + "id": 27, + "desc": "Called at the start of a puzzle script. Determines which puzzle engine to use.", + "params": [ + { "type": "string", "desc": "Engine name (from a predefined list)" } + ] + }, + "img_win": { + "id": 31, + "desc": "Seemingly denotes when a special top-screen image should be shown on the puzzle completion screen.", + "params": [{ "type": "int" }, { "type": "int" }] + }, + "puzzle_title": { + "id": 186, + "desc": "Defines the title of a puzzle. Called centrally as a list in `puzzletitle/qtitle.gds`.", + "params": [ + { "type": "int", "desc": "Puzzle ID" }, + { "type": "string", "desc": "Puzzle Title" } + ] + }, + "puzzle_picarats": { + "id": 195, + "desc": "Defines how much picarat a puzzle awards. Called centrally in `puzzletitle/pscript.gds`.", + "params": [ + { "type": "int", "desc": "Puzzle ID" }, + { "type": "int", "desc": "Value on first attempt" }, + { "type": "int", "desc": "Value on second attempt" }, + { "type": "int", "desc": "Value on all further attempts" } + ] + }, + "puzzle_info": { + "id": 220, + "desc": "Defines additional information about the puzzle displayed in the puzzle index. Some puzzle IDs ", + "params": [ + { "type": "int", "desc": "Puzzle ID" }, + { + "type": "string", + "desc": "Puzzle type (e.g. \"Write Answer\" or \"Draw Line\", but can be anything)" + }, + { + "type": "string", + "desc": "Puzzle original location (should be a recognizable map name, but can technically be anything, such as \"Layton's Challenges\")" + } + ] + }, + "circle_answer": { "id": 212 }, + "input_answer": { "id": 251 }, + "input_bg": { "id": 252 }, + "dog_unknown": { + "id": 194, + "desc": "Used in the Gizmo minigame, presumably to manage the placement of the dog parts", + "params": [ + { "type": "int", "desc": "Between 1-20, probably the part index?" }, + { "type": "int" }, + { "type": "int" }, + { "type": "int", "desc": "Always the same as param 1" }, + { "type": "int" }, + { "type": "int" } + ] + }, + "inn_hint_layton": { + "id": 202, + "desc": "Used in the Inn minigame to display the Professor's assessment of the room layout.", + "params": [ + { + "type": "string", + "desc": "The text to be displayed" + }, + { + "type": "int", + "desc": "Increments with the quality of the room (unsure about effect)" + }, + { + "type": "int", + "desc": "Always 0" + } + ] + }, + "inn_hint_luke": { + "id": 203, + "desc": "Used in the Inn minigame to display Luke's assessment of the room layout.", + "params": [ + { + "type": "string", + "desc": "The text to be displayed" + }, + { + "type": "int", + "desc": "Increments with the quality of the room (unsure about effect)" + }, + { + "type": "int", + "desc": "Always 0" + } + ] + }, + "inn_item": { + "id": 207, + "desc": "Used in the hotelscript to define the available Inn items. All used at the end for some reason.", + "params": [ + { "type": "int", "desc": "item ID" }, + { "type": "string", "desc": "Name of the item" } + ] + }, + "inn_item_layton": { + "id": 188, + "desc": "Used in the hotelscript, always followed by `inn_item_layton_comment` and `inn_item_layton_comment2`, likely to define Inn item placements in Layton's room.", + "params": [ + { "type": "int", "desc": "item ID" }, + { "type": "int" }, + { "type": "int" }, + { "type": "int" }, + { "type": "int" }, + { "type": "int" }, + { "type": "int" }, + { "type": "int" }, + { "type": "int" } + ] + }, + "inn_item_layton_comment": { + "id": 198, + "desc": "Used in the hotelscript after `inn_item_layton`. Defines the Professor's comment about an Inn minigame item.", + "params": [{ "type": "string", "desc": "The Professor's comment" }] + }, + "inn_item_layton_comment2": { + "id": 199, + "desc": "Used in the hotelscript after `inn_item_layton`. Defines the comment made by the professor if a second item of the same type is being added, along with the ID of the item that is the 'same type'.", + "params": [ + { "type": "string", "desc": "The Professor's comment" }, + { + "type": "int", + "desc": "The ID of the item that's considered a duplicate of this one's type, or -1 if there is no such item. (not sure which of the two comments is used in that case)" + } + ] + }, + "inn_item_luke": { + "id": 189, + "desc": "Used in the hotelscript, always followed by `inn_item_luke_comment` and `inn_item_luke_comment2`, likely to define Inn item placements in Luke's room." + }, + "inn_item_luke_comment": { + "id": 200, + "desc": "Used in the hotelscript with `inn_item_luke`. Defines Luke's comment about an Inn minigame item.", + "params": [{ "type": "string", "desc": "Luke's comment" }] + }, + "inn_item_luke_comment2": { + "id": 201, + "desc": "Used in the hotelscript with `inn_item_luke`. Defines the comment made by Luke if a second item of the same type is being added, along with the ID of the item that is the 'same type'.", + "params": [ + { "type": "string", "desc": "Luke's comment" }, + { + "type": "int", + "desc": "The ID of the item that's considered a duplicate of this one's type, or -1 if there is no such item. (not sure which of the two comments is used in that case)" + } + ] + }, + "nazobaba": { + "id": 229, + "desc": "Used in the Granny Riddleton script. One of these commands is used for every puzzle, possibly to define after which story progression point a puzzle gets sent to the cottage. All but one of the three unknown params are always -1, and sometimes all are (presumably for puzzles that never get sent to the cottage). Which value is present may determine the bottle color, and the value may determine after which progression point the puzzle shows up.", + "params": [ + { "type": "int", "desc": "Puzzle ID" }, + { "type": "int" }, + { "type": "int" }, + { "type": "int" } + ] + }, + + "slide2_grid_dimensions": { + "id": 164, + "desc": "Defines the grid dimensions in a 'Slide Puzzle 2'.", + "params": [ + { "type": "int", "desc": "grid width" }, + { "type": "int", "desc": "grid height" } + ] + }, + "slide2_grid_offset": { + "id": 163, + "desc": "Determines the offset of the grid's visual placement relative to the touch screen's top left corner (0,0). Used to align the puzzle itself with the background.", + "params": [ + { "type": "int", "desc": "x offset in px" }, + { "type": "int", "desc": "y offset in px" } + ] + }, + "slide2_grid_cellsize": { + "id": 165, + "desc": "Defines the visual scale of the grid by the width and height of the cells. This puzzle engine only allows square cells.", + "params": [ + { + "type": "int", + "desc": "width & height of the square grid cells in px" + } + ] + }, + "slide2_item": { + "id": 166, + "desc": "Place an item in the grid of a 'Slide Puzzle 2'", + "params": [ + { "type": "int", "desc": "x coordinate" }, + { "type": "int", "desc": "y coordinate" }, + { + "type": "int", + "desc": "unknown, but likely to do with properties such as draggability & obstacle behavior" + }, + { + "type": "string", + "desc": "Item sprite name, from `ani/slidepuzzle2.arc`. 'x' in particular has no image associated, and will show the background through (used for walls)." + } + ] + }, + "slide2_goalid": { + "id": 177, + "desc": "Defines a grid cell as the goal for a specific item type index. These indices seem to be attached to objects adhoc, but an object with each goal's specified index must be placed on all goal cells defined like this in order for the puzzle to solve.", + "params": [ + { "type": "int", "desc": "x coordinate" }, + { "type": "int", "desc": "y coordinate" }, + { "type": "int", "desc": "Item type index" } + ] + }, + "slide2_item_goalid": { + "id": 178, + "desc": "Following a `slide2_item` definition, attaches the given item type index to that object. Used to determine if the puzzle has been solved.", + "params": [ + { + "type": "int", + "desc": "Item type index (can seemingly be chosen adhoc to be anything)" + } + ] + }, + "slide2_item_goalcell": { + "id": 56, + "desc": "Following a `slide2_item` definition, sets a target location at which the cell needs to be placed for the puzzle to be solved." + }, + + "placetarget": { + "id": 94, + "desc": "Main definition of a 'Place Target' puzzle", + "params": [ + {"type": "int", "desc": "Target x position"}, + {"type": "int", "desc": "Target y position"}, + {"type": "string", "desc": "Sprite to be displayed at the touched location"}, + {"type": "unknown-2"}, + {"type": "int"} + ] + } +} diff --git a/formats/gds.py b/formats/gds.py index 9a6ac9a..cda15d6 100644 --- a/formats/gds.py +++ b/formats/gds.py @@ -3,6 +3,7 @@ import os import parse +import ast from utils import cli_file_pairs, foreach_file_pair from version import v @@ -12,30 +13,17 @@ def cli(): dir_path = "/".join(os.path.dirname(os.path.realpath(__file__).replace("\\", "/")).split("/")[:-1]) commands = json.load(open(f"{dir_path}/data/commands.json", encoding="utf-8")) -commands_i = {val: key for key, val in commands.items()} # Inverted version of commands - -class GDSModeException (Exception): - def __init__(self, mode): - self.mode = mode - super().__init__( - f"'{mode}' is not a valid mode for GDS.__init__(): must be one of 'bin', 'json', 'gda'") +commands_i = {val["id"]: key for key, val in commands.items() if "id" in val} # Inverted version of commands class GDS: - def __init__(self, file, mode="bin"): #modes: "bin"/"b", "json"/"j", "gda"/"a" - if mode == "bin" or mode == "b": - self.from_gds(file) - elif mode == "json" or mode == "j": - self.from_json(file) - elif mode == "gda" or mode == "a": - self.from_old(file) - else: - raise GDSModeException(mode) - - def from_gds(self, file): + def __init__(self, cmds = []): #modes: "bin"/"b", "json"/"j", "gda"/"a" + self.cmds = cmds + + @classmethod + def from_gds(Self, file): length = int.from_bytes(file[0:4], "little") if file[4:6] == b"\x0c\x00": - self.cmds = [] - return + return Self([]) cmd_data = file[6:length+4] cmds = [] @@ -91,30 +79,44 @@ def from_gds(self, file): else: raise Exception(f"GDS file error: Invalid or unsupported parameter type {hex(p_type)}!") - self.cmds = cmds + return Self(cmds) - def from_json (self, file): - self.cmds = json.loads(file)["data"] + @classmethod + def from_json (Self, file): + cmds = json.loads(file) #TODO: reject non-compatible json files + return Self(cmds) - def from_old (self, file): #TODO: make this, so gds_old can be completely removed + @classmethod + def from_gda (Self, file): #TODO: make this, so gds_old can be completely removed cmds = [] for line in file.split("\n"): - data = {} + line = line.strip() if line.startswith("#"): continue + if line == '': + continue + + data = {} line, strings = parse.remove_strings(line) line = line.rstrip().split(" ") cmd = line[0] - if cmd == "engine": - cmd = "0x1b" - elif cmd == "img_win": - cmd = "0x1f" + # TODO: handle inline comments, maybe + + if cmd in commands: + cmd = commands[cmd] + if "alias" in cmd: + cmd = commands[cmd["alias"]] + elif cmd.startswith("cmd_"): + cmd = int(cmd[4:]) + elif cmd.startswith("0x"): + cmd = int(cmd[2:], base=16) + else: + raise Exception(f"Unknown GDA command: {cmd}") - cmd = int(cmd[2:], base=16) params = [] for param in line[1:]: @@ -123,14 +125,24 @@ def from_old (self, file): #TODO: make this, so gds_old can be completely remove elif param.startswith("0x"): params.append({"type":"unknown-2", "data":int(param[2:], 16)}) elif param.startswith('"') and param.endswith('"'): - param = strings[int(param[1:-1])] + param = ast.literal_eval(f'"{strings[int(param[1:-1])]}"') params.append({"type":"string", "data":param}) + elif param.startswith("!6("): + params.append({"type":"unknown-6", "data":int(param[3:-1], 16)}) + elif param.startswith("!7("): + params.append({"type":"unknown-7", "data":int(param[3:-1], 16)}) + elif param.startswith("!8"): + params.append({"type":"unknown-8"}) + elif param.startswith("!9"): + params.append({"type":"unknown-9"}) + elif param.startswith("!b"): + params.append({"type":"unknown-b"}) else: raise Exception(f"Invalid GDA parameter: {param}") cmds.append({"command":cmd, "parameters":params}) - self.cmds = cmds + return Self(cmds) def __getitem__ (self, index): index = int(index) @@ -142,10 +154,10 @@ def to_json (self): def to_gds (self): out = b"\x00" * 2 for command in self.cmds: - if type(command["command"]) == int: - out += command["command"].to_bytes(2, "little") + if type(command["command"]["id"]) == int: + out += command["command"]["id"].to_bytes(2, "little") else: - out += commands[command["command"]].to_bytes(2, "little") + out += commands[command["command"]["id"]].to_bytes(2, "little") for param in command["parameters"]: if param["type"] == "int": out += b"\x01\x00" @@ -178,6 +190,38 @@ def to_gds (self): def to_bin (self): #alias return self.to_gds() + + def to_gda(self): + out = "" + for command in self.cmds: + if type(command["command"]) == int: + out += "0x"+command["command"].to_bytes(1, "little").hex() + else: + out += command['command'] + for param in command["parameters"]: + out += " " + if param["type"] == "int": + out += str(param["data"]) + elif param["type"] == "string": + out += repr(param["data"]) + elif param["type"] == "unknown-2": + out += hex(param["data"]) + elif param["type"] == "unknown-6": + b = param["data"].to_bytes(4, "little") + out += f"!6({b.hex()})" + elif param["type"] == "unknown-7": + b = param["data"].to_bytes(4, "little") + out += f"!7({b.hex()})" + elif param["type"] == "unknown-8": + out += "!8" + elif param["type"] == "unknown-9": + out += "!9" + elif param["type"] == "unknown-b": + out += "!b" + else: + raise Exception(f"GDA error: invalid or unsupported parameter type '{param['type']}'!") + out += "\n" + return out @cli.command( name="extract", @@ -215,8 +259,8 @@ def process(input, output): @cli.command( - name="create", - help="Converts a JSON to GDS.", + name="compilejson", + help="Converts a JSON document created by 'compilejson' to a GDS script file.", no_args_is_help = True ) @click.argument("input") @@ -225,21 +269,41 @@ def create_json(input, output): input = open(input, encoding="utf-8").read() output = open(output, "wb") - gds = GDS(input, "json") + gds = GDS.from_json(input) output.write(gds.to_bin()) output.close() @cli.command( - name="gdaimport", - help="Creates a GDS JSON file from the old GDA format.", + name="compile", + help="Generates a GDS binary script file from human-readable GDA files.", no_args_is_help = True ) @click.argument("input") @click.argument("output") def create_from_gda(input, output): input = open(input, encoding="utf-8").read() - output = open(output, "w", encoding="utf-8") + output = open(output, "wb") - gds = GDS(input, "gda") - output.write(gds.to_json()) + gds = GDS.from_gda(input) + output.write(gds.to_bin()) + output.close() + +@cli.command( + name="decompile", + help="Converts a GDS file into a human-readable GDA script format.", + no_args_is_help = True + ) +@click.argument("input") +@click.argument("output", required=False) +def create_to_gda(input, output = None): + if output is None: + output = input + if output.lower().endswith(".gds"): + output = output[:-4] + output = output + ".gda" + + input = open(input, "rb").read() + output = open(output, "w", encoding="utf-8") + gds = GDS.from_gds(input) + output.write(gds.to_gda()) output.close() From 6398b0b721b7a03ba67985bf12f314ee921c0dc5 Mon Sep 17 00:00:00 2001 From: ilonachan Date: Thu, 3 Oct 2024 21:01:04 +0200 Subject: [PATCH 02/17] Consolidated (de)compile options & added a format option instead; CLI can detect input file type automatically. Added yaml export type, and fixed a few bugs. --- formats/gds.py | 137 +++++++++++++++++++++++++++++++++++++------------ utils.py | 11 ++-- 2 files changed, 111 insertions(+), 37 deletions(-) diff --git a/formats/gds.py b/formats/gds.py index cda15d6..f3997cc 100644 --- a/formats/gds.py +++ b/formats/gds.py @@ -1,6 +1,9 @@ import click +import contextlib import json +import yaml import os +import sys import parse import ast @@ -16,7 +19,9 @@ def cli(): commands_i = {val["id"]: key for key, val in commands.items() if "id" in val} # Inverted version of commands class GDS: - def __init__(self, cmds = []): #modes: "bin"/"b", "json"/"j", "gda"/"a" + def __init__(self, cmds=None): #modes: "bin"/"b", "json"/"j", "gda"/"a" + if cmds is None: + cmds = [] self.cmds = cmds @classmethod @@ -83,10 +88,16 @@ def from_gds(Self, file): @classmethod def from_json (Self, file): - cmds = json.loads(file) + cmds = json.loads(file)["data"] #TODO: reject non-compatible json files return Self(cmds) + @classmethod + def from_yaml(Self, file): + cmds = yaml.safe_load(file)["data"] + #TODO: reject non-compatible yaml files + return Self(cmds) + @classmethod def from_gda (Self, file): #TODO: make this, so gds_old can be completely removed cmds = [] @@ -98,8 +109,6 @@ def from_gda (Self, file): #TODO: make this, so gds_old can be completely remove if line == '': continue - data = {} - line, strings = parse.remove_strings(line) line = line.rstrip().split(" ") cmd = line[0] @@ -148,14 +157,17 @@ def __getitem__ (self, index): index = int(index) return self.cmds[index] - def to_json (self): + def to_json(self): return json.dumps({"version": v, "data": self.cmds}, indent=4) + + def to_yaml(self): + return yaml.safe_dump({"version": v, "data": self.cmds}) def to_gds (self): out = b"\x00" * 2 for command in self.cmds: - if type(command["command"]["id"]) == int: - out += command["command"]["id"].to_bytes(2, "little") + if type(command["command"]) == int: + out += command["command"].to_bytes(2, "little") else: out += commands[command["command"]["id"]].to_bytes(2, "little") for param in command["parameters"]: @@ -258,52 +270,109 @@ def process(input, output): foreach_file_pair(pairs, process, quiet=quiet) -@cli.command( - name="compilejson", - help="Converts a JSON document created by 'compilejson' to a GDS script file.", - no_args_is_help = True - ) -@click.argument("input") -@click.argument("output") -def create_json(input, output): - input = open(input, encoding="utf-8").read() - output = open(output, "wb") - - gds = GDS.from_json(input) - output.write(gds.to_bin()) - output.close() @cli.command( name="compile", - help="Generates a GDS binary script file from human-readable GDA files.", no_args_is_help = True ) @click.argument("input") -@click.argument("output") -def create_from_gda(input, output): - input = open(input, encoding="utf-8").read() - output = open(output, "wb") +@click.argument("output", required=False, default = None) +@click.option("--format", "-f", required=False, default = None, multiple=False, help="The format of the input file. Will be inferred from the file ending or content if unset. Possible values: gda, json, yaml") +def compile(input, output, format): + """ + Generates a GDS binary from a human-readable script file. + """ + inpath = input + if format not in [None, "gda", "json", "yaml", "yml"]: + raise Exception(f"Unsupported input format: '{format}'") + + if format is None: + if inpath.lower().endswith(".gda"): + format = "gda" + elif inpath.lower().endswith(".json"): + format = "json" + elif inpath.lower().endswith(".yml") or inpath.lower().endswith(".yaml"): + format = "yaml" + + + if output is None: + output = inpath + if format == 'gda' and output.lower().endswith(".gda"): + output = output[:-4] + elif format == 'json' and output.lower().endswith(".json"): + output = output[:-5] + elif format in ['yaml', 'yml'] and output.lower().endswith(".yml"): + output = output[:-4] + elif format in ['yaml', 'yml'] and output.lower().endswith(".yaml"): + output = output[:-5] + output += ".gds" - gds = GDS.from_gda(input) + input = open(inpath, encoding="utf-8").read() + gds = None + with contextlib.suppress(Exception): + if format == 'gda': + gds = GDS.from_gda(input) + elif format == 'json': + gds = GDS.from_json(input) + elif format in ['yaml', 'yml']: + gds = GDS.from_yaml(input) + + if gds is None: + if format is not None: + # TODO: should this abort instead? + print(f"WARNING: Input file '{inpath}' did not have expected format '{format}'", file = sys.stderr) + # format not specified and couldn't be inferred, or file turns out not to have the correct format + # => try all the formats & see which one works (only one should be possible) + for f in ["json", "yaml", "gda"]: + with contextlib.suppress(Exception): + if f == 'gda': + gds = GDS.from_gda(input) + elif f == 'json': + gds = GDS.from_json(input) + elif f == 'yaml': + gds = GDS.from_yaml(input) + if gds is None: + raise Exception(f"File '{inpath}' couldn't be read: not a known file format" + +(f" (expected '{format}')" if format is not None else "")) + + output = open(output, "wb") output.write(gds.to_bin()) output.close() @cli.command( name="decompile", - help="Converts a GDS file into a human-readable GDA script format.", no_args_is_help = True ) @click.argument("input") -@click.argument("output", required=False) -def create_to_gda(input, output = None): +@click.argument("output", required=False, default = None) +@click.option("--format", "-f", default="gda", required=False, multiple=False, help="The format used for output. Possible values: gda (default), json, yaml") +def decompile(input, output, format): + """ + Convert a GDS file into a human-readable GDA script format. + """ + out_ending = "" + if format == 'gda': + out_ending = ".gda" + elif format == 'json': + out_ending = ".json" + elif format in ['yaml', 'yml']: + out_ending = ".yml" + else: + raise Exception(f"Unsupported output format: '{format}'") + if output is None: output = input if output.lower().endswith(".gds"): output = output[:-4] - output = output + ".gda" + output = output + out_ending input = open(input, "rb").read() - output = open(output, "w", encoding="utf-8") gds = GDS.from_gds(input) - output.write(gds.to_gda()) - output.close() + + with open(output, "w", encoding="utf-8") as output: + if format == 'gda': + output.write(gds.to_gda()) + elif format == 'json': + output.write(gds.to_json()) + elif format in ['yaml', 'yml']: + output.write(gds.to_yaml()) diff --git a/utils.py b/utils.py index 646e0f3..82194f0 100644 --- a/utils.py +++ b/utils.py @@ -1,6 +1,6 @@ import os -def cli_file_pairs(input = None, output = None, *, in_ending = None, out_ending = None, recursive = False): +def cli_file_pairs(input = None, output = None, *, in_ending = None, out_ending = None, filter_infer=None, recursive = False): """ Given the file path inputs to the various CLI commands, determines which input files should be operated on and mapped to which output files. @@ -39,7 +39,7 @@ def listfiles(path): continue yield os.path.join(path, f) - def filter_infer(input): + def default_filter_infer(input, force_accept=False): if in_ending is not None and not input.lower().endswith(in_ending): return None if out_ending is not None and input.lower().endswith(out_ending): @@ -53,15 +53,20 @@ def filter_infer(input): output += out_ending return output + if filter_infer is None: + filter_infer = default_filter_infer + input_dir = "" input_paths = [] + rel_pairs = None if os.path.isfile(input): input_dir, ip = os.path.split(input) input_paths = [ip] + rel_pairs = [(ip, filter_infer(ip, force_accept=True)) for ip in input_paths] else: input_dir = input input_paths = [os.path.relpath(f, input_dir) for f in listfiles(input)] - rel_pairs = [(ip, filter_infer(ip)) for ip in input_paths] + rel_pairs = [(ip, filter_infer(ip)) for ip in input_paths] rel_pairs = [(ip, op) for (ip, op) in rel_pairs if op is not None] if output is None: From c254ced85014a21b7e82c8d15cf30eca53f40620 Mon Sep 17 00:00:00 2001 From: ilonachan Date: Thu, 3 Oct 2024 21:01:04 +0200 Subject: [PATCH 03/17] Fully integrated the new path handling into the GDS commands. --- formats/gds.py | 496 ++++++++++++++++++++++++++++++++----------------- utils.py | 72 ++++--- 2 files changed, 375 insertions(+), 193 deletions(-) diff --git a/formats/gds.py b/formats/gds.py index f3997cc..1ac21a6 100644 --- a/formats/gds.py +++ b/formats/gds.py @@ -4,22 +4,32 @@ import yaml import os import sys +import collections import parse import ast from utils import cli_file_pairs, foreach_file_pair from version import v -@click.group(help="Script-like format, also used to store puzzle parameters.",options_metavar='') + +@click.group( + help="Script-like format, also used to store puzzle parameters.", options_metavar="" +) def cli(): pass -dir_path = "/".join(os.path.dirname(os.path.realpath(__file__).replace("\\", "/")).split("/")[:-1]) + +dir_path = "/".join( + os.path.dirname(os.path.realpath(__file__).replace("\\", "/")).split("/")[:-1] +) commands = json.load(open(f"{dir_path}/data/commands.json", encoding="utf-8")) -commands_i = {val["id"]: key for key, val in commands.items() if "id" in val} # Inverted version of commands +commands_i = { + val["id"]: key for key, val in commands.items() if "id" in val +} # Inverted version of commands + class GDS: - def __init__(self, cmds=None): #modes: "bin"/"b", "json"/"j", "gda"/"a" + def __init__(self, cmds=None): # modes: "bin"/"b", "json"/"j", "gda"/"a" if cmds is None: cmds = [] self.cmds = cmds @@ -29,7 +39,7 @@ def from_gds(Self, file): length = int.from_bytes(file[0:4], "little") if file[4:6] == b"\x0c\x00": return Self([]) - cmd_data = file[6:length+4] + cmd_data = file[6 : length + 4] cmds = [] cmd = None @@ -37,36 +47,65 @@ def from_gds(Self, file): c = 0 while True: if c >= length: - raise Exception("GDS file error: End of file reached with no 0xC command!") + raise Exception( + "GDS file error: End of file reached with no 0xC command!" + ) if cmd == None: - cmd = int.from_bytes(cmd_data[c:c+2], "little") + cmd = int.from_bytes(cmd_data[c : c + 2], "little") if cmd in commands_i: cmd = commands_i[cmd] c += 2 continue - p_type = int.from_bytes(cmd_data[c:c+2], "little") - + p_type = int.from_bytes(cmd_data[c : c + 2], "little") + if p_type == 0: - cmds.append({"command":cmd, "parameters":params}) + cmds.append({"command": cmd, "parameters": params}) cmd = None params = [] c += 2 elif p_type == 1: - params.append({"type": "int", "data": int.from_bytes(cmd_data[c+2:c+6], "little")}) + params.append( + { + "type": "int", + "data": int.from_bytes(cmd_data[c + 2 : c + 6], "little"), + } + ) c += 6 elif p_type == 2: - params.append({"type": "unknown-2", "data": int.from_bytes(cmd_data[c+2:c+6], "little")}) + params.append( + { + "type": "unknown-2", + "data": int.from_bytes(cmd_data[c + 2 : c + 6], "little"), + } + ) c += 6 elif p_type == 3: - str_len = int.from_bytes(cmd_data[c+2:c+4], "little") - params.append({"type": "string", "data": cmd_data[c+4:c+4+str_len].decode("ascii").rstrip("\x00")}) #TODO: JP/KO compatibility - c += str_len+4 + str_len = int.from_bytes(cmd_data[c + 2 : c + 4], "little") + params.append( + { + "type": "string", + "data": cmd_data[c + 4 : c + 4 + str_len] + .decode("ascii") + .rstrip("\x00"), + } + ) # TODO: JP/KO compatibility + c += str_len + 4 elif p_type == 6: - params.append({"type": "unknown-6", "data": int.from_bytes(cmd_data[c+2:c+6], "little")}) + params.append( + { + "type": "unknown-6", + "data": int.from_bytes(cmd_data[c + 2 : c + 6], "little"), + } + ) c += 6 elif p_type == 7: - params.append({"type": "unknown-7", "data": int.from_bytes(cmd_data[c+2:c+6], "little")}) + params.append( + { + "type": "unknown-7", + "data": int.from_bytes(cmd_data[c + 2 : c + 6], "little"), + } + ) c += 6 elif p_type == 8: params.append({"type": "unknown-8"}) @@ -74,39 +113,41 @@ def from_gds(Self, file): elif p_type == 9: params.append({"type": "unknown-9"}) c += 2 - elif p_type == 0xb: + elif p_type == 0xB: params.append({"type": "unknown-b"}) c += 2 - elif p_type == 0xc: - #cmd = hex(cmd) - cmds.append({"command":cmd, "parameters":params}) + elif p_type == 0xC: + # cmd = hex(cmd) + cmds.append({"command": cmd, "parameters": params}) break else: - raise Exception(f"GDS file error: Invalid or unsupported parameter type {hex(p_type)}!") - + raise Exception( + f"GDS file error: Invalid or unsupported parameter type {hex(p_type)}!" + ) + return Self(cmds) - + @classmethod - def from_json (Self, file): + def from_json(Self, file): cmds = json.loads(file)["data"] - #TODO: reject non-compatible json files + # TODO: reject non-compatible json files return Self(cmds) - + @classmethod def from_yaml(Self, file): cmds = yaml.safe_load(file)["data"] - #TODO: reject non-compatible yaml files + # TODO: reject non-compatible yaml files return Self(cmds) - + @classmethod - def from_gda (Self, file): #TODO: make this, so gds_old can be completely removed + def from_gda(Self, file): # TODO: make this, so gds_old can be completely removed cmds = [] - + for line in file.split("\n"): line = line.strip() if line.startswith("#"): continue - if line == '': + if line == "": continue line, strings = parse.remove_strings(line) @@ -125,51 +166,51 @@ def from_gda (Self, file): #TODO: make this, so gds_old can be completely remove cmd = int(cmd[2:], base=16) else: raise Exception(f"Unknown GDA command: {cmd}") - + params = [] for param in line[1:]: if param.isdigit(): - params.append({"type":"int", "data":int(param)}) + params.append({"type": "int", "data": int(param)}) elif param.startswith("0x"): - params.append({"type":"unknown-2", "data":int(param[2:], 16)}) + params.append({"type": "unknown-2", "data": int(param[2:], 16)}) elif param.startswith('"') and param.endswith('"'): param = ast.literal_eval(f'"{strings[int(param[1:-1])]}"') - params.append({"type":"string", "data":param}) + params.append({"type": "string", "data": param}) elif param.startswith("!6("): - params.append({"type":"unknown-6", "data":int(param[3:-1], 16)}) + params.append({"type": "unknown-6", "data": int(param[3:-1], 16)}) elif param.startswith("!7("): - params.append({"type":"unknown-7", "data":int(param[3:-1], 16)}) + params.append({"type": "unknown-7", "data": int(param[3:-1], 16)}) elif param.startswith("!8"): - params.append({"type":"unknown-8"}) + params.append({"type": "unknown-8"}) elif param.startswith("!9"): - params.append({"type":"unknown-9"}) + params.append({"type": "unknown-9"}) elif param.startswith("!b"): - params.append({"type":"unknown-b"}) + params.append({"type": "unknown-b"}) else: raise Exception(f"Invalid GDA parameter: {param}") - - cmds.append({"command":cmd, "parameters":params}) + + cmds.append({"command": cmd, "parameters": params}) return Self(cmds) - def __getitem__ (self, index): + def __getitem__(self, index): index = int(index) return self.cmds[index] - + def to_json(self): return json.dumps({"version": v, "data": self.cmds}, indent=4) def to_yaml(self): return yaml.safe_dump({"version": v, "data": self.cmds}) - - def to_gds (self): + + def to_gds(self): out = b"\x00" * 2 for command in self.cmds: if type(command["command"]) == int: out += command["command"].to_bytes(2, "little") else: - out += commands[command["command"]["id"]].to_bytes(2, "little") + out += command["command"]["id"].to_bytes(2, "little") for param in command["parameters"]: if param["type"] == "int": out += b"\x01\x00" @@ -179,8 +220,10 @@ def to_gds (self): out += param["data"].to_bytes(4, "little") elif param["type"] == "string": out += b"\x03\x00" - out += (len(param["data"])+1).to_bytes(2, "little") - out += param["data"].encode("ASCII") + b"\x00" #TODO: JP/KO compatibility + out += (len(param["data"]) + 1).to_bytes(2, "little") + out += ( + param["data"].encode("ASCII") + b"\x00" + ) # TODO: JP/KO compatibility elif param["type"] == "unknown-6": out += b"\x06\x00" out += param["data"].to_bytes(4, "little") @@ -194,22 +237,24 @@ def to_gds (self): elif param["type"] == "unknown-b": out += b"\x0b\x00" else: - raise Exception(f"GDS JSON error: Invalid or unsupported parameter type '{param['type']}'!") + raise Exception( + f"GDS JSON error: Invalid or unsupported parameter type '{param['type']}'!" + ) out += b"\x00\x00" out = out[:-2] + b"\x0c\x00" return len(out).to_bytes(4, "little") + out - - def to_bin (self): #alias + + def to_bin(self): # alias return self.to_gds() - + def to_gda(self): out = "" for command in self.cmds: if type(command["command"]) == int: - out += "0x"+command["command"].to_bytes(1, "little").hex() + out += "0x" + command["command"].to_bytes(1, "little").hex() else: - out += command['command'] + out += command["command"] for param in command["parameters"]: out += " " if param["type"] == "int": @@ -231,148 +276,261 @@ def to_gda(self): elif param["type"] == "unknown-b": out += "!b" else: - raise Exception(f"GDA error: invalid or unsupported parameter type '{param['type']}'!") + raise Exception( + f"GDA error: invalid or unsupported parameter type '{param['type']}'!" + ) out += "\n" return out -@cli.command( - name="extract", - no_args_is_help = False - ) + +@cli.command(name="compile", no_args_is_help=True) @click.argument("input", required=False, type=click.Path(exists=True)) @click.argument("output", required=False, type=click.Path(exists=False)) -@click.option("--recursive", "-r", is_flag=True, help="Recurse into subdirectories of the input directory to find more applicable files.") -@click.option("--quiet", "-q", is_flag=True, help="Suppress all output. By default, operations involving multiple files will show a progressbar.") -def unpack_json(input = None, output = None, recursive = False, quiet = False): +@click.option( + "--recursive", + "-r", + is_flag=True, + help="Recurse into subdirectories of the input directory to find more applicable files.", +) +@click.option( + "--quiet", + "-q", + is_flag=True, + help="Suppress all output. By default, operations involving multiple files will show a progressbar.", +) +@click.option( + "--overwrite/--no-overwrite", + "-o/-O", + default=True, + help="Whether existing files should be overwritten. Default: true", +) +@click.option( + "--format", + "-f", + required=False, + default=None, + multiple=False, + help="The format of the input file. Will be inferred from the file ending or content if unset. " + "If multiple file types would compile to the same output (but may not necessarily have the same content), " + "specify this to disambigute. Possible values: gda, json, yaml", +) +def compile( + input=None, output=None, recursive=False, quiet=False, format=None, overwrite=None +): """ - Converts the GDS script(s) at INPUT to JSON files at OUTPUT. + Compiles the human-readable script(s) at INPUT into the game's binary script files at OUTPUT. INPUT can be a single file or a directory (which obviously has to exist). In the latter case subfiles with the correct file ending will be processed. If unset, defaults to the current working directory. The meaning of OUTPUT may depend on INPUT: - If INPUT is a file, then OUTPUT is expected to be a file, unless it explicitly ends with a slash indicating a directory. - In this case, if unset OUTPUT will default to the INPUT filename with `.json` exchanged/appended. + In this case, if unset OUTPUT will default to the INPUT filename with `.gds` exchanged/appended. - Otherwise OUTPUT has to be a directory as well (or an error will be shown). In this case, if unset OUTPUT will default to the INPUT directory (which may itself default to the current working directory). In the file-to-file case, the paths are explicitly used as they are. Otherwise, if multiple input files were collected, or OUTPUT is a directory, - an output path is inferred for each input file by exchanging the `.gds` file ending for `.json`, or otherwise appending the `.json` file ending. + an output path is inferred for each input file by exchanging the input format's file ending for, or otherwise appending the `.gds` file ending. + + In the case where INPUT is a directory, if no format is specified, this command will collect files of all compatible types. Note that this can lead + to situations where multiple files would compile to the same output (e.g. `test.json` and `test.gda` would both be candidates for `test.gds`); + this command will NOT make a choice in this case, and instead ask to explicitly specify the format to be used. """ + + in_endings = [] + if format is None: + in_endings = [".gda", ".json", ".yaml", ".yml"] + elif format == "gda": + in_endings = [".gda"] + elif format == "json": + in_endings = [".json"] + elif format in ["yaml", "yml"]: + in_endings = [".yaml", ".yml"] + else: + raise Exception(f"Unsupported input format: '{format}'") + def process(input, output): - input = open(input, "rb").read() - output = open(output, "w", encoding="utf-8") - gds = GDS(input) - output.write(gds.to_json()) + inpath = input + input = open(inpath, "r", encoding="utf-8").read() + + format2 = format + if format2 is None: + if inpath.lower().endswith(".gda"): + format2 = "gda" + elif inpath.lower().endswith(".json"): + format2 = "json" + elif inpath.lower().endswith(".yml") or inpath.lower().endswith(".yaml"): + format2 = "yaml" + + gds = None + with contextlib.suppress(Exception): + if format2 == "gda": + gds = GDS.from_gda(input) + elif format2 == "json": + gds = GDS.from_json(input) + elif format2 in ["yaml", "yml"]: + gds = GDS.from_yaml(input) + + if gds is None: + if format2 is not None: + # TODO: should this abort instead? + print( + f"WARNING: Input file '{inpath}' did not have expected format '{format2}'", + file=sys.stderr, + ) + # format not specified and couldn't be inferred, or file turns out not to have the correct format + # => try all the formats & see which one works (only one should be possible) + for f in ["gda", "json", "yaml"]: + with contextlib.suppress(Exception): + if f == "gda": + gds = GDS.from_gda(input) + elif f == "json": + gds = GDS.from_json(input) + elif f == "yaml": + gds = GDS.from_yaml(input) + if gds is not None: + break + if gds is None: + raise Exception( + f"File '{inpath}' couldn't be read: not a known file format" + + (f" (expected '{format2}')" if format2 is not None else "") + ) + + output = open(output, "wb") + output.write(gds.to_bin()) output.close() - pairs = cli_file_pairs(input, output, in_ending=".gds", out_ending=".json", recursive=recursive) - foreach_file_pair(pairs, process, quiet=quiet) + pairs = cli_file_pairs( + input, output, in_endings=in_endings, out_ending=".gds", recursive=recursive + ) + duplicates = collections.defaultdict(list) + for ip, op in pairs: + duplicates[op].append(ip) + duplicates = {k: v for k, v in duplicates.items() if len(v) > 1} + if len(duplicates) > 0: + print( + f"ERROR: {len(duplicates)} {'files have' if len(duplicates) > 1 else 'file has'} multiple conflicting source files; please explicitly specify a format to determine which should be used.", + file=sys.stderr, + ) + for op, ips in duplicates.items(): + pathlist = ", ".join("'" + ip + "'" for ip in ips) + print(f"'{op}' could be compiled from {pathlist}", file=sys.stderr) + sys.exit(-1) + if not overwrite: + new_pairs = [] + existing = [] + for ip, op in pairs: + if os.path.exists(op): + existing.append(op) + else: + new_pairs.append((ip, op)) -@cli.command( - name="compile", - no_args_is_help = True - ) -@click.argument("input") -@click.argument("output", required=False, default = None) -@click.option("--format", "-f", required=False, default = None, multiple=False, help="The format of the input file. Will be inferred from the file ending or content if unset. Possible values: gda, json, yaml") -def compile(input, output, format): - """ - Generates a GDS binary from a human-readable script file. - """ - inpath = input - if format not in [None, "gda", "json", "yaml", "yml"]: - raise Exception(f"Unsupported input format: '{format}'") + if not quiet: + print(f"Skipping {len(existing)} existing output files.") - if format is None: - if inpath.lower().endswith(".gda"): - format = "gda" - elif inpath.lower().endswith(".json"): - format = "json" - elif inpath.lower().endswith(".yml") or inpath.lower().endswith(".yaml"): - format = "yaml" - - - if output is None: - output = inpath - if format == 'gda' and output.lower().endswith(".gda"): - output = output[:-4] - elif format == 'json' and output.lower().endswith(".json"): - output = output[:-5] - elif format in ['yaml', 'yml'] and output.lower().endswith(".yml"): - output = output[:-4] - elif format in ['yaml', 'yml'] and output.lower().endswith(".yaml"): - output = output[:-5] - output += ".gds" - - input = open(inpath, encoding="utf-8").read() - gds = None - with contextlib.suppress(Exception): - if format == 'gda': - gds = GDS.from_gda(input) - elif format == 'json': - gds = GDS.from_json(input) - elif format in ['yaml', 'yml']: - gds = GDS.from_yaml(input) - - if gds is None: - if format is not None: - # TODO: should this abort instead? - print(f"WARNING: Input file '{inpath}' did not have expected format '{format}'", file = sys.stderr) - # format not specified and couldn't be inferred, or file turns out not to have the correct format - # => try all the formats & see which one works (only one should be possible) - for f in ["json", "yaml", "gda"]: - with contextlib.suppress(Exception): - if f == 'gda': - gds = GDS.from_gda(input) - elif f == 'json': - gds = GDS.from_json(input) - elif f == 'yaml': - gds = GDS.from_yaml(input) - if gds is None: - raise Exception(f"File '{inpath}' couldn't be read: not a known file format" - +(f" (expected '{format}')" if format is not None else "")) - - output = open(output, "wb") - output.write(gds.to_bin()) - output.close() - -@cli.command( - name="decompile", - no_args_is_help = True - ) -@click.argument("input") -@click.argument("output", required=False, default = None) -@click.option("--format", "-f", default="gda", required=False, multiple=False, help="The format used for output. Possible values: gda (default), json, yaml") -def decompile(input, output, format): + pairs = new_pairs + + foreach_file_pair(pairs, process, quiet=quiet) + + +@cli.command(name="decompile", no_args_is_help=True) +@click.argument("input", required=False, type=click.Path(exists=True)) +@click.argument("output", required=False, type=click.Path(exists=False)) +@click.option( + "--recursive", + "-r", + is_flag=True, + help="Recurse into subdirectories of the input directory to find more applicable files.", +) +@click.option( + "--quiet", + "-q", + is_flag=True, + help="Suppress all output. By default, operations involving multiple files will show a progressbar.", +) +@click.option( + "--overwrite/--no-overwrite", + "-o/-O", + default=True, + help="Whether existing files should be overwritten. Default: true", +) +@click.option( + "--format", + "-f", + required=False, + multiple=False, + help="The format used for output. Possible values: gda (default), json, yaml", +) +def decompile( + input=None, output=None, recursive=False, quiet=False, format=None, overwrite=None +): """ - Convert a GDS file into a human-readable GDA script format. + Decompiles the GDS script(s) at INPUT into a human-readable text format at OUTPUT. + + INPUT can be a single file or a directory (which obviously has to exist). In the latter case subfiles with the correct file ending will be processed. + If unset, defaults to the current working directory. + + The meaning of OUTPUT may depend on INPUT: + - If INPUT is a file, then OUTPUT is expected to be a file, unless it explicitly ends with a slash indicating a directory. + In this case, if unset OUTPUT will default to the INPUT filename with `.json` exchanged/appended. + - Otherwise OUTPUT has to be a directory as well (or an error will be shown). + In this case, if unset OUTPUT will default to the INPUT directory (which may itself default to the current working directory). + + In the file-to-file case, the paths are explicitly used as they are. Otherwise, if multiple input files were collected, or OUTPUT is a directory, + an output path is inferred for each input file by exchanging the `.gds` file ending for `.json`, or otherwise appending the `.json` file ending. """ out_ending = "" - if format == 'gda': + if format == "gda" or format is None: out_ending = ".gda" - elif format == 'json': + elif format == "json": out_ending = ".json" - elif format in ['yaml', 'yml']: + elif format in ["yaml", "yml"]: out_ending = ".yml" else: raise Exception(f"Unsupported output format: '{format}'") - - if output is None: - output = input - if output.lower().endswith(".gds"): - output = output[:-4] - output = output + out_ending - - input = open(input, "rb").read() - gds = GDS.from_gds(input) - - with open(output, "w", encoding="utf-8") as output: - if format == 'gda': - output.write(gds.to_gda()) - elif format == 'json': - output.write(gds.to_json()) - elif format in ['yaml', 'yml']: - output.write(gds.to_yaml()) + + def process(input, output): + input = open(input, "rb").read() + gds = GDS.from_gds(input) + + nonlocal format + if format is None: + if output.lower().endswith(".gda"): + format = "gda" + elif output.lower().endswith(".json"): + format = "json" + elif output.lower().endswith(".yml") or output.lower().endswith(".yaml"): + format = "yaml" + else: + print( + f"WARNING: output format couldn't be inferred from filename '{output}'; using default (gda). To remove this warning, please explicitly specify a format.", + file=sys.stderr, + ) + + with open(output, "w", encoding="utf-8") as output: + if format == "gda": + output.write(gds.to_gda()) + elif format == "json": + output.write(gds.to_json()) + elif format in ["yaml", "yml"]: + output.write(gds.to_yaml()) + + pairs = cli_file_pairs( + input, output, in_endings=[".gds"], out_ending=out_ending, recursive=recursive + ) + if not overwrite: + new_pairs = [] + existing = [] + for ip, op in pairs: + if os.path.exists(op): + existing.append(op) + else: + new_pairs.append((ip, op)) + + if not quiet: + print(f"Skipping {len(existing)} existing output files.") + + pairs = new_pairs + foreach_file_pair(pairs, process, quiet=quiet) diff --git a/utils.py b/utils.py index 82194f0..a35c133 100644 --- a/utils.py +++ b/utils.py @@ -1,6 +1,15 @@ import os -def cli_file_pairs(input = None, output = None, *, in_ending = None, out_ending = None, filter_infer=None, recursive = False): + +def cli_file_pairs( + input=None, + output=None, + *, + in_endings=None, + out_ending=None, + filter_infer=None, + recursive=False, +): """ Given the file path inputs to the various CLI commands, determines which input files should be operated on and mapped to which output files. @@ -9,13 +18,13 @@ def cli_file_pairs(input = None, output = None, *, in_ending = None, out_ending - If the input is a directory, all files in this directory (with the file ending `in_ending`) will be separately treated as input paths. - If there is no input (which also means there's no output), the current working directory is used as both input and output (which should be fine since most commands produce files of a different ending). - + Assuming the input is a file, then if the output is the same this pair is used as-is. Otherwise, if the output isn't specified (or a directory) he output path is *inferred* like this: - If the input ends with the expected file ending, that is stripped and replaced with the output file ending. It's expected that if a user wants more control over this file extension, they should provide an output manually for each input file. - If the input has a different ending (or none at all), the output file ending is simply appended to the full name. - + This same inference is used if the input is a directory: here we have multiple input paths, for which the targets can not have been specified, and so the inference is applied to each of them separately. The output must be a directory (or the input directory itself if not specified) and all the output paths are calculated such that input paths relative to the input directory become output paths relative to the output directory @@ -24,13 +33,13 @@ def cli_file_pairs(input = None, output = None, *, in_ending = None, out_ending if input is None: input = "." - + if not os.path.exists(input): raise FileNotFoundError(input) - + def listfiles(path): if recursive: - for (dp, _, fn) in os.walk(path, topdown=True): + for dp, _, fn in os.walk(path, topdown=True): for f in fn: yield os.path.join(dp, f) else: @@ -38,24 +47,30 @@ def listfiles(path): if not os.path.isfile(os.path.join(path, f)): continue yield os.path.join(path, f) - + def default_filter_infer(input, force_accept=False): - if in_ending is not None and not input.lower().endswith(in_ending): + if in_endings is not None and not any( + input.lower().endswith(ie) for ie in in_endings + ): return None if out_ending is not None and input.lower().endswith(out_ending): return None - + output = input - if in_ending is not None and input.lower().endswith(in_ending): - output = input[:-len(in_ending)] + if in_endings is not None: + endings = [ie for ie in in_endings if input.lower().endswith(ie)] + if endings: + output = input[: -len(endings[0])] if out_ending is None: - raise ValueError("Can't infer output file names without a target file ending specified") + raise ValueError( + "Can't infer output file names without a target file ending specified" + ) output += out_ending return output - + if filter_infer is None: filter_infer = default_filter_infer - + input_dir = "" input_paths = [] rel_pairs = None @@ -68,31 +83,40 @@ def default_filter_infer(input, force_accept=False): input_paths = [os.path.relpath(f, input_dir) for f in listfiles(input)] rel_pairs = [(ip, filter_infer(ip)) for ip in input_paths] rel_pairs = [(ip, op) for (ip, op) in rel_pairs if op is not None] - + if output is None: output = input_dir - if os.path.isfile(input) and not os.path.isdir(output) and os.path.split(output)[1] != '': + if ( + os.path.isfile(input) + and not os.path.isdir(output) + and os.path.split(output)[1] != "" + ): return [(input, output)] - + if os.path.isfile(output): raise OSError(f"Output path exists but is not a directory: '{output}'") output_dir = output - - pairs = [(os.path.join(input_dir, ip), os.path.join(output_dir, op)) for (ip, op) in rel_pairs] + + pairs = [ + (os.path.join(input_dir, ip), os.path.join(output_dir, op)) + for (ip, op) in rel_pairs + ] return pairs -def foreach_file_pair(pairs, fn, quiet = False): + +def foreach_file_pair(pairs, fn, quiet=False): try: from tqdm import tqdm - if not quiet: + + if not quiet and len(pairs) > 5: progress = tqdm(pairs) - for (input, output) in progress: + for input, output in progress: progress.set_description(input) fn(input, output) return except ImportError: # TQDM isn't installed; just don't show a progress bar. pass - for (input, output) in pairs: - fn(input, output) \ No newline at end of file + for input, output in pairs: + fn(input, output) From 77cd3366c69a0c1b0923fb778b9e13917074ce28 Mon Sep 17 00:00:00 2001 From: ilonachan Date: Thu, 3 Oct 2024 21:01:04 +0200 Subject: [PATCH 04/17] updated some old comments --- formats/gds.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/formats/gds.py b/formats/gds.py index 1ac21a6..ad33094 100644 --- a/formats/gds.py +++ b/formats/gds.py @@ -29,7 +29,7 @@ def cli(): class GDS: - def __init__(self, cmds=None): # modes: "bin"/"b", "json"/"j", "gda"/"a" + def __init__(self, cmds=None): if cmds is None: cmds = [] self.cmds = cmds @@ -86,10 +86,10 @@ def from_gds(Self, file): { "type": "string", "data": cmd_data[c + 4 : c + 4 + str_len] - .decode("ascii") + .decode("ascii") # TODO: JP/KO compatibility .rstrip("\x00"), } - ) # TODO: JP/KO compatibility + ) c += str_len + 4 elif p_type == 6: params.append( From 0ae215cc58e781857a8e3e6ac8aee918ff0600df Mon Sep 17 00:00:00 2001 From: ilonachan Date: Thu, 3 Oct 2024 21:01:04 +0200 Subject: [PATCH 05/17] feat(gds): added ALL the metadata + harness to read & use it TODO: improve the parser to handle jumps and address labels TODO: create the GDA parsing and writing logic --- data/gds_commands/common.yml | 298 ++++++++++++++++++ data/gds_commands/event.yml | 336 ++++++++++++++++++++ data/gds_commands/fx.yml | 45 +++ data/gds_commands/help.md | 119 ++++++++ data/gds_commands/logic.yml | 45 +++ data/gds_commands/minigames/dog.yml | 10 + data/gds_commands/minigames/hotel.yml | 86 ++++++ data/gds_commands/puzzle/buttonlist.yml | 14 + data/gds_commands/puzzle/coin.yml | 20 ++ data/gds_commands/puzzle/common.yml | 208 +++++++++++++ data/gds_commands/puzzle/connect.yml | 12 + data/gds_commands/puzzle/cup.yml | 18 ++ data/gds_commands/puzzle/cut.yml | 40 +++ data/gds_commands/puzzle/drag.yml | 3 + data/gds_commands/puzzle/drawinput.yml | 38 +++ data/gds_commands/puzzle/match.yml | 6 + data/gds_commands/puzzle/queen.yml | 36 +++ data/gds_commands/puzzle/rivercross.yml | 34 +++ data/gds_commands/puzzle/scale.yml | 11 + data/gds_commands/puzzle/shape.yml | 29 ++ data/gds_commands/puzzle/slide.yml | 21 ++ data/gds_commands/puzzle/tile.yml | 46 +++ data/gds_commands/puzzle/trace.yml | 42 +++ data/gds_commands/room.yml | 259 ++++++++++++++++ formats/{gds.py => gds/__init__.py} | 92 +++--- formats/gds/_test/sub/subby.yaml | 7 + formats/gds/_test/test.yml | 39 +++ formats/gds/cmddef.py | 389 ++++++++++++++++++++++++ formats/gds/gds.py | 220 ++++++++++++++ formats/gds/model.py | 82 +++++ 30 files changed, 2564 insertions(+), 41 deletions(-) create mode 100644 data/gds_commands/common.yml create mode 100644 data/gds_commands/event.yml create mode 100644 data/gds_commands/fx.yml create mode 100644 data/gds_commands/help.md create mode 100644 data/gds_commands/logic.yml create mode 100644 data/gds_commands/minigames/dog.yml create mode 100644 data/gds_commands/minigames/hotel.yml create mode 100644 data/gds_commands/puzzle/buttonlist.yml create mode 100644 data/gds_commands/puzzle/coin.yml create mode 100644 data/gds_commands/puzzle/common.yml create mode 100644 data/gds_commands/puzzle/connect.yml create mode 100644 data/gds_commands/puzzle/cup.yml create mode 100644 data/gds_commands/puzzle/cut.yml create mode 100644 data/gds_commands/puzzle/drag.yml create mode 100644 data/gds_commands/puzzle/drawinput.yml create mode 100644 data/gds_commands/puzzle/match.yml create mode 100644 data/gds_commands/puzzle/queen.yml create mode 100644 data/gds_commands/puzzle/rivercross.yml create mode 100644 data/gds_commands/puzzle/scale.yml create mode 100644 data/gds_commands/puzzle/shape.yml create mode 100644 data/gds_commands/puzzle/slide.yml create mode 100644 data/gds_commands/puzzle/tile.yml create mode 100644 data/gds_commands/puzzle/trace.yml create mode 100644 data/gds_commands/room.yml rename formats/{gds.py => gds/__init__.py} (88%) create mode 100644 formats/gds/_test/sub/subby.yaml create mode 100644 formats/gds/_test/test.yml create mode 100755 formats/gds/cmddef.py create mode 100644 formats/gds/gds.py create mode 100644 formats/gds/model.py diff --git a/data/gds_commands/common.yml b/data/gds_commands/common.yml new file mode 100644 index 0000000..7f276bc --- /dev/null +++ b/data/gds_commands/common.yml @@ -0,0 +1,298 @@ +# a prefix prepended to each command name with a dot. +# ex: a command "hi" in a package with prefix "test" will be decompiled to "test.hi" +# By default the prefix is generated from the directory structure and package file name, +# we reset it explicitly here to get these functions into the common namespace. +prefix: null + +# Commands that we know the meaning of, reasonably well at least. +commands: + playSfx: + id: 0x10 + desc: Only possible candidate instruction for playing the jingle on the level5 logo screen. Uses a lookup table internally, + the sound IDs are irregular and look nonsensical. Needs more research + # This probably won't be queried programmatically, but it's important to admit our limitations. + uncertain: true + # This can either be an unnamed (or empty) list of parameters, or an ordered dict with parameter names marked. + # If omitted or null, the command takes no arguments. + params: + soundId: + type: int + desc: the ID of the sfx to be played + playSfxAny: + id: 0xf3 + uncertain: true + desc: Can play any sound from the list without indirection, I think. + params: + soundId: int + playSfx700: + id: 0x59 + desc: I'm not sure what's so special about SFX 700, and I can't even find it in the files! (also this command is unused.) + + setScriptType: + id: 0x51 + desc: > + Sets the "current" script type to the specified name. The currently running script will + continue, but once it completes this value hints to the engine what should happen next. + params: + scriptType: + # TODO: enum types? + type: string + desc: | + One of the following: + `Question`: Puzzle + `Room` + `Event` + `Movie` + `Event Movie` (TODO: what's the difference?) + `Reset`: Possibly returns to the title screen? + `Taiken`: ??? + `Challenge Mode`: Might have something to do with Layton's Challenges + `Staff Roll` + setResumeScriptType: + id: 0x52 + desc: > + Some script types aren't actually "scripts" but technical events; they cannot decide how + flow should continue once they're done. That's why for these events (things like puzzles, + event movies etc) you can additionally set a script type that should be set after that + event is done. + params: + scriptType: + type: string + desc: | + Supports only the following values (see `0x51 setScriptType`): + + `Question` + `Room` + `Event` + `Challenge Mode` + + spriteTop: + id: 0x7e + desc: Displays a sprite on the top screen; ordinarily all sprite/bg/ui operations apply to the touchscreen. + params: + xPos: int + yPos: int + spriteName: string + animName: string + setSpriteTopAnim: + id: 0x7f + params: + spriteId: int + animName: string + setSpriteTopPos: + id: 0x80 + params: + spriteId: int + xPos: int + yPos: int + spriteTopReveal: + id: 0x81 + params: + spriteId: int + spriteTopHide: + id: 0x82 + params: + spriteId: int + + spriteTopMoveMode: + id: 0x94 + uncertain: true + desc: + params: + actorId: int + moveMode: + type: string + desc: either "Normal" or "Move To Target" + + # Commands that we don't understand well enough to name yet. Their format is similar to known commands, except the ID is the name. + 0x09: + params: [] + 0x0a: + params: [] + 0x0d: + params: + - type: int + 0x0e: + params: [] + 0x0f: + params: [] + 0x11: + params: + unk_1: + type: int + 0x1c: + params: + - type: int + 0x4a: + condition: true + desc: checks if some global flag is false + 0x4b: + condition: true + desc: checks if some global flag (same as 0x4a) is true + 0x5f: + params: [] + 0x62: + desc: Sets some global variable. + params: + value: + type: int + desc: the value to set + 0x63: + desc: Checks the global variable set by `0x62` (could this be global story progress?) + condition: true + params: + value: + type: int + desc: the value to check against + 0x64: + desc: sets some global flag to true + 0x66: + desc: sets some global flag to false + 0x67: + desc: sets some global flag to true + 0x69: + params: + - type: int + 0x6a: + params: + - type: int + 0x6b: + params: + - type: int + 0x7a: + desc: Seems to be a wait instruction, that perhaps advances by n frames + params: [int] + 0x7b: + params: + - int + - int + - int + - int + 0x7c: + params: + - int + - int + - int + - int + 0x7d: + params: [] + 0x85: + desc: pushes a nonzero byte value to a (very short) list. + params: [int] + 0x86: + desc: checks if the list from `0x85` contains the given byte value + condition: true + params: [int] + 0x87: + params: [int] + 0x88: + params: [int] + 0x8d: + desc: checks if some global flag at the given location (out of 1024) is set + condition: true + params: + flagId: int + 0x8e: + desc: sets some global flag at the given location (out of 1024) + condition: true + params: + flagId: int + value: + type: bool + desc: true if either a nonzero number or the string "true", otherwise false + 0x93: + desc: sets some property on a top sprite + params: + spriteId: int + unk_2: int + 0x95: + desc: sets some point on a top sprite + params: + spriteId: int + xPos: int + yPos: int + 0x96: + desc: sets some property on a top sprite + params: + spriteId: int + unk_2: int + 0x9e: + params: [int] + 0xae: + params: + - string + - string + - int + - int + - int + - int + + 0xb6: + desc: > + Passing this function a format string will printf the current condition flag value into it. + However, nothing is done with that value. + params: + fmt: string + + 0xc0: + params: [int] + 0xc1: + condition: true + params: [] + 0xce: + params: [int] + + 0xdd: + desc: Selects a movie (of the emb%i pool) and sets script type to MOVIE2 + params: + movieId: int + + 0xde: + params: [] + 0xdf: + desc: noop + params: [] + 0xe0: + desc: noop + params: [] + 0xe1: + desc: noop + params: [] + 0xe2: + desc: noop + params: [] + 0xe3: + desc: noop + params: [] + 0xe9: + desc: sets a global value + params: [int] + 0xea: + desc: reads the global value set by `0xe9`, and if it's a valid event ID, set the curScriptType and curEvent appropriately (if it's -1, set to ROOM) + params: [] + 0xeb: + condition: true + # You have entered GStruct1 Zone + 0xec: + params: [int, int] + 0xed: + params: [int, int] + 0xee: + params: [int] + # + 0xef: + params: [int, int, int, int] + 0xf0: + params: [int, int, int] + 0xf1: + params: [int, int] + 0xf4: + params: [int] + 0xf6: + params: [] + 0xf7: + params: [] + 0xf8: + params: [] + 0xf9: + params: [] \ No newline at end of file diff --git a/data/gds_commands/event.yml b/data/gds_commands/event.yml new file mode 100644 index 0000000..d642215 --- /dev/null +++ b/data/gds_commands/event.yml @@ -0,0 +1,336 @@ +commands: + setCurrent: + id: 0x60 + desc: > + Sets the current event in global state. Once the current script is completed, + and the script type is set to "Event", that script will be executed. + param: + eventId: + type: int + + + textbox: + id: 0x98 + desc: > + Displays a textbox for the specified on-screen actor, pointing in the + direction indicated by that actor's sprite location. + params: + textId: + type: int + desc: > + the text to display, in textbox format. This is sourced from + etext/?/e???/e_t.txt. + actorId: + type: int + desc: the id (in initialization order) of the actor sprite saying the text. + textboxRight: + id: 0x9c + desc: > + Displays a textbox for the specified on-screen actor, pointing right. + params: + textId: + type: int + desc: > + the text to display, in textbox format. This is sourced from + etext/?/e???/e_t.txt. + actorId: + type: int + desc: the id (in initialization order) of the actor sprite saying the text. + textboxLeft: + id: 0x9d + desc: > + Displays a textbox for the specified on-screen actor, pointing left. + params: + textId: + type: int + desc: > + the text to display, in textbox format. This is sourced from + etext/?/e???/e_t.txt. + actorId: + type: int + desc: the id (in initialization order) of the actor sprite saying the text. + textboxMiddle: + id: 0xcd + desc: Displays a text box for the specified on-screen actor, with no arrow pointing. + + textboxNoSpeaker: + id: 0x9b + desc: Displays a textbox without a speaker on screen. + params: + textId: + type: int + desc: > + the text to display, in textbox format. This is sourced from + etext/?/e???/e_t.txt. + textboxNoSpeaker2: + id: 0x44 + desc: Displays a textbox without a speaker on screen. (I'm not sure what the difference between these two functions is) + params: + textId: + type: int + desc: > + the text to display, in textbox format. This is sourced from + etext/?/e???/e_t.txt. + textboxNoSpeakerRight: + id: 0x99 + desc: Displays a textbox pointing right, but without a speaker on screen. + params: + textId: + type: int + desc: > + the text to display, in textbox format. This is sourced from + etext/?/e???/e_t.txt. + textboxNoSpeakerLeft: + id: 0x9a + desc: Displays a textbox pointing left, but without a speaker on screen. + params: + textId: + type: int + desc: > + the text to display, in textbox format. This is sourced from + etext/?/e???/e_t.txt. + + textboxBgRight: + id: 0x45 + desc: Displays the background of a textbox pointing right, but doesn't print anything. + textboxBgLeft: + id: 0x46 + desc: Displays the background of a textbox pointing left, but doesn't print anything. + textboxBgNoSpeaker: + id: 0x65 + desc: Displays the background of a textbox pointing in no direction. + + setFlag2: + id: 0x55 + desc: > + Every event has a byte's worth of story flags to access, and this instruction sets byte 2 of + that to true. Maybe it's a simple unified way to tell if an event has already been seen + by the player. + + pickupItem: + id: 0x56 + desc: Shows an item pickup display for the given item ID, and adds it to the inventory. + params: + itemId: + type: int + desc: > + The item ID. It's likely the only valid ones are 1 ("some fish bones") and 3 ("a wristwatch"), + but more can *potentially* be added by providing stext/?/stext.pcm//p_%i.txt and a sprite + frame in ani/prize.spr + pickupGizmo: + id: 0xc4 + desc: Shows a pickup display for the given Gizmo ID + params: + gizmoId: int + pickupPainting: + id: 0xc5 + desc: Shows a pickup display for the given painting scrap ID + params: + scrapId: int + pickupHotel: + id: 0xcc + desc: Shows a pickup display for the given hotel item ID + params: + itemId: int + + + puzzleIndexPopup: + id: 0x57 + desc: Shows the notification that " has been added to the puzzle index". + + mysteryPopup: + id: 0xe8 + desc: Shows the notification that " has been added to your mysteries". + params: + mysteryId: + type: int + desc: this is actually bounds-checked to be between 1-10. The names are source from storytext.pcm//it_%i.txt + + promptSave: + id: 0x61 + desc: Shows a popup allowing the player to save. + + actorSprite: + id: 0x6c + desc: > + Initializes an actor sprite in the current engine. I'm not sure what exactly makes them special, + but they're used mostly in events to display the speakers (and their name tags); however they + also appear in the Drag Puzzle type. + params: + # For parameters that you seriously don't have anything else to say about, this is a nice shortcut + xPos: int + yPos: int + spriteName: + type: string + unk_4: + type: string + + actorSpriteHead: + id: 0x6d + desc: > + Many of the actors have their facial expression separated from their body, probably to save on + space since otherwise all permutations would have to be baked in separately. + params: + xPos: int + yPos: int + spriteName: string + animName: string + actorId: + type: int + desc: The ID (in declaration order) of the actor that this head belongs to. + + setActorAnim: + id: 0x6e + desc: Sets the specified actor's (body) animation to the specified name + params: + actorId: int + animName: string + setActorAnimHead: + id: 0x6f + desc: Sets the specified actor's head animation to the specified name + params: + actorId: int + animName: string + + setActorPos: + id: 0x70 + desc: Sets the specified actor's (body) position to the specified point + params: + # TODO: could the compiler be smart here and emit warnings depending on logic? + actorId: int + xPos: int + yPos: int + + revealActor: + id: 0x71 + uncertain: true + desc: Sets the specified actor to be visible(?) + params: + actorId: int + hideActor: + id: 0x72 + uncertain: true + desc: Sets the specified actor to be visible(?) + params: + actorId: int + + actorMoveMode: + id: 0x90 + uncertain: true + desc: + params: + actorId: int + moveMode: + type: string + desc: either "Normal" or "Move To Target" + + + currentMovie: + id: 0xd3 + desc: Sets the current movie ID. Once the current script completes, if the current script state is set to "Movie", that movie will play. + + + choice3: + id: 0xd5 + desc: Displays a three-option choice, like the Chelmey/DonPaolo deduction minigame. + choice2: + id: 0xdb + desc: Displays a two-option choice, like the endgame continue prompt. Might actually have logic specific to that. + choiceQuestion: + id: 0xd9 + desc: The question to be displayed for this choice + params: + question: + type: string + desc: Unlike most strings in this game, this one is hardcoded in the script, and therefore NOT localized! + choiceAnswer1: + id: 0xd6 + desc: Answer 1 of this choice + params: + answer: + type: string + desc: Just like the question, not localizable. + unk_2: int + choiceAnswer2: + id: 0xd7 + desc: Answer 2 of this choice + params: + answer: + type: string + desc: Just like the question, not localizable. + unk_2: int + choiceAnswer3: + id: 0xd8 + desc: Answer 3 of this choice + params: + answer: + type: string + desc: Just like the question, not localizable. + unk_2: int + choiceAnswerIs: + id: 0xda + condition: true + desc: Checks if the given answer ID (from 1-3) was chosen by the user + params: + answer: int + + + movieSub: + id: 0xfd + desc: Adds a subtitle to the upcoming movie sequence + params: + textId: int + startTime: float + duration: float + moviePlay: + id: 0xfe + desc: Plays the specified movie clip + params: + movieId: int + + 0x47: + params: + - type: int + 0x78: + desc: sets some float value for the specified actor and its head (rotation??) + params: + actorId: int + unk_2: float + 0x79: + desc: sets some float value for the specified actor and its head (rotation??) + params: + actorId: int + unk_2: float + 0x8f: + params: + actorId: int + unk_2: int + + 0x91: + params: + actorId: int + unk_2: int + 0x92: + params: + actorId: int + unk_2: int + 0x97: + params: + actorId: int + unk_2: int + + 0xaf: + params: [int] + 0xb0: + params: [int] + 0xb3: + params: [int] + 0xb4: + params: + actorId: int + unk_2: bool + + 0xb8: + desc: sets the eventFlag with the given ID to 1 + params: + eventFlagId: int \ No newline at end of file diff --git a/data/gds_commands/fx.yml b/data/gds_commands/fx.yml new file mode 100644 index 0000000..bfd4e86 --- /dev/null +++ b/data/gds_commands/fx.yml @@ -0,0 +1,45 @@ +commands: + fadeIn: + id: 0x01 + desc: Shows a fast fade in effect on both screens. + fadeIn2: + id: 0x02 + desc: Shows a slow fade in effect on both screens. + fadeInTop: + id: 0x03 + desc: Shows a fade in effect on the top screen. + fadeInBottom: + id: 0x04 + desc: Shows a fade in effect on the bottom screen. + fadeOut: + id: 0x05 + desc: Shows a fast fade out effect on both screens. + fadeOut2: + id: 0x06 + desc: Shows a slow fade out effect on both screens. + fadeOutTop: + id: 0x07 + desc: Shows a fade out effect on the top screen. + fadeOutBottom: + id: 0x08 + desc: Shows a fade out effect on the bottom screen. + displayImgBottom: + id: 0x0b + desc: Shows the specified image path on the bottom screen + params: + bgxName: + type: string + desc: The name of the image file, with bgx file ending (will look for a .arc file with that name under /bg) + unk_2: + type: int + desc: Seems to always be 2 (for now) + displayImgTop: + id: 0x0c + desc: Shows the specified image path on the top screen + params: + bgxName: + type: string + desc: The name of the image file, with bgx file ending (will look for a .arc file with that name under /bg) + unk_2: + type: int + desc: Seems to always be 3 (for now) diff --git a/data/gds_commands/help.md b/data/gds_commands/help.md new file mode 100644 index 0000000..9845b58 --- /dev/null +++ b/data/gds_commands/help.md @@ -0,0 +1,119 @@ +# GDS: The Scripting Language of Professor Layton + +Most complex events in the Professor Layton game(s) are driven by **GDS** (which may stand for **G**ame **D**ata **S**cript), +a binary scripting language that can trigger events in the currently running +script engine. This is used, among other things, to: + +- choose a puzzle engine, and initialize details of the puzzle specific to that engine +- define room layouts, i.e. entities, text objects, hint coins, exits,... +- write out dialogue sequences, including all NPC interactions +- provide localized data for certain minigames and use cases, where it is easier to read a single + script containing all the definitions into an existing engine, than to open a specific text file +- setup subtitles for the event movies +- It also defines what the logo sequence on bootup looks like. Yes, that is not hardcoded, + you can change the logo sequence! + +While far too many things are still hard-defined in code (such as Puzzle IDs), this makes +the Layton engine fairly data-driven and moddable! It's therefore extremely important to gain +an understanding of this scripting format, to take control of the engine in custom scenarios. + +This page describes the basic structure of GDS scripts, according to what we know now. This knowledge +was gained mostly from common-sense guesses about commands that were only used in specific scripts, +as well as a disassembly effort by [@ilonachan](https://github.com/ilonachan). The information here +*may* be limited to the European version of the game, but it seems unlikely the scripting language +would have been changed significantly between regional releases. + +This knowledge gives important insights about how **GDA** (which may stand for **G**ame **D**ata **A**ssembly), +a custom language designed to be an easy-to-read decompilation target for GDS, should be structured. + +## Structure + +In general, all scripts (after their 4-byte length header) are made up of *tokens* of specified type. +The type is defined in two bytes, though it never comes close to using that range: +all possible values (as far as we know so far) are: + +- `0` for executing commands, +- `1,2,3,4` for data parameters, +- `6,7` for jump addressing, +- `8,9,10` for logic operators, and +- `11` and `12` for loop behavior, +- `12` to end the script. + +`5` is technically handled in code, but never used and contains no data. All type values higher than 12 will cause the read function to fail and the program to behave unpredictably (including infinite loops). + +### Basic Commands + +A GDS script consists mostly of these: a type 0 token contains the command ID, and may be followed by multiple +type 1-4 parameter tokens; how many are read is determined by the command's logic itself. + +Nearly all of these commands have a static parameter list with fixed meaning, and these commands and their parameters are +defined for programmatic purposes by the YAML files in this directory (split into categories of relevance, because many +commands are only implemented in certain engine contexts). + +Certain commands have a more variable parameter list or parsing behavior than the regular ones: these commands are documented in YAML, but their parameter list is specially handled in [the gds parser module](../../formats/gds.py). Specific ones used for control flow are described further down. + +Commands may also modify a game-internal conditional flag, which will be described further below. + +### Data parameters + +The following are all the data type parameters handled: + +- `1`: a 4-byte integer +- `2`: a 4-byte floating point number +- `3`: a string, beginning with a 2-byte length, and then that many 1-byte "ASCII" characters (plus null terminator) +- `4`: unknown, but also read as a string + +Each command decides which types of data it will read; it will *not* check if the read parameter actually corresponds to that +type, so specifying incorrect types will cause unpredictable error behavior. + +### Conditional Jump Instructions + +At the core these are also implemented as type-0 instructions, but their "parameter" fetching logic differs significantly. +The most common example is the instruction ensemble `0x12 jumpUnless`, `0x17 jumpIfNotArrived`, and `0x16 jumpChainUnless`. +These would probably be more aptly described as `if`, `else` and `else if` (I haven't 100% confirmed that, but their behavior and use imply this). + +To define jump targets, the following tokens are used: + +- `6`: a 4-byte jump location within the binary script file. Always seems to point to the beginning of a type-7 token. +- `7`: a 4-byte jump source address within the binary file. Seems to always be the target of a single jump instruction, + and holds the address of its source. While this doesn't seem strictly required by the jump routine, + it makes reading and debugging scripts easier, and that's likely why they never bothered to remove it. + +`if` and `else if` are obviously not very useful without any conditions to check; for this purpose, a list of type-0 commands (with arguments) are executed right after them, and the conditional flag they returned is incorporated into the boolean expression (note that not all instructions modify the conditional flag, the ones that do are marked in our analysis files). +The conditional chain ends once a type-6 token with a jump address is encountered. + +To define complex conditional chains, the following flag tokens can be interspersed with instruction executions: + +- `8`: All following commands' condition output is treated as negated. + Note that while you'd think this flag would only apply to the next command, or be reset by toggling it, + this is not the case! Once set the flag persists, and there's no way to unset it (within the same conditional). + They probably never bothered to check this, because I only see this used to negate single conditions. +- `9`: Changes the mode of how multiple commands' flags are combined: ANDs their results. +- `10`: Changes the mode of how multiple commands' flags are combined: ORs their results. + +When writing your own scripts, note the following pitfalls: +- If no conditions are provided, `0x12` will always jump (because the default condition value is false) +- If neither flag 9 or 10 are specified, the result of a new instruction's condition will always overwrite all previous ones, + until either one is specified. +- The 8 flag may seem like a one-shot or toggle, but internally it actually can't be reset! Once used, all future instruction + results will be negated, and there is no easy way around this. Plan your conditionals around this fact, or nest if blocks! +- Using a command that does not produce a conditional output will simply leave the previous one untouched. This is fairly useless + because there is no short-circuiting, and contitional instructions normally don't modify state to prevent a command from running before the `if` block entirely. + But if you choose to do this, keep in mind that the previously negated condition will be negated again! In DISCARD mode this would undo the negate flag's effect (but that's pretty useless because ideally you're only in DISCARD mode in the first instruction), and in AND/OR modes it fixes the value to `false`/`true` respectively (which is also very useless). Therefore, ideally only condition instructions should be used in conditions. + +### Loop Instructions + +The instructions `0x14 repeatN` and `0x15 while` are assumed to be loops (hence why I named them that). `repeatN` takes a single number parameter for the amount of times to loop, and then simply ignores all tokens until it finds a jump address; `while` consumes a conditional in the same way as `if`, and re-executes it on each loop. + +The loop will execute commands until: +1. the GDS instruction pointer exceeds the loop exit address, in which case it will be moved back to the start of the loop (checking conditions) +2. it encounters a type-`11` token (a flag with no data), in which case it will break the loop by jumping to its end (in hex this type is recognizable as `0xb` for **b**reak) +3. it encounters a type-`12` token (a flag with no data), in which case it will abort the loop but leave the program counter as + it is. This is not equivalent to the `continue` of most programming languages, rather saying "we don't want to loop anymore, + but finish this final iteration". + This token is also always found at the end of a script, and I'm still looking for instances outside of that, meaning there is a chance this is simply a failsafe to make sure the loop doesn't continue beyond the script buffer. However, if this behavior seems useful, it doesn't look like using it would have any negative side effects. +4. the next instruction would be `0x4f return`, in which case the loop manually handles to respect the early return. + +## Future Work: A Turing Machine in Professor Layton 1 + +Most script engines only provide a limited number of flags to freely set and access, but maybe this is enough to do a fun proof-of-concept turing machine design. \ No newline at end of file diff --git a/data/gds_commands/logic.yml b/data/gds_commands/logic.yml new file mode 100644 index 0000000..78294fd --- /dev/null +++ b/data/gds_commands/logic.yml @@ -0,0 +1,45 @@ +prefix: null +commands: + if: + id: 0x12 + aliases: + - jumpUnless + desc: Executes a boolean expression consisting of instructions and logic flags, and jumps away if the condition is false (which is the default if no condition is provided). + complex: true + elif: + id: 0x16 + aliases: + - jumpUnlessTargetAnd + desc: If reached through regular program flow, always jumps. If reached from a `jump` instruction, executes the logic of `jump`. + complex: true + else: + id: 0x17 + aliases: + - jumpUnlessTarget + desc: If reached through regular program flow, always jumps. If reached from a `jump` instruction, simply continues. + repeatN: + id: 0x14 + desc: Takes an loop count (int) and a jump target (addr), and loops the instructions up to that target location that many times. + complex: true + while: + id: 0x15 + desc: Takes a boolean expression like `if`, as well as a jump target (addr), and loops the instructions up to that target location until the condition evaluates to false or a break token is encountered. + complex: true + + "true": + id: 0x19 + condition: true + desc: Always returns true + "false": + id: 0x1a + condition: false + desc: Always returns false + + return: + id: 0x4f + desc: Aborts the script by setting the instruction pointer past its end. + + 0x18: + condition: true + desc: Might have been a debug condition, because the logic to determine its value is just a function that returns false. Probably got compiled out. + \ No newline at end of file diff --git a/data/gds_commands/minigames/dog.yml b/data/gds_commands/minigames/dog.yml new file mode 100644 index 0000000..822dcae --- /dev/null +++ b/data/gds_commands/minigames/dog.yml @@ -0,0 +1,10 @@ +commands: + partdef: + id: 0xc2 + params: + - int + - int + - int + - int + - int + - int \ No newline at end of file diff --git a/data/gds_commands/minigames/hotel.yml b/data/gds_commands/minigames/hotel.yml new file mode 100644 index 0000000..7201fb2 --- /dev/null +++ b/data/gds_commands/minigames/hotel.yml @@ -0,0 +1,86 @@ +commands: + + item: + id: 0xcf + desc: Defines an inn item's name + params: + itemId: int + name: string + + itemLayton: + id: 0xbc + params: + itemId: int + xPos: int + yPos: int + unk_4: int + unk_5: int + unk_6: int + unk_7: int + unk_8: int + unk_9: int + itemComment1Layton: + id: 0xc6 + desc: > + The comment from Layton when placing the given item in his room. Must be called right after an + `0xbc itemLayton` command, and refers to that item. + params: + comment: string + itemComment2Layton: + id: 0xc7 + desc: > + The comment from Layton when placing the given item in his room, given that the specified item ID + is already in there. Used to point out duplications, as well as two things going together well. + Like `0xc6 itemComment1Layton`, has to follow after a `0xbc itemLayton` call. + params: + comment: string + otherItem: int + + itemLuke: + id: 0xbd + params: + - int + - int + - int + - int + - int + - int + - int + - int + - int + itemComment1Luke: + id: 0xc8 + desc: > + The comment from Luke when placing the given item in his room. Must be called right after an + `0xbd itemLuke` command, and refers to that item. + params: + comment: string + itemComment2Luke: + id: 0xc9 + desc: > + The comment from Luke when placing the given item in his room, given that the specified item ID + is already in there. Used to point out duplications, as well as two things going together well. + Like `0xc8 itemComment1Like`, has to follow after a `0xbd itemLuke` call. + params: + comment: string + otherItem: int + + + roomCommentLayton: + id: 0xca + desc: The comment from Layton, expressing his overall satisfaction with the room layout. + params: + comment: string + level: int + unused: + type: int + desc: always 0 + roomCommentLuke: + id: 0xcb + desc: The comment from Luke, expressing his overall satisfaction with the room layout. + params: + comment: string + level: int + unused: + type: int + desc: always 0 \ No newline at end of file diff --git a/data/gds_commands/puzzle/buttonlist.yml b/data/gds_commands/puzzle/buttonlist.yml new file mode 100644 index 0000000..9e2d536 --- /dev/null +++ b/data/gds_commands/puzzle/buttonlist.yml @@ -0,0 +1,14 @@ +commands: + count: + id: 0x1d + desc: Defines how many buttons there should be in the list. Note that the engine entirely controls button placement. + params: + count: + type: int + correct: + id: 0x1e + desc: Defines which of the buttons is the correct one. + params: + btnId: + type: int + desc: The index of the button that should be the correct answer. \ No newline at end of file diff --git a/data/gds_commands/puzzle/coin.yml b/data/gds_commands/puzzle/coin.yml new file mode 100644 index 0000000..c348058 --- /dev/null +++ b/data/gds_commands/puzzle/coin.yml @@ -0,0 +1,20 @@ +commands: + 0x25: + params: + - type: int + - type: int + - type: int + 0x26: + params: + - type: int + - type: int + 0x83: + params: + - int + - int + - int + 0x84: + params: + - int + - int + - int \ No newline at end of file diff --git a/data/gds_commands/puzzle/common.yml b/data/gds_commands/puzzle/common.yml new file mode 100644 index 0000000..be17408 --- /dev/null +++ b/data/gds_commands/puzzle/common.yml @@ -0,0 +1,208 @@ +prefix: "puzzle" +commands: + setCurrent: + id: 0x48 + desc: | + Sets the current puzzle ID in the global game state. If at the end of script execution the + global "current script type" is set to "question", the puzzle with this ID will be started. + params: + puzzleId: + type: int + isCurrentNotSolved: + id: 0x4c + condition: true + desc: Checks if the most recently (perhaps currently) attempted puzzle has NOT been solved + isCurrentSolved: + id: 0x4d + condition: true + desc: Checks if the most recently (perhaps currently) attempted puzzle has been solved + + isSolved: + id: 0x54 + condition: true + desc: Checks if the specified puzzle ID has been solved (puzzle flag 2) + params: + puzzleId: int + countEnoughSolved: + id: 0x77 + condition: true + desc: > + Counts how many puzzles the player has solved, and returns true if that number is greater + than or equal to the given threshold. + params: + threshold: int + + engine: + id: 0x1b + desc: Loads a puzzle engine with the specified name, and continues the script in that context. + params: + engineName: + type: string + desc: the name of the engine to load + imgWin: + id: 0x1f + desc: Selects the image to be displayed when the puzzle is done, probably. + uncertain: true + params: + id: + type: int + desc: seems to always be the current puzzle ID + unk_2: + type: int + desc: seems to always be 0 + slideItemGoalTile: + id: 0x38 + desc: For puzzles that involve sliding objects into correct locations (slide, drag, shape (unused)) define the location that would be an object's goal. + params: + xPos: + type: int + yPos: + type: int + drawLineColor: + id: 0x42 + desc: | + Many puzzle types give the user the ability to draw lines freehand or between points; this command sets the RGB color of those lines (with 5bit channel precision). + + A full list of those puzzle types: trace, traceButton, drawInput2, cut, drag. + params: + r: + type: int + g: + type: int + b: + type: int + + button: + id: 0x5d + desc: For the puzzle types OnOff and FreeButton, this adds a button with the specified sprite texture at the given location. + params: + xPos: + type: int + yPos: + type: int + spriteName: + type: string + desc: The sprite used for the button. As with all button sprites, it should define a depressed state in the animation "shadow". + correct: + type: bool|int + desc: Whether this button should be (de)pressed in order for the puzzle to solve. + unk_5: + type: int + + placeTarget: + id: 0x5e + desc: The sole defining command for the PlaceTarget puzzle type. + params: + unk_1: + type: int + unk_2: + type: int + unk_3: + type: string + unk_4: + type: float + unk_5: + type: int + + gridOffset: + id: 0xa3 + desc: For grid-based puzzles, determines the offset in pixels that grid should have on the screen. + params: + xPos: int + yPos: int + gridDimensions: + id: 0xa4 + desc: For grid-based puzzles, determines the dimensions of the grid in tiles. This also initializes the grid. + params: + width: int + height: int + gridTileSize: + id: 0xa5 + desc: For grid-based puzzles, determines the size in pixels that each grid cell should have. + + + title: + id: 0xba + desc: > + Used in the technical script/puzzletitle/?/qtitle.gds, to determine the localized name + of all puzzles at once. This script is read, for example, by the Puzzle Index and Wifi Puzzle list. + params: + puzzleId: + type: int + desc: the ID of the puzzle to set the name for + name: string + picarat: + id: 0xc3 + desc: Defines how many picarat are awarded for each attempt. Stored in script/pcarot/pscript.gds + params: + puzzleId: int + first: + type: int + desc: The amount awarded solving on first try + second: + type: int + desc: The amount awarded after one error + other: + type: int + desc: The amount awarded after the second failed attempt, on all subsequent tries + indexInfo: + id: 0xdc + desc: Loads information for the puzzle index + params: + puzzleId: int + type: + type: string + desc: There is no fixed list of options, but the length may not exceed 64 characters. + location: + type: string + desc: There is no fixed list of options, but the length may not exceed 64 characters. + + + 0x20: + params: + - type: int + 0x27: + desc: Used by the match, scale, coin and cut puzzle types. + params: + - type: int + 0x2b: + desc: Used by the match and tileRotate puzzle types, so it's likely a way to define the initial positions for movable and rotatable pieces. + params: + - type: int + - type: int + - type: float + - type: float + - type: float + # ok so apparently it can happen that an instruction checks if the next token + # is of the type it expects, and if not it uses a default value instead. + optional: true + default: 30 + 0x39: + desc: For puzzles that involve tiles in a grid (tile, tileRotate, shape (unused)) do ??? + params: + unk_1: + type: float + + 0x49: + condition: true + desc: checks some kind of flag for the current puzzle (puzzle flags mask 1) + 0x58: + condition: true + desc: checks some kind of flag for the current puzzle (puzzle flags mask 1); for regular puzzles at least. + 0x4e: + condition: true + desc: checks some kind of flag for the current puzzle (puzzle flags mask 4) + 0xb7: + desc: sets all the flags for the given puzzle ID to true (1, 4, and 2 ie "solved") + params: + puzzleId: int + + 0xaa: + desc: used by all the tile-based puzzles + params: [int] + + + 0x00: + desc: DO NOT USE! The function pointer slot for this command is never populated, trying to call it will probably crash! + 0xff: + desc: DO NOT USE! The function pointer slot for this command is never populated, trying to call it will probably crash! \ No newline at end of file diff --git a/data/gds_commands/puzzle/connect.yml b/data/gds_commands/puzzle/connect.yml new file mode 100644 index 0000000..d17db3c --- /dev/null +++ b/data/gds_commands/puzzle/connect.yml @@ -0,0 +1,12 @@ +commands: + 0x28: + params: + - type: int + - type: int + - type: int + - type: int + 0x29: + params: + - type: int + - type: int + - type: int \ No newline at end of file diff --git a/data/gds_commands/puzzle/cup.yml b/data/gds_commands/puzzle/cup.yml new file mode 100644 index 0000000..aa18b1c --- /dev/null +++ b/data/gds_commands/puzzle/cup.yml @@ -0,0 +1,18 @@ +commands: + cup: + id: 0x3a + desc: Adds a new cup object. + params: + - type: string + - type: int + - type: int + - type: int + - type: int + - type: int + color: + id: 0xbf + desc: Sets the color of the liquid + params: + r: int + g: int + b: int \ No newline at end of file diff --git a/data/gds_commands/puzzle/cut.yml b/data/gds_commands/puzzle/cut.yml new file mode 100644 index 0000000..93269bb --- /dev/null +++ b/data/gds_commands/puzzle/cut.yml @@ -0,0 +1,40 @@ +commands: + gridFillType: + id: 0x9f + params: + xPos: int + yPos: int + width: int + height: int + value: int + gridSetType3: + id: 0xa0 + desc: Sets the type of the given cell in the grid to 3 (whatever that means). In the CutPuzzle this + means a source/target for cut lines. + params: + xPos: int + yPos: int + solutionCut: + id: 0xa1 + desc: Adds a cut between two grid(!) points to the solution. + params: + xPos1: int + yPos1: int + xPos2: int + yPos2: int + + 0xa2: + 0xac: + desc: sets the color of the cut line once it's done + uncertain: true + param: + - int + - int + - int + 0xad: + desc: sets the color of the cut line while it's still being drawn + uncertain: true + param: + - int + - int + - int \ No newline at end of file diff --git a/data/gds_commands/puzzle/drag.yml b/data/gds_commands/puzzle/drag.yml new file mode 100644 index 0000000..f04e4a7 --- /dev/null +++ b/data/gds_commands/puzzle/drag.yml @@ -0,0 +1,3 @@ +commands: + 0xf2: + params: [float] \ No newline at end of file diff --git a/data/gds_commands/puzzle/drawinput.yml b/data/gds_commands/puzzle/drawinput.yml new file mode 100644 index 0000000..c7ebea6 --- /dev/null +++ b/data/gds_commands/puzzle/drawinput.yml @@ -0,0 +1,38 @@ +commands: + setup: + id: 0xa8 + desc: Sets up the type and amount of draw inputs there should be + params: + inputType: int + numInputs: int + + 0x24: + params: + - type: int + - type: int + - type: int + 0xa7: + params: + - int + - string + 0xa9: + params: + - int + - string + 0xb9: + params: + inputId: int + unk_2: + type: int + desc: possibly the target value of the puzzle solution + 0xf5: + desc: seems to reset the inputs? + 0xfa: + desc: only in type2 + params: [int, int, int, int] + 0xfb: + desc: only in type2 + params: [int, string] + 0xfc: + desc: setInputBg? only in type2 + params: [string] \ No newline at end of file diff --git a/data/gds_commands/puzzle/match.yml b/data/gds_commands/puzzle/match.yml new file mode 100644 index 0000000..462d35e --- /dev/null +++ b/data/gds_commands/puzzle/match.yml @@ -0,0 +1,6 @@ +commands: + 0x2a: + params: + - type: int + - type: int + - type: float \ No newline at end of file diff --git a/data/gds_commands/puzzle/queen.yml b/data/gds_commands/puzzle/queen.yml new file mode 100644 index 0000000..1c82e4c --- /dev/null +++ b/data/gds_commands/puzzle/queen.yml @@ -0,0 +1,36 @@ +commands: + defineBoard: + id: 0x3b + desc: Defines base parameters for the overall board of the Queen puzzle. + params: + screenX: + type: int + desc: The pixel offset of the top left corner of the grid from the left edge of the screen + screenY: + type: int + desc: The pixel offset of the top left corner of the grid from the top edge of the screen + boardSize: + type: int + desc: The size of the board in cells + queenCount: + id: 0x3c + desc: Sets the number of queens the player needs to place + params: + count: + type: int + fixedQueen: + id: 0x3d + desc: Places an immovable (golden) queen on the board in the specified tile location + params: + tileX: + type: int + tileY: + type: int + partialMode: + id: 0x3e + desc: "Determines the \"partial mode\" setting, which is used in 'Too Many Queens 3': there are less queens than tile width, but they still need to block the entire grid." + params: + enable: + # bool is one special datatype that must be converted into another type before saving, and is just separate for convenience in GDA scripts. + type: bool|int + desc: Set to true in order to make a puzzle with the ruleset of 'Too Many Queens 3' diff --git a/data/gds_commands/puzzle/rivercross.yml b/data/gds_commands/puzzle/rivercross.yml new file mode 100644 index 0000000..720badf --- /dev/null +++ b/data/gds_commands/puzzle/rivercross.yml @@ -0,0 +1,34 @@ +commands: + chicken: + id: 0x31 + desc: Adds a chicken to the puzzle's play field. + params: + xPos: int + yPos: int + wolf: + id: 0x32 + desc: Adds a wolf to the puzzle's play field. + params: + xPos: int + yPos: int + man: + id: 0x89 + desc: Adds a man to the puzzle's play field. + params: + xPos: int + yPos: int + cabbage: + id: 0x8a + desc: Adds a man to the puzzle's play field. + params: + xPos: int + yPos: int + sheep: + id: 0x8b + desc: Adds a man to the puzzle's play field. + params: + xPos: int + yPos: int + + 0x8c: + params: [int] \ No newline at end of file diff --git a/data/gds_commands/puzzle/scale.yml b/data/gds_commands/puzzle/scale.yml new file mode 100644 index 0000000..a103013 --- /dev/null +++ b/data/gds_commands/puzzle/scale.yml @@ -0,0 +1,11 @@ +commands: + 0x2d: + params: + - type: int + 0x2e: + params: [] + 0x2f: + params: [] + 0x30: + params: [] + \ No newline at end of file diff --git a/data/gds_commands/puzzle/shape.yml b/data/gds_commands/puzzle/shape.yml new file mode 100644 index 0000000..f0d27fd --- /dev/null +++ b/data/gds_commands/puzzle/shape.yml @@ -0,0 +1,29 @@ +# This puzzle type seems to be unused, which means I can't infer anything from usage. +commands: + shape: + id: 0x33 + desc: Adds a shape object (max 8) + point: + id: 0x34 + desc: Adds a point to the last created shape object (max 16) + params: + xPos: + type: int + desc: x-position of the point + yPos: + type: int + desc: y-position of the point (orientation unclear) + + + 0x35: + params: + - type: int + - type: int + - type: int + 0x36: + params: + - type: int + - type: int + 0x37: + params: + - type: float \ No newline at end of file diff --git a/data/gds_commands/puzzle/slide.yml b/data/gds_commands/puzzle/slide.yml new file mode 100644 index 0000000..304a690 --- /dev/null +++ b/data/gds_commands/puzzle/slide.yml @@ -0,0 +1,21 @@ +commands: + item: + id: 0xa6 + desc: Adds a slidable item into the grid + params: + xPos: int + yPos: int + unk_3: int + imgName: + type: string + desc: The "animation" name representing this item's image. Sourced from either ani/slidepuzzle.spr or ani/slidepuzzle2.spr + tileGroup: + id: 0xb1 + params: + xPos: int + yPos: int + group: int + itemGroup: + id: 0xb2 + params: + group: int \ No newline at end of file diff --git a/data/gds_commands/puzzle/tile.yml b/data/gds_commands/puzzle/tile.yml new file mode 100644 index 0000000..a396f5c --- /dev/null +++ b/data/gds_commands/puzzle/tile.yml @@ -0,0 +1,46 @@ +commands: + tile: + id: 0x73 + desc: Adds a tile to the puzzle field. A reset-linked slot is automatically created. + params: + xPos: int + yPos: int + spriteName: string + animId: + type: int + slot: + id: 0x74 + desc: Adds a slot to the puzzle field; tiles can only be dropped in slots. + params: + xPos: int + yPos: int + assign: + id: 0x75 + desc: > + Assigns the given tile ID (in declaration order) to the given slot ID. To solve the puzzle + it is checked whether one of these assignments is satisfied; note particularly that you can add + multiple, which will result in the puzzle having multiple solutions. + params: + tileId: int + slotId: int + rotButton: + id: 0xb5 + desc: > + For the non-rotate tile puzzles, there may be buttons on the side to rotate the pieces + (there's actually only one example of this in the entire game, Puzzle #116p112) + params: + xPos: int + yPos: int + width: int + height: int + + + 0x76: + desc: Most likely sets the solution count for a tile puzzle. + params: + - int + 0xbb: + desc: pushes a truth value into a list + params: [bool] + 0xbe: + desc: sets some field to 0 \ No newline at end of file diff --git a/data/gds_commands/puzzle/trace.yml b/data/gds_commands/puzzle/trace.yml new file mode 100644 index 0000000..0e04077 --- /dev/null +++ b/data/gds_commands/puzzle/trace.yml @@ -0,0 +1,42 @@ +commands: + traceCenter: + id: 0x3f + desc: Places the centerpoint of the shape to be traced. I'm not yet sure what the significance of that is, because it seems very lax. + params: + xPos: + type: int + yPos: + type: int + inPoint: + id: 0x40 + desc: Adds an Inpoint, a point that needs to be inside the traced shape for the puzzle to complete. + params: + xPos: + type: int + yPos: + type: int + outPoint: + id: 0x41 + desc: Adds an Outpoint, a point that mustn't be inside the traced shape for the puzzle to complete. + params: + xPos: + type: int + yPos: + type: int + + + circleAnswer: + id: 0xd4 + desc: In the TraceButton (ie Circle Answer) puzzles, used to define a possible circle target + params: + xPos: int + yPos: int + radius?: float + unk_4: bool + + 0xd0: + params: + - int + - int + - int + - int \ No newline at end of file diff --git a/data/gds_commands/room.yml b/data/gds_commands/room.yml new file mode 100644 index 0000000..a542d6d --- /dev/null +++ b/data/gds_commands/room.yml @@ -0,0 +1,259 @@ +commands: + isCurrent: + id: 0x13 + condition: true + desc: Returns true if the engine's current room is equal to the provided ID. Can be used anywhere. + params: + roomId: + type: int + desc: the room ID to be tested against + + setCurrent: + id: 0x53 + desc: > + Sets the engine's current room ID to the specified value. This takes effect after the script + completes, the next time the script type is set to 'Event'. + params: + roomId: + type: int + + exit: + id: 0x22 + desc: Defines an exit the user can click on to change rooms. + params: + exitType: + type: int + desc: Value from 0-5. Not sure what zero does, but 5 is a door in the scene, and 1-4 stands for the screen edges starting left ccw. + xPos: + type: int + desc: x position of the sprite + yPos: + type: int + desc: y position of the sprite + unk_4: + type: int + unk_5: + type: int + exitRoom: + type: int + desc: Value from 0-255. Sets the room the player will move to when they take this exit + unk_7: + type: int + unk_8: + type: int + exitSound: + id: 0x5b + desc: Defines which footsteps sound is used on room transition + params: + exitId: + type: int + desc: the ID of the exit (in declaration order) + soundId: + type: int + desc: Can be any number, but only 1 (echoy footsteps) and 2 (creaky door) have special handling + + sprite: + id: 0x23 + desc: > + Defines a tiny sprite used in the room, from the "room_%i_%i.arc" files. Never actually used, + but the sprites look like sweet cut content. + uncertain: true + params: + xPos: + type: int + yPos: + type: int + bgsprite: + id: 0x5c + desc: Adds a non-interactive background sprite to the room, for animated room background effects. + params: + xPos: + type: int + yPos: + type: int + spriteName: + type: string + desc: path to the sprite to be used. + + textObj: + id: 0x43 + desc: Defines a Text Object, i.e. an area of the screen where clicking on it opens a brief flavor text box from one of the characters. + params: + speaker: + type: int + desc: > + 1 for Layton, 2 for Luke. Probably corresponds to the frames in ani/room_tobjp.spr, + which also includes the hint coin, but doesn't include anything else; my assumption is + that using a number outside of these two (and maybe 3) will break something. It's worth + testing how adding frames to the sprite file will affect things... + # TODO: something really cool here would be compound datatypes + # ex: + # params: + # ... + # bbox: + # type: BBox(int xPos, int yPos, int width, int height) + # + # And in a GDA script this could be written as `BBox(1,2,3,4)` to be more readable. + xPos: + type: int + yPos: + type: int + width: + type: int + height: + type: int + textId: + type: int + desc: The id of the text to be displayed. They are sourced from room/tobj/?/tobj.pcm//t_%i.txt + unk_7: + type: int + desc: Appears unused in code, and seems to always be 1 + + npcObject: + id: 0x50 + desc: > + Adds an interactable object to the room. This is used for all NPCs, and some other + items such as the crumpled notes and newspapers. + params: + xPos: + type: int + yPos: + type: int + width: + type: int + height: + type: int + spriteId: + type: int + desc: The sprite that will be placed at the given location. It only becomes interactable within the defined bounding box. Sourced from ani/obj_%i.spr with default animation. + eventId: + type: int + desc: The event that will be triggered when the player taps on the object + + eventTrigger: + id: 0x21 + desc: > + Unused command that does something similar to `0x87 npcObject`, but without registering a sprite. + Internally this method is called by that one, because it is responsible for determining the popup + icon on tap (though this is hardcoded now to always be the exclamation point). + params: + xPos: + type: int + yPos: + type: int + width: + type: int + height: + type: int + spriteName: + type: string + desc: the name of the sprite container holding the sprite that should popup when the trigger is tapped + animName: + type: string + desc: Within the sprite container, the name of the animation to be displayed. + eventId: + type: int + desc: The event that will be started when the trigger is tapped + + hintcoin: + id: 0x68 + desc: Adds a hint coin to the room. + params: + coinId: + type: int + desc: Every hint coin has a unique ID, presumably to allow saving which ones you've collected. + xPos: + type: int + yPos: + type: int + width: + type: int + height: + type: int + unk_6: + type: int + desc: appears to be unused. + + + mapInfo: + id: 0x5a + desc: > + As far as I can tell, this is used to setup info on the top screen (& probably for the save + process). Can also be used from events to briefly change scenes. + params: + roomId: + type: int + desc: the current room. Mostly useful when called from event scripts. + mapId: + type: int + desc: the ID of the top screen map image to be used + unk_3: + type: int + unk_4: + type: int + unk_5: + type: int + unk_6: + type: int + + + nazobabaBottle: + id: 0xe4 + desc: Only used in the Nazobaba (Granny Riddleton) house. Almost certainly registers the bottles that puzzles may appear in. + params: + xPos: int + yPos: int + width: int + height: int + unused: int + nazobabaAwards: + id: 0xe5 + desc: > + Registers a puzzle in the puzzle index, and awards a puzzle's assigned item, when solved from Nazobaba house + (since there is no story event to give you that item normally). Value -1 means no item given in that category. All + these commands are placed in script/nazobaba/babascript.gds in a list, but only the command corresponding to the + current puzzle ID will actually execute. (Runs in Event context only) + uncertain: true + params: + puzzleId: int + hotelId: int + gizmoId: int + paintingId: int + nazobabaGiveAward: + id: 0xe6 + desc: Responsible for calling babascript.gds (see `0xe5 nazobabaAwards`) + nazobabaSend: + id: 0xe7 + uncertain: true + desc: | + Sends the specified puzzle to the Nazobaba house, if it wasn't already solved. Also writes all those puzzle names somewhere, but not their IDs. + It seems these are called in bulk in certain otherwise empty scripts, which in turn are probably called in spots where puzzles would vanish. + params: + puzzleId: int + + + 0x2c: + desc: sets some kind of flag at the given index + params: + - type: int + + 0xab: + params: + - int + + 0xd1: + params: + - int + - int + - int + - bool + - int + - int + 0xd2: + params: + - int + - int + - int + - bool + - int + - int \ No newline at end of file diff --git a/formats/gds.py b/formats/gds/__init__.py similarity index 88% rename from formats/gds.py rename to formats/gds/__init__.py index ad33094..5e365d0 100644 --- a/formats/gds.py +++ b/formats/gds/__init__.py @@ -6,6 +6,7 @@ import sys import collections +import struct import parse import ast from utils import cli_file_pairs, foreach_file_pair @@ -68,14 +69,14 @@ def from_gds(Self, file): params.append( { "type": "int", - "data": int.from_bytes(cmd_data[c + 2 : c + 6], "little"), + "data": struct.unpack('>f', cmd_data[c + 2 : c + 6])[0], } ) c += 6 elif p_type == 2: params.append( { - "type": "unknown-2", + "type": "float", "data": int.from_bytes(cmd_data[c + 2 : c + 6], "little"), } ) @@ -92,9 +93,10 @@ def from_gds(Self, file): ) c += str_len + 4 elif p_type == 6: + # address within the gds file range. usually fits in a short, but can be an int. params.append( { - "type": "unknown-6", + "type": "taddr", "data": int.from_bytes(cmd_data[c + 2 : c + 6], "little"), } ) @@ -102,22 +104,26 @@ def from_gds(Self, file): elif p_type == 7: params.append( { - "type": "unknown-7", + "type": "saddr", "data": int.from_bytes(cmd_data[c + 2 : c + 6], "little"), } ) c += 6 elif p_type == 8: - params.append({"type": "unknown-8"}) + params.append({"type": "not"}) c += 2 elif p_type == 9: - params.append({"type": "unknown-9"}) + params.append({"type": "and"}) + c += 2 + elif p_type == 0xA: + params.append({"type": "or"}) c += 2 elif p_type == 0xB: - params.append({"type": "unknown-b"}) + # break + params.append({"type": "break"}) c += 2 elif p_type == 0xC: - # cmd = hex(cmd) + # eof cmds.append({"command": cmd, "parameters": params}) break else: @@ -160,7 +166,7 @@ def from_gda(Self, file): # TODO: make this, so gds_old can be completely remov cmd = commands[cmd] if "alias" in cmd: cmd = commands[cmd["alias"]] - elif cmd.startswith("cmd_"): + elif cmd.startswith("cmd_") or cmd.startswith("unk_"): cmd = int(cmd[4:]) elif cmd.startswith("0x"): cmd = int(cmd[2:], base=16) @@ -173,20 +179,22 @@ def from_gda(Self, file): # TODO: make this, so gds_old can be completely remov if param.isdigit(): params.append({"type": "int", "data": int(param)}) elif param.startswith("0x"): - params.append({"type": "unknown-2", "data": int(param[2:], 16)}) + params.append({"type": "float", "data": float(param)}) elif param.startswith('"') and param.endswith('"'): param = ast.literal_eval(f'"{strings[int(param[1:-1])]}"') params.append({"type": "string", "data": param}) - elif param.startswith("!6("): - params.append({"type": "unknown-6", "data": int(param[3:-1], 16)}) - elif param.startswith("!7("): - params.append({"type": "unknown-7", "data": int(param[3:-1], 16)}) - elif param.startswith("!8"): - params.append({"type": "unknown-8"}) - elif param.startswith("!9"): - params.append({"type": "unknown-9"}) - elif param.startswith("!b"): - params.append({"type": "unknown-b"}) + elif param.startswith("@"): + params.append({"type": "taddr", "data": int(param[1:], 16)}) + elif param.startswith("$"): + params.append({"type": "saddr", "data": int(param[1:], 16)}) + elif param.startswith("NOT"): + params.append({"type": "not"}) + elif param.startswith("AND"): + params.append({"type": "and"}) + elif param.startswith("OR"): + params.append({"type": "or"}) + elif param.startswith("BREAK"): + params.append({"type": "break"}) else: raise Exception(f"Invalid GDA parameter: {param}") @@ -215,26 +223,28 @@ def to_gds(self): if param["type"] == "int": out += b"\x01\x00" out += param["data"].to_bytes(4, "little") - elif param["type"] == "unknown-2": + elif param["type"] == "float": out += b"\x02\x00" - out += param["data"].to_bytes(4, "little") + out += struct.pack('>f', param["data"]) elif param["type"] == "string": out += b"\x03\x00" out += (len(param["data"]) + 1).to_bytes(2, "little") out += ( param["data"].encode("ASCII") + b"\x00" ) # TODO: JP/KO compatibility - elif param["type"] == "unknown-6": + elif param["type"] == "taddr": out += b"\x06\x00" out += param["data"].to_bytes(4, "little") - elif param["type"] == "unknown-7": + elif param["type"] == "saddr": out += b"\x07\x00" out += param["data"].to_bytes(4, "little") - elif param["type"] == "unknown-8": + elif param["type"] == "not": out += b"\x08\x00" - elif param["type"] == "unknown-9": + elif param["type"] == "and": out += b"\x09\x00" - elif param["type"] == "unknown-b": + elif param["type"] == "or": + out += b"\x0a\x00" + elif param["type"] == "break": out += b"\x0b\x00" else: raise Exception( @@ -261,20 +271,20 @@ def to_gda(self): out += str(param["data"]) elif param["type"] == "string": out += repr(param["data"]) - elif param["type"] == "unknown-2": - out += hex(param["data"]) - elif param["type"] == "unknown-6": - b = param["data"].to_bytes(4, "little") - out += f"!6({b.hex()})" - elif param["type"] == "unknown-7": - b = param["data"].to_bytes(4, "little") - out += f"!7({b.hex()})" - elif param["type"] == "unknown-8": - out += "!8" - elif param["type"] == "unknown-9": - out += "!9" - elif param["type"] == "unknown-b": - out += "!b" + elif param["type"] == "float": + out += str(param["data"]) + elif param["type"] == "taddr": + out += f"@{param['data'].hex()}" + elif param["type"] == "saddr": + out += f"${param['data'].hex()}" + elif param["type"] == "not": + out += "NOT" + elif param["type"] == "and": + out += "AND" + elif param["type"] == "or": + out += "OR" + elif param["type"] == "break": + out += "BREAK" else: raise Exception( f"GDA error: invalid or unsupported parameter type '{param['type']}'!" diff --git a/formats/gds/_test/sub/subby.yaml b/formats/gds/_test/sub/subby.yaml new file mode 100644 index 0000000..3db6426 --- /dev/null +++ b/formats/gds/_test/sub/subby.yaml @@ -0,0 +1,7 @@ +version: 0.1 +prefix: "nope." + +commands: + hi: + id: 0x24 + desc: hi \ No newline at end of file diff --git a/formats/gds/_test/test.yml b/formats/gds/_test/test.yml new file mode 100644 index 0000000..e511dfb --- /dev/null +++ b/formats/gds/_test/test.yml @@ -0,0 +1,39 @@ +version: 0.1 +prefix: null +context: all + +commands: + 0x01: + name: test + desc: Test Command + condition: false + uncertain: false + params: + - int + - type: int + name: testparam + desc: some number + uncertain: false + test2: + id: 0x02 + desc: Test Command 2 + context: ["event", "puzzle"] + aliases: + - tc2 + condition: true + uncertain: true + params: + unk_1: string + unk_2: + type: int + uncertain: true + optional: true + +groups: + ouchie: + commands: + nice: + id: 0x69 + desc: nice + 0x74: + 0x75: \ No newline at end of file diff --git a/formats/gds/cmddef.py b/formats/gds/cmddef.py new file mode 100755 index 0000000..1583dd4 --- /dev/null +++ b/formats/gds/cmddef.py @@ -0,0 +1,389 @@ +#!/usr/bin/env python3 + +import yaml +import os +from dataclasses import dataclass, field +from dacite import from_dict +from typing import Optional, List, Union + +import logging + +logger = logging.getLogger("flora.debug.gds.cmddef") + + +@dataclass +class GDSCommandParam: + cmd: int + """The command for which this is a parameter""" + idx: int + """The index of the parameter in order""" + type: str + """ + The data type of the command. Real builtin GDS datatypes are: + int (1) + float (2) + string (3) + longstr (4, never used) + + Our GDA framework also defines some auxiliary datatypes: + bool: backed by either an int (true = nonzero) or a string (true = "true") + bool|int: backed by an int (true = nonzero + + TODO: Compound datatypes may make the code more understandable, by combining semantically + related parameters into a single one: + pt(x: int, y: int) + rect(x: int, y: int, w: int, h: int) + ... + """ + name: Optional[str] = None + """A descriptive name for the parameter. Can optionally be written in the script.""" + desc: Optional[str] = None + """ + Description of what the parameter means. + """ + uncertain: bool = False + """ + Purely informative: whether research is conclusive about the meaning of the parameter. + The fact that the parameter exists is NOT uncertain, but its meaning may not be clear yet. + Parameters without a name are normally uncertain, even when they don't mark it explicitly. + """ + optional: bool = False + """ + A small number of commands have optional parameters at the end of their argument list: + they check if the next token (see general info about GDS) has the expected type, and if + not stops reading the argument list. + """ + + +@dataclass +class GDSCommand: + id: int + """ID of the command, as it appears in binary scripts""" + + name: Optional[str] = None + """Human-readable name, as it would appear in the decompiled assembly""" + aliases: list[str] = field(default_factory=lambda: []) + """ + Other names this command may be recognized by when compiling scripts, + for example a naming convention from a previous decompiler version + """ + + desc: Optional[str] = None + """ + Description of what the command does, where it's used etc. + """ + uncertain: bool = False + """ + Purely informative: whether the research is conclusive about the meaning, purpose, functionality + etc of this command. Note that the structural properties below are obvious to read off from + disassembly, and are therefore not subject to this; it purely qualifies our understanding + of the command's meaning and usage. + """ + + condition: bool = False + """ + True if the command sets the condition flag. Those commands usually don't have side effects. + """ + context: list[str] = field(default_factory=lambda: ["all"]) + """ + (list of) context name(s) where the command is defined. The game contains many different + script engines called in different contexts, and while they all run the same type of script, + most commands are noops in engines they weren't meant to be used in. + + The special string "all" denotes that a command is implemented everywhere (for example, + if it doesn't proxy through the script engine itself but directly to the sound system etc.) + """ + + params: list[GDSCommandParam] = field(default_factory=lambda: []) + """ + List of parameters the command accepts, if it's a simple command with a fixed parameter list. + """ + complex: bool = False + """ + Set to true for special structural commands that don't have a fixed parameter list, + but instead need to be parsed in a more complex way. The logic for that will be looked + up in Python code. + """ + + file: Optional[str] = None + """ + The file path where the command is defined; for debug purposes + """ + + +CONTEXTS = ["all", "event", "room", "puzzle"] + + +def load_group( + data: dict, + group_name: str = None, + parent_prefix: str = None, + parent_context: Union[str, List[str]] = None, + filename: str = None, +) -> List[GDSCommand]: # sourcery skip: remove-pass-body + """ + Reads a single group object from the command definition YAML files. That group might be the top-level file itself. + """ + if "prefix" in data: + prefix = data["prefix"] + else: + prefix = parent_prefix or "" + if group_name is not None: + prefix += f"{group_name}." + + context = data["context"] if "context" in data else parent_context + commands = data["commands"] + if isinstance(commands, dict): + + def command_kv(k, v): + if v is None: + v = {} + cid = v.get("id") + name = v.get("name") + if isinstance(k, int): + if cid is not None: + if k != cid: + raise ValueError( + "id field specified in a GDSCommand listed by id, and the two values didn't match" + ) + logger.warning( + "id field specified in a GDSCommand listed by id is redundant" + ) + cid = k + elif isinstance(k, str): + if name is not None: + if k != name: + raise ValueError( + "name field specified in a GDSCommand listed by name, and the two values didn't match" + ) + logger.warning( + "name field specified in a GDSCommand listed by name is redundant" + ) + # TODO: handle hex ID in string? + name = k + else: + raise ValueError( + "GDSCommands can only be listed by their name or numeric id" + ) + v["id"] = cid + v["name"] = name + return v + + commands = [command_kv(k, v) for k, v in commands.items()] + + def process_command(c): + if c.get("id") is None: + raise ValueError("GDSCommand must define an id") + + if "context" not in c: + c["context"] = context or "all" + if isinstance(c["context"], str): + c["context"] = [c["context"]] + if "prefix" not in c: + c["prefix"] = prefix + if c["prefix"] is not None and c["name"] is not None: + c["name"] = c["prefix"] + c["name"] + + params = c.get("params") or [] + if isinstance(params, dict): + + def param_kv(k, v): + # have to do that here already, otherwise I don't have any object to put the name into + if isinstance(v, str): + v = {"type": v} + if isinstance(k, int): + # specifying a number here means we don't want a name + pass + else: + if "name" in v: + # pylint: disable=logging-not-lazy + logger.warning( + "name field specified in a parameter that was listed by name" + + (", and did not match" if k != v["name"] else "") + ) + + v["name"] = k + return v + + params = [param_kv(k, v) for k, v in params.items()] + + def process_param(idx, p): + if isinstance(p, str): + p = {"type": p} + p["cmd"] = c["id"] + p["idx"] = idx + + return from_dict(data_class=GDSCommandParam, data=p) + + c["params"] = [process_param(i, p) for i, p in enumerate(params)] + + c["file"] = filename + + return from_dict(data_class=GDSCommand, data=c) + + commands = [process_command(v) for v in commands] + + groups = data.get("groups") + if isinstance(groups, dict): + for k, v in groups.items(): + commands.extend(load_group(v, k, prefix, context, filename)) + elif groups is not None: + logger.warning( + "groups key has to be a dictionary mapping a group name to a command list" + ) + + return commands + + +def load_file(filename: str, root: str) -> list[GDSCommand]: + """ + Reads a single command definition YAML file. Default prefix is the directory structure relative to the root, if specified. + """ + with open(filename, encoding="utf8") as f: + data = yaml.safe_load(f) + + # version = data.get("version") + + if root is not None: + cur = os.path.splitext(os.path.relpath(filename, root))[0] + cur, group_name = os.path.split(cur) + parent_prefix = "" + while cur != "": + cur, seg = os.path.split(cur) + parent_prefix = f"{seg}.{parent_prefix}" + else: + parent_prefix = None + + return load_group(data, group_name, parent_prefix, None, filename) + + +def load_cmdrepo(root: str) -> list[GDSCommand]: + """ + Reads all the command definitions from the YAML files in the specified directory, and returns them as a list of data objects. + Only very basic sanity checking internal to each command definition is performed by this point. + """ + commands = [] + for r, _dirs, files in os.walk(root): + for f in files: + if f.lower().endswith("yml") or f.lower().endswith("yaml"): + path = os.path.join(r, f) + commands.extend(load_file(path, root)) + return commands + + +def build_maps( + cmds: list[GDSCommand], +) -> tuple[dict[int, GDSCommand], dict[str, GDSCommand]]: + """ + Structures a list of commands into a bidirectional lookup table by command ID and human-readable name. + """ + by_id = {} + by_name = {} + + for cmd in cmds: + if cmd.id in by_id: + raise ValueError(f"Command id {hex(cmd.id)} defined twice") + by_id[cmd.id] = cmd + + if cmd.name in by_name: + existing = by_name[cmd.name] + if not isinstance(existing, GDSCommand): + other = '", "'.join(c.name for c in existing["ALIAS_CONFLICT"]) + logger.warning( + 'Command name "%s" is already used as alias by commands "%s". ' + "The former will take precedence. Please make sure to update any GDA files decompiled " + "with a previous version of Flora.", + cmd.name, + other, + ) + elif existing.name != cmd.name: + logger.warning( + 'Command "%s" has alias "%s", which is also a command name. ' + "The latter will take precedence. Please make sure to update any GDA files decompiled " + "with a previous version of Flora.", + existing.name, + cmd.name, + ) + else: + raise ValueError(f'Command name "{cmd.name}" defined twice') + + if cmd.name is None: + continue + by_name[cmd.name] = cmd + for alias in cmd.aliases: + if alias in by_name: + existing = by_name[alias] + if not isinstance(existing, GDSCommand): + existing["ALIAS_CONFLICT"].append(cmd) + elif existing.name == alias: + logger.warning( + 'Command "%s" has alias "%s", which is also a command name. ' + "The latter will take precedence. Please make sure to update any GDA files decompiled " + "with a previous version of Flora.", + cmd.name, + alias, + ) + else: + by_name[alias] = {"ALIAS_CONFLICT": [existing, cmd]} + else: + by_name[alias] = cmd + + for k, v in by_name.items(): + if not isinstance(v, GDSCommand): + other = '", "'.join(c.name for c in existing["ALIAS_CONFLICT"]) + logger.warning( + 'Commands "{other}" all define alias "%s"; none of them will be registered.', + k, + ) + del by_name[k] + + return (by_id, by_name) + + +COMMANDS_BYNAME = {} +COMMANDS_BYID = {} + +CMDDEF_ROOT = "data/gds_commands/" + + +def init_commands(root: str = None): + global COMMANDS_BYID, COMMANDS_BYNAME # pylint: disable=global-statement + if root is None: + root = CMDDEF_ROOT + try: + commands = load_cmdrepo(root) + COMMANDS_BYID, COMMANDS_BYNAME = build_maps(commands) + except ValueError as e: + logger.error(e) + print( + "An error occured when reading command definitions from metadata. Your installation of Flora might be corrupted. If you're sure it's not, " + "you can file an issue on GitHub: https://github.com/patataofcourse/Flora" + ) + return + + if undefined_commands := [i for i in range(0x100) if i not in COMMANDS_BYID]: + logger.warning( + "The definition for the following commands is missing:\n%s\nThis may cause significant soundness issues in the (de)compiler!" + "Please make sure to correctly identify and define the parameter counts for these commands.", + ", ".join(hex(i) for i in undefined_commands), + ) + print( + "An error occured when reading command definitions from metadata. Your installation of Flora might be corrupted. If you're sure it's not, " + "you can file an issue on GitHub: https://github.com/patataofcourse/Flora" + ) + + +# sourcery skip: hoist-statement-from-if +if __name__ == "__main__": + logging.basicConfig() + + # TODO: unit tests just to make sure this function's understanding of syntax is still correct + # initGlobalCommands("formats/gds/_test") + init_commands() + + import pprint + + pprint.pp(COMMANDS_BYID) + pprint.pp(COMMANDS_BYNAME) +else: + init_commands() \ No newline at end of file diff --git a/formats/gds/gds.py b/formats/gds/gds.py new file mode 100644 index 0000000..73e2030 --- /dev/null +++ b/formats/gds/gds.py @@ -0,0 +1,220 @@ +from typing import Tuple, List +import struct + +from .model import ( + GDSProgram, + GDSToken, + GDSElement, + GDSInvocation, + GDSLoopInvocation, + GDSIfInvocation, + GDSValue, + GDSAddress, + GDSConditionToken +) +from .cmddef import COMMANDS_BYID, GDSCommand + +from tagged_union import _ +from tagged_union import match + + +def read_condition(data: bytes, cursor: int) -> Tuple[int, List[GDSConditionToken], GDSAddress]: + return (cursor, [], GDSAddress(1234)) + + +def read_if(data: bytes, cursor: int, cmdobj: GDSCommand) -> Tuple[int, GDSIfInvocation]: + cursor, cond, addr = read_condition(data, cursor) + + return (cursor, GDSIfInvocation(command=cmdobj, condition=cond, target_addr=addr, args = [])) + + +def read_elif(data: bytes, cursor: int, cmdobj: GDSCommand) -> Tuple[int, GDSIfInvocation]: + data, res = read_if(data, cursor, cmdobj) + res.elseif = True + return (cursor, res) + + +def read_repeatN(data: bytes, cursor: int, cmdobj: GDSCommand) -> Tuple[int, GDSLoopInvocation]: + cursor, cntt = read_token(data, cursor) + if cntt not in GDSToken.int: + raise ValueError( + f"Unexpected parameter token type: should have been int, token was {arg}" + ) + cnt = cntt() + + cursor, saddrt = read_token(data, cursor) + while saddrt not in GDSToken.saddr: + cursor, saddrt = read_token(data, cursor) + if saddrt in GDSToken.fileend: + raise ValueError("repeatN: encountered EOF looking for jump address") + addr = saddrt() + + return (cursor, GDSLoopInvocation(command=cmdobj, condition=cnt, target_addr=GDSAddress(addr), args=[])) + + +def read_while(data: bytes, cursor: int, cmdobj: GDSCommand) -> Tuple[int, GDSLoopInvocation]: + cursor, cond, addr = read_condition(data, cursor) + + return (cursor, GDSLoopInvocation(command=cmdobj, condition=cond, target_addr=addr, args=[])) + + +def read_simple( + data: bytes, cursor: int, cmdobj: GDSCommand +) -> Tuple[int, GDSInvocation]: + args = [] + for param in cmdobj.params: + cursor, arg = read_token(data, cursor) + val, reqtypes = match( + arg, + { + GDSToken.int: lambda val: ( + GDSValue.int(val), + ["int", "bool", "bool|int"], + ), + GDSToken.float: lambda val: (GDSValue.float(val), ["float"]), + GDSToken.str: lambda val: (GDSValue.str(val), ["string", "bool"]), + GDSToken.longstr: lambda val: (GDSValue.longstr(val), ["longstr"]), + _: lambda: (None, []), + }, + ) + if param.type not in reqtypes: + raise ValueError( + f"Unexpected parameter token type: should have been {param.type}, token was {arg}" + ) + + args.append(val) + return (cursor, GDSInvocation(command=cmdobj, args=args)) + + +READ_COMPLEX = { + "if": read_if, + "elif": read_elif, + "repeatN": read_repeatN, + "while": read_while, +} + + +def read_command(data: bytes, cursor: int, token: GDSToken) -> Tuple[int, GDSInvocation]: + if token not in GDSToken.command: + raise ValueError("Expected instruction") + cmdid = token() + cmdobj = COMMANDS_BYID.get(cmdid) + if cmdobj is None: + raise ValueError(f"Command {cmdid} not defined") + + if cmdobj.complex: + return ( + READ_COMPLEX.get(cmdobj.name) or READ_COMPLEX.get(cmdobj.id) + )(data, cursor) + + args = [] + for param in cmdobj.params: + cursor, arg = read_token(data, cursor) + val, reqtypes = match( + arg, + { + GDSToken.int: lambda val: ( + GDSValue.int(val), + ["int", "bool", "bool|int"], + ), + GDSToken.float: lambda val: ( + GDSValue.float(val), + ["float"], + ), + GDSToken.str: lambda val: ( + GDSValue.str(val), + ["string", "bool"], + ), + GDSToken.longstr: lambda val: ( + GDSValue.longstr(val), + ["longstr"], + ), + _: lambda: (None, []), + }, + ) + if param.type not in reqtypes: + raise ValueError( + f"Unexpected parameter token type: should have been {param.type}, token was {arg}" + ) + + args.append(val) + return GDSInvocation(command=cmdobj, args=args) + + + + +def read_token(data: bytes, cursor: int) -> Tuple[int, GDSToken]: + p_type = int.from_bytes(data[cursor : cursor + 2], "little") + if p_type == 0: + cmd = int.from_bytes(data[cursor + 2 : cursor + 4], "little") + return (cursor + 4, GDSToken.command(cmd)) + if p_type == 1: + val = int.from_bytes(data[cursor + 2 : cursor + 6], "little") + return (cursor + 6, GDSToken.int(val)) + if p_type == 2: + val = struct.unpack(">f", data[cursor + 2 : cursor + 6]) + return (cursor + 6, GDSToken.float(val)) + if p_type == 3: + str_len = int.from_bytes(data[cursor + 2 : cursor + 4], "little") + if str_len > 64: + print( + "WARN: string literal is too long (max is 64 bytes); this may lead to errors in the game." + ) + val = ( + data[cursor + 4 : cursor + 4 + str_len] + .decode("ascii") # TODO: JP/KO compatibility + .rstrip("\x00") + ) + return (cursor + 4 + str_len, GDSToken.str(val)) + if p_type == 4: + str_len = int.from_bytes(data[cursor + 2 : cursor + 4], "little") + val = ( + data[cursor + 4 : cursor + 4 + str_len] + .decode("ascii") # TODO: JP/KO compatibility + .rstrip("\x00") + ) + return (cursor + 4 + str_len, GDSToken.longstr(val)) + if p_type == 6: + addr = int.from_bytes(data[cursor + 2 : cursor + 6], "little") + return (cursor + 6, GDSToken.saddr(addr)) + if p_type == 7: + addr = int.from_bytes(data[cursor + 2 : cursor + 6], "little") + return (cursor + 6, GDSToken.taddr(addr)) + if p_type == 8: + return (cursor + 2, GDSToken.NOT()) + if p_type == 9: + return (cursor + 2, GDSToken.AND()) + if p_type == 10: + return (cursor + 2, GDSToken.OR()) + if p_type == 11: + return (cursor + 2, GDSToken.BREAK()) + if p_type == 12: + return (cursor + 2, GDSToken.fileend()) + if p_type == 5: + return (cursor + 2, GDSToken.unused5()) + raise ValueError("Invalid GDS token type") + + +def read_gds(data: bytes, path: str = None) -> GDSProgram: + length = int.from_bytes(data[:4], "little") + cursor = 4 + cmds = [] + + cursor, cur_token = read_token(data, cursor) + # TODO + while cur_token not in GDSToken.fileend: + if cur_token in GDSToken.command: + cursor, cmd = read_command(data, cursor, cur_token) + cmds.append(cmd) + elif cur_token in GDSToken.taddr: + cmds.append( + GDSElement.target_label(GDSJump(source=cur_token(), target=cursor - 4)) + ) + elif cur_token in GDSToken.BREAK: + cmds.append(GDSElement.BREAK()) + else: + raise ValueError("Unexpected token type") + + cursor, cur_token = read_token(data, cursor) + + return GDSProgram(context="all", path=path, tokens=cmds) diff --git a/formats/gds/model.py b/formats/gds/model.py new file mode 100644 index 0000000..c1e99a7 --- /dev/null +++ b/formats/gds/model.py @@ -0,0 +1,82 @@ +from typing import List, Optional, NewType, Union +from dataclasses import dataclass, field +# pylint: disable=unused-wildcard-import,wildcard-import +from tagged_union import * + +from .cmddef import GDSCommand + + +GDSAddress = NewType('GDSAddress', int) + + +@dataclass +class GDSInvocation: + command: GDSCommand + args: List["GDSValue"] + + +@tagged_union +class GDSConditionToken: + command = GDSInvocation + NOT = Unit + AND = Unit + OR = Unit + + +@dataclass +class GDSIfInvocation(GDSInvocation): + condition: List[GDSConditionToken] + elseif: bool = False + target_addr: GDSAddress + + +@dataclass +class GDSLoopInvocation(GDSInvocation): + condition: Union[List[GDSConditionToken] | int] + target_addr: GDSAddress + + +@tagged_union +class GDSValue: + int = int + float = float + str = str + longstr = str + + +@dataclass +class GDSJump: + source: GDSAddress + target: GDSAddress + + +@tagged_union +class GDSElement: + command = GDSInvocation + target_label = GDSJump + BREAK = Unit + + +@dataclass +class GDSProgram: + context: str + path: Optional[str] = None + tokens: List[GDSElement] = field(default_factory=list) + + +@tagged_union +class GDSToken: + command = int + int = int + float = float + str = str + longstr = str + unused5 = Unit + # the source address is the one pointing to the target + saddr = GDSAddress + taddr = GDSAddress + NOT = Unit + AND = Unit + OR = Unit + BREAK = Unit + fileend = Unit \ No newline at end of file From f925d366fa307937022f9d92d19fac48c4410b96 Mon Sep 17 00:00:00 2001 From: ilonachan Date: Thu, 3 Oct 2024 21:01:04 +0200 Subject: [PATCH 06/17] doc(gds): added GDA spec, improved model --- doc/gds/gda.md | 144 +++++++++++++++ data/gds_commands/help.md => doc/gds/gds.md | 0 formats/gds/cmddef.py | 12 +- formats/gds/model.py | 193 ++++++++++++++++---- 4 files changed, 307 insertions(+), 42 deletions(-) create mode 100644 doc/gds/gda.md rename data/gds_commands/help.md => doc/gds/gds.md (100%) diff --git a/doc/gds/gda.md b/doc/gds/gda.md new file mode 100644 index 0000000..1bc6c2c --- /dev/null +++ b/doc/gds/gda.md @@ -0,0 +1,144 @@ +# GDA: a human-friendly decompilation of GDS + +With how significant [GDS](./gds.md) is for controlling this game, obviously modders would be interested in modifying or writing these script files themselves. But that's pretty difficult to do with a binary format that has location-sensitive jump markers... of course we could build a visual editor, but that's extremely +out of scope for Flora. + +Incidentally, this problem is identical to the one of writing actual real-world executable programs in machine code. And the easiest solution people came up with was **Assembly**: a slightly more human-readable format, where instructions weren't just numbers but had understandable (or at least decipherable) names, and crucially, label and jump locations would be assembled automatically. While most of us write programs in high-level, sometimes even interpreted languages, Assembly is also still widely used! + +And so it only makes sense that, in order to make both the reading and the creation of GDS scripts easier, Flora should include a (dis)assembler. The special dialect we use is called **GDA**, which may stand for **G**ame **D**ata **A**ssembly. This doc describes our specification of the format's syntax. + +## Structure + +The GDA specification and compiler were both written by the Flora devs, based on knowledge from the decompilation of the game and its scripts. It's a hopefully good attempt at a readable scripting language matching the feature set of GDS. + +A GDA script consists of a few elements: + +- An (optional but recommended) version comment at the beginning +- Instructions with parameters +- Jump labels referred to by the jump instructions +- Optional comments + +### Comments + +At any point in the file, in a new line or after an instruction in the same line, you may use the `#` character to have GDA ignore the entire rest of the line. This is used purely for your own documentation purposes. Unfortunately when re-decompiling a GDS file, the compiler doesn't yet have a way to keep these comments in place. + +A special comment of the form `#!version 1.0` can be placed as the first non-empty line of a GDA file, to mark the version of the language that the script was written in. This is useful for the compiler, because it allows disambiguating commands whose names have changed across Flora/GDA versions (due to improved understanding and research). + +### Instructions + +An instruction is referred to by its name, which comes from a predefined list of commands. Since the instructions in GDS are named neither in the data files nor in the assembly, we took our best guess to name them descriptively and accurately. But as our understanding evolves (or mods add new capabilities to GDS), these names may change over time. + +You can ask Flora to print help for a certain command (by name or hex id), or list all known name (and parameter) assignments. + +### Parameters + +There are four types of parameter values usable in a script: +- Integers, which are just written as regular numbers. Negative numbers are allowed, and sometimes used as sentinel values (markers that denote a special case and are not meant to be interpreted as a number, usually -1). Remember that the value of these numbers needs to fit into 4 bytes, which means it must be in the (inclusive) range from -2^31 to 2^31-1 (alternatively, if you know the number is treated as unsigned positive, you can go from 0 to 2^32-1). + > TODO: many commands have a far smaller number range, this should be denoted in their type! + + You are also allowed to write them in different bases, by using the prefixes `0x` for hex, and `0b` for binary. We don't support writing numbers in octal, what the hell. (This means you can add leading zeroes without worrying.) +- Floats, which are also written as numbers, but which contain a decimal point or are written in scientific notation. If you want to force an integer to be a float, just write it like `1234.` (the decimal part is implied zero). Likewise you can omit the integer part for numbers like `.5`. Scientific notation works like it does in most languages, by separating the mantisse and exponent with an `e`. +- Strings, which may contain up to 63 standard ASCII characters, and are enclosed in either single or double quotes. Common escape sequences like line breaks (`\n`), arbitrary ASCII (`\x12`) or escaping the quotes (`\"`) or escape character (`\\`) are supported; but keep in mind that this is separate from the escape mechanisms some instructions use internally (for example, "break page" or "change expression" commands inlined in dialogue text). Backslash escaping is understood and processed only by the GDA compiler. + > Technical note: If you want to know why the max length isn't a power of two when 64 is right there: that's because the buffer itself is 64 bytes long, but must keep room for a single null byte at the end of the string. GDA automatically takes care of this for you. +- Long Strings, which work exactly like regular Strings, but may be (practically) as long as you wish! ...they're also completely unused in the game, and you can't use them in places where a regular string is required (maybe someone could make a mod that makes more or even all GDS commands accept them in place of strings). The only reason they even exist seems to be because `0x21 eventTrigger`'s filename seems to have been very long at some point, but it is now simply a format string derived in code (and also completely unused). If you still want to use one, just prepend `l` to the front of a regular string literal `l'like so'`. + > TODO: Check other language versions of this game. Perhaps the Asian versions needed more string length since their character set is larger. + +### Jump Labels + +A jump label denotes a location in the GDS binary file, right before the next instruction after it. They are used for jumps, and physically represented in code as a number pointing to the location that jumps to them (there is only one at all times). + +You can define one (or more) jump labels by name in a location in GDA code, by beginning the line with an `@` symbol and writing space-separated label names: +``` +@JumpLoc_1 JumpLoc2 +print "this is the next instruction after the jump" +``` + +Referencing a label is equally simple, by just writing `@` before its label name. These references can be used as another type of function parameter: +``` +jump @Label +print "This gets skipped" +@Label +print "This gets printed" +``` + +The GDA compiler will take care of converting these names into their actual binary file locations for you. + +In the degenerate case that a jump address doesn't correspond to a physical label at its target location (which seems possible, since that is technically completely unnecessary), this syntax wouldn't produce identical results as its input... So when defining a jump label, you can prepend `!` to (each of) the names, indicating that no label should be created in the binary file. This is mostly for completeness, and you shouldn't ever need to do this in your own scripts, so it's best if you don't. + +### `if` statements + +In practice there are only a handful of statements which accept an address as a parameter, and all of them do it via special handling: `if`, `elif`, `else`, `repeatN` and `while`. If handled naïvely as described above, the if statement would look roughly like this: + +``` +if @else_1 + +@else_1 +elif @else_2 + +@else_2 +else @endif_1 + +@endif + +``` + +But that's unnecessarily cluttered, so instead there's special syntax for if blocks that more resembles how other languages do it: + +``` +if { + +} elif { + +} else { + +} + +``` + +In detail, the behavior of the commands is as follows: +- `if` will evaluate a condition, and jump if it is `false` (if there is no condition, that's the default). If after jumping it encounters an `elif` as the next instruction, it'll evaluate that condition too and again jump if it is false. If it encounters an `else` as the next instruction, it will always jump to the location it specifies. +- `elif` and `else`, when encountered outside the context of an `if` instruction being executed, will simply discard all tokens before the jump address and always jump there. + +Because of the way these functions play together, none of them can always be used as a pure jump instruction without side effects; but the compiler may make an effort to let you know when that did happen. + +It is important to note that while all scripts in the base game have well-structured `if` blocks, this may not necessarily be the case for third-party scripts. The main problems arise when non-condition instructions, in particular other jumps, are placed inside a condition block: +- An `if` statement may jump out of the condition block, which is technically fine, but can't be represented with the nice block syntax. In that case we fall back to the basic syntax. +- Similarly, a `loop` may have its end outside of the condition block, and while this will almost certainly crash the game, the basic syntax of the loop section below can be used as fallback. + +> TODO: A big problem happens when the condition of an `elif` statement contains a jump: when reached from `if` the condition will be evaluated and the next address read, but when encountered regularly the first address (of the inner if) will be used instead! This actually can't be represented at all, I think... We could solve this by simply refusing to (de)compile files that are so badly malformed, but we need to detect it. + +### loop statements + +Similarly, loops in their simplest form would be written like this: +``` +while : @endwhile_1 +# or +#repeatN : @endwhile_1 + +@endwhile_1 + +``` + +And likewise, in regular cases this might be simplified to: +``` +while { +# or +#repeatN { + +} + +``` + +It's possible that there isn't actually a degenerate case here, since the loop statement seems to only really be usable for its intended purpose. The only possible edge case is when the loop doesn't end with a physical jump label, which shouldn't matter and is addressed by using `@!endwhile_1` as described above. + +## Compiler + +The Compiler and Decompiler are part of Flora, and executable using `flora gds (de)compile` respectively. Its goal is to produce a GDA script that, when reassembled into GDS, produces an identical binary. (The opposite is certainly impossible, since comments are lost, but the logic should obviously be identical.) + +If a text file doesn't seem to be a valid GDA script, the compiler will report (hopefully helpful) errors. It will *not* output a faulty binary file, or overwrite/delete existing GDS files with invalid ones. + +If a `.gds` file doesn't seem to be a valid script binary, the decompiler will report so. All GDS files in the base game are valid scripts, so if you encounter this while unpacking a fresh ROM, it may be corrupted. This is not guaranteed for third-party files, but all scripts compiled by Flora should be valid as well; if that is not the case, this is a bug you should report to us. + +### Versioning + +The current version of the GDA language and compiler can be found using `flora gds -v`. This version mostly refers to the command list, of which instruction and parameter names may change in the future; these changes will be documented in the resource files, to ensure the compiler still understands what to do when encountering a script of an older version. It can only recognize this by the script's version comment, which is why you should always include it and match it to your current compiler version. (The decompiler does this automatically.) diff --git a/data/gds_commands/help.md b/doc/gds/gds.md similarity index 100% rename from data/gds_commands/help.md rename to doc/gds/gds.md diff --git a/formats/gds/cmddef.py b/formats/gds/cmddef.py index 1583dd4..2d23ffc 100755 --- a/formats/gds/cmddef.py +++ b/formats/gds/cmddef.py @@ -1,13 +1,15 @@ #!/usr/bin/env python3 -import yaml +from typing import Optional, List, Union, Set import os from dataclasses import dataclass, field -from dacite import from_dict -from typing import Optional, List, Union import logging +import yaml +from dacite import from_dict + + logger = logging.getLogger("flora.debug.gds.cmddef") @@ -84,7 +86,7 @@ class GDSCommand: """ True if the command sets the condition flag. Those commands usually don't have side effects. """ - context: list[str] = field(default_factory=lambda: ["all"]) + context: Set[str] = field(default_factory=lambda: {"all"}) """ (list of) context name(s) where the command is defined. The game contains many different script engines called in different contexts, and while they all run the same type of script, @@ -378,7 +380,7 @@ def init_commands(root: str = None): logging.basicConfig() # TODO: unit tests just to make sure this function's understanding of syntax is still correct - # initGlobalCommands("formats/gds/_test") + # init_commands("formats/gds/_test") init_commands() import pprint diff --git a/formats/gds/model.py b/formats/gds/model.py index c1e99a7..2fa8e97 100644 --- a/formats/gds/model.py +++ b/formats/gds/model.py @@ -1,82 +1,201 @@ -from typing import List, Optional, NewType, Union +from typing import List, Optional, NewType, Union, Set from dataclasses import dataclass, field # pylint: disable=unused-wildcard-import,wildcard-import from tagged_union import * from .cmddef import GDSCommand - GDSAddress = NewType('GDSAddress', int) +@tagged_union +class GDSValue: + """ + A value usable as a parameter in a GDSInvocation. + """ + int = int + float = float + str = str + longstr = str + @dataclass class GDSInvocation: + """ + The specific invocation of a GDS commmand, with the given parameter values. + """ command: GDSCommand - args: List["GDSValue"] + args: List[GDSValue] @tagged_union class GDSConditionToken: + """ + A token that appears in the condition for a flow statement. + """ command = GDSInvocation NOT = Unit AND = Unit OR = Unit +@dataclass +class GDSJumpTarget: + """ + A jump address used by flow instructions in GDS scripts. + """ + label: str + """ + The name of the label to be jumped to. + """ + + @dataclass class GDSIfInvocation(GDSInvocation): condition: List[GDSConditionToken] + block: Optional[List["GDSElement"]] elseif: bool = False - target_addr: GDSAddress + elze: bool = False + target: GDSJumpTarget @dataclass class GDSLoopInvocation(GDSInvocation): condition: Union[List[GDSConditionToken] | int] - target_addr: GDSAddress - - -@tagged_union -class GDSValue: - int = int - float = float - str = str - longstr = str + block: Optional[List["GDSElement"]] + target: GDSJumpTarget @dataclass -class GDSJump: - source: GDSAddress - target: GDSAddress +class GDSLabel: + """ + A target label from a GDS script file. + """ + name: str + """ + The name of the label + """ + present: bool = True + """ + Whether the label actually physically exists, or was inserted as the target + of a jump that didn't point to a target label + """ + loc: Optional[int] = None + """ + The location encoded in the physically stored label word. + Only set if it differs from what would be expected, i.e. the location + of a jump address pointing to the label. + """ @tagged_union class GDSElement: + """ + An entry in a GDS script file. + """ command = GDSInvocation - target_label = GDSJump + label = GDSLabel BREAK = Unit @dataclass -class GDSProgram: - context: str - path: Optional[str] = None - tokens: List[GDSElement] = field(default_factory=list) +class GDSContext: + """ + The context that was determined for a GDS script, either manually or by context narrowing. + + TODO: find the best way to represent conflicts + """ + manual_name: Optional[str] = None + """ + A manual context set on construction, which will never change on narrowing + """ + candidates: Set[str] = field(default_factory=lambda: ["all"]) + """ + A list of candidate contexts, if any are still applicable. This list will be empty if narrowing caused a conflict. + """ + conflicts: List[GDSCommand] = field(default_factory=list) + """ + Whenever an instruction does not match any of the candidate contexts, the offending command is recorded here, and its + context candidates added to the global candidate list. + """ + + def narrow(self, cmd: GDSCommand) -> bool: + """ + Checks if the given command is compatible with the currently assumed candidates, and narrows the list if so. + Otherwise, records the conflict and returns false. + """ + intersection = GDSContext.intersection([self.manual_name] if self.manual_name else self.candidates, cmd.context) + if not intersection: + self.conflicts.append(cmd) + if not self.manual_name: + self.candidates = GDSContext.union(self.candidates, cmd.context) + return False + if not self.manual_name: + self.candidates = intersection + return True + + @staticmethod + def intersection(n1: Union[str, Set[str]], n2: Union[str, Set[str]]) -> Set[str]: + if isinstance(n1, list): + res = set() + for n in n1: + res.update(GDSContext.intersection(n, n2)) + return res + if isinstance(n2, list): + res = set() + for n in n2: + res.update(GDSContext.intersection(n1, n)) + return res + + # sourcery skip: assign-if-exp, reintroduce-else + if n1 == n2: + return {n1} + if n2.startswith(f"{n1}."): + return {n2} + if n1.startswith(f"{n2}."): + return {n1} + return set() + + @staticmethod + def union(n1: Union[str, Set[str]], n2: Union[str, Set[str]]) -> Set[str]: + if isinstance(n1, list): + res = set() + for n in n1: + res.update(GDSContext.intersection(n, n2)) + return res + if isinstance(n2, list): + res = set() + for n in n2: + res.update(GDSContext.intersection(n1, n)) + return res + + # sourcery skip: assign-if-exp, reintroduce-else + if n1 == n2: + return {n1} + if n2.startswith(f"{n1}."): + return {n1} + if n1.startswith(f"{n2}."): + return {n2} + return set() -@tagged_union -class GDSToken: - command = int - int = int - float = float - str = str - longstr = str - unused5 = Unit - # the source address is the one pointing to the target - saddr = GDSAddress - taddr = GDSAddress - NOT = Unit - AND = Unit - OR = Unit - BREAK = Unit - fileend = Unit \ No newline at end of file +@dataclass +class GDSProgram: + """ + The program contained in a GDS script file. + """ + context: Optional[GDSContext] = None + """ + The execution context of the script, or any candidates for it. + """ + path: Optional[str] = None + """ + The file path this script was loaded from, if any. + """ + elements: List[GDSElement] = field(default_factory=list) + """ + The instructions or other flow elements in the script. + """ + labels: List[str] = field(default_factory=list) + """ + A list of all the labels present in the script. May not technically be necessary. + """ From 992c8c8d348423889767b0b0053711b498b46ab7 Mon Sep 17 00:00:00 2001 From: ilonachan Date: Thu, 3 Oct 2024 21:01:04 +0200 Subject: [PATCH 07/17] wip --- formats/gds/gds.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/formats/gds/gds.py b/formats/gds/gds.py index 73e2030..7af1bfd 100644 --- a/formats/gds/gds.py +++ b/formats/gds/gds.py @@ -1,9 +1,11 @@ from typing import Tuple, List import struct +# pylint: disable=unused-wildcard-import,wildcard-import +from tagged_union import * + from .model import ( GDSProgram, - GDSToken, GDSElement, GDSInvocation, GDSLoopInvocation, @@ -14,6 +16,30 @@ ) from .cmddef import COMMANDS_BYID, GDSCommand + + +@tagged_union +class GDSToken: + """ + Raw token + """ + command = int + int = int + float = float + str = str + longstr = str + unused5 = Unit + # the source address is the one pointing to the target + saddr = GDSAddress + taddr = GDSAddress + NOT = Unit + AND = Unit + OR = Unit + BREAK = Unit + fileend = Unit + + + from tagged_union import _ from tagged_union import match @@ -142,7 +168,6 @@ def read_command(data: bytes, cursor: int, token: GDSToken) -> Tuple[int, GDSInv - def read_token(data: bytes, cursor: int) -> Tuple[int, GDSToken]: p_type = int.from_bytes(data[cursor : cursor + 2], "little") if p_type == 0: From 4638ef7e9dbc73c863d864d3bdf77bb18877ae19 Mon Sep 17 00:00:00 2001 From: ilonachan Date: Thu, 3 Oct 2024 21:01:35 +0200 Subject: [PATCH 08/17] feat(gds): The decompiler works! Aside from some patches for mistakes in the original game scripts, it perfectly decompiles all scripts and reassembles then into the same binaries. --- data/.gitignore | 1 + data/gds_commands/event.yml | 13 +- data/gds_commands/logic.yml | 1 + data/gds_commands/puzzle/coin.yml | 1 - data/gds_commands/puzzle/common.yml | 16 +- data/gds_commands/puzzle/cut.yml | 4 +- data/gds_commands/room.yml | 2 - formats/__init__.py | 1 + formats/gds/__init__.py | 546 ------------------ formats/gds/cmddef.py | 9 +- formats/gds/gds.py | 820 ++++++++++++++++++++++------ formats/gds/model.py | 58 +- formats/gds/old.py | 541 ++++++++++++++++++ formats/gds/patch.py | 76 +++ formats/gds/test_cmddef.py | 0 formats/gds/test_gds.py | 126 +++++ main.py | 2 +- utils.py | 74 ++- 18 files changed, 1531 insertions(+), 760 deletions(-) create mode 100644 data/.gitignore create mode 100644 formats/gds/old.py create mode 100644 formats/gds/patch.py create mode 100644 formats/gds/test_cmddef.py create mode 100644 formats/gds/test_gds.py diff --git a/data/.gitignore b/data/.gitignore new file mode 100644 index 0000000..d5440b6 --- /dev/null +++ b/data/.gitignore @@ -0,0 +1 @@ +game_root/** \ No newline at end of file diff --git a/data/gds_commands/event.yml b/data/gds_commands/event.yml index d642215..d4ca3a0 100644 --- a/data/gds_commands/event.yml +++ b/data/gds_commands/event.yml @@ -4,7 +4,7 @@ commands: desc: > Sets the current event in global state. Once the current script is completed, and the script type is set to "Event", that script will be executed. - param: + params: eventId: type: int @@ -52,6 +52,15 @@ commands: textboxMiddle: id: 0xcd desc: Displays a text box for the specified on-screen actor, with no arrow pointing. + params: + textId: + type: int + desc: > + the text to display, in textbox format. This is sourced from + etext/?/e???/e_t.txt. + actorId: + type: int + desc: the id (in initialization order) of the actor sprite saying the text. textboxNoSpeaker: id: 0x9b @@ -228,6 +237,8 @@ commands: currentMovie: id: 0xd3 desc: Sets the current movie ID. Once the current script completes, if the current script state is set to "Movie", that movie will play. + params: + movieId: int choice3: diff --git a/data/gds_commands/logic.yml b/data/gds_commands/logic.yml index 78294fd..8be922c 100644 --- a/data/gds_commands/logic.yml +++ b/data/gds_commands/logic.yml @@ -17,6 +17,7 @@ commands: aliases: - jumpUnlessTarget desc: If reached through regular program flow, always jumps. If reached from a `jump` instruction, simply continues. + complex: true repeatN: id: 0x14 desc: Takes an loop count (int) and a jump target (addr), and loops the instructions up to that target location that many times. diff --git a/data/gds_commands/puzzle/coin.yml b/data/gds_commands/puzzle/coin.yml index c348058..931dfbf 100644 --- a/data/gds_commands/puzzle/coin.yml +++ b/data/gds_commands/puzzle/coin.yml @@ -3,7 +3,6 @@ commands: params: - type: int - type: int - - type: int 0x26: params: - type: int diff --git a/data/gds_commands/puzzle/common.yml b/data/gds_commands/puzzle/common.yml index be17408..041b948 100644 --- a/data/gds_commands/puzzle/common.yml +++ b/data/gds_commands/puzzle/common.yml @@ -1,4 +1,4 @@ -prefix: "puzzle" +prefix: "puzzle." commands: setCurrent: id: 0x48 @@ -119,6 +119,8 @@ commands: gridTileSize: id: 0xa5 desc: For grid-based puzzles, determines the size in pixels that each grid cell should have. + params: + sizePx: int title: @@ -171,12 +173,14 @@ commands: - type: int - type: int - type: float + # ok so apparently it can happen that an instruction checks if the next token + # is of the type it expects, and if not it uses a default value instead. - type: float + optional: true + default: 16.0 - type: float - # ok so apparently it can happen that an instruction checks if the next token - # is of the type it expects, and if not it uses a default value instead. optional: true - default: 30 + default: 30.0 0x39: desc: For puzzles that involve tiles in a grid (tile, tileRotate, shape (unused)) do ??? params: @@ -188,7 +192,9 @@ commands: desc: checks some kind of flag for the current puzzle (puzzle flags mask 1) 0x58: condition: true - desc: checks some kind of flag for the current puzzle (puzzle flags mask 1); for regular puzzles at least. + desc: checks some kind of flag for the given puzzle (puzzle flags mask 1); for regular puzzles at least. + params: + - int 0x4e: condition: true desc: checks some kind of flag for the current puzzle (puzzle flags mask 4) diff --git a/data/gds_commands/puzzle/cut.yml b/data/gds_commands/puzzle/cut.yml index 93269bb..66c55a0 100644 --- a/data/gds_commands/puzzle/cut.yml +++ b/data/gds_commands/puzzle/cut.yml @@ -27,14 +27,14 @@ commands: 0xac: desc: sets the color of the cut line once it's done uncertain: true - param: + params: - int - int - int 0xad: desc: sets the color of the cut line while it's still being drawn uncertain: true - param: + params: - int - int - int \ No newline at end of file diff --git a/data/gds_commands/room.yml b/data/gds_commands/room.yml index a542d6d..492135f 100644 --- a/data/gds_commands/room.yml +++ b/data/gds_commands/room.yml @@ -193,8 +193,6 @@ commands: type: int unk_5: type: int - unk_6: - type: int nazobabaBottle: diff --git a/formats/__init__.py b/formats/__init__.py index fca0634..134510f 100644 --- a/formats/__init__.py +++ b/formats/__init__.py @@ -1 +1,2 @@ from formats import gds, bg, pcm, puzzle, ndsrom, compression +from formats.gds import old \ No newline at end of file diff --git a/formats/gds/__init__.py b/formats/gds/__init__.py index 5e365d0..e69de29 100644 --- a/formats/gds/__init__.py +++ b/formats/gds/__init__.py @@ -1,546 +0,0 @@ -import click -import contextlib -import json -import yaml -import os -import sys -import collections - -import struct -import parse -import ast -from utils import cli_file_pairs, foreach_file_pair -from version import v - - -@click.group( - help="Script-like format, also used to store puzzle parameters.", options_metavar="" -) -def cli(): - pass - - -dir_path = "/".join( - os.path.dirname(os.path.realpath(__file__).replace("\\", "/")).split("/")[:-1] -) -commands = json.load(open(f"{dir_path}/data/commands.json", encoding="utf-8")) -commands_i = { - val["id"]: key for key, val in commands.items() if "id" in val -} # Inverted version of commands - - -class GDS: - def __init__(self, cmds=None): - if cmds is None: - cmds = [] - self.cmds = cmds - - @classmethod - def from_gds(Self, file): - length = int.from_bytes(file[0:4], "little") - if file[4:6] == b"\x0c\x00": - return Self([]) - cmd_data = file[6 : length + 4] - cmds = [] - - cmd = None - params = [] - c = 0 - while True: - if c >= length: - raise Exception( - "GDS file error: End of file reached with no 0xC command!" - ) - if cmd == None: - cmd = int.from_bytes(cmd_data[c : c + 2], "little") - if cmd in commands_i: - cmd = commands_i[cmd] - c += 2 - continue - - p_type = int.from_bytes(cmd_data[c : c + 2], "little") - - if p_type == 0: - cmds.append({"command": cmd, "parameters": params}) - cmd = None - params = [] - c += 2 - elif p_type == 1: - params.append( - { - "type": "int", - "data": struct.unpack('>f', cmd_data[c + 2 : c + 6])[0], - } - ) - c += 6 - elif p_type == 2: - params.append( - { - "type": "float", - "data": int.from_bytes(cmd_data[c + 2 : c + 6], "little"), - } - ) - c += 6 - elif p_type == 3: - str_len = int.from_bytes(cmd_data[c + 2 : c + 4], "little") - params.append( - { - "type": "string", - "data": cmd_data[c + 4 : c + 4 + str_len] - .decode("ascii") # TODO: JP/KO compatibility - .rstrip("\x00"), - } - ) - c += str_len + 4 - elif p_type == 6: - # address within the gds file range. usually fits in a short, but can be an int. - params.append( - { - "type": "taddr", - "data": int.from_bytes(cmd_data[c + 2 : c + 6], "little"), - } - ) - c += 6 - elif p_type == 7: - params.append( - { - "type": "saddr", - "data": int.from_bytes(cmd_data[c + 2 : c + 6], "little"), - } - ) - c += 6 - elif p_type == 8: - params.append({"type": "not"}) - c += 2 - elif p_type == 9: - params.append({"type": "and"}) - c += 2 - elif p_type == 0xA: - params.append({"type": "or"}) - c += 2 - elif p_type == 0xB: - # break - params.append({"type": "break"}) - c += 2 - elif p_type == 0xC: - # eof - cmds.append({"command": cmd, "parameters": params}) - break - else: - raise Exception( - f"GDS file error: Invalid or unsupported parameter type {hex(p_type)}!" - ) - - return Self(cmds) - - @classmethod - def from_json(Self, file): - cmds = json.loads(file)["data"] - # TODO: reject non-compatible json files - return Self(cmds) - - @classmethod - def from_yaml(Self, file): - cmds = yaml.safe_load(file)["data"] - # TODO: reject non-compatible yaml files - return Self(cmds) - - @classmethod - def from_gda(Self, file): # TODO: make this, so gds_old can be completely removed - cmds = [] - - for line in file.split("\n"): - line = line.strip() - if line.startswith("#"): - continue - if line == "": - continue - - line, strings = parse.remove_strings(line) - line = line.rstrip().split(" ") - cmd = line[0] - - # TODO: handle inline comments, maybe - - if cmd in commands: - cmd = commands[cmd] - if "alias" in cmd: - cmd = commands[cmd["alias"]] - elif cmd.startswith("cmd_") or cmd.startswith("unk_"): - cmd = int(cmd[4:]) - elif cmd.startswith("0x"): - cmd = int(cmd[2:], base=16) - else: - raise Exception(f"Unknown GDA command: {cmd}") - - params = [] - - for param in line[1:]: - if param.isdigit(): - params.append({"type": "int", "data": int(param)}) - elif param.startswith("0x"): - params.append({"type": "float", "data": float(param)}) - elif param.startswith('"') and param.endswith('"'): - param = ast.literal_eval(f'"{strings[int(param[1:-1])]}"') - params.append({"type": "string", "data": param}) - elif param.startswith("@"): - params.append({"type": "taddr", "data": int(param[1:], 16)}) - elif param.startswith("$"): - params.append({"type": "saddr", "data": int(param[1:], 16)}) - elif param.startswith("NOT"): - params.append({"type": "not"}) - elif param.startswith("AND"): - params.append({"type": "and"}) - elif param.startswith("OR"): - params.append({"type": "or"}) - elif param.startswith("BREAK"): - params.append({"type": "break"}) - else: - raise Exception(f"Invalid GDA parameter: {param}") - - cmds.append({"command": cmd, "parameters": params}) - - return Self(cmds) - - def __getitem__(self, index): - index = int(index) - return self.cmds[index] - - def to_json(self): - return json.dumps({"version": v, "data": self.cmds}, indent=4) - - def to_yaml(self): - return yaml.safe_dump({"version": v, "data": self.cmds}) - - def to_gds(self): - out = b"\x00" * 2 - for command in self.cmds: - if type(command["command"]) == int: - out += command["command"].to_bytes(2, "little") - else: - out += command["command"]["id"].to_bytes(2, "little") - for param in command["parameters"]: - if param["type"] == "int": - out += b"\x01\x00" - out += param["data"].to_bytes(4, "little") - elif param["type"] == "float": - out += b"\x02\x00" - out += struct.pack('>f', param["data"]) - elif param["type"] == "string": - out += b"\x03\x00" - out += (len(param["data"]) + 1).to_bytes(2, "little") - out += ( - param["data"].encode("ASCII") + b"\x00" - ) # TODO: JP/KO compatibility - elif param["type"] == "taddr": - out += b"\x06\x00" - out += param["data"].to_bytes(4, "little") - elif param["type"] == "saddr": - out += b"\x07\x00" - out += param["data"].to_bytes(4, "little") - elif param["type"] == "not": - out += b"\x08\x00" - elif param["type"] == "and": - out += b"\x09\x00" - elif param["type"] == "or": - out += b"\x0a\x00" - elif param["type"] == "break": - out += b"\x0b\x00" - else: - raise Exception( - f"GDS JSON error: Invalid or unsupported parameter type '{param['type']}'!" - ) - out += b"\x00\x00" - out = out[:-2] + b"\x0c\x00" - - return len(out).to_bytes(4, "little") + out - - def to_bin(self): # alias - return self.to_gds() - - def to_gda(self): - out = "" - for command in self.cmds: - if type(command["command"]) == int: - out += "0x" + command["command"].to_bytes(1, "little").hex() - else: - out += command["command"] - for param in command["parameters"]: - out += " " - if param["type"] == "int": - out += str(param["data"]) - elif param["type"] == "string": - out += repr(param["data"]) - elif param["type"] == "float": - out += str(param["data"]) - elif param["type"] == "taddr": - out += f"@{param['data'].hex()}" - elif param["type"] == "saddr": - out += f"${param['data'].hex()}" - elif param["type"] == "not": - out += "NOT" - elif param["type"] == "and": - out += "AND" - elif param["type"] == "or": - out += "OR" - elif param["type"] == "break": - out += "BREAK" - else: - raise Exception( - f"GDA error: invalid or unsupported parameter type '{param['type']}'!" - ) - out += "\n" - return out - - -@cli.command(name="compile", no_args_is_help=True) -@click.argument("input", required=False, type=click.Path(exists=True)) -@click.argument("output", required=False, type=click.Path(exists=False)) -@click.option( - "--recursive", - "-r", - is_flag=True, - help="Recurse into subdirectories of the input directory to find more applicable files.", -) -@click.option( - "--quiet", - "-q", - is_flag=True, - help="Suppress all output. By default, operations involving multiple files will show a progressbar.", -) -@click.option( - "--overwrite/--no-overwrite", - "-o/-O", - default=True, - help="Whether existing files should be overwritten. Default: true", -) -@click.option( - "--format", - "-f", - required=False, - default=None, - multiple=False, - help="The format of the input file. Will be inferred from the file ending or content if unset. " - "If multiple file types would compile to the same output (but may not necessarily have the same content), " - "specify this to disambigute. Possible values: gda, json, yaml", -) -def compile( - input=None, output=None, recursive=False, quiet=False, format=None, overwrite=None -): - """ - Compiles the human-readable script(s) at INPUT into the game's binary script files at OUTPUT. - - INPUT can be a single file or a directory (which obviously has to exist). In the latter case subfiles with the correct file ending will be processed. - If unset, defaults to the current working directory. - - The meaning of OUTPUT may depend on INPUT: - - If INPUT is a file, then OUTPUT is expected to be a file, unless it explicitly ends with a slash indicating a directory. - In this case, if unset OUTPUT will default to the INPUT filename with `.gds` exchanged/appended. - - Otherwise OUTPUT has to be a directory as well (or an error will be shown). - In this case, if unset OUTPUT will default to the INPUT directory (which may itself default to the current working directory). - - In the file-to-file case, the paths are explicitly used as they are. Otherwise, if multiple input files were collected, or OUTPUT is a directory, - an output path is inferred for each input file by exchanging the input format's file ending for, or otherwise appending the `.gds` file ending. - - In the case where INPUT is a directory, if no format is specified, this command will collect files of all compatible types. Note that this can lead - to situations where multiple files would compile to the same output (e.g. `test.json` and `test.gda` would both be candidates for `test.gds`); - this command will NOT make a choice in this case, and instead ask to explicitly specify the format to be used. - """ - - in_endings = [] - if format is None: - in_endings = [".gda", ".json", ".yaml", ".yml"] - elif format == "gda": - in_endings = [".gda"] - elif format == "json": - in_endings = [".json"] - elif format in ["yaml", "yml"]: - in_endings = [".yaml", ".yml"] - else: - raise Exception(f"Unsupported input format: '{format}'") - - def process(input, output): - inpath = input - input = open(inpath, "r", encoding="utf-8").read() - - format2 = format - if format2 is None: - if inpath.lower().endswith(".gda"): - format2 = "gda" - elif inpath.lower().endswith(".json"): - format2 = "json" - elif inpath.lower().endswith(".yml") or inpath.lower().endswith(".yaml"): - format2 = "yaml" - - gds = None - with contextlib.suppress(Exception): - if format2 == "gda": - gds = GDS.from_gda(input) - elif format2 == "json": - gds = GDS.from_json(input) - elif format2 in ["yaml", "yml"]: - gds = GDS.from_yaml(input) - - if gds is None: - if format2 is not None: - # TODO: should this abort instead? - print( - f"WARNING: Input file '{inpath}' did not have expected format '{format2}'", - file=sys.stderr, - ) - # format not specified and couldn't be inferred, or file turns out not to have the correct format - # => try all the formats & see which one works (only one should be possible) - for f in ["gda", "json", "yaml"]: - with contextlib.suppress(Exception): - if f == "gda": - gds = GDS.from_gda(input) - elif f == "json": - gds = GDS.from_json(input) - elif f == "yaml": - gds = GDS.from_yaml(input) - if gds is not None: - break - if gds is None: - raise Exception( - f"File '{inpath}' couldn't be read: not a known file format" - + (f" (expected '{format2}')" if format2 is not None else "") - ) - - output = open(output, "wb") - output.write(gds.to_bin()) - output.close() - - pairs = cli_file_pairs( - input, output, in_endings=in_endings, out_ending=".gds", recursive=recursive - ) - - duplicates = collections.defaultdict(list) - for ip, op in pairs: - duplicates[op].append(ip) - duplicates = {k: v for k, v in duplicates.items() if len(v) > 1} - if len(duplicates) > 0: - print( - f"ERROR: {len(duplicates)} {'files have' if len(duplicates) > 1 else 'file has'} multiple conflicting source files; please explicitly specify a format to determine which should be used.", - file=sys.stderr, - ) - for op, ips in duplicates.items(): - pathlist = ", ".join("'" + ip + "'" for ip in ips) - print(f"'{op}' could be compiled from {pathlist}", file=sys.stderr) - sys.exit(-1) - - if not overwrite: - new_pairs = [] - existing = [] - for ip, op in pairs: - if os.path.exists(op): - existing.append(op) - else: - new_pairs.append((ip, op)) - - if not quiet: - print(f"Skipping {len(existing)} existing output files.") - - pairs = new_pairs - - foreach_file_pair(pairs, process, quiet=quiet) - - -@cli.command(name="decompile", no_args_is_help=True) -@click.argument("input", required=False, type=click.Path(exists=True)) -@click.argument("output", required=False, type=click.Path(exists=False)) -@click.option( - "--recursive", - "-r", - is_flag=True, - help="Recurse into subdirectories of the input directory to find more applicable files.", -) -@click.option( - "--quiet", - "-q", - is_flag=True, - help="Suppress all output. By default, operations involving multiple files will show a progressbar.", -) -@click.option( - "--overwrite/--no-overwrite", - "-o/-O", - default=True, - help="Whether existing files should be overwritten. Default: true", -) -@click.option( - "--format", - "-f", - required=False, - multiple=False, - help="The format used for output. Possible values: gda (default), json, yaml", -) -def decompile( - input=None, output=None, recursive=False, quiet=False, format=None, overwrite=None -): - """ - Decompiles the GDS script(s) at INPUT into a human-readable text format at OUTPUT. - - INPUT can be a single file or a directory (which obviously has to exist). In the latter case subfiles with the correct file ending will be processed. - If unset, defaults to the current working directory. - - The meaning of OUTPUT may depend on INPUT: - - If INPUT is a file, then OUTPUT is expected to be a file, unless it explicitly ends with a slash indicating a directory. - In this case, if unset OUTPUT will default to the INPUT filename with `.json` exchanged/appended. - - Otherwise OUTPUT has to be a directory as well (or an error will be shown). - In this case, if unset OUTPUT will default to the INPUT directory (which may itself default to the current working directory). - - In the file-to-file case, the paths are explicitly used as they are. Otherwise, if multiple input files were collected, or OUTPUT is a directory, - an output path is inferred for each input file by exchanging the `.gds` file ending for `.json`, or otherwise appending the `.json` file ending. - """ - out_ending = "" - if format == "gda" or format is None: - out_ending = ".gda" - elif format == "json": - out_ending = ".json" - elif format in ["yaml", "yml"]: - out_ending = ".yml" - else: - raise Exception(f"Unsupported output format: '{format}'") - - def process(input, output): - input = open(input, "rb").read() - gds = GDS.from_gds(input) - - nonlocal format - if format is None: - if output.lower().endswith(".gda"): - format = "gda" - elif output.lower().endswith(".json"): - format = "json" - elif output.lower().endswith(".yml") or output.lower().endswith(".yaml"): - format = "yaml" - else: - print( - f"WARNING: output format couldn't be inferred from filename '{output}'; using default (gda). To remove this warning, please explicitly specify a format.", - file=sys.stderr, - ) - - with open(output, "w", encoding="utf-8") as output: - if format == "gda": - output.write(gds.to_gda()) - elif format == "json": - output.write(gds.to_json()) - elif format in ["yaml", "yml"]: - output.write(gds.to_yaml()) - - pairs = cli_file_pairs( - input, output, in_endings=[".gds"], out_ending=out_ending, recursive=recursive - ) - if not overwrite: - new_pairs = [] - existing = [] - for ip, op in pairs: - if os.path.exists(op): - existing.append(op) - else: - new_pairs.append((ip, op)) - - if not quiet: - print(f"Skipping {len(existing)} existing output files.") - - pairs = new_pairs - foreach_file_pair(pairs, process, quiet=quiet) diff --git a/formats/gds/cmddef.py b/formats/gds/cmddef.py index 2d23ffc..19c025c 100755 --- a/formats/gds/cmddef.py +++ b/formats/gds/cmddef.py @@ -9,6 +9,8 @@ import yaml from dacite import from_dict +from utils import RESOURCES + logger = logging.getLogger("flora.debug.gds.cmddef") @@ -29,7 +31,8 @@ class GDSCommandParam: Our GDA framework also defines some auxiliary datatypes: bool: backed by either an int (true = nonzero) or a string (true = "true") - bool|int: backed by an int (true = nonzero + bool|int: backed by an int (true = nonzero) + bool|string: backed by a string (true = "true") TODO: Compound datatypes may make the code more understandable, by combining semantically related parameters into a single one: @@ -181,6 +184,7 @@ def process_command(c): c["context"] = context or "all" if isinstance(c["context"], str): c["context"] = [c["context"]] + c["context"] = set(c["context"]) if "prefix" not in c: c["prefix"] = prefix if c["prefix"] is not None and c["name"] is not None: @@ -255,6 +259,7 @@ def load_file(filename: str, root: str) -> list[GDSCommand]: parent_prefix = f"{seg}.{parent_prefix}" else: parent_prefix = None + group_name = None return load_group(data, group_name, parent_prefix, None, filename) @@ -345,7 +350,7 @@ def build_maps( COMMANDS_BYNAME = {} COMMANDS_BYID = {} -CMDDEF_ROOT = "data/gds_commands/" +CMDDEF_ROOT = os.path.join(RESOURCES, "gds_commands") def init_commands(root: str = None): diff --git a/formats/gds/gds.py b/formats/gds/gds.py index 7af1bfd..06bd1b1 100644 --- a/formats/gds/gds.py +++ b/formats/gds/gds.py @@ -1,8 +1,10 @@ -from typing import Tuple, List +from typing import Tuple, List, Optional, Callable, TypeVar, Mapping, Union +from dataclasses import dataclass, field import struct +import functools # pylint: disable=unused-wildcard-import,wildcard-import -from tagged_union import * +from utils import tagged_union, TU, match from .model import ( GDSProgram, @@ -12,234 +14,704 @@ GDSIfInvocation, GDSValue, GDSAddress, - GDSConditionToken + GDSConditionToken, + GDSLabel, + GDSJumpAddress, + GDSContext, ) from .cmddef import COMMANDS_BYID, GDSCommand - @tagged_union -class GDSToken: +class GDSTokenValue: """ - Raw token + Raw token """ - command = int - int = int - float = float - str = str - longstr = str - unused5 = Unit + + command: TU[int] + int: TU[int] + float: TU[float] + str: TU[str] + longstr: TU[str] + unused5: TU[None] # the source address is the one pointing to the target - saddr = GDSAddress - taddr = GDSAddress - NOT = Unit - AND = Unit - OR = Unit - BREAK = Unit - fileend = Unit + saddr: TU[GDSAddress] + taddr: TU[GDSAddress] + NOT: TU[None] + AND: TU[None] + OR: TU[None] + BREAK: TU[None] + fileend: TU[None] +@dataclass(kw_only=True) +class BinaryLocalized: + loc: int -from tagged_union import _ -from tagged_union import match +@dataclass(kw_only=True) +class GDSToken(BinaryLocalized): + val: GDSTokenValue -def read_condition(data: bytes, cursor: int) -> Tuple[int, List[GDSConditionToken], GDSAddress]: - return (cursor, [], GDSAddress(1234)) +@dataclass(kw_only=True) +class LabelUse(BinaryLocalized): + loc: int + addr: int + use: Optional[str] = None + primary: bool = False -def read_if(data: bytes, cursor: int, cmdobj: GDSCommand) -> Tuple[int, GDSIfInvocation]: - cursor, cond, addr = read_condition(data, cursor) - - return (cursor, GDSIfInvocation(command=cmdobj, condition=cond, target_addr=addr, args = [])) +@dataclass(kw_only=True) +class LabelToken(BinaryLocalized): + loc: int + addr: Optional[int] + pointsto: Optional[LabelUse] = None -def read_elif(data: bytes, cursor: int, cmdobj: GDSCommand) -> Tuple[int, GDSIfInvocation]: - data, res = read_if(data, cursor, cmdobj) - res.elseif = True - return (cursor, res) +@dataclass(kw_only=True) +class BinaryCommand(GDSInvocation, BinaryLocalized): + loc: int = None -def read_repeatN(data: bytes, cursor: int, cmdobj: GDSCommand) -> Tuple[int, GDSLoopInvocation]: - cursor, cntt = read_token(data, cursor) - if cntt not in GDSToken.int: - raise ValueError( - f"Unexpected parameter token type: should have been int, token was {arg}" - ) - cnt = cntt() - cursor, saddrt = read_token(data, cursor) - while saddrt not in GDSToken.saddr: - cursor, saddrt = read_token(data, cursor) - if saddrt in GDSToken.fileend: - raise ValueError("repeatN: encountered EOF looking for jump address") - addr = saddrt() +@dataclass(kw_only=True) +class BinaryIfCommand(GDSIfInvocation, BinaryLocalized): + loc: int = None - return (cursor, GDSLoopInvocation(command=cmdobj, condition=cnt, target_addr=GDSAddress(addr), args=[])) +@dataclass(kw_only=True) +class BinaryLoopCommand(GDSLoopInvocation, BinaryLocalized): + loc: int = None -def read_while(data: bytes, cursor: int, cmdobj: GDSCommand) -> Tuple[int, GDSLoopInvocation]: - cursor, cond, addr = read_condition(data, cursor) - - return (cursor, GDSLoopInvocation(command=cmdobj, condition=cond, target_addr=addr, args=[])) +F = TypeVar("F", bound=Callable) -def read_simple( - data: bytes, cursor: int, cmdobj: GDSCommand -) -> Tuple[int, GDSInvocation]: + +def prevcursor(fn: F) -> F: + """ + For the CompilerState methods, records the cursor before the function call + so it can be restored for peeking functionality + """ + + @functools.wraps(fn) + def wrapper(self, *args, **kwargs): + prev = self.cursor + val = fn(self, *args, **kwargs) + self.prev_cursor = prev + return val + + return wrapper + + +@dataclass(kw_only=True) +class DecompilerState: + data: bytes + len: int + cursor: int + prev_cursor: int + elements: List[GDSElement] = field(default_factory=lambda: []) + labels: Mapping[int, List[Union[LabelUse, LabelToken]]] = field( + default_factory=dict + ) + context: GDSContext = field(default_factory=GDSContext) + + def __init__(self, data: bytes): + self.data = data + self.len = int.from_bytes(data[:4], "little") + self.cursor = 4 + self.prev_cursor = 4 + self.elements = [] + self.labels = {} + self.context = GDSContext() + + @prevcursor + def read_bytes(self, l: int) -> bytes: + val = self.data[self.cursor : self.cursor + l] + self.cursor += l + return val + + @prevcursor + def read_int(self, l: int) -> int: + return int.from_bytes(self.read_bytes(l), "little") + + @prevcursor + def read_token(self) -> GDSToken: + loc = self.cursor + p_type = self.read_int(2) + val = None + if p_type == 0: + cmd = self.read_int(2) + val = GDSTokenValue.command(cmd) + elif p_type == 1: + val = self.read_int(4) + val = GDSTokenValue.int(val) + elif p_type == 2: + val = struct.unpack(" 64: + # print( + # "WARN: string literal is too long (max is 64 bytes); this may lead to errors in the game." + # ) + val = ( + self.read_bytes(str_len) + .decode("ascii") # TODO: JP/KO compatibility + .rstrip("\x00") + ) + val = GDSTokenValue.str(val) + elif p_type == 4: + str_len = self.read_int(2) + val = ( + self.read_bytes(str_len) + .decode("ascii") # TODO: JP/KO compatibility + .rstrip("\x00") + ) + val = GDSTokenValue.longstr(val) + elif p_type == 6: + addr = self.read_int(4) + val = GDSTokenValue.saddr(addr) + elif p_type == 7: + addr = self.read_int(4) + val = GDSTokenValue.taddr(addr) + elif p_type == 8: + val = GDSTokenValue.NOT() + elif p_type == 9: + val = GDSTokenValue.AND() + elif p_type == 10: + val = GDSTokenValue.OR() + elif p_type == 11: + val = GDSTokenValue.BREAK() + elif p_type == 12: + val = GDSTokenValue.fileend() + elif p_type == 5: + val = GDSTokenValue.unused5() + else: + raise ValueError("Invalid GDS token type") + return GDSToken(loc=loc, val=val) + + def read_label(self, token: GDSToken) -> LabelToken: + if token.val not in GDSTokenValue.taddr: + raise ValueError("Not a target label") + ltoken = LabelToken(loc=token.loc, addr=GDSAddress(token.val())) + + if (ltoken.loc + 2) not in self.labels: + self.labels[ltoken.loc + 2] = [] + self.labels[ltoken.loc + 2].append(ltoken) + self.elements.append(GDSElement.label(ltoken)) + + return ltoken + + def read_address(self, token: GDSToken, use: str = "") -> LabelUse: + if token.val not in GDSTokenValue.saddr: + raise ValueError("Not a jump address") + luse = LabelUse(loc=token.loc, addr=GDSAddress(token.val()), use=use) + + if luse.addr not in self.labels: + self.labels[luse.addr] = [] + self.labels[luse.addr].append(luse) + + return luse + + def name_labels(self): + label_names = {} + label_prefix_counter: Mapping[str, int] = {} + + for addr, labels in self.labels.items(): + target = None + sources = [] + use = None + for l in labels: + if isinstance(l, LabelToken): + target = l + else: + sources.append(l) + use = l.use if use is None or use == l.use else "" + if use is None: + use = "" + if target is None: + i = 0 + for x in self.elements: + if x.loc >= addr + 4: + break + i += 1 + target = LabelToken(loc=addr - 2, addr=None) + labels.insert(target) + self.elements.insert(i, target) + + # name the label according to its use + prefix = f"{use}_" if use != "" else "" + if prefix not in label_prefix_counter: + label_prefix_counter[prefix] = 1 + counter = label_prefix_counter[prefix] + label_prefix_counter[prefix] += 1 + name = prefix + str(counter) + label_names[addr] = name + + # set which address is the primary one, if any + for s in sources: + if s.loc + 2 == target.addr: + s.primary = True + target.pointsto = s + break + + # # Fold if/elif/else and loop blocks if possible + # # i.e. the corresponding label is only used by the one ifelseloop jump, + # # and it jumps forward so a block even makes sense + # # TODO: I need to combine all the labels before a statement together + # if len(sources) == 1 and sources[0].addr >= sources[0].loc+6: + # pass + return label_names + + +def read_gds(data: bytes, path: str = None) -> GDSProgram: + ctx = DecompilerState(data) + + cur_token = ctx.read_token() + # TODO + while cur_token.val not in GDSTokenValue.fileend: + if cur_token.val in GDSTokenValue.command: + ctx.elements.append(GDSElement.command(read_command(ctx, cur_token))) + elif cur_token.val in GDSTokenValue.taddr: + ctx.read_label(cur_token) + elif cur_token.val in GDSTokenValue.BREAK: + b = GDSElement.BREAK() + b.loc: int = cur_token.loc + ctx.elements.append(b) + else: + # The game technically allows this, but it's meaningless nonsense so for now we forbid it + raise ValueError("Unexpected token type") + + cur_token = ctx.read_token() + + label_names = ctx.name_labels() + + oldlabel_indices = [ + (i, el()) for i, el in enumerate(ctx.elements) if el in GDSElement.label + ] + olduse_indices = [ + (el, el().target) + for el in ctx.elements + if el in GDSElement.command + and isinstance(el(), (BinaryIfCommand, BinaryLoopCommand)) + and isinstance(el().target, LabelUse) + ] + + labels = {} + for addr, name in label_names.items(): + lst = [] + for i in ctx.labels[addr]: + if isinstance(i, LabelToken): + newlabel = GDSLabel( + name=name, + present=i.addr is not None, + loc=i.addr if i.pointsto is None else None, + ) + for idx, el in oldlabel_indices: + if el == i: + ctx.elements[idx] = GDSElement.label(newlabel) + break + lst.append(newlabel) + elif isinstance(i, LabelUse): + newuse = GDSJumpAddress(label=name, primary=i.primary) + for el, olduse in olduse_indices: + if olduse == i: + el().target = newuse + break + lst.append(newuse) + else: + raise TypeError() + labels[name] = lst + + return GDSProgram( + context=ctx.context, path=path, elements=ctx.elements, labels=labels + ) + + +def read_condition( + ctx: DecompilerState, use: str = "" +) -> Tuple[List[GDSConditionToken], LabelUse]: + cur_token = ctx.read_token() + cond = [] + while cur_token.val not in GDSTokenValue.saddr: + if cur_token.val in GDSTokenValue.NOT: + cond.append(GDSConditionToken.NOT()) + elif cur_token.val in GDSTokenValue.AND: + cond.append(GDSConditionToken.AND()) + elif cur_token.val in GDSTokenValue.OR: + cond.append(GDSConditionToken.OR()) + elif cur_token.val in GDSTokenValue.command: + cond.append(GDSConditionToken.command(read_command(ctx, cur_token))) + else: + raise ValueError("Unexpected token type") + cur_token = ctx.read_token() + + addr = ctx.read_address(cur_token, use) + + return cond, addr + + +def read_command(ctx: GDSCommand, token: GDSToken) -> BinaryCommand: + if token.val not in GDSTokenValue.command: + raise ValueError("Expected instruction") + + cmdid = token.val() + cmdobj = COMMANDS_BYID.get(cmdid) + if cmdobj is None: + raise ValueError(f"Command {cmdid} not defined") + + if cmdobj.complex: + return (READ_COMPLEX.get(cmdobj.name) or READ_COMPLEX.get(cmdobj.id))( + ctx, cmdobj + ) + + cmd = read_simple(ctx, cmdobj) + cmd.loc = token.loc + return cmd + + +def read_simple(ctx: DecompilerState, cmdobj: GDSCommand) -> BinaryCommand: args = [] for param in cmdobj.params: - cursor, arg = read_token(data, cursor) - val, reqtypes = match( - arg, + arg = ctx.read_token() + options = match( + arg.val, { - GDSToken.int: lambda val: ( - GDSValue.int(val), - ["int", "bool", "bool|int"], - ), - GDSToken.float: lambda val: (GDSValue.float(val), ["float"]), - GDSToken.str: lambda val: (GDSValue.str(val), ["string", "bool"]), - GDSToken.longstr: lambda val: (GDSValue.longstr(val), ["longstr"]), - _: lambda: (None, []), + GDSTokenValue.int: lambda val: [ + (GDSValue.int(val), ["int"]), + ( + GDSValue.bool(val), + ["bool", "bool|int"], + ), + ], + GDSTokenValue.float: lambda val: [(GDSValue.float(val), ["float"])], + GDSTokenValue.str: lambda val: [ + (GDSValue.str(val), ["string"]), + ( + GDSValue.bool(val), + ["bool", "bool|string"], + ), + ], + GDSTokenValue.longstr: lambda val: [ + (GDSValue.longstr(val), ["longstr"]) + ], + ...: lambda: [], }, ) - if param.type not in reqtypes: - raise ValueError( - f"Unexpected parameter token type: should have been {param.type}, token was {arg}" - ) + val = next((v for v, reqtypes in options if param.type in reqtypes), None) + if val is None: + if not param.optional: + raise ValueError( + f"Unexpected parameter token type: should have been {param.type}, token was {arg}" + ) + val = None + ctx.cursor = ctx.prev_cursor args.append(val) - return (cursor, GDSInvocation(command=cmdobj, args=args)) + return BinaryCommand(command=cmdobj, args=args) + + +def read_if(ctx: DecompilerState, cmdobj: GDSCommand) -> BinaryIfCommand: + cond, addr = read_condition(ctx, "if") + + # this diagnostic is literally just objectively wrong + # pylint: disable=unexpected-keyword-arg + return BinaryIfCommand( + command=cmdobj, + args=[], + condition=cond, + block=[], + elze=False, + elseif=False, + target=addr, + ) + + +def read_elif(ctx: DecompilerState, cmdobj: GDSCommand) -> BinaryIfCommand: + cond, addr = read_condition(ctx, "elif") + + # this diagnostic is literally just objectively wrong + # pylint: disable=unexpected-keyword-arg + return BinaryIfCommand( + command=cmdobj, + args=[], + condition=cond, + block=[], + elze=False, + elseif=True, + target=addr, + ) + + +def read_else(ctx: DecompilerState, cmdobj: GDSCommand) -> BinaryIfCommand: + addr = None + cur_token = ctx.read_token() + while cur_token.val not in GDSTokenValue.saddr: + cur_token = ctx.read_token() + addr = ctx.read_address(cur_token, "else") + + # this diagnostic is literally just objectively wrong + # pylint: disable=unexpected-keyword-arg + return BinaryIfCommand( + command=cmdobj, + args=[], + condition=None, + block=[], + elze=True, + elseif=False, + target=addr, + loc=None, + ) + + +def read_repeatN(ctx: DecompilerState, cmdobj: GDSCommand) -> BinaryLoopCommand: + cntt = ctx.read_token() + if cntt.val not in GDSTokenValue.int: + raise ValueError( + f"Unexpected parameter token type: should have been int, token was {cntt}" + ) + cnt = cntt.val() + + saddrt = ctx.read_token() + while saddrt.val not in GDSTokenValue.saddr: + if saddrt.val in GDSTokenValue.fileend: + raise ValueError("repeatN: encountered EOF looking for jump address") + saddrt = ctx.read_token() + addr = ctx.read_address(saddrt, "loop") + + # this diagnostic is literally just objectively wrong + # pylint: disable=unexpected-keyword-arg + return BinaryLoopCommand( + command=cmdobj, condition=cnt, target=addr, args=[], block=[] + ) + + +def read_while(ctx: DecompilerState, cmdobj: GDSCommand) -> BinaryLoopCommand: + cond, addr = read_condition(ctx, "loop") + + # this diagnostic is literally just objectively wrong + # pylint: disable=unexpected-keyword-arg + return BinaryLoopCommand( + command=cmdobj, condition=cond, target=addr, args=[], block=[] + ) READ_COMPLEX = { "if": read_if, "elif": read_elif, + "else": read_else, "repeatN": read_repeatN, "while": read_while, } -def read_command(data: bytes, cursor: int, token: GDSToken) -> Tuple[int, GDSInvocation]: - if token not in GDSToken.command: - raise ValueError("Expected instruction") - cmdid = token() - cmdobj = COMMANDS_BYID.get(cmdid) - if cmdobj is None: - raise ValueError(f"Command {cmdid} not defined") - - if cmdobj.complex: - return ( - READ_COMPLEX.get(cmdobj.name) or READ_COMPLEX.get(cmdobj.id) - )(data, cursor) - - args = [] - for param in cmdobj.params: - cursor, arg = read_token(data, cursor) - val, reqtypes = match( - arg, +@dataclass(kw_only=True) +class CompilerState: + data: bytearray + elements: List[GDSElement] + labels: Mapping[str, List[Union[GDSLabel, GDSJumpAddress]]] + label_locs: Mapping[str, Tuple[int, GDSLabel]] + use_locs: Mapping[str, List[Tuple[int, GDSJumpAddress]]] + + def __init__(self, prog: GDSProgram): + self.data = bytearray() + # self.context = prog.context + self.elements = prog.elements + self.labels = prog.labels + self.label_locs = {} + self.use_locs = {} + + def write_bytes(self, b: bytes): + self.data += b + + def write_token(self, token: GDSTokenValue): + els = match( + token, { - GDSToken.int: lambda val: ( - GDSValue.int(val), - ["int", "bool", "bool|int"], + GDSTokenValue.command: lambda val: ( + (0).to_bytes(2, "little"), + val.to_bytes(2, "little"), + ), + GDSTokenValue.int: lambda val: ( + (1).to_bytes(2, "little"), + val.to_bytes(4, "little"), + ), + GDSTokenValue.float: lambda val: ( + (2).to_bytes(2, "little"), + struct.pack(" Tuple[int, GDSToken]: - p_type = int.from_bytes(data[cursor : cursor + 2], "little") - if p_type == 0: - cmd = int.from_bytes(data[cursor + 2 : cursor + 4], "little") - return (cursor + 4, GDSToken.command(cmd)) - if p_type == 1: - val = int.from_bytes(data[cursor + 2 : cursor + 6], "little") - return (cursor + 6, GDSToken.int(val)) - if p_type == 2: - val = struct.unpack(">f", data[cursor + 2 : cursor + 6]) - return (cursor + 6, GDSToken.float(val)) - if p_type == 3: - str_len = int.from_bytes(data[cursor + 2 : cursor + 4], "little") - if str_len > 64: - print( - "WARN: string literal is too long (max is 64 bytes); this may lead to errors in the game." + def write_command(self, cmd: GDSInvocation): + self.write_token(GDSTokenValue.command((cmd.command).id)) + + if cmd.command.complex: + (WRITE_COMPLEX.get(cmd.command.name) or WRITE_COMPLEX.get(cmd.command.id))( + self, cmd ) - val = ( - data[cursor + 4 : cursor + 4 + str_len] - .decode("ascii") # TODO: JP/KO compatibility - .rstrip("\x00") - ) - return (cursor + 4 + str_len, GDSToken.str(val)) - if p_type == 4: - str_len = int.from_bytes(data[cursor + 2 : cursor + 4], "little") - val = ( - data[cursor + 4 : cursor + 4 + str_len] - .decode("ascii") # TODO: JP/KO compatibility - .rstrip("\x00") - ) - return (cursor + 4 + str_len, GDSToken.longstr(val)) - if p_type == 6: - addr = int.from_bytes(data[cursor + 2 : cursor + 6], "little") - return (cursor + 6, GDSToken.saddr(addr)) - if p_type == 7: - addr = int.from_bytes(data[cursor + 2 : cursor + 6], "little") - return (cursor + 6, GDSToken.taddr(addr)) - if p_type == 8: - return (cursor + 2, GDSToken.NOT()) - if p_type == 9: - return (cursor + 2, GDSToken.AND()) - if p_type == 10: - return (cursor + 2, GDSToken.OR()) - if p_type == 11: - return (cursor + 2, GDSToken.BREAK()) - if p_type == 12: - return (cursor + 2, GDSToken.fileend()) - if p_type == 5: - return (cursor + 2, GDSToken.unused5()) - raise ValueError("Invalid GDS token type") + return + write_simple(self, cmd) -def read_gds(data: bytes, path: str = None) -> GDSProgram: - length = int.from_bytes(data[:4], "little") - cursor = 4 - cmds = [] + def write_label(self, label: GDSLabel): + if not label.present: + return - cursor, cur_token = read_token(data, cursor) - # TODO - while cur_token not in GDSToken.fileend: - if cur_token in GDSToken.command: - cursor, cmd = read_command(data, cursor, cur_token) - cmds.append(cmd) - elif cur_token in GDSToken.taddr: - cmds.append( - GDSElement.target_label(GDSJump(source=cur_token(), target=cursor - 4)) + backptr = label.loc + label_loc = len(self.data) + 6 + + for use_loc, addr in self.use_locs.get(label.name) or []: + if addr.primary: + backptr = use_loc + self.data[use_loc - 4 : use_loc] = label_loc.to_bytes(4, "little") + + self.write_token(GDSTokenValue.taddr(GDSAddress(backptr or 0))) + self.label_locs[label.name] = (label_loc, label) + + def write_addr(self, addr: GDSJumpAddress): + label_loc = None + use_loc = len(self.data) + 6 + + if addr.label in self.label_locs: + label_loc, _ = self.label_locs[addr.label] + if addr.primary: + self.data[label_loc - 4 : label_loc] = use_loc.to_bytes(4, "little") + + self.write_token(GDSTokenValue.saddr(GDSAddress(label_loc or 0))) + + if addr.label not in self.use_locs: + self.use_locs[addr.label] = [] + self.use_locs[addr.label].append((use_loc, addr)) + + +def write_gds(prog: GDSProgram) -> bytes: + ctx = CompilerState(prog) + + for el in ctx.elements: + if el in GDSElement.BREAK: + ctx.write_token(GDSTokenValue.BREAK()) + elif el in GDSElement.command: + ctx.write_command(el()) + elif el in GDSElement.label: + ctx.write_label(el()) + + ctx.write_token(GDSTokenValue.fileend()) + return len(ctx.data).to_bytes(4, 'little') + bytes(ctx.data) + + +def write_simple(ctx: CompilerState, cmd: GDSInvocation): + args = cmd.args + if len(args) < len(cmd.command.params): + args += [None] * len(cmd.command.params)-len(args) + for arg, param in zip(args, cmd.command.params): + if arg is None: + if param.optional: + continue + else: + raise ValueError("Too few parameters") + options = match( + arg, + { + GDSValue.int: lambda val: [ + (GDSTokenValue.int(val), ["int", "bool", "bool|int"]), + ], + GDSValue.float: lambda val: [(GDSTokenValue.float(val), ["float"])], + GDSValue.str: lambda val: [ + (GDSTokenValue.str(val), ["string", "bool", "bool|string"]), + ], + GDSValue.longstr: lambda val: [ + (GDSTokenValue.longstr(val), ["longstr"]) + ], + GDSValue.bool: lambda val: + [(GDSTokenValue.int(val), ["bool", "bool|int"])] if isinstance(val, int) + else + [(GDSTokenValue.str(val), ["bool", "bool|string"])] if isinstance(val, str) + else + [ + (GDSTokenValue.int(1 if val else 0), ["bool", "bool|int"]), + ( + GDSTokenValue.str("true" if val else "false"), + ["bool", "bool|string"], + ), + ] + , + ...: lambda: [], + }, + ) + tok = next((v for v, reqtypes in options if param.type in reqtypes), None) + if tok is None: + raise ValueError( + f"Unexpected parameter type: should have been {param.type}, value was {arg}" ) - elif cur_token in GDSToken.BREAK: - cmds.append(GDSElement.BREAK()) + + ctx.write_token(tok) + + +def write_condition(ctx: CompilerState, cond: List[GDSConditionToken]): + for tok in cond: + if tok in GDSConditionToken.NOT: + ctx.write_token(GDSTokenValue.NOT()) + elif tok in GDSConditionToken.AND: + ctx.write_token(GDSTokenValue.AND()) + elif tok in GDSConditionToken.OR: + ctx.write_token(GDSTokenValue.OR()) + elif tok in GDSConditionToken.command: + ctx.write_command(tok()) else: raise ValueError("Unexpected token type") - cursor, cur_token = read_token(data, cursor) - return GDSProgram(context="all", path=path, tokens=cmds) +def write_if(ctx: CompilerState, cmd: GDSIfInvocation): + write_condition(ctx, cmd.condition) + ctx.write_addr(cmd.target) + + +def write_else(ctx: CompilerState, cmd: GDSIfInvocation): + # write_condition(ctx, cmd.condition) + ctx.write_addr(cmd.target) + + +def write_repeatN(ctx: CompilerState, cmd: GDSLoopInvocation): + ctx.write_token(GDSTokenValue.int(cmd.condition)) + ctx.write_addr(cmd.target) + +def write_while(ctx: CompilerState, cmd: GDSLoopInvocation): + write_condition(ctx, cmd.condition) + ctx.write_addr(cmd.target) + + +WRITE_COMPLEX = { + "if": write_if, + "elif": write_if, + "else": write_else, + "repeatN": write_repeatN, + "while": write_while, +} diff --git a/formats/gds/model.py b/formats/gds/model.py index 2fa8e97..7f9355e 100644 --- a/formats/gds/model.py +++ b/formats/gds/model.py @@ -1,7 +1,7 @@ -from typing import List, Optional, NewType, Union, Set +from typing import List, Optional, NewType, Union, Set, Mapping from dataclasses import dataclass, field # pylint: disable=unused-wildcard-import,wildcard-import -from tagged_union import * +from utils import tagged_union, TU from .cmddef import GDSCommand @@ -12,13 +12,15 @@ class GDSValue: """ A value usable as a parameter in a GDSInvocation. """ - int = int - float = float - str = str - longstr = str + int: TU[int] + float: TU[float] + str: TU[str] + longstr: TU[str] + # (value, represented_as_string) + bool: TU[Union[bool, int, str]] -@dataclass +@dataclass(kw_only=True) class GDSInvocation: """ The specific invocation of a GDS commmand, with the given parameter values. @@ -32,14 +34,14 @@ class GDSConditionToken: """ A token that appears in the condition for a flow statement. """ - command = GDSInvocation - NOT = Unit - AND = Unit - OR = Unit + command: TU[GDSInvocation] + NOT: TU[None] + AND: TU[None] + OR: TU[None] -@dataclass -class GDSJumpTarget: +@dataclass(kw_only=True) +class GDSJumpAddress: """ A jump address used by flow instructions in GDS scripts. """ @@ -47,25 +49,31 @@ class GDSJumpTarget: """ The name of the label to be jumped to. """ + primary: bool = True + """ + Whether the label this address points to actually points back to this address + instance. Only one of the pointers to a single label can have this flag set + (but normally, there is also a 1:1 correspondence between addresses and labels) + """ -@dataclass +@dataclass(kw_only=True) class GDSIfInvocation(GDSInvocation): condition: List[GDSConditionToken] + target: GDSJumpAddress block: Optional[List["GDSElement"]] elseif: bool = False elze: bool = False - target: GDSJumpTarget -@dataclass +@dataclass(kw_only=True) class GDSLoopInvocation(GDSInvocation): - condition: Union[List[GDSConditionToken] | int] + condition: Union[List[GDSConditionToken], int] block: Optional[List["GDSElement"]] - target: GDSJumpTarget + target: GDSJumpAddress -@dataclass +@dataclass(kw_only=True) class GDSLabel: """ A target label from a GDS script file. @@ -92,12 +100,12 @@ class GDSElement: """ An entry in a GDS script file. """ - command = GDSInvocation - label = GDSLabel - BREAK = Unit + command: TU[GDSInvocation] + label: TU[GDSLabel] + BREAK: TU[None] -@dataclass +@dataclass(kw_only=True) class GDSContext: """ The context that was determined for a GDS script, either manually or by context narrowing. @@ -178,7 +186,7 @@ def union(n1: Union[str, Set[str]], n2: Union[str, Set[str]]) -> Set[str]: return set() -@dataclass +@dataclass(kw_only=True) class GDSProgram: """ The program contained in a GDS script file. @@ -195,7 +203,7 @@ class GDSProgram: """ The instructions or other flow elements in the script. """ - labels: List[str] = field(default_factory=list) + labels: Mapping[str, List[Union[GDSLabel, GDSJumpAddress]]] = field(default_factory=list) """ A list of all the labels present in the script. May not technically be necessary. """ diff --git a/formats/gds/old.py b/formats/gds/old.py new file mode 100644 index 0000000..9a15311 --- /dev/null +++ b/formats/gds/old.py @@ -0,0 +1,541 @@ +import click +import contextlib +import json +import yaml +import os +import sys +import collections + +import struct +import parse +import ast +from utils import cli_file_pairs, foreach_file_pair, RESOURCES +from version import v + + +@click.group( + help="Script-like format, also used to store puzzle parameters.", options_metavar="" +) +def cli(): + pass + +commands = json.load(open(os.path.join(RESOURCES,"commands.json"), encoding="utf-8")) +commands_i = { + val["id"]: key for key, val in commands.items() if "id" in val +} # Inverted version of commands + +class GDS: + def __init__(self, cmds=None): + if cmds is None: + cmds = [] + self.cmds = cmds + + @classmethod + def from_gds(Self, file): + length = int.from_bytes(file[0:4], "little") + if file[4:6] == b"\x0c\x00": + return Self([]) + cmd_data = file[6 : length + 4] + cmds = [] + + cmd = None + params = [] + c = 0 + while True: + if c >= length: + raise Exception( + "GDS file error: End of file reached with no 0xC command!" + ) + if cmd == None: + cmd = int.from_bytes(cmd_data[c : c + 2], "little") + if cmd in commands_i: + cmd = commands_i[cmd] + c += 2 + continue + + p_type = int.from_bytes(cmd_data[c : c + 2], "little") + + if p_type == 0: + cmds.append({"command": cmd, "parameters": params}) + cmd = None + params = [] + c += 2 + elif p_type == 1: + params.append( + { + "type": "int", + "data": struct.unpack('>f', cmd_data[c + 2 : c + 6])[0], + } + ) + c += 6 + elif p_type == 2: + params.append( + { + "type": "float", + "data": int.from_bytes(cmd_data[c + 2 : c + 6], "little"), + } + ) + c += 6 + elif p_type == 3: + str_len = int.from_bytes(cmd_data[c + 2 : c + 4], "little") + params.append( + { + "type": "string", + "data": cmd_data[c + 4 : c + 4 + str_len] + .decode("ascii") # TODO: JP/KO compatibility + .rstrip("\x00"), + } + ) + c += str_len + 4 + elif p_type == 6: + # address within the gds file range. usually fits in a short, but can be an int. + params.append( + { + "type": "taddr", + "data": int.from_bytes(cmd_data[c + 2 : c + 6], "little"), + } + ) + c += 6 + elif p_type == 7: + params.append( + { + "type": "saddr", + "data": int.from_bytes(cmd_data[c + 2 : c + 6], "little"), + } + ) + c += 6 + elif p_type == 8: + params.append({"type": "not"}) + c += 2 + elif p_type == 9: + params.append({"type": "and"}) + c += 2 + elif p_type == 0xA: + params.append({"type": "or"}) + c += 2 + elif p_type == 0xB: + # break + params.append({"type": "break"}) + c += 2 + elif p_type == 0xC: + # eof + cmds.append({"command": cmd, "parameters": params}) + break + else: + raise Exception( + f"GDS file error: Invalid or unsupported parameter type {hex(p_type)}!" + ) + + return Self(cmds) + + @classmethod + def from_json(Self, file): + cmds = json.loads(file)["data"] + # TODO: reject non-compatible json files + return Self(cmds) + + @classmethod + def from_yaml(Self, file): + cmds = yaml.safe_load(file)["data"] + # TODO: reject non-compatible yaml files + return Self(cmds) + + @classmethod + def from_gda(Self, file): # TODO: make this, so gds_old can be completely removed + cmds = [] + + for line in file.split("\n"): + line = line.strip() + if line.startswith("#"): + continue + if line == "": + continue + + line, strings = parse.remove_strings(line) + line = line.rstrip().split(" ") + cmd = line[0] + + # TODO: handle inline comments, maybe + + if cmd in commands: + cmd = commands[cmd] + if "alias" in cmd: + cmd = commands[cmd["alias"]] + elif cmd.startswith("cmd_") or cmd.startswith("unk_"): + cmd = int(cmd[4:]) + elif cmd.startswith("0x"): + cmd = int(cmd[2:], base=16) + else: + raise Exception(f"Unknown GDA command: {cmd}") + + params = [] + + for param in line[1:]: + if param.isdigit(): + params.append({"type": "int", "data": int(param)}) + elif param.startswith("0x"): + params.append({"type": "float", "data": float(param)}) + elif param.startswith('"') and param.endswith('"'): + param = ast.literal_eval(f'"{strings[int(param[1:-1])]}"') + params.append({"type": "string", "data": param}) + elif param.startswith("@"): + params.append({"type": "taddr", "data": int(param[1:], 16)}) + elif param.startswith("$"): + params.append({"type": "saddr", "data": int(param[1:], 16)}) + elif param.startswith("NOT"): + params.append({"type": "not"}) + elif param.startswith("AND"): + params.append({"type": "and"}) + elif param.startswith("OR"): + params.append({"type": "or"}) + elif param.startswith("BREAK"): + params.append({"type": "break"}) + else: + raise Exception(f"Invalid GDA parameter: {param}") + + cmds.append({"command": cmd, "parameters": params}) + + return Self(cmds) + + def __getitem__(self, index): + index = int(index) + return self.cmds[index] + + def to_json(self): + return json.dumps({"version": v, "data": self.cmds}, indent=4) + + def to_yaml(self): + return yaml.safe_dump({"version": v, "data": self.cmds}) + + def to_gds(self): + out = b"\x00" * 2 + for command in self.cmds: + if type(command["command"]) == int: + out += command["command"].to_bytes(2, "little") + else: + out += command["command"]["id"].to_bytes(2, "little") + for param in command["parameters"]: + if param["type"] == "int": + out += b"\x01\x00" + out += param["data"].to_bytes(4, "little") + elif param["type"] == "float": + out += b"\x02\x00" + out += struct.pack('>f', param["data"]) + elif param["type"] == "string": + out += b"\x03\x00" + out += (len(param["data"]) + 1).to_bytes(2, "little") + out += ( + param["data"].encode("ASCII") + b"\x00" + ) # TODO: JP/KO compatibility + elif param["type"] == "taddr": + out += b"\x06\x00" + out += param["data"].to_bytes(4, "little") + elif param["type"] == "saddr": + out += b"\x07\x00" + out += param["data"].to_bytes(4, "little") + elif param["type"] == "not": + out += b"\x08\x00" + elif param["type"] == "and": + out += b"\x09\x00" + elif param["type"] == "or": + out += b"\x0a\x00" + elif param["type"] == "break": + out += b"\x0b\x00" + else: + raise Exception( + f"GDS JSON error: Invalid or unsupported parameter type '{param['type']}'!" + ) + out += b"\x00\x00" + out = out[:-2] + b"\x0c\x00" + + return len(out).to_bytes(4, "little") + out + + def to_bin(self): # alias + return self.to_gds() + + def to_gda(self): + out = "" + for command in self.cmds: + if type(command["command"]) == int: + out += "0x" + command["command"].to_bytes(1, "little").hex() + else: + out += command["command"] + for param in command["parameters"]: + out += " " + if param["type"] == "int": + out += str(param["data"]) + elif param["type"] == "string": + out += repr(param["data"]) + elif param["type"] == "float": + out += str(param["data"]) + elif param["type"] == "taddr": + out += f"@{param['data'].hex()}" + elif param["type"] == "saddr": + out += f"${param['data'].hex()}" + elif param["type"] == "not": + out += "NOT" + elif param["type"] == "and": + out += "AND" + elif param["type"] == "or": + out += "OR" + elif param["type"] == "break": + out += "BREAK" + else: + raise Exception( + f"GDA error: invalid or unsupported parameter type '{param['type']}'!" + ) + out += "\n" + return out + + +@cli.command(name="compile", no_args_is_help=True) +@click.argument("input", required=False, type=click.Path(exists=True)) +@click.argument("output", required=False, type=click.Path(exists=False)) +@click.option( + "--recursive", + "-r", + is_flag=True, + help="Recurse into subdirectories of the input directory to find more applicable files.", +) +@click.option( + "--quiet", + "-q", + is_flag=True, + help="Suppress all output. By default, operations involving multiple files will show a progressbar.", +) +@click.option( + "--overwrite/--no-overwrite", + "-o/-O", + default=True, + help="Whether existing files should be overwritten. Default: true", +) +@click.option( + "--format", + "-f", + required=False, + default=None, + multiple=False, + help="The format of the input file. Will be inferred from the file ending or content if unset. " + "If multiple file types would compile to the same output (but may not necessarily have the same content), " + "specify this to disambigute. Possible values: gda, json, yaml", +) +def compile( + input=None, output=None, recursive=False, quiet=False, format=None, overwrite=None +): + """ + Compiles the human-readable script(s) at INPUT into the game's binary script files at OUTPUT. + + INPUT can be a single file or a directory (which obviously has to exist). In the latter case subfiles with the correct file ending will be processed. + If unset, defaults to the current working directory. + + The meaning of OUTPUT may depend on INPUT: + - If INPUT is a file, then OUTPUT is expected to be a file, unless it explicitly ends with a slash indicating a directory. + In this case, if unset OUTPUT will default to the INPUT filename with `.gds` exchanged/appended. + - Otherwise OUTPUT has to be a directory as well (or an error will be shown). + In this case, if unset OUTPUT will default to the INPUT directory (which may itself default to the current working directory). + + In the file-to-file case, the paths are explicitly used as they are. Otherwise, if multiple input files were collected, or OUTPUT is a directory, + an output path is inferred for each input file by exchanging the input format's file ending for, or otherwise appending the `.gds` file ending. + + In the case where INPUT is a directory, if no format is specified, this command will collect files of all compatible types. Note that this can lead + to situations where multiple files would compile to the same output (e.g. `test.json` and `test.gda` would both be candidates for `test.gds`); + this command will NOT make a choice in this case, and instead ask to explicitly specify the format to be used. + """ + + in_endings = [] + if format is None: + in_endings = [".gda", ".json", ".yaml", ".yml"] + elif format == "gda": + in_endings = [".gda"] + elif format == "json": + in_endings = [".json"] + elif format in ["yaml", "yml"]: + in_endings = [".yaml", ".yml"] + else: + raise Exception(f"Unsupported input format: '{format}'") + + def process(input, output): + inpath = input + input = open(inpath, "r", encoding="utf-8").read() + + format2 = format + if format2 is None: + if inpath.lower().endswith(".gda"): + format2 = "gda" + elif inpath.lower().endswith(".json"): + format2 = "json" + elif inpath.lower().endswith(".yml") or inpath.lower().endswith(".yaml"): + format2 = "yaml" + + gds = None + with contextlib.suppress(Exception): + if format2 == "gda": + gds = GDS.from_gda(input) + elif format2 == "json": + gds = GDS.from_json(input) + elif format2 in ["yaml", "yml"]: + gds = GDS.from_yaml(input) + + if gds is None: + if format2 is not None: + # TODO: should this abort instead? + print( + f"WARNING: Input file '{inpath}' did not have expected format '{format2}'", + file=sys.stderr, + ) + # format not specified and couldn't be inferred, or file turns out not to have the correct format + # => try all the formats & see which one works (only one should be possible) + for f in ["gda", "json", "yaml"]: + with contextlib.suppress(Exception): + if f == "gda": + gds = GDS.from_gda(input) + elif f == "json": + gds = GDS.from_json(input) + elif f == "yaml": + gds = GDS.from_yaml(input) + if gds is not None: + break + if gds is None: + raise Exception( + f"File '{inpath}' couldn't be read: not a known file format" + + (f" (expected '{format2}')" if format2 is not None else "") + ) + + output = open(output, "wb") + output.write(gds.to_bin()) + output.close() + + pairs = cli_file_pairs( + input, output, in_endings=in_endings, out_ending=".gds", recursive=recursive + ) + + duplicates = collections.defaultdict(list) + for ip, op in pairs: + duplicates[op].append(ip) + duplicates = {k: v for k, v in duplicates.items() if len(v) > 1} + if len(duplicates) > 0: + print( + f"ERROR: {len(duplicates)} {'files have' if len(duplicates) > 1 else 'file has'} multiple conflicting source files; please explicitly specify a format to determine which should be used.", + file=sys.stderr, + ) + for op, ips in duplicates.items(): + pathlist = ", ".join("'" + ip + "'" for ip in ips) + print(f"'{op}' could be compiled from {pathlist}", file=sys.stderr) + sys.exit(-1) + + if not overwrite: + new_pairs = [] + existing = [] + for ip, op in pairs: + if os.path.exists(op): + existing.append(op) + else: + new_pairs.append((ip, op)) + + if not quiet: + print(f"Skipping {len(existing)} existing output files.") + + pairs = new_pairs + + foreach_file_pair(pairs, process, quiet=quiet) + + +@cli.command(name="decompile", no_args_is_help=True) +@click.argument("input", required=False, type=click.Path(exists=True)) +@click.argument("output", required=False, type=click.Path(exists=False)) +@click.option( + "--recursive", + "-r", + is_flag=True, + help="Recurse into subdirectories of the input directory to find more applicable files.", +) +@click.option( + "--quiet", + "-q", + is_flag=True, + help="Suppress all output. By default, operations involving multiple files will show a progressbar.", +) +@click.option( + "--overwrite/--no-overwrite", + "-o/-O", + default=True, + help="Whether existing files should be overwritten. Default: true", +) +@click.option( + "--format", + "-f", + required=False, + multiple=False, + help="The format used for output. Possible values: gda (default), json, yaml", +) +def decompile( + input=None, output=None, recursive=False, quiet=False, format=None, overwrite=None +): + """ + Decompiles the GDS script(s) at INPUT into a human-readable text format at OUTPUT. + + INPUT can be a single file or a directory (which obviously has to exist). In the latter case subfiles with the correct file ending will be processed. + If unset, defaults to the current working directory. + + The meaning of OUTPUT may depend on INPUT: + - If INPUT is a file, then OUTPUT is expected to be a file, unless it explicitly ends with a slash indicating a directory. + In this case, if unset OUTPUT will default to the INPUT filename with `.json` exchanged/appended. + - Otherwise OUTPUT has to be a directory as well (or an error will be shown). + In this case, if unset OUTPUT will default to the INPUT directory (which may itself default to the current working directory). + + In the file-to-file case, the paths are explicitly used as they are. Otherwise, if multiple input files were collected, or OUTPUT is a directory, + an output path is inferred for each input file by exchanging the `.gds` file ending for `.json`, or otherwise appending the `.json` file ending. + """ + out_ending = "" + if format == "gda" or format is None: + out_ending = ".gda" + elif format == "json": + out_ending = ".json" + elif format in ["yaml", "yml"]: + out_ending = ".yml" + else: + raise Exception(f"Unsupported output format: '{format}'") + + def process(input, output): + input = open(input, "rb").read() + gds = GDS.from_gds(input) + + nonlocal format + if format is None: + if output.lower().endswith(".gda"): + format = "gda" + elif output.lower().endswith(".json"): + format = "json" + elif output.lower().endswith(".yml") or output.lower().endswith(".yaml"): + format = "yaml" + else: + print( + f"WARNING: output format couldn't be inferred from filename '{output}'; using default (gda). To remove this warning, please explicitly specify a format.", + file=sys.stderr, + ) + + with open(output, "w", encoding="utf-8") as output: + if format == "gda": + output.write(gds.to_gda()) + elif format == "json": + output.write(gds.to_json()) + elif format in ["yaml", "yml"]: + output.write(gds.to_yaml()) + + pairs = cli_file_pairs( + input, output, in_endings=[".gds"], out_ending=out_ending, recursive=recursive + ) + if not overwrite: + new_pairs = [] + existing = [] + for ip, op in pairs: + if os.path.exists(op): + existing.append(op) + else: + new_pairs.append((ip, op)) + + if not quiet: + print(f"Skipping {len(existing)} existing output files.") + + pairs = new_pairs + foreach_file_pair(pairs, process, quiet=quiet) diff --git a/formats/gds/patch.py b/formats/gds/patch.py new file mode 100644 index 0000000..4f21cb6 --- /dev/null +++ b/formats/gds/patch.py @@ -0,0 +1,76 @@ +# TODO: what to do with these? +# Do we just provide a baseline patch internally? Or do I expand the GDA language +# to accommodate for these mistakes? +PATCHES = { + # These scripts use `0x12 if` in a condition, in places where that doesn't seem legal. + # The game would likely crash if this inner if's condition was ever true. The intention + # seems to have been to write `if ... and if ...`, which hints at the nature of the scripting + # language used in the development process! But is still incorrect. + "data/script/rooms/room4_param.gds": [(0x2B1, b"\0\0\x12\0", b"\x09\0\x09\0")], + "data/script/rooms/room13_in.gds": [(0x5C, b"\0\0\x12\0", b"\x09\0\x09\0")], + # Similarly someone might forget that "elseif" is its own command, and write "else if" instead. + # It's actually possible that this doesn't cause an error though! + "data/script/rooms/room12_in.gds": [ + (0x127, b"\0\0\x17\0\0\0\x12\0", b"\0\0\x16\0\x09\0\x09\0") + ], + # I have no idea about these. Maybe stuff got cut? + "data/script/rooms/room23_in.gds": [ + (0x18, b"\0\0\x12\0\0\0\x8d\0", b"\0\0\xdf\0\0\0\xdf\0") + ], + "data/script/rooms/room24_in.gds": [ + (0x18, b"\0\0\x12\0\0\0\x8d\0", b"\0\0\xdf\0\0\0\xdf\0") + ], + # The code says that the second parameter of 0x78 is a float, but it's called with an + # int here... Apparently in their language it was very easy to accidentally write an integer + # instead of a float, and there were no compile-time checks for that. + "data/script/event/e49.gds": [ + (0x24D, b"\x01\0\xfa\xff\xff\xff", b"\x02\0\xc0\0\xc0\0"), + (0x25D, b"\x01\0\xfa\xff\xff\xff", b"\x02\0\xc0\0\xc0\0"), + ], + "data/script/event/e126.gds": [(0x398, b"\x01", b"\x02")], + "data/script/event/e276.gds": [(0x1B4, b"\x01", b"\x02")], + "data/script/event/e233.gds": [(0x1F8, b"\x01", b"\x02")], + "data/script/event/e42.gds": [(0x1C3, b"\x01", b"\x02")], +} +""" +A list of patches to the vanilla scripts, correcting some common mistakes(?). + +For some of these it's more obvious than for others that they are incorrect; an example being +command arguments that should be floats but are integers (likely because of a forgotten decimal point). +There are also cases of extremely nonstandard overlapping use of multiple if statements, which would +crash the game in certain cases, but look like they are obviously meant to mean something correct. +""" + + +def patch(data: bytes, filepath: str) -> bytes: + if filepath not in PATCHES: + return data + + data = bytearray(data) + + for start, old, new in PATCHES[filepath]: + if data[start : start + len(old)] != old: + print( + f"WARN: {filepath}: patch at {hex(start)} incorrect: expected {old}, was {data[start:start+len(old)]}" + ) + continue + data[start : start + len(old)] = new + + return bytes(data) + + +def unpatch(data: bytes, filepath: str) -> bytes: + if filepath not in PATCHES: + return data + + data = bytearray(data) + + for start, old, new in PATCHES[filepath]: + if data[start : start + len(new)] != new: + print( + f"WARN: {filepath}: patch at {hex(start)} incorrect: expected {new}, was {data[start:start+len(new)]}" + ) + continue + data[start : start + len(new)] = old + + return bytes(data) \ No newline at end of file diff --git a/formats/gds/test_cmddef.py b/formats/gds/test_cmddef.py new file mode 100644 index 0000000..e69de29 diff --git a/formats/gds/test_gds.py b/formats/gds/test_gds.py new file mode 100644 index 0000000..3c6c812 --- /dev/null +++ b/formats/gds/test_gds.py @@ -0,0 +1,126 @@ +import os +import pprint + +import unittest + +from .gds import read_gds, write_gds +from .patch import patch +from utils import RESOURCES + +def print_hexdump(data: bytes): + res = "" + padstart = 4 + for i in range(padstart): + if i % 16 == 0: + res += "\n" + elif i % 4 == 0: + res += " " + res += "## " + + i = padstart + for b in data: + if i % 16 == 0: + res += "\n" + elif i % 4 == 0: + res += " " + + if b < 16: + res += "0" + res += hex(b)[2:] + res += " " + i += 1 + return res + + +BASE = os.path.join(RESOURCES, "game_root/lt1_eu") + + +def test_file(filepath: str, base: str): + # sourcery skip: use-fstring-for-concatenation + + with open(os.path.join(base, filepath), "rb") as f: + data = f.read() + data = patch(data, filepath) + try: + prog = read_gds(data, filepath) + except Exception as e: + print(f"ERR: {filepath}: could not decompile: {e}") + return + try: + data2 = write_gds(prog) + except Exception as e: + print(f"ERR: {filepath}: could not recompile: {e}") + return + if data != data2: + print( + f"ERR: {filepath}: Recompiled contents not identical. Result written as {filepath}2 for comparison." + ) + with open(os.path.join(base, filepath + "2"), "wb") as f: + f.write(data2) + elif os.path.exists(os.path.join(base, filepath + "2")): + os.remove(os.path.join(base, filepath + "2")) + + +# TODO: make this a unit test! +def test_all(base: str): + for root, _subdirs, files in os.walk(os.path.join(base, "data/script")): + for f in files: + if not f.endswith(".gds"): + continue + path = os.path.relpath(os.path.join(root, f), base) + test_file(path, base) + +# test_all(BASE) +# test_file("data/script/qscript/q4_param.gds", BASE) + +class TestGDSVanillaFiles(unittest.TestCase): + def test_all_files(self): + """ + Checks if the decompiler is able to disassemble and reassemble each of the unmodified original game scripts. + This baseline sanity check ensures that merely running `decompile` and then `compile` does not corrupt + game files. + + Note that for this test to be possible, original files from the LT1 EU version .nds ROM need to be + extracted/symlinked into `{flora root dir}/data/game_root/lt1_eu/`, such that the folder contains the + subfolders `data`, `dwc` and `ftc`. Eventually Flora should support this, but for now you can use Tinke to do it. + """ + base = BASE +# sourcery skip: no-conditionals-in-tests + if not os.path.exists(base) or not os.path.isdir(base): + self.skipTest("Could not find vanilla game files under \"data/game_root/lt1_eu\". Please make sure " + "to extract and either copy or symlink the entire contents of a valid LT1 EU version .nds ROM " + "into that location.") +# sourcery skip: no-loop-in-tests + for root, _subdirs, files in os.walk(os.path.join(base, "data/script")): + for f in files: + if not f.endswith(".gds"): + continue + path = os.path.relpath(os.path.join(root, f), base) + with self.subTest(msg=f"Check file {path}"): + self.single_file(path, base) + + def single_file(self, filepath: str, base: str): + with open(os.path.join(base, filepath), "rb") as f: + data = f.read() + data = patch(data, filepath) + try: + prog = read_gds(data, filepath) + except Exception as e: + self.fail(f"Decompile: {e}") + raise e + try: + data2 = write_gds(prog) + except Exception as e: + self.fail(f"Recompile: {e}") + raise e + + self.assertEqual(data, data2, "Recompiled result is not identical") + # if data != data2: + # print( + # f"ERR: {filepath}: Recompiled contents not identical. Result written as {filepath}2 for comparison." + # ) + # with open(os.path.join(base, filepath + "2"), "wb") as f: + # f.write(data2) + # elif os.path.exists(os.path.join(base, filepath + "2")): + # os.remove(os.path.join(base, filepath + "2")) + diff --git a/main.py b/main.py index 2c52fa0..72a8fd1 100644 --- a/main.py +++ b/main.py @@ -10,7 +10,7 @@ def cli(): pass -cli.add_command(formats.gds.cli, "gds") +cli.add_command(formats.gds.old.cli, "gds") cli.add_command(formats.bg.cli, "bg") cli.add_command(formats.pcm.cli, "pcm") cli.add_command(formats.puzzle.cli, "puzzle") diff --git a/utils.py b/utils.py index a35c133..f731a1b 100644 --- a/utils.py +++ b/utils.py @@ -1,5 +1,5 @@ import os - +from typing import TypeVar, Generic, Mapping, get_type_hints, get_origin, Callable, Union, Any def cli_file_pairs( input=None, @@ -120,3 +120,75 @@ def foreach_file_pair(pairs, fn, quiet=False): pass for input, output in pairs: fn(input, output) + +T = TypeVar('T') +class TU(Generic[T]): + union: type + name: str + + def __init__(self, union: type, name: str): + self.union = union + self.name = name + + def __call__(self, val=None) -> "TUI[T]": + # assert isinstance(val, T) + return TUI(self, val) + + def __contains__(self, other): + return other.variant == self if isinstance(other, TUI) else False + + # def __eq__(self, other): + # return self.union == other.union and self.name == other.name + + def __hash__(self): + return hash((self.union, self.name)) + + def __repr__(self): + return f"{self.union.__name__}.{self.name}" + +class TUI(Generic[T]): + variant: TU[T] + value: T + + def __init__(self, variant: TU[T], value: T): + self.variant = variant + self.value = value + + def __call__(self) -> T: + return self.value + + def __eq__(self, other): + return self.variant == other.variant and self.value == other.value + + def __hash__(self): + return hash((self.variant, self.value)) + + def __repr__(self): + return f"{repr(self.variant)}({repr(self.value)})" + +def tagged_union(cls: type): + hints = get_type_hints(cls) + members = [(k,v) for (k,v) in hints.items() if get_origin(v) == TU] + + for (name, t) in members: + setattr(cls, name, t(cls, name)) + + return cls + +class Test: + a: TU[int] + b: TU[str] + +R = TypeVar('R') +def match(union: TUI, fns: Mapping[Union[TU, type(...)], Callable[[Any],R]]) -> R: + ellipsis_fn = None + for k,v in fns.items(): + if k is Ellipsis: + ellipsis_fn = v + continue + if union in k: + return v(union()) + if ellipsis_fn is not None: + return ellipsis_fn() + +RESOURCES = os.path.join(os.path.dirname(__file__), "data") From 661840e03a620e22988fc50c33bd65b3da3f7bf7 Mon Sep 17 00:00:00 2001 From: ilonachan Date: Thu, 3 Oct 2024 21:01:35 +0200 Subject: [PATCH 09/17] feat(gds): The writing frontend for GDA works! It successfully writes scripts in a nice format, with indentations too! --- formats/gds/cmddef.py | 2 + formats/gds/gda.py | 186 +++++++++++++++++++++++++++++++ formats/gds/gds.py | 240 ++++++++++++++++++++++++++++++---------- formats/gds/model.py | 4 +- formats/gds/test_gda.py | 43 +++++++ utils.py | 17 ++- 6 files changed, 429 insertions(+), 63 deletions(-) create mode 100644 formats/gds/gda.py create mode 100644 formats/gds/test_gda.py diff --git a/formats/gds/cmddef.py b/formats/gds/cmddef.py index 19c025c..7cb5483 100755 --- a/formats/gds/cmddef.py +++ b/formats/gds/cmddef.py @@ -25,6 +25,7 @@ class GDSCommandParam: """ The data type of the command. Real builtin GDS datatypes are: int (1) + (also usable as uint, short, ushort, byte, ubyte; TODO: document which parameters use which) float (2) string (3) longstr (4, never used) @@ -40,6 +41,7 @@ class GDSCommandParam: rect(x: int, y: int, w: int, h: int) ... """ + name: Optional[str] = None """A descriptive name for the parameter. Can optionally be written in the script.""" desc: Optional[str] = None diff --git a/formats/gds/gda.py b/formats/gds/gda.py new file mode 100644 index 0000000..f82fdd4 --- /dev/null +++ b/formats/gds/gda.py @@ -0,0 +1,186 @@ +""" +Provides the `read_gda` and `write_gda` methods to read/write a `GDSProgram` (see the `model` module) from/to a a human-readable GDA text file (buffer). + +Requires the command definitions provided by `cmddef` for type checking and parameter data, as well as (possibly in the future) documentation features. +""" + +from typing import Optional, List +from dataclasses import dataclass + +from .model import ( + GDSProgram, + GDSElement, + GDSLabel, + GDSInvocation, + GDSValue, + GDSIfInvocation, + GDSLoopInvocation, + GDSConditionToken +) + + +def read_gda(data: str, path: Optional[str] = None) -> GDSProgram: + pass + + +@dataclass(kw_only=True) +class WriterContext: + filename: Optional[str] = None + result: str = "" + indent: int = 0 + + def write(self, s: str): + self.result += s + def nl(self): + self.result += '\n' + ' '*self.indent + def insert_comment(self, comment: str): + self.result, latest_line = self.result.rsplit("\n", 1) + self.nl() + self.write(f"# {comment}\n") + self.write(latest_line) + + +def write_gda(prog: GDSProgram, filename: Optional[str] = None) -> str: + ctx = WriterContext(filename=filename) + version = "0.1" + ctx.write(f"#!version {version}\n") + ctx.write(f"# {filename}\n") + + for el in prog.elements: + ctx.nl() + write_element(ctx, el) + return ctx.result + + +def write_element(ctx: WriterContext, el: GDSElement): + if el in GDSElement.BREAK: + ctx.write("break") + elif el in GDSElement.label: + lbl: GDSLabel = el() + ctx.write(f'{"@" if lbl.present else "@!"}{lbl.name}') + if lbl.loc is not None: + ctx.write(f"({lbl.loc})") + elif el in GDSElement.command: + cmd: GDSInvocation = el() + ctx.write(cmd.command.name or hex(cmd.command.id)) + if cmd.command.complex: + ( + WRITE_COMPLEX.get(cmd.command.name) + or WRITE_COMPLEX.get(cmd.command.id) + )(ctx, cmd) + else: + write_simple(ctx, cmd) + +def write_simple(ctx: WriterContext, cmd: GDSInvocation): + if len(cmd.args) == 0: + return + ctx.write(" (") + first = True + for arg, param in zip(cmd.args, cmd.command.params): + if first: + first = False + else: + ctx.write(" ") + if arg in GDSValue.bool: + val = arg() + if isinstance(val, str): + arg = GDSValue.str(val) + elif isinstance(val, int): + arg = GDSValue.int(val) + else: + ctx.write("true" if arg else "false") + if arg in GDSValue.int: + val: int = arg() + bytelen = ( + 4 + if param.type.endswith("int") + else 2 if param.type.endswith("short") else 1 + ) + if not param.type.startswith("u") and val >= 2 ** (bytelen * 8 - 1): + val -= 2 ** (bytelen * 8 - 1) + ctx.write(str(val)) + elif arg in GDSValue.float: + val: float = arg() + ctx.write(str(val)) + elif arg in GDSValue.str or arg in GDSValue.longstr: + val: str = arg() + if param.type == "longstr": + ctx.write("l") + ctx.write(repr(val)) + ctx.write(")") + +def write_condition(ctx: WriterContext, cond: List[GDSConditionToken]): + first = True + for tok in cond: + if first: + first = False + else: + ctx.write(" ") + if tok in GDSConditionToken.NOT: + ctx.write("not") + elif tok in GDSConditionToken.AND: + ctx.write("and") + elif tok in GDSConditionToken.OR: + ctx.write("or") + elif tok in GDSConditionToken.command: + cmd: GDSInvocation = tok() + ctx.write(cmd.command.name or hex(cmd.command.id)) + if cmd.command.complex: + ( + WRITE_COMPLEX.get(cmd.command.name) + or WRITE_COMPLEX.get(cmd.command.id) + )(ctx, cmd) + else: + write_simple(ctx, cmd) + +def write_block(ctx: WriterContext, cond: List[GDSElement]): + ctx.write(" {") + ctx.indent += 2 + for el in cond: + ctx.nl() + write_element(ctx, el) + ctx.indent -= 2 + ctx.nl() + ctx.write("}") + +def write_if(ctx: WriterContext, cmd: GDSIfInvocation): + ctx.write(" ") + write_condition(ctx, cmd.condition) + ctx.write(":") + if cmd.target is not None: + ctx.write(f' {"@" if cmd.target.primary else "@!"}{cmd.target.label}') + if cmd.block is not None: + write_block(ctx, cmd.block) + +def write_else(ctx: WriterContext, cmd: GDSIfInvocation): + ctx.write(":") + if cmd.target is not None: + ctx.write(f' {"@" if cmd.target.primary else "@!"}{cmd.target.label}') + if cmd.block is not None: + write_block(ctx, cmd.block) + +def write_repeatN(ctx: WriterContext, cmd: GDSLoopInvocation): + count: int = cmd.condition + ctx.write(f" {count}:") + if cmd.target is not None: + ctx.write(f' {"@" if cmd.target.primary else "@!"}{cmd.target.label}') + if cmd.block is not None: + write_block(ctx, cmd.block) + +def write_while(ctx: WriterContext, cmd: GDSLoopInvocation): + ctx.write(" ") + write_condition(ctx, cmd.condition) + ctx.write(":") + if cmd.target is not None: + ctx.write(f' {"@" if cmd.target.primary else "@!"}{cmd.target.label}') + if cmd.block is not None: + write_block(ctx, cmd.block) + + +WRITE_COMPLEX = { + "if": write_if, + "elif": write_if, + "else": write_else, + "repeatN": write_repeatN, + "while": write_while, +} diff --git a/formats/gds/gds.py b/formats/gds/gds.py index 06bd1b1..ba5c0b9 100644 --- a/formats/gds/gds.py +++ b/formats/gds/gds.py @@ -1,10 +1,16 @@ +""" +Provides the `read_gds` and `write_gds` methods to read/write a `GDSProgram` (see the `model` module) from/to a binary GDS file (buffer). + +Requires the command definitions provided by `cmddef` to know parameter counts and behavior +""" + from typing import Tuple, List, Optional, Callable, TypeVar, Mapping, Union from dataclasses import dataclass, field import struct import functools # pylint: disable=unused-wildcard-import,wildcard-import -from utils import tagged_union, TU, match +from utils import tagged_union, TU, match, nested_break from .model import ( GDSProgram, @@ -264,6 +270,117 @@ def name_labels(self): # pass return label_names + def collapse_blocks(self, block: List[GDSElement]) -> List[GDSElement]: + sid = 0 + while sid < len(block): + with nested_break() as continue_outer: + el = block[sid] + if el not in GDSElement.command or not isinstance( + el(), (GDSIfInvocation, GDSLoopInvocation) + ): + raise continue_outer # also very convenient for preserving a cleanup block in a continue + + block_cmd: Union[GDSIfInvocation, GDSLoopInvocation] = el() + + source: LabelUse = block_cmd.target + if source is None: + raise continue_outer + target: LabelToken = None + for label in self.labels[source.addr]: + if isinstance(label, LabelUse) and label is not source: + # more than one thing jumps here, + # that's not pro-level + raise continue_outer + if isinstance(label, LabelToken): + target = label + if target.loc <= source.loc: + # backwards jumps are VERY not pro-level + raise continue_outer + + # Cut out the frame between these two tokens and put it in a block in the command right before. + # Also remove the labels from the map + tid = sid + 1 + while tid < len(block) and not ( + block[tid] in GDSElement.label and block[tid]() is target + ): + tid += 1 + if tid == len(block): + # I believe the only way this can happen at this point, + # is if the jump target is outside of the current block. Which is perfect! + raise continue_outer + + # excludes the command in front and the target label + sub_block = block[sid + 1 : tid] + block = block[: sid + 1] + block[tid + 1 :] + block_cmd.target = None + block_cmd.block = sub_block + + block_cmd.block = self.collapse_blocks(sub_block) + sid += 1 + return block + + def finalize_labels( + self, block: List[GDSElement], label_names: Mapping[int, str], labels=None + ) -> Mapping[str, Union[GDSLabel, GDSAddress]]: + if labels is None: + labels = {} + + oldlabel_indices = [ + (i, el()) for i, el in enumerate(block) if el in GDSElement.label + ] + olduse_indices = [ + (el, el().target) + for el in block + if el in GDSElement.command + and isinstance(el(), (BinaryIfCommand, BinaryLoopCommand)) + and isinstance(el().target, LabelUse) + ] + + for addr, name in label_names.items(): + if name not in labels: + labels[name] = [] + for i in self.labels[addr]: + if isinstance(i, LabelToken): + newlabel = GDSLabel( + name=name, + present=i.addr is not None, + loc=i.addr if i.pointsto is None else None, + ) + found = False + idx = None + for idx, el in oldlabel_indices: + if el == i: + found = True + break + if found: + block[idx] = GDSElement.label(newlabel) + labels[name].append(newlabel) + elif isinstance(i, LabelUse): + newuse = GDSJumpAddress(label=name, primary=i.primary) + found = False + el = None + for el, olduse in olduse_indices: + if olduse == i: + found = True + break + if found: + el().target = newuse + labels[name].append(newuse) + else: + raise TypeError() + + for el in block: + if el not in GDSElement.command or not isinstance( + el(), (BinaryIfCommand, BinaryLoopCommand) + ): + continue + block_cmd = el() + if block_cmd.block is None: + continue + self.finalize_labels(block_cmd.block, label_names, labels) + + return labels + def read_gds(data: bytes, path: str = None) -> GDSProgram: ctx = DecompilerState(data) @@ -287,42 +404,9 @@ def read_gds(data: bytes, path: str = None) -> GDSProgram: label_names = ctx.name_labels() - oldlabel_indices = [ - (i, el()) for i, el in enumerate(ctx.elements) if el in GDSElement.label - ] - olduse_indices = [ - (el, el().target) - for el in ctx.elements - if el in GDSElement.command - and isinstance(el(), (BinaryIfCommand, BinaryLoopCommand)) - and isinstance(el().target, LabelUse) - ] - - labels = {} - for addr, name in label_names.items(): - lst = [] - for i in ctx.labels[addr]: - if isinstance(i, LabelToken): - newlabel = GDSLabel( - name=name, - present=i.addr is not None, - loc=i.addr if i.pointsto is None else None, - ) - for idx, el in oldlabel_indices: - if el == i: - ctx.elements[idx] = GDSElement.label(newlabel) - break - lst.append(newlabel) - elif isinstance(i, LabelUse): - newuse = GDSJumpAddress(label=name, primary=i.primary) - for el, olduse in olduse_indices: - if olduse == i: - el().target = newuse - break - lst.append(newuse) - else: - raise TypeError() - labels[name] = lst + ctx.elements = ctx.collapse_blocks(ctx.elements) + + labels = ctx.finalize_labels(ctx.elements, label_names) return GDSProgram( context=ctx.context, path=path, elements=ctx.elements, labels=labels @@ -379,7 +463,10 @@ def read_simple(ctx: DecompilerState, cmdobj: GDSCommand) -> BinaryCommand: arg.val, { GDSTokenValue.int: lambda val: [ - (GDSValue.int(val), ["int"]), + ( + GDSValue.int(val), + ["int", "uint", "short", "ushort", "byte", "ubyte"], + ), ( GDSValue.bool(val), ["bool", "bool|int"], @@ -399,7 +486,16 @@ def read_simple(ctx: DecompilerState, cmdobj: GDSCommand) -> BinaryCommand: ...: lambda: [], }, ) - val = next((v for v, reqtypes in options if param.type in reqtypes), None) + # pylint: disable=not-callable + val = next( + ( + v + for v, reqtypes in options + if (isinstance(reqtypes, list) and param.type in reqtypes) + or (callable(reqtypes) and reqtypes(param.type)) + ), + None, + ) if val is None: if not param.optional: raise ValueError( @@ -421,7 +517,7 @@ def read_if(ctx: DecompilerState, cmdobj: GDSCommand) -> BinaryIfCommand: command=cmdobj, args=[], condition=cond, - block=[], + block=None, elze=False, elseif=False, target=addr, @@ -437,7 +533,7 @@ def read_elif(ctx: DecompilerState, cmdobj: GDSCommand) -> BinaryIfCommand: command=cmdobj, args=[], condition=cond, - block=[], + block=None, elze=False, elseif=True, target=addr, @@ -457,7 +553,7 @@ def read_else(ctx: DecompilerState, cmdobj: GDSCommand) -> BinaryIfCommand: command=cmdobj, args=[], condition=None, - block=[], + block=None, elze=True, elseif=False, target=addr, @@ -622,15 +718,15 @@ def write_gds(prog: GDSProgram) -> bytes: ctx.write_command(el()) elif el in GDSElement.label: ctx.write_label(el()) - + ctx.write_token(GDSTokenValue.fileend()) - return len(ctx.data).to_bytes(4, 'little') + bytes(ctx.data) + return len(ctx.data).to_bytes(4, "little") + bytes(ctx.data) def write_simple(ctx: CompilerState, cmd: GDSInvocation): args = cmd.args if len(args) < len(cmd.command.params): - args += [None] * len(cmd.command.params)-len(args) + args += [None] * len(cmd.command.params) - len(args) for arg, param in zip(args, cmd.command.params): if arg is None: if param.optional: @@ -641,7 +737,19 @@ def write_simple(ctx: CompilerState, cmd: GDSInvocation): arg, { GDSValue.int: lambda val: [ - (GDSTokenValue.int(val), ["int", "bool", "bool|int"]), + ( + GDSTokenValue.int(val), + [ + "int", + "uint", + "short", + "ushort", + "byte", + "ubyte", + "bool", + "bool|int", + ], + ), ], GDSValue.float: lambda val: [(GDSTokenValue.float(val), ["float"])], GDSValue.str: lambda val: [ @@ -650,23 +758,34 @@ def write_simple(ctx: CompilerState, cmd: GDSInvocation): GDSValue.longstr: lambda val: [ (GDSTokenValue.longstr(val), ["longstr"]) ], - GDSValue.bool: lambda val: - [(GDSTokenValue.int(val), ["bool", "bool|int"])] if isinstance(val, int) - else - [(GDSTokenValue.str(val), ["bool", "bool|string"])] if isinstance(val, str) - else - [ - (GDSTokenValue.int(1 if val else 0), ["bool", "bool|int"]), - ( - GDSTokenValue.str("true" if val else "false"), - ["bool", "bool|string"], - ), - ] - , + GDSValue.bool: lambda val: ( + [(GDSTokenValue.int(val), ["bool", "bool|int"])] + if isinstance(val, int) + else ( + [(GDSTokenValue.str(val), ["bool", "bool|string"])] + if isinstance(val, str) + else [ + (GDSTokenValue.int(1 if val else 0), ["bool", "bool|int"]), + ( + GDSTokenValue.str("true" if val else "false"), + ["bool", "bool|string"], + ), + ] + ) + ), ...: lambda: [], }, ) - tok = next((v for v, reqtypes in options if param.type in reqtypes), None) + # pylint: disable=not-callable + tok = next( + ( + v + for v, reqtypes in options + if (isinstance(reqtypes, list) and param.type in reqtypes) + or (callable(reqtypes) and reqtypes(param.type)) + ), + None, + ) if tok is None: raise ValueError( f"Unexpected parameter type: should have been {param.type}, value was {arg}" @@ -703,6 +822,7 @@ def write_repeatN(ctx: CompilerState, cmd: GDSLoopInvocation): ctx.write_token(GDSTokenValue.int(cmd.condition)) ctx.write_addr(cmd.target) + def write_while(ctx: CompilerState, cmd: GDSLoopInvocation): write_condition(ctx, cmd.condition) ctx.write_addr(cmd.target) diff --git a/formats/gds/model.py b/formats/gds/model.py index 7f9355e..f9485c0 100644 --- a/formats/gds/model.py +++ b/formats/gds/model.py @@ -61,7 +61,7 @@ class GDSJumpAddress: class GDSIfInvocation(GDSInvocation): condition: List[GDSConditionToken] target: GDSJumpAddress - block: Optional[List["GDSElement"]] + block: Optional[List["GDSElement"]] = None elseif: bool = False elze: bool = False @@ -69,7 +69,7 @@ class GDSIfInvocation(GDSInvocation): @dataclass(kw_only=True) class GDSLoopInvocation(GDSInvocation): condition: Union[List[GDSConditionToken], int] - block: Optional[List["GDSElement"]] + block: Optional[List["GDSElement"]] = None target: GDSJumpAddress diff --git a/formats/gds/test_gda.py b/formats/gds/test_gda.py new file mode 100644 index 0000000..2d0c483 --- /dev/null +++ b/formats/gds/test_gda.py @@ -0,0 +1,43 @@ +import os +import pprint + +import unittest + +import tqdm + +from .gds import read_gds, write_gds +from .gda import read_gda, write_gda +from .patch import patch + +from utils import RESOURCES + + +def test_file(gdspath: str, base: str): + with open(os.path.join(base, gdspath), "rb") as gdsf: + gds = gdsf.read() + gds = patch(gds, gdspath) + prog = read_gds(gds, gdspath) + gda = write_gda(prog, gdspath) + gdapath = gdspath.replace(".gds", ".gda") + with open(os.path.join(base, gdapath), "w", encoding="utf8") as gdaf: + gdaf.write(gda) + + +BASE = os.path.join(RESOURCES, "game_root/lt1_eu") + + +def test_all(base: str): + for f in tqdm.tqdm( + os.path.join(root, f) + for root, _subdirs, files in os.walk(os.path.join(base, "data/script")) + for f in files + ): + if not f.endswith(".gds"): + continue + path = os.path.relpath(f, base) + test_file(path, base) + + +if __name__ == "__main__": + # test_file("data/script/event/e6.gds", BASE) + test_all(BASE) diff --git a/utils.py b/utils.py index f731a1b..c90d689 100644 --- a/utils.py +++ b/utils.py @@ -1,5 +1,8 @@ import os -from typing import TypeVar, Generic, Mapping, get_type_hints, get_origin, Callable, Union, Any +from contextlib import contextmanager, suppress +from typing import (Any, Callable, Generic, Mapping, TypeVar, Union, + get_origin, get_type_hints) + def cli_file_pairs( input=None, @@ -192,3 +195,15 @@ def match(union: TUI, fns: Mapping[Union[TU, type(...)], Callable[[Any],R]]) -> return ellipsis_fn() RESOURCES = os.path.join(os.path.dirname(__file__), "data") + + +@contextmanager +def nested_break(): + """ + Raise the returned value to instantly bail out of the with block. + Replaces loop labels for breaking out of nested iterations. + """ + class NestedBreakException(Exception): + pass + with suppress(NestedBreakException): + yield NestedBreakException From 800a7095faca7b01fbf5a3652599d79e1b618708 Mon Sep 17 00:00:00 2001 From: ilonachan Date: Thu, 3 Oct 2024 21:01:35 +0200 Subject: [PATCH 10/17] fix(gds): float values in decompiled GDA scripts are now displayed rounded to the lowest number of decimal points that still produces identical byte data. --- formats/gds/gda.py | 3 ++- utils.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/formats/gds/gda.py b/formats/gds/gda.py index f82fdd4..fa92f3a 100644 --- a/formats/gds/gda.py +++ b/formats/gds/gda.py @@ -18,6 +18,7 @@ GDSConditionToken ) +from utils import round_perfect def read_gda(data: str, path: Optional[str] = None) -> GDSProgram: pass @@ -101,7 +102,7 @@ def write_simple(ctx: WriterContext, cmd: GDSInvocation): ctx.write(str(val)) elif arg in GDSValue.float: val: float = arg() - ctx.write(str(val)) + ctx.write(str(round_perfect(val))) elif arg in GDSValue.str or arg in GDSValue.longstr: val: str = arg() if param.type == "longstr": diff --git a/utils.py b/utils.py index c90d689..b572fd5 100644 --- a/utils.py +++ b/utils.py @@ -2,6 +2,7 @@ from contextlib import contextmanager, suppress from typing import (Any, Callable, Generic, Mapping, TypeVar, Union, get_origin, get_type_hints) +import struct def cli_file_pairs( @@ -207,3 +208,16 @@ class NestedBreakException(Exception): pass with suppress(NestedBreakException): yield NestedBreakException + +def round_places(x: float, places: int=0) -> float: + return round(x * (10**places)) / (10**places) + +def round_perfect(x: float) -> float: + for i in range(1, 8): + y = round_places(x, i) + # If the rounded form is equivalent in bits to the full version, + # just use this simpler rounded version. + if struct.unpack(" Date: Thu, 3 Oct 2024 21:01:35 +0200 Subject: [PATCH 11/17] wip --- formats/gds/cmddef.py | 4 ++-- formats/gds/gda.py | 42 +++++++++++++++++++++++++++++++++++++----- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/formats/gds/cmddef.py b/formats/gds/cmddef.py index 7cb5483..7726284 100755 --- a/formats/gds/cmddef.py +++ b/formats/gds/cmddef.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -from typing import Optional, List, Union, Set +from typing import Optional, List, Union, Set, Any, Mapping import os from dataclasses import dataclass, field @@ -395,4 +395,4 @@ def init_commands(root: str = None): pprint.pp(COMMANDS_BYID) pprint.pp(COMMANDS_BYNAME) else: - init_commands() \ No newline at end of file + init_commands() diff --git a/formats/gds/gda.py b/formats/gds/gda.py index fa92f3a..0b3c0b3 100644 --- a/formats/gds/gda.py +++ b/formats/gds/gda.py @@ -7,6 +7,9 @@ from typing import Optional, List from dataclasses import dataclass + +from utils import round_perfect +from .cmddef import GDSCommand from .model import ( GDSProgram, GDSElement, @@ -18,8 +21,6 @@ GDSConditionToken ) -from utils import round_perfect - def read_gda(data: str, path: Optional[str] = None) -> GDSProgram: pass @@ -36,9 +37,10 @@ def nl(self): self.result += '\n' + ' '*self.indent def insert_comment(self, comment: str): self.result, latest_line = self.result.rsplit("\n", 1) - self.nl() - self.write(f"# {comment}\n") - self.write(latest_line) + for line in comment.splitlines(): + self.nl() + self.write(f"# {line}") + self.write("\n"+latest_line) def write_gda(prog: GDSProgram, filename: Optional[str] = None) -> str: @@ -185,3 +187,33 @@ def write_while(ctx: WriterContext, cmd: GDSLoopInvocation): "repeatN": write_repeatN, "while": write_while, } + + + +def format_comment(comment: str, filename: str, args: List[GDSValue], cmd: GDSCommand): + from parsy import forward_declaration, regex, seq, string, alt + + str_part = regex(r'[^$]') + str_part_fvar = regex(r'[^$}:]') + str_part_expr = regex(r'[^$)]') + fvar_esc = string(r'}}') | string(r'::') + expr_esc = string(r'))') + + format_args = regex(r"0\d+") + fvar = forward_declaration() + expr = forward_declaration() + str_var = string('$') >> ( + string('{') >> fvar << string('}') + | string('(') >> expr << string(')') + ) + + varname = regex(r"\w+") + + fvar_expr = (str_part_fvar | fvar_esc | str_var).many().concat() + fvar.become( seq ( (varname) , string(":") >> format_args | None ) ) + expr.become( (str_part_expr | expr_esc | str_var).many().concat() ) + + str_ = (str_part | str_var).many().concat() + + # TODO: parse variables like $TEST or ${1:03} and $(filepath) + pass \ No newline at end of file From a2391904afcced1514b7724f3efc1aa826acb25d Mon Sep 17 00:00:00 2001 From: ilonachan Date: Thu, 3 Oct 2024 21:01:35 +0200 Subject: [PATCH 12/17] wip (seems to work, but I want to make changes to values, which requires changes to tagged-enums) --- formats/gds/gda.py | 73 +++++++++++++++++++++++++++++++++++---- requirements-dev.txt | 0 requirements-optional.txt | 1 + requirements.txt | 3 ++ 4 files changed, 70 insertions(+), 7 deletions(-) create mode 100644 requirements-dev.txt create mode 100644 requirements-optional.txt diff --git a/formats/gds/gda.py b/formats/gds/gda.py index 0b3c0b3..098ca90 100644 --- a/formats/gds/gda.py +++ b/formats/gds/gda.py @@ -4,7 +4,8 @@ Requires the command definitions provided by `cmddef` for type checking and parameter data, as well as (possibly in the future) documentation features. """ -from typing import Optional, List +import contextlib +from typing import Any, Optional, List from dataclasses import dataclass @@ -191,15 +192,65 @@ def write_while(ctx: WriterContext, cmd: GDSLoopInvocation): def format_comment(comment: str, filename: str, args: List[GDSValue], cmd: GDSCommand): - from parsy import forward_declaration, regex, seq, string, alt + import re + def get_var(name: str) -> Any: + if name.lower() == "lang": + return "en" + if name.lower() == "eventid": + match: re.Match = re.match(r"/data/script/event/e(\d+).gd[as]", filename) + if match: + return int(match.group(1)) + else: + return "?" + with contextlib.suppress(ValueError): + arg_id = int(name) - 1 + if arg_id < 0 or arg_id >= len(args): + return "?" + + arg = args[arg_id] + + + + return "?" + + def format_value(value: Any, modifiers: List[str]) -> str: + for m in modifiers: + if not m.startswith("r"): + continue + step, max_, *_ = m[1:].split("<=") + [None] + if not isinstance(value, int): + value = "?" + break + step = int(step) + value = (value // step) * step + if max_ is not None: + value = min(value, int(max_)) + + for m in modifiers: + if not m.startswith("0"): + continue + l = int(m[1:]) + if not isinstance(value, int): + value = "?" * l + break + value = str(value) + value = "0" * max(0, l - len(value)) + value + + return str(value) + + def readfile(path: str) -> str: + return f"test file content\n({path})" + + from parsy import forward_declaration, regex, seq, string str_part = regex(r'[^$]') + str_esc = string(r'$$') str_part_fvar = regex(r'[^$}:]') str_part_expr = regex(r'[^$)]') fvar_esc = string(r'}}') | string(r'::') expr_esc = string(r'))') - format_args = regex(r"0\d+") + format_args = regex(r"0\d+") | regex(r"r\d+(<=\d+)?") fvar = forward_declaration() expr = forward_declaration() str_var = string('$') >> ( @@ -210,10 +261,18 @@ def format_comment(comment: str, filename: str, args: List[GDSValue], cmd: GDSCo varname = regex(r"\w+") fvar_expr = (str_part_fvar | fvar_esc | str_var).many().concat() - fvar.become( seq ( (varname) , string(":") >> format_args | None ) ) - expr.become( (str_part_expr | expr_esc | str_var).many().concat() ) + fvar.become( seq ( varname.map(get_var) , (string(":") >> format_args).many() ).map(lambda a: format_value(a[0], a[1])) ) + expr.become( (str_part_expr | expr_esc | str_var).many().concat().map(readfile) ) - str_ = (str_part | str_var).many().concat() + str_ = (str_part | str_esc | str_var).many().concat() # TODO: parse variables like $TEST or ${1:03} and $(filepath) - pass \ No newline at end of file + return str_.parse(comment) + +if __name__ == "__main__": + EXAMPLE = """ + /data/etext/${lang}/e${eventid:r100<=300:03}{pcm}/e${eventid}_t${1}.txt: + + $(/data/etext/${lang}/e${eventid:r100<=300:03}{pcm}/e${eventid}_t${1}.txt) + """ + print(format_comment(EXAMPLE, "/data/script/event/e324.gds", [], None)) \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..e69de29 diff --git a/requirements-optional.txt b/requirements-optional.txt new file mode 100644 index 0000000..fa9cf06 --- /dev/null +++ b/requirements-optional.txt @@ -0,0 +1 @@ +tqdm \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 7389c82..5a305e6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,6 @@ click pillow ndspy +parsy +pyyaml +dacite \ No newline at end of file From 3e81c66b48b9bcf1975c11171c57df20ffa77768 Mon Sep 17 00:00:00 2001 From: ilonachan Date: Thu, 3 Oct 2024 21:01:35 +0200 Subject: [PATCH 13/17] feat(gds): Parser for GDA is done, can de- and re-compile all vanilla GDS files into identical binaries (except patches) --- data/gds_commands/event.yml | 40 +- data/gds_commands/minigames/hotel.yml | 12 +- formats/gds/__init__.py | 3 + formats/gds/cmddef.py | 13 +- formats/gds/gda.py | 538 +++++++++++++++++++++----- formats/gds/gds.py | 283 ++++++-------- formats/gds/model.py | 26 +- formats/gds/old.py | 9 +- formats/gds/preview.py | 69 ++++ formats/gds/test_gda.py | 71 +++- formats/gds/value.py | 512 ++++++++++++++++++++++++ main.py | 4 +- utils.py | 123 +++--- version.py | 2 +- 14 files changed, 1339 insertions(+), 366 deletions(-) create mode 100644 formats/gds/preview.py create mode 100644 formats/gds/value.py diff --git a/data/gds_commands/event.yml b/data/gds_commands/event.yml index d4ca3a0..1ac6fc9 100644 --- a/data/gds_commands/event.yml +++ b/data/gds_commands/event.yml @@ -14,6 +14,10 @@ commands: desc: > Displays a textbox for the specified on-screen actor, pointing in the direction indicated by that actor's sprite location. + comment: | + /data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt: + + $(/data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt) params: textId: type: int @@ -27,6 +31,10 @@ commands: id: 0x9c desc: > Displays a textbox for the specified on-screen actor, pointing right. + comment: | + /data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt: + + $(/data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt) params: textId: type: int @@ -40,6 +48,10 @@ commands: id: 0x9d desc: > Displays a textbox for the specified on-screen actor, pointing left. + comment: | + /data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt: + + $(/data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt) params: textId: type: int @@ -52,6 +64,10 @@ commands: textboxMiddle: id: 0xcd desc: Displays a text box for the specified on-screen actor, with no arrow pointing. + comment: | + /data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt: + + $(/data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt) params: textId: type: int @@ -65,6 +81,10 @@ commands: textboxNoSpeaker: id: 0x9b desc: Displays a textbox without a speaker on screen. + comment: | + /data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt: + + $(/data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt) params: textId: type: int @@ -74,6 +94,10 @@ commands: textboxNoSpeaker2: id: 0x44 desc: Displays a textbox without a speaker on screen. (I'm not sure what the difference between these two functions is) + comment: | + /data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt: + + $(/data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt) params: textId: type: int @@ -83,6 +107,10 @@ commands: textboxNoSpeakerRight: id: 0x99 desc: Displays a textbox pointing right, but without a speaker on screen. + comment: | + /data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt: + + $(/data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt) params: textId: type: int @@ -92,6 +120,10 @@ commands: textboxNoSpeakerLeft: id: 0x9a desc: Displays a textbox pointing left, but without a speaker on screen. + comment: | + /data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt: + + $(/data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt) params: textId: type: int @@ -252,14 +284,14 @@ commands: desc: The question to be displayed for this choice params: question: - type: string + type: string(64) desc: Unlike most strings in this game, this one is hardcoded in the script, and therefore NOT localized! choiceAnswer1: id: 0xd6 desc: Answer 1 of this choice params: answer: - type: string + type: string(64) desc: Just like the question, not localizable. unk_2: int choiceAnswer2: @@ -267,7 +299,7 @@ commands: desc: Answer 2 of this choice params: answer: - type: string + type: string(64) desc: Just like the question, not localizable. unk_2: int choiceAnswer3: @@ -275,7 +307,7 @@ commands: desc: Answer 3 of this choice params: answer: - type: string + type: string(64) desc: Just like the question, not localizable. unk_2: int choiceAnswerIs: diff --git a/data/gds_commands/minigames/hotel.yml b/data/gds_commands/minigames/hotel.yml index 7201fb2..89758e3 100644 --- a/data/gds_commands/minigames/hotel.yml +++ b/data/gds_commands/minigames/hotel.yml @@ -25,7 +25,7 @@ commands: The comment from Layton when placing the given item in his room. Must be called right after an `0xbc itemLayton` command, and refers to that item. params: - comment: string + comment: string(82) itemComment2Layton: id: 0xc7 desc: > @@ -33,7 +33,7 @@ commands: is already in there. Used to point out duplications, as well as two things going together well. Like `0xc6 itemComment1Layton`, has to follow after a `0xbc itemLayton` call. params: - comment: string + comment: string(82) otherItem: int itemLuke: @@ -54,7 +54,7 @@ commands: The comment from Luke when placing the given item in his room. Must be called right after an `0xbd itemLuke` command, and refers to that item. params: - comment: string + comment: string(82) itemComment2Luke: id: 0xc9 desc: > @@ -62,7 +62,7 @@ commands: is already in there. Used to point out duplications, as well as two things going together well. Like `0xc8 itemComment1Like`, has to follow after a `0xbd itemLuke` call. params: - comment: string + comment: string(82) otherItem: int @@ -70,7 +70,7 @@ commands: id: 0xca desc: The comment from Layton, expressing his overall satisfaction with the room layout. params: - comment: string + comment: string(75) level: int unused: type: int @@ -79,7 +79,7 @@ commands: id: 0xcb desc: The comment from Luke, expressing his overall satisfaction with the room layout. params: - comment: string + comment: string(75) level: int unused: type: int diff --git a/formats/gds/__init__.py b/formats/gds/__init__.py index e69de29..994d8c2 100644 --- a/formats/gds/__init__.py +++ b/formats/gds/__init__.py @@ -0,0 +1,3 @@ +import formats.gds.cmddef as cmddef + +cmddef.init_commands() \ No newline at end of file diff --git a/formats/gds/cmddef.py b/formats/gds/cmddef.py index 7726284..eba6736 100755 --- a/formats/gds/cmddef.py +++ b/formats/gds/cmddef.py @@ -10,6 +10,7 @@ from dacite import from_dict from utils import RESOURCES +import formats.gds.value as value logger = logging.getLogger("flora.debug.gds.cmddef") @@ -21,7 +22,7 @@ class GDSCommandParam: """The command for which this is a parameter""" idx: int """The index of the parameter in order""" - type: str + type: "value.GDSValueType" """ The data type of the command. Real builtin GDS datatypes are: int (1) @@ -86,6 +87,12 @@ class GDSCommand: disassembly, and are therefore not subject to this; it purely qualifies our understanding of the command's meaning and usage. """ + comment: Optional[str] = None + """ + A format string for a comment that should be placed above every execution of this command in + decompiled scripts. Useful to give the reader additional context, such as the path to a file + referenced by an abstract numeric parameter, or even the contents of said file! + """ condition: bool = False """ @@ -220,6 +227,8 @@ def process_param(idx, p): p = {"type": p} p["cmd"] = c["id"] p["idx"] = idx + + p["type"] = value.parse_type(p["type"]) return from_dict(data_class=GDSCommandParam, data=p) @@ -394,5 +403,3 @@ def init_commands(root: str = None): pprint.pp(COMMANDS_BYID) pprint.pp(COMMANDS_BYNAME) -else: - init_commands() diff --git a/formats/gds/gda.py b/formats/gds/gda.py index 098ca90..042c8e2 100644 --- a/formats/gds/gda.py +++ b/formats/gds/gda.py @@ -5,51 +5,391 @@ """ import contextlib -from typing import Any, Optional, List -from dataclasses import dataclass +from typing import Any, Optional, List, Mapping, Union +from dataclasses import dataclass, field +from parsy import ( + forward_declaration, + regex, + seq, + string, + generate, + eof, + Parser, + success, + fail, + peek, + Result as ParseResult, +) -from utils import round_perfect -from .cmddef import GDSCommand +import formats.gds.value as value +import formats.gds.cmddef as cmddef from .model import ( + GDSJumpAddress, GDSProgram, GDSElement, GDSLabel, GDSInvocation, - GDSValue, GDSIfInvocation, GDSLoopInvocation, - GDSConditionToken + GDSConditionToken, +) + +# === PARSER === + + +@Parser +def peek_parser_state(stream, index): + print(stream[index:]) + return ParseResult.success(index, None) + + +ws = regex(r"[^\S\r\n]+") +nl = regex(r"\n\s*") | eof.result("\n") +comment = string("#") >> regex(r".*") << nl + + +def has_nl(res: str) -> Parser: + return success(res) if "\n" in res else fail("newline required") + + +# TODO: make at least one nl required here +ws_nl = ( + ((regex(r"\n\s*") | ws | comment.map(lambda v: v + "\n")).many()) + .concat() + .bind(has_nl) +) +lexeme = lambda p: p << ws.optional() + +lparen = lexeme(string("(")) +rparen = lexeme(string(")")) +lbrace = lexeme(string("{")) +rbrace = lexeme(string("}")) +colon = lexeme(string(":")) +comma = lexeme(string(",")) +eqsign = lexeme(string("=")) + +BREAK = lexeme(string("break")) +NOT = lexeme(string("not")) +AND = lexeme(string("and")) +OR = lexeme(string("or")) + +meta_comment = ( + lexeme(string("#!")) + >> ( + string("version") + >> ws + >> lexeme(regex(r"[\w\.]+")).map(lambda v: ("version", v)) + ) + | regex(r".*").map(lambda v: (None, v)) << nl ) + +command_name = lexeme( + regex(r"[a-zA-Z_][\w\.]*") | regex(r"0x[0-9a-fA-F]{1,2}").map(lambda i: int(i, 16)) +) + + +@generate +def parse_condition(): + conds = [] + while True: + kw = yield ( + NOT.result(GDSConditionToken.NOT()) + | AND.result(GDSConditionToken.AND()) + | OR.result(GDSConditionToken.OR()) + ).optional() + if kw is not None: + conds.append(kw) + continue + cmd = yield parse_command.map(GDSConditionToken.command).optional() + if cmd is None: + break + conds.append(cmd) + return conds + + +@generate +def parse_label(): + yield string("@") + present = yield string("!").result(False).optional(True) + name = yield lexeme(regex("[\w]+")) + loc = None + if (yield lparen.optional()) is not None: + loc = yield lexeme( + regex(r"0x[0-9a-fA-F]+").map(lambda v: int(v, 16)) | regex(r"\d+").map(int) + ).optional() + yield rparen + + return GDSLabel(name=name, present=present, loc=loc) + + +@generate +def parse_addr(): + yield string("@") + primary = yield string("!").result(False).optional(True) + label = yield lexeme(regex("[\w]+")) + return GDSJumpAddress(label=label, primary=primary) + + +@generate +def parse_block(): + yield lbrace + yield ws_nl.optional() + block = [] + while True: + el = yield parse_element | peek(rbrace).result(None) | eof.result(None) + if el is None: + break + block.append(el) + yield ws_nl + yield rbrace + return block + + +def parse_simple(cmdobj: cmddef.GDSCommand) -> Parser: + # pylint: disable=redefined-outer-name + @generate + def parse_simple(): + if next((p for p in cmdobj.params if not p.optional), None) is None: + if (yield lparen.result(False).optional(True)): + return GDSInvocation(command=cmdobj, args=[]) + + yield lparen + + args = [] + for param in cmdobj.params: + p = param.type.parser() + if param.optional: + p = p.optional() + value = yield lexeme(p) + if value is None and not param.optional: + raise ValueError(f"Could not read parameter {param.name}") + args.append(value) + + yield rparen + + return GDSInvocation(command=cmdobj, args=args) + + return parse_simple + + +def parse_if(cmdobj: cmddef.GDSCommand) -> Parser: + # pylint: disable=redefined-outer-name + @generate + def parse_if(): + condition = yield parse_condition + + yield colon + + target = yield parse_addr.optional() + block = None + if target is None: + block = yield parse_block + + return GDSIfInvocation( + command=cmdobj, + args=[], + condition=condition, + target=target, + block=block, + elseif=cmdobj.id == 0x16, + elze=False, + ) + + return parse_if + + +def parse_else(cmdobj: cmddef.GDSCommand) -> Parser: + # pylint: disable=redefined-outer-name + @generate + def parse_else(): + yield colon + + target = yield parse_addr.optional() + block = None + if target is None: + block = yield parse_block + + return GDSIfInvocation( + command=cmdobj, + args=[], + condition=None, + target=target, + block=block, + elseif=False, + elze=True, + ) + + return parse_else + + +def parse_repeatN(cmdobj: cmddef.GDSCommand) -> Parser: + # pylint: disable=redefined-outer-name + @generate + def parse_repeatN(): + condition = yield value.GDSIntType(4, True).parser() + + yield colon + + target = yield parse_addr.optional() + block = None + if target is None: + block = yield parse_block + + return GDSLoopInvocation( + command=cmdobj, + args=[], + condition=condition, + target=target, + block=block, + ) + + return parse_repeatN + + +def parse_while(cmdobj: cmddef.GDSCommand) -> Parser: + # pylint: disable=redefined-outer-name + @generate + def parse_while(): + condition = yield parse_condition + + yield colon + + target = yield parse_addr.optional() + block = None + if target is None: + block = yield parse_block + + return GDSLoopInvocation( + command=cmdobj, + args=[], + condition=condition, + target=target, + block=block, + ) + + return parse_while + + +@generate +def parse_command(): + name = yield command_name + cmdobj = cmddef.COMMANDS_BYNAME.get(name) or cmddef.COMMANDS_BYID.get(name) + if cmdobj is None: + raise ValueError(f"Command {name} not defined") + + if cmdobj.complex: + cmd = yield PARSE_COMPLEX.get(cmdobj.name)(cmdobj) or PARSE_COMPLEX.get( + cmdobj.id + )(cmdobj) + else: + cmd = yield parse_simple(cmdobj) + + return cmd + + +PARSE_COMPLEX = { + "if": parse_if, + "elif": parse_if, + "else": parse_else, + "repeatN": parse_repeatN, + "while": parse_while, +} + + +@generate +def parse_element(): + kw = yield BREAK.result(GDSElement.BREAK()).optional() + if kw is not None: + return kw + label = yield parse_label.optional() + if label is not None: + return GDSElement.label(label) + cmd = yield parse_command + return GDSElement.command(cmd) + + +@generate +def parse_gda_source(): + version = yield meta_comment.map( + lambda v: v[1] if v[0] == "version" else None + ).optional() + + elements = [] + while True: + yield ws_nl.many() + yield ws.optional() + # yield peek_parser_state + el = yield parse_element | eof.result(None) + if el is None: + break + elements.append(el) + yield ws_nl | eof + return (version, elements) + + def read_gda(data: str, path: Optional[str] = None) -> GDSProgram: - pass + version, elements = parse_gda_source.parse(data) + + def collect_labels(elements: List[GDSElement], labels: Mapping[str, List[Union[GDSLabel, GDSJumpAddress]]] = None): + if labels is None: + labels = {} + for el in elements: + if el in GDSElement.label: + name = el().name + if name not in labels: + labels[name] = [] + labels[name].append(el()) + elif el in GDSElement.command and isinstance(el(), (GDSIfInvocation, GDSLoopInvocation)): + block_cmd = el() + if block_cmd.target is not None: + name = block_cmd.target.label + if name not in labels: + labels[name] = [] + labels[name].append(block_cmd.target) + + collect_labels(block_cmd.block, labels) + return labels + + labels = collect_labels(elements) + + return GDSProgram(path=path, elements=elements, labels=labels) + + +# === WRITER === @dataclass(kw_only=True) class WriterContext: filename: Optional[str] = None + workdir: Optional[str] = None result: str = "" indent: int = 0 def write(self, s: str): self.result += s + def nl(self): - self.result += '\n' + ' '*self.indent + self.result += "\n" + " " * self.indent + + # pylint: disable=redefined-outer-name def insert_comment(self, comment: str): self.result, latest_line = self.result.rsplit("\n", 1) for line in comment.splitlines(): self.nl() self.write(f"# {line}") - self.write("\n"+latest_line) + self.write("\n" + latest_line) -def write_gda(prog: GDSProgram, filename: Optional[str] = None) -> str: - ctx = WriterContext(filename=filename) +def write_gda( + prog: GDSProgram, filename: Optional[str] = None, workdir: Optional[str] = None +) -> str: + ctx = WriterContext(filename=filename, workdir=workdir) version = "0.1" ctx.write(f"#!version {version}\n") ctx.write(f"# {filename}\n") - + for el in prog.elements: ctx.nl() write_element(ctx, el) @@ -68,51 +408,31 @@ def write_element(ctx: WriterContext, el: GDSElement): cmd: GDSInvocation = el() ctx.write(cmd.command.name or hex(cmd.command.id)) if cmd.command.complex: - ( - WRITE_COMPLEX.get(cmd.command.name) - or WRITE_COMPLEX.get(cmd.command.id) - )(ctx, cmd) + (WRITE_COMPLEX.get(cmd.command.name) or WRITE_COMPLEX.get(cmd.command.id))( + ctx, cmd + ) else: write_simple(ctx, cmd) + def write_simple(ctx: WriterContext, cmd: GDSInvocation): if len(cmd.args) == 0: return ctx.write(" (") - first = True - for arg, param in zip(cmd.args, cmd.command.params): - if first: - first = False - else: - ctx.write(" ") - if arg in GDSValue.bool: - val = arg() - if isinstance(val, str): - arg = GDSValue.str(val) - elif isinstance(val, int): - arg = GDSValue.int(val) - else: - ctx.write("true" if arg else "false") - if arg in GDSValue.int: - val: int = arg() - bytelen = ( - 4 - if param.type.endswith("int") - else 2 if param.type.endswith("short") else 1 - ) - if not param.type.startswith("u") and val >= 2 ** (bytelen * 8 - 1): - val -= 2 ** (bytelen * 8 - 1) - ctx.write(str(val)) - elif arg in GDSValue.float: - val: float = arg() - ctx.write(str(round_perfect(val))) - elif arg in GDSValue.str or arg in GDSValue.longstr: - val: str = arg() - if param.type == "longstr": - ctx.write("l") - ctx.write(repr(val)) + ctx.write(" ".join(arg.write() for arg in cmd.args if arg is not None)) ctx.write(")") + if cmd.command.comment is not None: + ctx.insert_comment( + format_comment( + cmd.command.comment, + CommentContext( + args=cmd.args, filename=ctx.filename, workdir=ctx.workdir + ), + ) + ) + + def write_condition(ctx: WriterContext, cond: List[GDSConditionToken]): first = True for tok in cond: @@ -137,6 +457,7 @@ def write_condition(ctx: WriterContext, cond: List[GDSConditionToken]): else: write_simple(ctx, cmd) + def write_block(ctx: WriterContext, cond: List[GDSElement]): ctx.write(" {") ctx.indent += 2 @@ -147,6 +468,7 @@ def write_block(ctx: WriterContext, cond: List[GDSElement]): ctx.nl() ctx.write("}") + def write_if(ctx: WriterContext, cmd: GDSIfInvocation): ctx.write(" ") write_condition(ctx, cmd.condition) @@ -156,6 +478,7 @@ def write_if(ctx: WriterContext, cmd: GDSIfInvocation): if cmd.block is not None: write_block(ctx, cmd.block) + def write_else(ctx: WriterContext, cmd: GDSIfInvocation): ctx.write(":") if cmd.target is not None: @@ -163,6 +486,7 @@ def write_else(ctx: WriterContext, cmd: GDSIfInvocation): if cmd.block is not None: write_block(ctx, cmd.block) + def write_repeatN(ctx: WriterContext, cmd: GDSLoopInvocation): count: int = cmd.condition ctx.write(f" {count}:") @@ -171,6 +495,7 @@ def write_repeatN(ctx: WriterContext, cmd: GDSLoopInvocation): if cmd.block is not None: write_block(ctx, cmd.block) + def write_while(ctx: WriterContext, cmd: GDSLoopInvocation): ctx.write(" ") write_condition(ctx, cmd.condition) @@ -189,90 +514,105 @@ def write_while(ctx: WriterContext, cmd: GDSLoopInvocation): "while": write_while, } +import os + +@dataclass +class CommentContext: + filename: Optional[str] = None + workdir: Optional[str] = None + args: List[value.GDSValue] = field(default_factory=list) + omit_file_contents: bool = False + lang: Optional[str] = "en" -def format_comment(comment: str, filename: str, args: List[GDSValue], cmd: GDSCommand): +# pylint: disable=redefined-outer-name +def format_comment(comment: str, ctx: CommentContext): + if ctx.filename is not None and os.path.isabs(ctx.filename): + ctx.filename = os.path.relpath(ctx.filename, "/") + # sourcery skip: assign-if-exp import re + def get_var(name: str) -> Any: if name.lower() == "lang": - return "en" + return ctx.lang or "??" if name.lower() == "eventid": - match: re.Match = re.match(r"/data/script/event/e(\d+).gd[as]", filename) + if ctx.filename is None: + return "?" + match: re.Match = re.match(r"data/script/event/e(\d+).gd[as]", ctx.filename) if match: return int(match.group(1)) - else: - return "?" + return "?" with contextlib.suppress(ValueError): arg_id = int(name) - 1 - if arg_id < 0 or arg_id >= len(args): + if arg_id < 0 or arg_id >= len(ctx.args): return "?" - - arg = args[arg_id] - - + arg: value.GDSValue = ctx.args[arg_id] + return arg.value + return "?" - def format_value(value: Any, modifiers: List[str]) -> str: + def format_value(val: Any, modifiers: List[str]) -> str: for m in modifiers: if not m.startswith("r"): continue step, max_, *_ = m[1:].split("<=") + [None] - if not isinstance(value, int): - value = "?" + if not isinstance(val, int): + val = "?" break step = int(step) - value = (value // step) * step + val = (val // step) * step if max_ is not None: - value = min(value, int(max_)) - + val = min(val, int(max_)) + for m in modifiers: if not m.startswith("0"): continue l = int(m[1:]) - if not isinstance(value, int): - value = "?" * l + if not isinstance(val, int): + val = "?" * l break - value = str(value) - value = "0" * max(0, l - len(value)) + value - - return str(value) + val = str(val) + val = "0" * max(0, l - len(val)) + val + + return str(val) def readfile(path: str) -> str: - return f"test file content\n({path})" + if path is None or ctx.workdir is None or ctx.omit_file_contents: + return "<...>" + if os.path.isabs(path): + path = os.path.relpath(path, "/") + realpath = os.path.join(ctx.workdir, path) + try: + with open(realpath, encoding="utf8") as f: + return f.read() + except FileNotFoundError: + return "" + + str_part = regex(r"[^$]") + str_esc = string(r"$$") + # str_part_fvar = regex(r"[^$}:]") + str_part_expr = regex(r"[^$)]") + # fvar_esc = string(r"}}") | string(r"::") + expr_esc = string(r"))") - from parsy import forward_declaration, regex, seq, string - - str_part = regex(r'[^$]') - str_esc = string(r'$$') - str_part_fvar = regex(r'[^$}:]') - str_part_expr = regex(r'[^$)]') - fvar_esc = string(r'}}') | string(r'::') - expr_esc = string(r'))') - format_args = regex(r"0\d+") | regex(r"r\d+(<=\d+)?") fvar = forward_declaration() expr = forward_declaration() - str_var = string('$') >> ( - string('{') >> fvar << string('}') - | string('(') >> expr << string(')') + str_var = string("$") >> ( + string("{") >> fvar << string("}") | string("(") >> expr << string(")") ) - + varname = regex(r"\w+") - - fvar_expr = (str_part_fvar | fvar_esc | str_var).many().concat() - fvar.become( seq ( varname.map(get_var) , (string(":") >> format_args).many() ).map(lambda a: format_value(a[0], a[1])) ) - expr.become( (str_part_expr | expr_esc | str_var).many().concat().map(readfile) ) - - str_ = (str_part | str_esc | str_var).many().concat() - - # TODO: parse variables like $TEST or ${1:03} and $(filepath) - return str_.parse(comment) -if __name__ == "__main__": - EXAMPLE = """ - /data/etext/${lang}/e${eventid:r100<=300:03}{pcm}/e${eventid}_t${1}.txt: + # fvar_expr = (str_part_fvar | fvar_esc | str_var).many().concat() + fvar.become( + seq(varname.map(get_var), (string(":") >> format_args).many()).map( + lambda a: format_value(a[0], a[1]) + ) + ) + expr.become((str_part_expr | expr_esc | str_var).many().concat().map(readfile)) - $(/data/etext/${lang}/e${eventid:r100<=300:03}{pcm}/e${eventid}_t${1}.txt) - """ - print(format_comment(EXAMPLE, "/data/script/event/e324.gds", [], None)) \ No newline at end of file + str_ = (str_part | str_esc | str_var).many().concat() + + return str_.parse(comment) diff --git a/formats/gds/gds.py b/formats/gds/gds.py index ba5c0b9..c488181 100644 --- a/formats/gds/gds.py +++ b/formats/gds/gds.py @@ -12,20 +12,9 @@ # pylint: disable=unused-wildcard-import,wildcard-import from utils import tagged_union, TU, match, nested_break -from .model import ( - GDSProgram, - GDSElement, - GDSInvocation, - GDSLoopInvocation, - GDSIfInvocation, - GDSValue, - GDSAddress, - GDSConditionToken, - GDSLabel, - GDSJumpAddress, - GDSContext, -) -from .cmddef import COMMANDS_BYID, GDSCommand +import formats.gds.model as model +import formats.gds.cmddef as cmddef +import formats.gds.value as value @tagged_union @@ -41,8 +30,8 @@ class GDSTokenValue: longstr: TU[str] unused5: TU[None] # the source address is the one pointing to the target - saddr: TU[GDSAddress] - taddr: TU[GDSAddress] + saddr: TU[model.GDSAddress] + taddr: TU[model.GDSAddress] NOT: TU[None] AND: TU[None] OR: TU[None] @@ -76,17 +65,17 @@ class LabelToken(BinaryLocalized): @dataclass(kw_only=True) -class BinaryCommand(GDSInvocation, BinaryLocalized): +class BinaryCommand(model.GDSInvocation, BinaryLocalized): loc: int = None @dataclass(kw_only=True) -class BinaryIfCommand(GDSIfInvocation, BinaryLocalized): +class BinaryIfCommand(model.GDSIfInvocation, BinaryLocalized): loc: int = None @dataclass(kw_only=True) -class BinaryLoopCommand(GDSLoopInvocation, BinaryLocalized): +class BinaryLoopCommand(model.GDSLoopInvocation, BinaryLocalized): loc: int = None @@ -115,11 +104,11 @@ class DecompilerState: len: int cursor: int prev_cursor: int - elements: List[GDSElement] = field(default_factory=lambda: []) + elements: List[model.GDSElement] = field(default_factory=lambda: []) labels: Mapping[int, List[Union[LabelUse, LabelToken]]] = field( default_factory=dict ) - context: GDSContext = field(default_factory=GDSContext) + context: model.GDSContext = field(default_factory=model.GDSContext) def __init__(self, data: bytes): self.data = data @@ -128,7 +117,7 @@ def __init__(self, data: bytes): self.prev_cursor = 4 self.elements = [] self.labels = {} - self.context = GDSContext() + self.context = model.GDSContext() @prevcursor def read_bytes(self, l: int) -> bytes: @@ -200,19 +189,19 @@ def read_token(self) -> GDSToken: def read_label(self, token: GDSToken) -> LabelToken: if token.val not in GDSTokenValue.taddr: raise ValueError("Not a target label") - ltoken = LabelToken(loc=token.loc, addr=GDSAddress(token.val())) + ltoken = LabelToken(loc=token.loc, addr=model.GDSAddress(token.val())) if (ltoken.loc + 2) not in self.labels: self.labels[ltoken.loc + 2] = [] self.labels[ltoken.loc + 2].append(ltoken) - self.elements.append(GDSElement.label(ltoken)) + self.elements.append(model.GDSElement.label(ltoken)) return ltoken def read_address(self, token: GDSToken, use: str = "") -> LabelUse: if token.val not in GDSTokenValue.saddr: raise ValueError("Not a jump address") - luse = LabelUse(loc=token.loc, addr=GDSAddress(token.val()), use=use) + luse = LabelUse(loc=token.loc, addr=model.GDSAddress(token.val()), use=use) if luse.addr not in self.labels: self.labels[luse.addr] = [] @@ -270,17 +259,17 @@ def name_labels(self): # pass return label_names - def collapse_blocks(self, block: List[GDSElement]) -> List[GDSElement]: + def collapse_blocks(self, block: List[model.GDSElement]) -> List[model.GDSElement]: sid = 0 while sid < len(block): with nested_break() as continue_outer: el = block[sid] - if el not in GDSElement.command or not isinstance( - el(), (GDSIfInvocation, GDSLoopInvocation) + if el not in model.GDSElement.command or not isinstance( + el(), (model.GDSIfInvocation, model.GDSLoopInvocation) ): raise continue_outer # also very convenient for preserving a cleanup block in a continue - block_cmd: Union[GDSIfInvocation, GDSLoopInvocation] = el() + block_cmd: Union[model.GDSIfInvocation, model.GDSLoopInvocation] = el() source: LabelUse = block_cmd.target if source is None: @@ -301,7 +290,7 @@ def collapse_blocks(self, block: List[GDSElement]) -> List[GDSElement]: # Also remove the labels from the map tid = sid + 1 while tid < len(block) and not ( - block[tid] in GDSElement.label and block[tid]() is target + block[tid] in model.GDSElement.label and block[tid]() is target ): tid += 1 if tid == len(block): @@ -314,24 +303,24 @@ def collapse_blocks(self, block: List[GDSElement]) -> List[GDSElement]: block = block[: sid + 1] + block[tid + 1 :] block_cmd.target = None block_cmd.block = sub_block - + block_cmd.block = self.collapse_blocks(sub_block) sid += 1 return block def finalize_labels( - self, block: List[GDSElement], label_names: Mapping[int, str], labels=None - ) -> Mapping[str, Union[GDSLabel, GDSAddress]]: + self, block: List[model.GDSElement], label_names: Mapping[int, str], labels=None + ) -> Mapping[str, Union[model.GDSLabel, model.GDSAddress]]: if labels is None: labels = {} oldlabel_indices = [ - (i, el()) for i, el in enumerate(block) if el in GDSElement.label + (i, el()) for i, el in enumerate(block) if el in model.GDSElement.label ] olduse_indices = [ (el, el().target) for el in block - if el in GDSElement.command + if el in model.GDSElement.command and isinstance(el(), (BinaryIfCommand, BinaryLoopCommand)) and isinstance(el().target, LabelUse) ] @@ -341,7 +330,7 @@ def finalize_labels( labels[name] = [] for i in self.labels[addr]: if isinstance(i, LabelToken): - newlabel = GDSLabel( + newlabel = model.GDSLabel( name=name, present=i.addr is not None, loc=i.addr if i.pointsto is None else None, @@ -353,10 +342,10 @@ def finalize_labels( found = True break if found: - block[idx] = GDSElement.label(newlabel) + block[idx] = model.GDSElement.label(newlabel) labels[name].append(newlabel) elif isinstance(i, LabelUse): - newuse = GDSJumpAddress(label=name, primary=i.primary) + newuse = model.GDSJumpAddress(label=name, primary=i.primary) found = False el = None for el, olduse in olduse_indices: @@ -370,7 +359,7 @@ def finalize_labels( raise TypeError() for el in block: - if el not in GDSElement.command or not isinstance( + if el not in model.GDSElement.command or not isinstance( el(), (BinaryIfCommand, BinaryLoopCommand) ): continue @@ -382,18 +371,18 @@ def finalize_labels( return labels -def read_gds(data: bytes, path: str = None) -> GDSProgram: +def read_gds(data: bytes, path: str = None) -> model.GDSProgram: ctx = DecompilerState(data) cur_token = ctx.read_token() # TODO while cur_token.val not in GDSTokenValue.fileend: if cur_token.val in GDSTokenValue.command: - ctx.elements.append(GDSElement.command(read_command(ctx, cur_token))) + ctx.elements.append(model.GDSElement.command(read_command(ctx, cur_token))) elif cur_token.val in GDSTokenValue.taddr: ctx.read_label(cur_token) elif cur_token.val in GDSTokenValue.BREAK: - b = GDSElement.BREAK() + b = model.GDSElement.BREAK() b.loc: int = cur_token.loc ctx.elements.append(b) else: @@ -408,25 +397,25 @@ def read_gds(data: bytes, path: str = None) -> GDSProgram: labels = ctx.finalize_labels(ctx.elements, label_names) - return GDSProgram( + return model.GDSProgram( context=ctx.context, path=path, elements=ctx.elements, labels=labels ) def read_condition( ctx: DecompilerState, use: str = "" -) -> Tuple[List[GDSConditionToken], LabelUse]: +) -> Tuple[List[model.GDSConditionToken], LabelUse]: cur_token = ctx.read_token() cond = [] while cur_token.val not in GDSTokenValue.saddr: if cur_token.val in GDSTokenValue.NOT: - cond.append(GDSConditionToken.NOT()) + cond.append(model.GDSConditionToken.NOT()) elif cur_token.val in GDSTokenValue.AND: - cond.append(GDSConditionToken.AND()) + cond.append(model.GDSConditionToken.AND()) elif cur_token.val in GDSTokenValue.OR: - cond.append(GDSConditionToken.OR()) + cond.append(model.GDSConditionToken.OR()) elif cur_token.val in GDSTokenValue.command: - cond.append(GDSConditionToken.command(read_command(ctx, cur_token))) + cond.append(model.GDSConditionToken.command(read_command(ctx, cur_token))) else: raise ValueError("Unexpected token type") cur_token = ctx.read_token() @@ -436,12 +425,12 @@ def read_condition( return cond, addr -def read_command(ctx: GDSCommand, token: GDSToken) -> BinaryCommand: +def read_command(ctx: DecompilerState, token: GDSToken) -> BinaryCommand: if token.val not in GDSTokenValue.command: raise ValueError("Expected instruction") cmdid = token.val() - cmdobj = COMMANDS_BYID.get(cmdid) + cmdobj = cmddef.COMMANDS_BYID.get(cmdid) if cmdobj is None: raise ValueError(f"Command {cmdid} not defined") @@ -455,47 +444,12 @@ def read_command(ctx: GDSCommand, token: GDSToken) -> BinaryCommand: return cmd -def read_simple(ctx: DecompilerState, cmdobj: GDSCommand) -> BinaryCommand: +def read_simple(ctx: DecompilerState, cmdobj: "cmddef.GDSCommand") -> BinaryCommand: args = [] for param in cmdobj.params: + arg = ctx.read_token() - options = match( - arg.val, - { - GDSTokenValue.int: lambda val: [ - ( - GDSValue.int(val), - ["int", "uint", "short", "ushort", "byte", "ubyte"], - ), - ( - GDSValue.bool(val), - ["bool", "bool|int"], - ), - ], - GDSTokenValue.float: lambda val: [(GDSValue.float(val), ["float"])], - GDSTokenValue.str: lambda val: [ - (GDSValue.str(val), ["string"]), - ( - GDSValue.bool(val), - ["bool", "bool|string"], - ), - ], - GDSTokenValue.longstr: lambda val: [ - (GDSValue.longstr(val), ["longstr"]) - ], - ...: lambda: [], - }, - ) - # pylint: disable=not-callable - val = next( - ( - v - for v, reqtypes in options - if (isinstance(reqtypes, list) and param.type in reqtypes) - or (callable(reqtypes) and reqtypes(param.type)) - ), - None, - ) + val = param.type.from_token(arg.val) if val is None: if not param.optional: raise ValueError( @@ -508,7 +462,7 @@ def read_simple(ctx: DecompilerState, cmdobj: GDSCommand) -> BinaryCommand: return BinaryCommand(command=cmdobj, args=args) -def read_if(ctx: DecompilerState, cmdobj: GDSCommand) -> BinaryIfCommand: +def read_if(ctx: DecompilerState, cmdobj: "cmddef.GDSCommand") -> BinaryIfCommand: cond, addr = read_condition(ctx, "if") # this diagnostic is literally just objectively wrong @@ -524,7 +478,7 @@ def read_if(ctx: DecompilerState, cmdobj: GDSCommand) -> BinaryIfCommand: ) -def read_elif(ctx: DecompilerState, cmdobj: GDSCommand) -> BinaryIfCommand: +def read_elif(ctx: DecompilerState, cmdobj: "cmddef.GDSCommand") -> BinaryIfCommand: cond, addr = read_condition(ctx, "elif") # this diagnostic is literally just objectively wrong @@ -540,8 +494,7 @@ def read_elif(ctx: DecompilerState, cmdobj: GDSCommand) -> BinaryIfCommand: ) -def read_else(ctx: DecompilerState, cmdobj: GDSCommand) -> BinaryIfCommand: - addr = None +def read_else(ctx: DecompilerState, cmdobj: "cmddef.GDSCommand") -> BinaryIfCommand: cur_token = ctx.read_token() while cur_token.val not in GDSTokenValue.saddr: cur_token = ctx.read_token() @@ -561,7 +514,7 @@ def read_else(ctx: DecompilerState, cmdobj: GDSCommand) -> BinaryIfCommand: ) -def read_repeatN(ctx: DecompilerState, cmdobj: GDSCommand) -> BinaryLoopCommand: +def read_repeatN(ctx: DecompilerState, cmdobj: "cmddef.GDSCommand") -> BinaryLoopCommand: cntt = ctx.read_token() if cntt.val not in GDSTokenValue.int: raise ValueError( @@ -583,7 +536,7 @@ def read_repeatN(ctx: DecompilerState, cmdobj: GDSCommand) -> BinaryLoopCommand: ) -def read_while(ctx: DecompilerState, cmdobj: GDSCommand) -> BinaryLoopCommand: +def read_while(ctx: DecompilerState, cmdobj: "cmddef.GDSCommand") -> BinaryLoopCommand: cond, addr = read_condition(ctx, "loop") # this diagnostic is literally just objectively wrong @@ -605,12 +558,12 @@ def read_while(ctx: DecompilerState, cmdobj: GDSCommand) -> BinaryLoopCommand: @dataclass(kw_only=True) class CompilerState: data: bytearray - elements: List[GDSElement] - labels: Mapping[str, List[Union[GDSLabel, GDSJumpAddress]]] - label_locs: Mapping[str, Tuple[int, GDSLabel]] - use_locs: Mapping[str, List[Tuple[int, GDSJumpAddress]]] + elements: List[model.GDSElement] + labels: Mapping[str, List[Union[model.GDSLabel, model.GDSJumpAddress]]] + label_locs: Mapping[str, Tuple[int, model.GDSLabel]] + use_locs: Mapping[str, List[Tuple[int, model.GDSJumpAddress]]] - def __init__(self, prog: GDSProgram): + def __init__(self, prog: model.GDSProgram): self.data = bytearray() # self.context = prog.context self.elements = prog.elements @@ -666,7 +619,7 @@ def write_token(self, token: GDSTokenValue): for el in els: self.data += el - def write_command(self, cmd: GDSInvocation): + def write_command(self, cmd: model.GDSInvocation): self.write_token(GDSTokenValue.command((cmd.command).id)) if cmd.command.complex: @@ -677,7 +630,7 @@ def write_command(self, cmd: GDSInvocation): write_simple(self, cmd) - def write_label(self, label: GDSLabel): + def write_label(self, label: model.GDSLabel): if not label.present: return @@ -689,10 +642,10 @@ def write_label(self, label: GDSLabel): backptr = use_loc self.data[use_loc - 4 : use_loc] = label_loc.to_bytes(4, "little") - self.write_token(GDSTokenValue.taddr(GDSAddress(backptr or 0))) + self.write_token(GDSTokenValue.taddr(model.GDSAddress(backptr or 0))) self.label_locs[label.name] = (label_loc, label) - def write_addr(self, addr: GDSJumpAddress): + def write_addr(self, addr: model.GDSJumpAddress): label_loc = None use_loc = len(self.data) + 6 @@ -701,29 +654,49 @@ def write_addr(self, addr: GDSJumpAddress): if addr.primary: self.data[label_loc - 4 : label_loc] = use_loc.to_bytes(4, "little") - self.write_token(GDSTokenValue.saddr(GDSAddress(label_loc or 0))) + self.write_token(GDSTokenValue.saddr(model.GDSAddress(label_loc or 0))) if addr.label not in self.use_locs: self.use_locs[addr.label] = [] self.use_locs[addr.label].append((use_loc, addr)) + + def write_block(self, block: List[model.GDSElement]): + name = None + i = 0 + while name is None or name in self.labels: + i += 1 + name = f"block_{i}" + use = model.GDSJumpAddress(label=name, primary=True) + target = model.GDSLabel(name=name, present=True, loc=None) + self.labels[name] = [use, target] + + self.write_addr(use) + for el in block: + if el in model.GDSElement.BREAK: + self.write_token(GDSTokenValue.BREAK()) + elif el in model.GDSElement.command: + self.write_command(el()) + elif el in model.GDSElement.label: + self.write_label(el()) + self.write_label(target) -def write_gds(prog: GDSProgram) -> bytes: +def write_gds(prog: model.GDSProgram) -> bytes: ctx = CompilerState(prog) for el in ctx.elements: - if el in GDSElement.BREAK: + if el in model.GDSElement.BREAK: ctx.write_token(GDSTokenValue.BREAK()) - elif el in GDSElement.command: + elif el in model.GDSElement.command: ctx.write_command(el()) - elif el in GDSElement.label: + elif el in model.GDSElement.label: ctx.write_label(el()) ctx.write_token(GDSTokenValue.fileend()) return len(ctx.data).to_bytes(4, "little") + bytes(ctx.data) -def write_simple(ctx: CompilerState, cmd: GDSInvocation): +def write_simple(ctx: CompilerState, cmd: model.GDSInvocation): args = cmd.args if len(args) < len(cmd.command.params): args += [None] * len(cmd.command.params) - len(args) @@ -733,59 +706,9 @@ def write_simple(ctx: CompilerState, cmd: GDSInvocation): continue else: raise ValueError("Too few parameters") - options = match( - arg, - { - GDSValue.int: lambda val: [ - ( - GDSTokenValue.int(val), - [ - "int", - "uint", - "short", - "ushort", - "byte", - "ubyte", - "bool", - "bool|int", - ], - ), - ], - GDSValue.float: lambda val: [(GDSTokenValue.float(val), ["float"])], - GDSValue.str: lambda val: [ - (GDSTokenValue.str(val), ["string", "bool", "bool|string"]), - ], - GDSValue.longstr: lambda val: [ - (GDSTokenValue.longstr(val), ["longstr"]) - ], - GDSValue.bool: lambda val: ( - [(GDSTokenValue.int(val), ["bool", "bool|int"])] - if isinstance(val, int) - else ( - [(GDSTokenValue.str(val), ["bool", "bool|string"])] - if isinstance(val, str) - else [ - (GDSTokenValue.int(1 if val else 0), ["bool", "bool|int"]), - ( - GDSTokenValue.str("true" if val else "false"), - ["bool", "bool|string"], - ), - ] - ) - ), - ...: lambda: [], - }, - ) + # pylint: disable=not-callable - tok = next( - ( - v - for v, reqtypes in options - if (isinstance(reqtypes, list) and param.type in reqtypes) - or (callable(reqtypes) and reqtypes(param.type)) - ), - None, - ) + tok = arg.as_token() if tok is None: raise ValueError( f"Unexpected parameter type: should have been {param.type}, value was {arg}" @@ -794,38 +717,50 @@ def write_simple(ctx: CompilerState, cmd: GDSInvocation): ctx.write_token(tok) -def write_condition(ctx: CompilerState, cond: List[GDSConditionToken]): +def write_condition(ctx: CompilerState, cond: List[model.GDSConditionToken]): for tok in cond: - if tok in GDSConditionToken.NOT: + if tok in model.GDSConditionToken.NOT: ctx.write_token(GDSTokenValue.NOT()) - elif tok in GDSConditionToken.AND: + elif tok in model.GDSConditionToken.AND: ctx.write_token(GDSTokenValue.AND()) - elif tok in GDSConditionToken.OR: + elif tok in model.GDSConditionToken.OR: ctx.write_token(GDSTokenValue.OR()) - elif tok in GDSConditionToken.command: + elif tok in model.GDSConditionToken.command: ctx.write_command(tok()) else: raise ValueError("Unexpected token type") -def write_if(ctx: CompilerState, cmd: GDSIfInvocation): +def write_if(ctx: CompilerState, cmd: model.GDSIfInvocation): write_condition(ctx, cmd.condition) - ctx.write_addr(cmd.target) + if cmd.target is not None: + ctx.write_addr(cmd.target) + elif cmd.block is not None: + ctx.write_block(cmd.block) -def write_else(ctx: CompilerState, cmd: GDSIfInvocation): +def write_else(ctx: CompilerState, cmd: model.GDSIfInvocation): # write_condition(ctx, cmd.condition) - ctx.write_addr(cmd.target) + if cmd.target is not None: + ctx.write_addr(cmd.target) + elif cmd.block is not None: + ctx.write_block(cmd.block) -def write_repeatN(ctx: CompilerState, cmd: GDSLoopInvocation): +def write_repeatN(ctx: CompilerState, cmd: model.GDSLoopInvocation): ctx.write_token(GDSTokenValue.int(cmd.condition)) - ctx.write_addr(cmd.target) + if cmd.target is not None: + ctx.write_addr(cmd.target) + elif cmd.block is not None: + ctx.write_block(cmd.block) -def write_while(ctx: CompilerState, cmd: GDSLoopInvocation): +def write_while(ctx: CompilerState, cmd: model.GDSLoopInvocation): write_condition(ctx, cmd.condition) - ctx.write_addr(cmd.target) + if cmd.target is not None: + ctx.write_addr(cmd.target) + elif cmd.block is not None: + ctx.write_block(cmd.block) WRITE_COMPLEX = { diff --git a/formats/gds/model.py b/formats/gds/model.py index f9485c0..0f9f009 100644 --- a/formats/gds/model.py +++ b/formats/gds/model.py @@ -1,32 +1,20 @@ from typing import List, Optional, NewType, Union, Set, Mapping from dataclasses import dataclass, field # pylint: disable=unused-wildcard-import,wildcard-import -from utils import tagged_union, TU -from .cmddef import GDSCommand +from utils import tagged_union, TU +import formats.gds.value as value +import formats.gds.cmddef as cmddef GDSAddress = NewType('GDSAddress', int) -@tagged_union -class GDSValue: - """ - A value usable as a parameter in a GDSInvocation. - """ - int: TU[int] - float: TU[float] - str: TU[str] - longstr: TU[str] - # (value, represented_as_string) - bool: TU[Union[bool, int, str]] - - @dataclass(kw_only=True) class GDSInvocation: """ The specific invocation of a GDS commmand, with the given parameter values. """ - command: GDSCommand - args: List[GDSValue] + command: "cmddef.GDSCommand" + args: List["value.GDSValue"] @tagged_union @@ -120,13 +108,13 @@ class GDSContext: """ A list of candidate contexts, if any are still applicable. This list will be empty if narrowing caused a conflict. """ - conflicts: List[GDSCommand] = field(default_factory=list) + conflicts: List["cmddef.GDSCommand"] = field(default_factory=list) """ Whenever an instruction does not match any of the candidate contexts, the offending command is recorded here, and its context candidates added to the global candidate list. """ - def narrow(self, cmd: GDSCommand) -> bool: + def narrow(self, cmd: "cmddef.GDSCommand") -> bool: """ Checks if the given command is compatible with the currently assumed candidates, and narrows the list if so. Otherwise, records the conflict and returns false. diff --git a/formats/gds/old.py b/formats/gds/old.py index 9a15311..96da98d 100644 --- a/formats/gds/old.py +++ b/formats/gds/old.py @@ -7,10 +7,11 @@ import collections import struct -import parse import ast + +import version +import parse from utils import cli_file_pairs, foreach_file_pair, RESOURCES -from version import v @click.group( @@ -202,10 +203,10 @@ def __getitem__(self, index): return self.cmds[index] def to_json(self): - return json.dumps({"version": v, "data": self.cmds}, indent=4) + return json.dumps({"version": version.__version__, "data": self.cmds}, indent=4) def to_yaml(self): - return yaml.safe_dump({"version": v, "data": self.cmds}) + return yaml.safe_dump({"version": version.__version__, "data": self.cmds}) def to_gds(self): out = b"\x00" * 2 diff --git a/formats/gds/preview.py b/formats/gds/preview.py new file mode 100644 index 0000000..8ac33d7 --- /dev/null +++ b/formats/gds/preview.py @@ -0,0 +1,69 @@ +""" +This module (eventually) contains a small GDS simulator, suitable for piecing together a visually +explorable representation of room layouts, a structured view of dialogue events, and an overview +of the script's logical control flow. + +TODO: WIP, and no priority at the moment +""" + +from .model import GDSProgram + +def preview_room(prog: GDSProgram): + """ + Executes the provided room script, by building the room in a small view window + like the game itself would, but with many introspection tools. + + The program should consist of + 1. a main view in which the content visible on the bottom screen is assembled. + All items placed here should be represented by their actual animated sprites in game, + and bounding boxes should be shown. Clicking on an item should show technical details, + such as exactly what type of item it is, or in the case of triggers, what event they cause. + This information also includes the conditions under which the object is included in the scene. + 2. a topscreen view displaying what map information is being set by the room script, and possibly + other adhoc sprites like the objectives arrows. Again, bounding boxes where relevant. + 3. a sidebar that lists all items placed in the room by category, also split by what conditions need + to be required to make the item appear. + + This information could all just be an informational display, but optionally, certain properties like the + positions, names and events of items could be editable. A "save" functionality should write the result of + these edits to a new GDA script. + """ + pass + +def preview_event(prog: GDSProgram): + """ + Executes the provided event script, by playing through the sequence in a small view window + like the game itself would, but with many introspection tools. + + The program should consist of + 1. a main view in which actors and text bubbles are displayed like in the game. + Items placed here should be represented by their actual animated sprites in game, + and clickable bounding boxes shown to display technical details of placement and animation logic. + 2. (optionally, depending on what the event can do) a topscreen view displaying the results of special + logic displaying information on the top screen. + 3. a sidebar listing all events in the dialogue tree in order, highlighting the currently displayed step, + allowing the user to click on previous ones to rewind. Should also display the logic instructions + determining which parts of the event are played, but allow the user to override them and see + all parts of the dialogue. + + This information could all just be an informational display, but optionally, certain properties like the + positions, count and animations of the actors, as well as the dialogue text, could be editable. A "save" + functionality should write the result of these edits to a new GDA script. + """ + pass + +def preview_puzzle(prog: GDSProgram): + """ + Least likely to actually get done: executes the provided puzzle script. These seldom have much interactivity, + but they're extremely contextual on the engine used. + + The program should consist of + 1. a main view displaying the assembled touchscreen view a player would be presented with in-game, + with animations and bounding boxes for important elements (and grid line hints where applicable). + 2. a "topscreen view" which may not need to be completely accurate, instead listing all crucial information + about the puzzle such as title, description, picarat counts, even the hints in a separate tab. + 3. a sidebar with debug information about the internal state of the puzzle engine, and the ability + to show/hide technical objects used for puzzle grading that would otherwise clutter the screen. + + Of course this view is highly dependent on the puzzle engine, which makes it difficult to conceptualize at this point. + """ \ No newline at end of file diff --git a/formats/gds/test_gda.py b/formats/gds/test_gda.py index 2d0c483..fe9a932 100644 --- a/formats/gds/test_gda.py +++ b/formats/gds/test_gda.py @@ -6,21 +6,40 @@ import tqdm from .gds import read_gds, write_gds -from .gda import read_gda, write_gda +import formats.gds.gda as gda from .patch import patch +from .value import * from utils import RESOURCES def test_file(gdspath: str, base: str): with open(os.path.join(base, gdspath), "rb") as gdsf: - gds = gdsf.read() - gds = patch(gds, gdspath) - prog = read_gds(gds, gdspath) - gda = write_gda(prog, gdspath) + gdsb = gdsf.read() + gdsb = patch(gdsb, gdspath) + prog = read_gds(gdsb, gdspath) + gdas = gda.write_gda(prog, gdspath, base) gdapath = gdspath.replace(".gds", ".gda") with open(os.path.join(base, gdapath), "w", encoding="utf8") as gdaf: - gdaf.write(gda) + gdaf.write(gdas) + + try: + reread = gda.read_gda(gdas, gdapath) + except Exception as e: + print(f"ERR: {gdapath}: could not read decompiled script: {e}") + return + try: + recompiled = write_gds(reread) + except Exception as e: + print(f"ERR: {gdapath}: could not recompile script: {e}") + return + + if recompiled != gdsb: + print( + f"ERR: {gdspath}: Recompiled contents not identical. Result written as {gdspath}2 for comparison." + ) + with open(os.path.join(base, f"{gdspath}2"), "wb") as gdsf: + gdsf.write(recompiled) BASE = os.path.join(RESOURCES, "game_root/lt1_eu") @@ -38,6 +57,44 @@ def test_all(base: str): test_file(path, base) +def test_comment(): + EXAMPLE = """ + /data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt: + + $(/data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt) + """ + print( + gda.format_comment( + EXAMPLE, + gda.CommentContext( + args=[GDSIntValue(7, GDSIntType())], + filename="/data/script/event/e6.gds", + workdir=BASE, + ), + ) + ) + + +def test_parsers(): + print(gda.parse_element.parse("if not 0x49:{ # test \n0x49 # test2\n#test3\n}")) + + +def test_readfile(gdapath: str, base: str): + with open(os.path.join(base, gdapath), "r", encoding="utf8") as gdaf: + gdas = gdaf.read() + + prog = gda.read_gda(gdas, gdapath) + compiled = write_gds(prog) + + gdspath = gdapath.replace(".gda", ".gds") + with open(os.path.join(base, gdspath), "rb") as gdsf: + gdsb = gdsf.read() + gdsb = patch(gdsb, gdspath) + + assert gdsb == compiled + + if __name__ == "__main__": - # test_file("data/script/event/e6.gds", BASE) + # test_file("data/script/event/e49.gds", BASE) test_all(BASE) + # test_parsers() diff --git a/formats/gds/value.py b/formats/gds/value.py new file mode 100644 index 0000000..1ea5077 --- /dev/null +++ b/formats/gds/value.py @@ -0,0 +1,512 @@ +""" +Defines the way values and their types are defined and handled in GDS and GDA. +""" + +from abc import ABC, abstractmethod +from typing import Any, Union, Optional, Literal, List +from dataclasses import dataclass + +from parsy import regex, string, Parser, seq, success, fail, peek + +import formats.gds.gds as gds +from utils import round_perfect + +TYPES: List["GDSValueType"] = [] + + +def parse_type(descriptor: str) -> Optional["GDSValueType"]: + """ + Given a type descriptor (used internally in the GDS command definition files), + creates a corresponding type object, or returns None if the descriptor is invalid. + + See the individual type classes registered in `TYPES` for information on valid + descriptor formats + """ + for t in TYPES: + ty = t.parse_type(descriptor) + if ty is not None: + return ty + return None + + +class GDSValueType(ABC): + """ + A GDS Value Type. Defines how a value of this type is created from + a GDS token or a string parsed in a GDA file. + """ + @classmethod + @abstractmethod + def parse_type(cls, descriptor: str) -> Optional["GDSValueType"]: + """ + Parses the descriptor and returns the corresponding type object + if the descriptor corresponds to a variant of this type, + otherwise returns None to signal the next candidate type should be tried. + """ + + @abstractmethod + def from_token(self, tok: gds.GDSTokenValue) -> "GDSValue": + """ + Parses a value of this type from a GDSToken(Value), or returns + None if there is no way to correspond that token to a value of this type. + """ + + @abstractmethod + def parser(self) -> Parser: + """ + Returns a `parsy` parser that attempts to read a value of this type from a GDA script, + and if so returns a value instance of this type. + """ + + +class GDSValue(ABC): + """ + A GDS Value, of a specific `GDSValueType`. Defines how a value + is printed, either in Python code or in a GDA script, + and conversion to other value types / GDS tokens. + """ + type: GDSValueType + value: Any + + @abstractmethod + def as_token(self) -> gds.GDSTokenValue: + """ + Converts the value into a GDSToken(Value). There should + rarely be a reason for this to fail, + but if it does this function should return None. + """ + + @abstractmethod + def __format__(self, format_spec: Optional[str] = None) -> str: + pass + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({format(self)})" + + def write(self) -> str: + """ + Returns a string representation of this value, as it would be written in a GDA script. + Separate from Python's format and repr mechanisms, though it corresponds more closely + to the latter; however some representations in GDA would not be valid repr strings in Python + (specifically long strings). + """ + return format(self) + + +@dataclass +class GDSIntType(GDSValueType): + """ + An integer type, with a fixed byte length, and a marker whether it is intended to be + read as an unsigned integer in the game. + """ + bytelen: int = 4 + unsigned: bool = True + + @classmethod + def parse_type(cls, descriptor: str) -> "GDSIntType": + """ + Valid descriptors are: + int(n): n bytes, signed + uint(n): n bytes, unsigned + + or alternatively the shorthands: + int / uint: 4 bytes, optionally unsigned + short / ushort: 2 bytes, optionally unsigned + byte / ubyte: 1 byte, optionally unsigned + + Note that all int values are still written as 4 little endian bytes in GDS files, + but only a subset of these values is actually read by the game. Note also that + this hard limits the byte length to 4 at most. + """ + unsigned = False + if descriptor.startswith("u"): + unsigned = True + descriptor = descriptor[1:] + + if descriptor.startswith("int"): + if descriptor == "int": + bytelen = 4 + elif descriptor[3] == "(": + bytelen = int(descriptor[4:-1]) + else: + return None + elif descriptor == "short": + bytelen = 2 + elif descriptor == "byte": + bytelen = 1 + else: + return None + return GDSIntType(bytelen=bytelen, unsigned=unsigned) + + def from_token(self, tok: gds.GDSTokenValue) -> "GDSIntValue": + if tok not in gds.GDSTokenValue.int: + return None + val = tok() + if not self.unsigned and val >= (256**self.bytelen)/2: + val -= 256**self.bytelen + return GDSIntValue(val, self) + + def parser(self) -> Parser: + """ + Valid integer literals can include a sign (+-), and can be written in base 10 (default) or + base 16/2 (prefixed by 0x or 0b respectively). Note that if a dot is found at the end of + the literal, parsing will fail, as this designates the value as a float instead. + """ + def parser_map( + val: int, lit_fmt: Literal["hex", "bin", "dec"] = "dec" + ) -> "GDSIntValue": + if self.unsigned: + if val < 0: + new_val = (256**self.bytelen) + val + if new_val >= 0: + print( + f"WARN: assigned negative value {val} to uint; this will be converted " + f"to the two's complement in {self.bytelen} bytes ({new_val})" + ) + val = new_val + else: + print( + f"WARN: assigned negative value {val} to uint; this value is out of " + f"bounds for a {self.bytelen}-byte int, and cannot be represented in " + "two's complement. This WILL lead to errors!" + ) + if val >= 256**self.bytelen: + print( + f"WARN: value {val} is out of bounds for {self.bytelen}-byte uint; " + "the value will be stored in full (truncated to 4 bytes due to " + f"technical limitations), but only the least significant {self.bytelen} " + "bytes will be read." + ) + elif val * 2 < -(256**self.bytelen) or val * 2 >= 256**self.bytelen: + print( + f"WARN: value {val} is out of bounds for {self.bytelen}-byte int; " + "the value will be stored in full (truncated to 4 bytes due to " + f"technical limitations), but only the least significant {self.bytelen} " + "bytes will be read." + ) + + return GDSIntValue(val, self, lit_fmt=lit_fmt) + + return ( + regex(r"[+-]?0x[0-9a-fA-F]+").map(lambda i: parser_map(int(i, 16), "hex")) + | regex(r"[+-]?0b[01]+").map(lambda i: parser_map(int(i, 2), "bin")) + | regex(r"[+-]?[0-9]+").map(lambda i: parser_map(int(i), "dec")) + ) << peek(regex(r"[^\.]?")) + + +TYPES.append(GDSIntType) + + +class GDSIntValue(GDSValue): + """ + An integer value. If written in a non-decimal base in a GDA script, this information is + preserved here (but not after being written to a GDS binary). + """ + value: int + lit_fmt: Literal["hex", "bin", "dec"] = "dec" + + def __init__( + self, value: int, ty: GDSIntType, lit_fmt: Literal["hex", "bin", "dec"] = "dec" + ): + self.type = ty + self.value = value + self.lit_fmt = lit_fmt + + def as_token(self) -> gds.GDSToken: + val = self.value + if val < 0: + val += 256**self.type.bytelen + return gds.GDSTokenValue.int(val) + + def __format__(self, format_spec: Optional[str] = None) -> str: + # sourcery skip: assign-if-exp, reintroduce-else + if self.lit_fmt == "hex": + return hex(self.value) + if self.lit_fmt == "bin": + return bin(self.value) + return str(self.value) + + +class GDSFloatType(GDSValueType): + """ + A float type, of which there are no variations. It matches the IEEE-754 32-bit float + specification, though it is stored in little-endian byte order in GDS files. + """ + @classmethod + def parse_type(cls, descriptor: str) -> "GDSFloatType": + return GDSFloatType() if descriptor == "float" else None + + def from_token(self, tok: gds.GDSTokenValue) -> "GDSFloatValue": + return GDSFloatValue(tok(), self) if tok in gds.GDSTokenValue.float else None + + def parser(self) -> Parser: + """ + Valid float literals can include a sign (+-), and MUST include a period as well + as a nonempty int literal to either side of it. Omitting the other will denote it as 0, + therefore `123.` is a convenient way to write integers explicitly as floats. + Literals may also include an exponent. + + TODO: allow writing integers here? Since we already know where the program expects + a float due to the command list, we could make a convenient conversion here... + """ + return regex(r"[+-]?([0-9]+\.|[0-9]*\.[0-9])[0-9]*([eE][+-]?[0-9]+)?").map( + lambda f: GDSFloatValue(float(f), self) + ) + + +TYPES.append(GDSFloatType) + + +class GDSFloatValue(GDSValue): + """ + A float value. The only interesting aspect is the way it's printed in GDA files: + since IEEE-754 introduces nasty rounding errors even for simple float values, + we correct this by attempting to round to the float with the least decimal digits + which still produces the same value as the original. + """ + value: float + + def __init__(self, value: float, ty: GDSFloatType): + self.type = ty + self.value = value + + def as_token(self) -> gds.GDSToken: + return gds.GDSTokenValue.float(self.value) + + def __format__(self, format_spec: Optional[str] = None) -> str: + return str(round_perfect(self.value)) + + +@dataclass +class GDSStringType(GDSValueType): + """ + A string type, with a maximum buffer length, and a flag indicating whether the game expects + a "long string". The main difference between the two is that a long string is stored in a + different buffer, meaning the two types can NOT be substituted for each other! + """ + maxlen: int = 63 + longstr: bool = False + + @classmethod + def parse_type(cls, descriptor: str) -> "GDSStringType": + """ + Valid descriptors are: str, string, longstr (for long strings) + optionally followed by the buffer length in bytes / characters. + """ + # NOTE: if the game uses a 64-char buffer internally, remember the null terminator + # also needs to fit! The length of the corresponding GDS type should then be 63. + # TODO: I haven't yet determined the game's actual buffer size for both string types... + if descriptor.startswith("string"): + longstr = False + descriptor = descriptor[6:] + elif descriptor.startswith("str"): + longstr = False + descriptor = descriptor[3:] + elif descriptor.startswith("longstr"): + longstr = True + descriptor = descriptor[7:] + else: + return None + if descriptor == "": + maxlen = 63 + elif descriptor[0] == "(": + maxlen = int(descriptor[1:-1]) + else: + return None + return GDSStringType(maxlen=maxlen, longstr=longstr) + + def from_token(self, tok: gds.GDSTokenValue) -> "GDSStringValue": + # TODO: this should allow me to make things more flexible later + if self.longstr: + return ( + GDSStringValue(tok(), self) + if tok in gds.GDSTokenValue.longstr + else None + ) + else: + return GDSStringValue(tok(), self) if tok in gds.GDSTokenValue.str else None + + def parser(self) -> Parser: + """ + Valid literals can be delimited by double quotes or single quotes; + the chosed quote type will need to be escaped if it appears in the literal. + Generally most Python string escape sequences should be supported. + + Long strings are denoted by prepending a single "l" to the front of the quoted literal. + """ + def parser_map(args: List[Any]) -> Parser: + is_longstr: bool = args[0] + text: str = args[1] + if is_longstr != self.longstr: + return fail( + f'A {"long" if is_longstr else "regular"} string cannot be ' + f'used in place of a {"long" if self.longstr else "regular"} string' + ) + if len(text) > self.maxlen: + print( + f"WARN: String {repr(text)} is too long ({len(text)}, max is {self.maxlen})" + ) + return success(GDSStringValue(text, self)) + + string_part_sq = regex(r"[^'\\]+") + string_part_dq = regex(r'[^"\\]+') + string_esc = string("\\") >> ( + string("\\") + | string("/") + | string('"') + | string("'") + | string("b").result("\b") + | string("f").result("\f") + | string("n").result("\n") + | string("r").result("\r") + | string("t").result("\t") + | regex(r"u[0-9a-fA-F]{4}").map(lambda s: chr(int(s[1:], 16))) + ) + quoted = ( + string('"') >> (string_part_dq | string_esc).many().concat() << string('"') + ) | (string("'") >> (string_part_sq | string_esc).many().concat() << string("'")) + + return seq(regex("l?").map(lambda s: s != ""), quoted).bind(parser_map) + + +TYPES.append(GDSStringType) + + +class GDSStringValue(GDSValue): + """ + A string value, also denoting whether it was defined as a long string. + This is currently redundant, since the GDSStringType already determines + whether a long string is expected; however, in the future interoperability + of the types might be possible. + """ + value: str + longstr: bool + + def __init__(self, value: str, ty: GDSStringType): + self.type = ty + self.value = value + self.longstr = ty.longstr + + def as_token(self) -> gds.GDSToken: + if self.longstr: + return gds.GDSTokenValue.longstr(self.value) + return gds.GDSTokenValue.str(self.value) + + def __format__(self, format_spec: Optional[str] = None) -> str: + return ("l" if self.longstr else "") + repr(self.value) + + +@dataclass +class GDSBoolType(GDSValueType): + """ + A boolean type. This type is virtual, meaning it doesn't actually exist in the GDS + file format; however it is a useful abstraction for writing GDA scripts, and can be easily + converted into the GDS format (though this process may be lossy). + """ + force_rep: Optional[Literal["int", "str"]] = None + + @classmethod + def parse_type(cls, descriptor: str) -> "GDSBoolType": + """ + Valid descriptors are: + bool|int : a bool backed by an integer token. Considered true if the value is != 0. + bool|string : a bool backed by a string token. Considered true exactly if the value + is the string "true" (WARNING: this is very specific and in particular, + case-sensitive. Especially for Python developers, accidentally writing + "True" and having the game recognize it as false is a serious pitfall! + For this reason, we discourage use of this type in its string form.) + bool : a bool that can be backed by either an integer or a string token; + the game will check both and uses whichever is applicable! + """ + if descriptor == "bool": + return GDSBoolType() + elif descriptor == "bool|int": + return GDSBoolType(force_rep="int") + elif descriptor == "bool|string": + return GDSBoolType(force_rep="str") + else: + return None + + def from_token(self, tok: gds.GDSTokenValue) -> "GDSBoolValue": + if (tok in gds.GDSTokenValue.int and self.force_rep != "str") or ( + tok in gds.GDSTokenValue.str and self.force_rep != "int" + ): + return GDSBoolValue(tok(), self) + return None + + def parser(self) -> Any: + """ + `true` and `false` are always valid literals, and will be converted to the required + backing token type (default is int). Otherwise, if the backing type allows it, + either a string or an int literal can be used instead. + + As all values other than 0/1 or "true"/"false" are flattened into a truth value by + the game, this is only possible to preserve binary parity between decompiled-then-recompiled + original scripts (which sometimes show a preference for either type despite the command + supporting both). + """ + p = regex(r"true|false").map(lambda b: GDSBoolValue(b == "true", self)) + if self.force_rep != "int": + p |= GDSStringType().parser().map(lambda v: GDSBoolValue(v.value, self)) + if self.force_rep != "str": + p |= GDSIntType().parser().map(lambda v: GDSBoolValue(v.value, self)) + return p + + +TYPES.append(GDSBoolType) + + +class GDSBoolValue(GDSValue): + """ + A boolean value, backed either by an actual boolean, or an int or string. + Note that if the GDSBoolType declares a forced backing type, but the value + here is encoded as the opposite backing type, the value will be converted + in a lossy manner (this doesn't matter for the game though.) + """ + value: Union[bool, int, str] + + def __init__(self, value: bool, ty: GDSBoolType): + self.type = ty + self.value = value + + def as_token(self) -> gds.GDSToken: + preferred_rep = self.type.force_rep or None + value = self.value + if isinstance(value, int): + if preferred_rep == "str": + if value not in ["true", "false"]: + print( + f"WARN: assigning string value {repr(value)} (false) to int-backed bool; " + "this may lose information, but this information would be ignored " + "by the game anyway." + ) + value = value != 0 + else: + preferred_rep = "int" + elif isinstance(value, str): + if preferred_rep == "int": + if value not in [0, 1]: + print( + f"WARN: assigning int value {value} (true) to string-backed bool; " + "this may lose information, but this information would be ignored " + "by the game anyway." + ) + value = value == "true" + else: + preferred_rep = "str" + + if preferred_rep is None: + preferred_rep = "int" + + if preferred_rep == "int": + if isinstance(value, bool): + value = 1 if value else 0 + return gds.GDSTokenValue.int(value) + elif preferred_rep == "str": + if isinstance(value, bool): + value = "true" if value else "false" + return gds.GDSTokenValue.str(value) + # unreachable + return None + + def __format__(self, format_spec: Optional[str] = None) -> str: + return repr(self.value) diff --git a/main.py b/main.py index 72a8fd1..29f50e6 100644 --- a/main.py +++ b/main.py @@ -1,12 +1,12 @@ import click import formats -from version import v +import version CONTEXT_SETTINGS = dict(help_option_names = ['--help', '-h', '-?']) @click.group(name="flora", context_settings=CONTEXT_SETTINGS) -@click.version_option(v, '--version', '-v', prog_name="flora", message=f"Flora v{v} by patataofcourse") +@click.version_option(version.__version__, '--version', '-v', prog_name="flora", message=f"Flora v{version.__version__} by patataofcourse") def cli(): pass diff --git a/utils.py b/utils.py index b572fd5..973d3ef 100644 --- a/utils.py +++ b/utils.py @@ -1,18 +1,28 @@ import os from contextlib import contextmanager, suppress -from typing import (Any, Callable, Generic, Mapping, TypeVar, Union, - get_origin, get_type_hints) +from typing import ( + Any, + Callable, + Generic, + Mapping, + TypeVar, + Union, + Optional, + get_origin, + get_type_hints, + List, +) import struct def cli_file_pairs( - input=None, - output=None, + ipaths: Optional[str] = None, + opaths: Optional[str] = None, *, - in_endings=None, - out_ending=None, - filter_infer=None, - recursive=False, + in_endings: Optional[List[str]] = None, + out_ending: Optional[str] = None, + filter_infer: Callable[[str, bool], bool] = None, + recursive: bool = False, ): """ Given the file path inputs to the various CLI commands, determines which input files should be operated on and mapped to which output files. @@ -35,11 +45,11 @@ def cli_file_pairs( (i.e. `in/a/b/c.input` becomes `out/a/b/c.output`). If the output exists but isn't a directory, the CLI exits with an error explaining how this doesn't make sense. """ - if input is None: - input = "." + if ipaths is None: + ipaths = "." - if not os.path.exists(input): - raise FileNotFoundError(input) + if not os.path.exists(ipaths): + raise FileNotFoundError(ipaths) def listfiles(path): if recursive: @@ -52,19 +62,25 @@ def listfiles(path): continue yield os.path.join(path, f) - def default_filter_infer(input, force_accept=False): - if in_endings is not None and not any( - input.lower().endswith(ie) for ie in in_endings + def default_filter_infer(ipath, force_accept=False): + if ( + in_endings is not None + and not any(ipath.lower().endswith(ie) for ie in in_endings) + and not force_accept ): return None - if out_ending is not None and input.lower().endswith(out_ending): + if ( + out_ending is not None + and ipath.lower().endswith(out_ending) + and not force_accept + ): return None - output = input + output = ipath if in_endings is not None: - endings = [ie for ie in in_endings if input.lower().endswith(ie)] + endings = [ie for ie in in_endings if ipath.lower().endswith(ie)] if endings: - output = input[: -len(endings[0])] + output = ipath[: -len(endings[0])] if out_ending is None: raise ValueError( "Can't infer output file names without a target file ending specified" @@ -78,25 +94,25 @@ def default_filter_infer(input, force_accept=False): input_dir = "" input_paths = [] rel_pairs = None - if os.path.isfile(input): - input_dir, ip = os.path.split(input) + if os.path.isfile(ipaths): + input_dir, ip = os.path.split(ipaths) input_paths = [ip] - rel_pairs = [(ip, filter_infer(ip, force_accept=True)) for ip in input_paths] + rel_pairs = [(ip, filter_infer(ip, True)) for ip in input_paths] else: - input_dir = input - input_paths = [os.path.relpath(f, input_dir) for f in listfiles(input)] - rel_pairs = [(ip, filter_infer(ip)) for ip in input_paths] + input_dir = ipaths + input_paths = [os.path.relpath(f, input_dir) for f in listfiles(ipaths)] + rel_pairs = [(ip, filter_infer(ip, False)) for ip in input_paths] rel_pairs = [(ip, op) for (ip, op) in rel_pairs if op is not None] - if output is None: + if opaths is None: output = input_dir if ( - os.path.isfile(input) + os.path.isfile(ipaths) and not os.path.isdir(output) and os.path.split(output)[1] != "" ): - return [(input, output)] + return [(ipaths, output)] if os.path.isfile(output): raise OSError(f"Output path exists but is not a directory: '{output}'") @@ -110,22 +126,23 @@ def default_filter_infer(input, force_accept=False): def foreach_file_pair(pairs, fn, quiet=False): - try: + # If TQDM isn't installed, continue as if --quiet was specified + with suppress(ImportError): from tqdm import tqdm if not quiet and len(pairs) > 5: progress = tqdm(pairs) - for input, output in progress: - progress.set_description(input) - fn(input, output) + for ipath, opath in progress: + progress.set_description(ipath) + fn(ipath, opath) return - except ImportError: - # TQDM isn't installed; just don't show a progress bar. - pass - for input, output in pairs: - fn(input, output) + for ipath, opath in pairs: + fn(ipath, opath) + + +T = TypeVar("T") + -T = TypeVar('T') class TU(Generic[T]): union: type name: str @@ -150,6 +167,7 @@ def __hash__(self): def __repr__(self): return f"{self.union.__name__}.{self.name}" + class TUI(Generic[T]): variant: TU[T] value: T @@ -170,23 +188,28 @@ def __hash__(self): def __repr__(self): return f"{repr(self.variant)}({repr(self.value)})" + def tagged_union(cls: type): hints = get_type_hints(cls) - members = [(k,v) for (k,v) in hints.items() if get_origin(v) == TU] - - for (name, t) in members: + members = [(k, v) for (k, v) in hints.items() if get_origin(v) == TU] + + for name, t in members: setattr(cls, name, t(cls, name)) - + return cls + class Test: a: TU[int] b: TU[str] -R = TypeVar('R') -def match(union: TUI, fns: Mapping[Union[TU, type(...)], Callable[[Any],R]]) -> R: + +R = TypeVar("R") + + +def match(union: TUI, fns: Mapping[Union[TU, type(...)], Callable[[Any], R]]) -> R: ellipsis_fn = None - for k,v in fns.items(): + for k, v in fns.items(): if k is Ellipsis: ellipsis_fn = v continue @@ -195,6 +218,8 @@ def match(union: TUI, fns: Mapping[Union[TU, type(...)], Callable[[Any],R]]) -> if ellipsis_fn is not None: return ellipsis_fn() + +# TODO: use importlib.resources instead RESOURCES = os.path.join(os.path.dirname(__file__), "data") @@ -204,14 +229,18 @@ def nested_break(): Raise the returned value to instantly bail out of the with block. Replaces loop labels for breaking out of nested iterations. """ + class NestedBreakException(Exception): pass + with suppress(NestedBreakException): yield NestedBreakException -def round_places(x: float, places: int=0) -> float: + +def round_places(x: float, places: int = 0) -> float: return round(x * (10**places)) / (10**places) + def round_perfect(x: float) -> float: for i in range(1, 8): y = round_places(x, i) @@ -220,4 +249,4 @@ def round_perfect(x: float) -> float: if struct.unpack(" Date: Fri, 4 Oct 2024 11:47:06 +0200 Subject: [PATCH 14/17] feat(gds): integrated the new GDS system into the CLI. Dumping to JSON/YAML is no longer possible, but also not desirable for a DSL representation. --- data/gds_commands/common.yml | 14 +- formats/__init__.py | 1 - formats/gds/__init__.py | 203 ++++++++++++- formats/gds/dump.py | 78 +++++ formats/gds/old.py | 542 ----------------------------------- formats/gds/test_gda.py | 91 ++++-- formats/gds/test_gds.py | 26 +- formats/gds/value.py | 52 +++- main.py | 2 +- utils.py | 14 +- 10 files changed, 426 insertions(+), 597 deletions(-) create mode 100644 formats/gds/dump.py delete mode 100644 formats/gds/old.py diff --git a/data/gds_commands/common.yml b/data/gds_commands/common.yml index 7f276bc..14a4427 100644 --- a/data/gds_commands/common.yml +++ b/data/gds_commands/common.yml @@ -227,12 +227,18 @@ commands: - int - int - 0xb6: + storyscript: + id: 0xb6 desc: > - Passing this function a format string will printf the current condition flag value into it. - However, nothing is done with that value. + In the debug storyscript, registers a starting point from which (presumably) playtesters + could jump into the game without having to replay large earlier parts of it. params: - fmt: string + desc: + type: string + desc: The description of the starting point + scriptPath: + type: string + desc: The path to the script executed to setup the game state to the specified starting point 0xc0: params: [int] diff --git a/formats/__init__.py b/formats/__init__.py index 134510f..fca0634 100644 --- a/formats/__init__.py +++ b/formats/__init__.py @@ -1,2 +1 @@ from formats import gds, bg, pcm, puzzle, ndsrom, compression -from formats.gds import old \ No newline at end of file diff --git a/formats/gds/__init__.py b/formats/gds/__init__.py index 994d8c2..b355049 100644 --- a/formats/gds/__init__.py +++ b/formats/gds/__init__.py @@ -1,3 +1,204 @@ +import click +import contextlib +import collections +import sys +import os + import formats.gds.cmddef as cmddef +import formats.gds.gds as gds +import formats.gds.gda as gda +import formats.gds.patch as patch +from utils import cli_file_pairs, foreach_file_pair + +cmddef.init_commands() + + +@click.group( + help="Script-like format, also used to store puzzle parameters.", options_metavar="" +) +def cli(): + pass + + +@cli.command(name="compile", no_args_is_help=True) +@click.argument("input", required=False, type=click.Path(exists=True)) +@click.argument("output", required=False, type=click.Path(exists=False)) +@click.option( + "--recursive", + "-r", + is_flag=True, + help="Recurse into subdirectories of the input directory to find more applicable files.", +) +@click.option( + "--quiet", + "-q", + is_flag=True, + help="Suppress all output. By default, operations involving multiple files will show a progressbar.", +) +@click.option( + "--overwrite/--no-overwrite", + "-o/-O", + default=True, + help="Whether existing files should be overwritten. Default: true", +) +@click.option( + "--workspace", + "-w", + type=click.Path(dir_okay=True, file_okay=False, exists=True), + help="The path to the workspace directory, i.e. the root folder into which the NDS ROM was extracted. " + "Paths in scripts are relative to this. If unset, defaults to the current working directory (usually not what you want).", +) +def gds_compile(input=None, output=None, recursive=False, quiet=False, overwrite=None, workspace=None): + """ + Compiles the human-readable script(s) at INPUT into the game's binary script files at OUTPUT. + + INPUT can be a single file or a directory (which obviously has to exist). In the latter case subfiles with the correct file ending will be processed. + If unset, defaults to the current working directory. + + The meaning of OUTPUT may depend on INPUT: + - If INPUT is a file, then OUTPUT is expected to be a file, unless it explicitly ends with a slash indicating a directory. + In this case, if unset OUTPUT will default to the INPUT filename with `.gds` exchanged/appended. + - Otherwise OUTPUT has to be a directory as well (or an error will be shown). + In this case, if unset OUTPUT will default to the INPUT directory (which may itself default to the current working directory). + + In the file-to-file case, the paths are explicitly used as they are. Otherwise, if multiple input files were collected, or OUTPUT is a directory, + an output path is inferred for each input file by exchanging the input format's file ending for, or otherwise appending the `.gds` file ending. + + In the case where INPUT is a directory, if no format is specified, this command will collect all files with ending `.gda`. + """ + workdir = workspace or os.getcwd() + + def process(inpath, outpath): + with open(inpath, "r", encoding="utf-8") as inf: + filename = os.path.relpath(inpath, workdir) + prog = gda.read_gda(inf.read(), filename) + + with open(outpath, "wb") as outf: + outf.write(gds.write_gds(prog)) + + pairs = cli_file_pairs( + input, output, in_endings=[".gda"], out_ending=".gds", recursive=recursive + ) + + duplicates = collections.defaultdict(list) + for ip, op in pairs: + duplicates[op].append(ip) + duplicates = {k: v for k, v in duplicates.items() if len(v) > 1} + if duplicates: + print( + f"ERROR: {len(duplicates)} {'files have' if len(duplicates) > 1 else 'file has'} multiple conflicting source files; please explicitly specify a format to determine which should be used.", + file=sys.stderr, + ) + for op, ips in duplicates.items(): + pathlist = ", ".join(f"'{ip}'" for ip in ips) + print(f"'{op}' could be compiled from {pathlist}", file=sys.stderr) + sys.exit(-1) + + if not overwrite: + new_pairs = [] + existing = [] + for ip, op in pairs: + if os.path.exists(op): + existing.append(op) + else: + new_pairs.append((ip, op)) + + if not quiet: + print(f"Skipping {len(existing)} existing output files.") + + pairs = new_pairs + + foreach_file_pair(pairs, process, quiet=quiet) + + +@cli.command(name="decompile", no_args_is_help=True) +@click.argument("input", required=False, type=click.Path(exists=True)) +@click.argument("output", required=False, type=click.Path(exists=False)) +@click.option( + "--recursive", + "-r", + is_flag=True, + help="Recurse into subdirectories of the input directory to find more applicable files.", +) +@click.option( + "--quiet", + "-q", + is_flag=True, + help="Suppress all output. By default, operations involving multiple files will show a progressbar.", +) +@click.option( + "--overwrite/--no-overwrite", + "-o/-O", + default=True, + help="Whether existing files should be overwritten. Default: true", +) +@click.option( + "--workspace", + "-w", + type=click.Path(dir_okay=True, file_okay=False, exists=True), + help="The path to the workspace directory, i.e. the root folder into which the NDS ROM was extracted. " + "Paths in scripts are relative to this. If unset, defaults to the current working directory (usually not what you want).", +) +@click.option( + "--patches/--no-patches", + "-p/-P", + default=True, + help="Whether to apply 'baseline patches' that fix known errors in the vanilla scripts. Which patch to apply is determined " + "by the filename relative to the workspace directory. Default: true; note that some of the vanilla scripts WILL NOT PARSE without " + "these patches, so when in doubt, keep it enabled." +) +def gds_decompile( + input=None, output=None, recursive=False, quiet=False, overwrite=None, workspace=None, patches=None +): + """ + Decompiles the GDS script(s) at INPUT into the human-readable GDA script format at OUTPUT. + + INPUT can be a single file or a directory (which obviously has to exist). In the latter case subfiles with the correct file ending will be processed. + If unset, defaults to the current working directory. + + The meaning of OUTPUT may depend on INPUT: + - If INPUT is a file, then OUTPUT is expected to be a file, unless it explicitly ends with a slash indicating a directory. + In this case, if unset OUTPUT will default to the INPUT filename with `.gda` exchanged/appended. + - Otherwise OUTPUT has to be a directory as well (or an error will be shown). + In this case, if unset OUTPUT will default to the INPUT directory (which may itself default to the current working directory). + + In the file-to-file case, the paths are explicitly used as they are. Otherwise, if multiple input files were collected, or OUTPUT is a directory, + an output path is inferred for each input file by exchanging the `.gds` file ending for `.gda`, or otherwise appending the `.gda` file ending. + """ + # TODO: once/if workspaces are a thing, use auto-discovery to place the file in the workspace correctly + workdir = workspace or os.getcwd() + + def process(inpath, outpath): + try: + filepath = os.path.relpath(inpath, workdir) + with open(inpath, "rb") as inf: + indata = inf.read() + if patches: + indata = patch.patch(indata, filepath) + prog = gds.read_gds(indata, inpath) + + with open(outpath, "w", encoding="utf-8") as outf: + # TODO: determine the working directory? somehow?? + outf.write(gda.write_gda(prog, filepath, workdir)) + except KeyboardInterrupt: + raise + except Exception as e: + print(f"ERR: {inpath}: could not decompile: {e}") + + pairs = cli_file_pairs( + input, output, in_endings=[".gds"], out_ending=".gda", recursive=recursive + ) + if not overwrite: + new_pairs = [] + existing = [] + for ip, op in pairs: + if os.path.exists(op): + existing.append(op) + else: + new_pairs.append((ip, op)) + + if not quiet: + print(f"Skipping {len(existing)} existing output files.") -cmddef.init_commands() \ No newline at end of file + pairs = new_pairs + foreach_file_pair(pairs, process, quiet=quiet) diff --git a/formats/gds/dump.py b/formats/gds/dump.py new file mode 100644 index 0000000..0735cce --- /dev/null +++ b/formats/gds/dump.py @@ -0,0 +1,78 @@ +""" +Legacy support for writing a read GDS program to JSON or YAML. + +This doesn't seem very useful now, since the format and internal +representation have changed drastically. I still wanted to add +it just in case. +""" + +import json +from typing import List, Dict + +import version +import formats.gds.model as model + + +def dump_command(cmd: model.GDSInvocation) -> Dict: + # sourcery skip: remove-redundant-pass + obj = { + "command": cmd.command.name or cmd.command.id, + } + + if isinstance(cmd, (model.GDSIfInvocation, model.GDSLoopInvocation)): + if cmd.condition is not None: + if isinstance(cmd.condition, int): + obj["loop_cnt"] = cmd.condition + else: + cond = [] + + for c in cmd.condition: + if c in model.GDSConditionToken.AND: + cond.append({"flag": "and"}) + elif c in model.GDSConditionToken.OR: + cond.append({"flag": "or"}) + elif c in model.GDSConditionToken.NOT: + cond.append({"flag": "not"}) + elif c in model.GDSConditionToken.command: + cond.append(dump_command(c())) + else: + # unreachable + pass + + obj["condition"] = cond + if cmd.block is not None: + obj["block"] = dump_block(cmd.block) + + else: + obj["args"] = [ + {"type": arg.type.describe(), "value": arg.value} for arg in cmd.args + ] + + return obj + + +def dump_block(block: List[model.GDSElement]) -> List: + els = [] + + for el in block: + if el in model.GDSElement.command: + els.append(dump_command(el())) + elif el in model.GDSElement.BREAK: + els.append({"flag": "break"}) + elif el in model.GDSElement.label: + lbl = {"name": el().name, "present": el().present} + if el().loc is not None: + lbl["loc"] = el().loc + els.append({"label": lbl}) + + return els + + +def dump_json(prog: model.GDSProgram) -> str: + return json.dumps( + {"version": version.__version__, "data": dump_block(prog.elements)} + ) + + +def load_json(json: str) -> model.GDSProgram: + pass diff --git a/formats/gds/old.py b/formats/gds/old.py deleted file mode 100644 index 96da98d..0000000 --- a/formats/gds/old.py +++ /dev/null @@ -1,542 +0,0 @@ -import click -import contextlib -import json -import yaml -import os -import sys -import collections - -import struct -import ast - -import version -import parse -from utils import cli_file_pairs, foreach_file_pair, RESOURCES - - -@click.group( - help="Script-like format, also used to store puzzle parameters.", options_metavar="" -) -def cli(): - pass - -commands = json.load(open(os.path.join(RESOURCES,"commands.json"), encoding="utf-8")) -commands_i = { - val["id"]: key for key, val in commands.items() if "id" in val -} # Inverted version of commands - -class GDS: - def __init__(self, cmds=None): - if cmds is None: - cmds = [] - self.cmds = cmds - - @classmethod - def from_gds(Self, file): - length = int.from_bytes(file[0:4], "little") - if file[4:6] == b"\x0c\x00": - return Self([]) - cmd_data = file[6 : length + 4] - cmds = [] - - cmd = None - params = [] - c = 0 - while True: - if c >= length: - raise Exception( - "GDS file error: End of file reached with no 0xC command!" - ) - if cmd == None: - cmd = int.from_bytes(cmd_data[c : c + 2], "little") - if cmd in commands_i: - cmd = commands_i[cmd] - c += 2 - continue - - p_type = int.from_bytes(cmd_data[c : c + 2], "little") - - if p_type == 0: - cmds.append({"command": cmd, "parameters": params}) - cmd = None - params = [] - c += 2 - elif p_type == 1: - params.append( - { - "type": "int", - "data": struct.unpack('>f', cmd_data[c + 2 : c + 6])[0], - } - ) - c += 6 - elif p_type == 2: - params.append( - { - "type": "float", - "data": int.from_bytes(cmd_data[c + 2 : c + 6], "little"), - } - ) - c += 6 - elif p_type == 3: - str_len = int.from_bytes(cmd_data[c + 2 : c + 4], "little") - params.append( - { - "type": "string", - "data": cmd_data[c + 4 : c + 4 + str_len] - .decode("ascii") # TODO: JP/KO compatibility - .rstrip("\x00"), - } - ) - c += str_len + 4 - elif p_type == 6: - # address within the gds file range. usually fits in a short, but can be an int. - params.append( - { - "type": "taddr", - "data": int.from_bytes(cmd_data[c + 2 : c + 6], "little"), - } - ) - c += 6 - elif p_type == 7: - params.append( - { - "type": "saddr", - "data": int.from_bytes(cmd_data[c + 2 : c + 6], "little"), - } - ) - c += 6 - elif p_type == 8: - params.append({"type": "not"}) - c += 2 - elif p_type == 9: - params.append({"type": "and"}) - c += 2 - elif p_type == 0xA: - params.append({"type": "or"}) - c += 2 - elif p_type == 0xB: - # break - params.append({"type": "break"}) - c += 2 - elif p_type == 0xC: - # eof - cmds.append({"command": cmd, "parameters": params}) - break - else: - raise Exception( - f"GDS file error: Invalid or unsupported parameter type {hex(p_type)}!" - ) - - return Self(cmds) - - @classmethod - def from_json(Self, file): - cmds = json.loads(file)["data"] - # TODO: reject non-compatible json files - return Self(cmds) - - @classmethod - def from_yaml(Self, file): - cmds = yaml.safe_load(file)["data"] - # TODO: reject non-compatible yaml files - return Self(cmds) - - @classmethod - def from_gda(Self, file): # TODO: make this, so gds_old can be completely removed - cmds = [] - - for line in file.split("\n"): - line = line.strip() - if line.startswith("#"): - continue - if line == "": - continue - - line, strings = parse.remove_strings(line) - line = line.rstrip().split(" ") - cmd = line[0] - - # TODO: handle inline comments, maybe - - if cmd in commands: - cmd = commands[cmd] - if "alias" in cmd: - cmd = commands[cmd["alias"]] - elif cmd.startswith("cmd_") or cmd.startswith("unk_"): - cmd = int(cmd[4:]) - elif cmd.startswith("0x"): - cmd = int(cmd[2:], base=16) - else: - raise Exception(f"Unknown GDA command: {cmd}") - - params = [] - - for param in line[1:]: - if param.isdigit(): - params.append({"type": "int", "data": int(param)}) - elif param.startswith("0x"): - params.append({"type": "float", "data": float(param)}) - elif param.startswith('"') and param.endswith('"'): - param = ast.literal_eval(f'"{strings[int(param[1:-1])]}"') - params.append({"type": "string", "data": param}) - elif param.startswith("@"): - params.append({"type": "taddr", "data": int(param[1:], 16)}) - elif param.startswith("$"): - params.append({"type": "saddr", "data": int(param[1:], 16)}) - elif param.startswith("NOT"): - params.append({"type": "not"}) - elif param.startswith("AND"): - params.append({"type": "and"}) - elif param.startswith("OR"): - params.append({"type": "or"}) - elif param.startswith("BREAK"): - params.append({"type": "break"}) - else: - raise Exception(f"Invalid GDA parameter: {param}") - - cmds.append({"command": cmd, "parameters": params}) - - return Self(cmds) - - def __getitem__(self, index): - index = int(index) - return self.cmds[index] - - def to_json(self): - return json.dumps({"version": version.__version__, "data": self.cmds}, indent=4) - - def to_yaml(self): - return yaml.safe_dump({"version": version.__version__, "data": self.cmds}) - - def to_gds(self): - out = b"\x00" * 2 - for command in self.cmds: - if type(command["command"]) == int: - out += command["command"].to_bytes(2, "little") - else: - out += command["command"]["id"].to_bytes(2, "little") - for param in command["parameters"]: - if param["type"] == "int": - out += b"\x01\x00" - out += param["data"].to_bytes(4, "little") - elif param["type"] == "float": - out += b"\x02\x00" - out += struct.pack('>f', param["data"]) - elif param["type"] == "string": - out += b"\x03\x00" - out += (len(param["data"]) + 1).to_bytes(2, "little") - out += ( - param["data"].encode("ASCII") + b"\x00" - ) # TODO: JP/KO compatibility - elif param["type"] == "taddr": - out += b"\x06\x00" - out += param["data"].to_bytes(4, "little") - elif param["type"] == "saddr": - out += b"\x07\x00" - out += param["data"].to_bytes(4, "little") - elif param["type"] == "not": - out += b"\x08\x00" - elif param["type"] == "and": - out += b"\x09\x00" - elif param["type"] == "or": - out += b"\x0a\x00" - elif param["type"] == "break": - out += b"\x0b\x00" - else: - raise Exception( - f"GDS JSON error: Invalid or unsupported parameter type '{param['type']}'!" - ) - out += b"\x00\x00" - out = out[:-2] + b"\x0c\x00" - - return len(out).to_bytes(4, "little") + out - - def to_bin(self): # alias - return self.to_gds() - - def to_gda(self): - out = "" - for command in self.cmds: - if type(command["command"]) == int: - out += "0x" + command["command"].to_bytes(1, "little").hex() - else: - out += command["command"] - for param in command["parameters"]: - out += " " - if param["type"] == "int": - out += str(param["data"]) - elif param["type"] == "string": - out += repr(param["data"]) - elif param["type"] == "float": - out += str(param["data"]) - elif param["type"] == "taddr": - out += f"@{param['data'].hex()}" - elif param["type"] == "saddr": - out += f"${param['data'].hex()}" - elif param["type"] == "not": - out += "NOT" - elif param["type"] == "and": - out += "AND" - elif param["type"] == "or": - out += "OR" - elif param["type"] == "break": - out += "BREAK" - else: - raise Exception( - f"GDA error: invalid or unsupported parameter type '{param['type']}'!" - ) - out += "\n" - return out - - -@cli.command(name="compile", no_args_is_help=True) -@click.argument("input", required=False, type=click.Path(exists=True)) -@click.argument("output", required=False, type=click.Path(exists=False)) -@click.option( - "--recursive", - "-r", - is_flag=True, - help="Recurse into subdirectories of the input directory to find more applicable files.", -) -@click.option( - "--quiet", - "-q", - is_flag=True, - help="Suppress all output. By default, operations involving multiple files will show a progressbar.", -) -@click.option( - "--overwrite/--no-overwrite", - "-o/-O", - default=True, - help="Whether existing files should be overwritten. Default: true", -) -@click.option( - "--format", - "-f", - required=False, - default=None, - multiple=False, - help="The format of the input file. Will be inferred from the file ending or content if unset. " - "If multiple file types would compile to the same output (but may not necessarily have the same content), " - "specify this to disambigute. Possible values: gda, json, yaml", -) -def compile( - input=None, output=None, recursive=False, quiet=False, format=None, overwrite=None -): - """ - Compiles the human-readable script(s) at INPUT into the game's binary script files at OUTPUT. - - INPUT can be a single file or a directory (which obviously has to exist). In the latter case subfiles with the correct file ending will be processed. - If unset, defaults to the current working directory. - - The meaning of OUTPUT may depend on INPUT: - - If INPUT is a file, then OUTPUT is expected to be a file, unless it explicitly ends with a slash indicating a directory. - In this case, if unset OUTPUT will default to the INPUT filename with `.gds` exchanged/appended. - - Otherwise OUTPUT has to be a directory as well (or an error will be shown). - In this case, if unset OUTPUT will default to the INPUT directory (which may itself default to the current working directory). - - In the file-to-file case, the paths are explicitly used as they are. Otherwise, if multiple input files were collected, or OUTPUT is a directory, - an output path is inferred for each input file by exchanging the input format's file ending for, or otherwise appending the `.gds` file ending. - - In the case where INPUT is a directory, if no format is specified, this command will collect files of all compatible types. Note that this can lead - to situations where multiple files would compile to the same output (e.g. `test.json` and `test.gda` would both be candidates for `test.gds`); - this command will NOT make a choice in this case, and instead ask to explicitly specify the format to be used. - """ - - in_endings = [] - if format is None: - in_endings = [".gda", ".json", ".yaml", ".yml"] - elif format == "gda": - in_endings = [".gda"] - elif format == "json": - in_endings = [".json"] - elif format in ["yaml", "yml"]: - in_endings = [".yaml", ".yml"] - else: - raise Exception(f"Unsupported input format: '{format}'") - - def process(input, output): - inpath = input - input = open(inpath, "r", encoding="utf-8").read() - - format2 = format - if format2 is None: - if inpath.lower().endswith(".gda"): - format2 = "gda" - elif inpath.lower().endswith(".json"): - format2 = "json" - elif inpath.lower().endswith(".yml") or inpath.lower().endswith(".yaml"): - format2 = "yaml" - - gds = None - with contextlib.suppress(Exception): - if format2 == "gda": - gds = GDS.from_gda(input) - elif format2 == "json": - gds = GDS.from_json(input) - elif format2 in ["yaml", "yml"]: - gds = GDS.from_yaml(input) - - if gds is None: - if format2 is not None: - # TODO: should this abort instead? - print( - f"WARNING: Input file '{inpath}' did not have expected format '{format2}'", - file=sys.stderr, - ) - # format not specified and couldn't be inferred, or file turns out not to have the correct format - # => try all the formats & see which one works (only one should be possible) - for f in ["gda", "json", "yaml"]: - with contextlib.suppress(Exception): - if f == "gda": - gds = GDS.from_gda(input) - elif f == "json": - gds = GDS.from_json(input) - elif f == "yaml": - gds = GDS.from_yaml(input) - if gds is not None: - break - if gds is None: - raise Exception( - f"File '{inpath}' couldn't be read: not a known file format" - + (f" (expected '{format2}')" if format2 is not None else "") - ) - - output = open(output, "wb") - output.write(gds.to_bin()) - output.close() - - pairs = cli_file_pairs( - input, output, in_endings=in_endings, out_ending=".gds", recursive=recursive - ) - - duplicates = collections.defaultdict(list) - for ip, op in pairs: - duplicates[op].append(ip) - duplicates = {k: v for k, v in duplicates.items() if len(v) > 1} - if len(duplicates) > 0: - print( - f"ERROR: {len(duplicates)} {'files have' if len(duplicates) > 1 else 'file has'} multiple conflicting source files; please explicitly specify a format to determine which should be used.", - file=sys.stderr, - ) - for op, ips in duplicates.items(): - pathlist = ", ".join("'" + ip + "'" for ip in ips) - print(f"'{op}' could be compiled from {pathlist}", file=sys.stderr) - sys.exit(-1) - - if not overwrite: - new_pairs = [] - existing = [] - for ip, op in pairs: - if os.path.exists(op): - existing.append(op) - else: - new_pairs.append((ip, op)) - - if not quiet: - print(f"Skipping {len(existing)} existing output files.") - - pairs = new_pairs - - foreach_file_pair(pairs, process, quiet=quiet) - - -@cli.command(name="decompile", no_args_is_help=True) -@click.argument("input", required=False, type=click.Path(exists=True)) -@click.argument("output", required=False, type=click.Path(exists=False)) -@click.option( - "--recursive", - "-r", - is_flag=True, - help="Recurse into subdirectories of the input directory to find more applicable files.", -) -@click.option( - "--quiet", - "-q", - is_flag=True, - help="Suppress all output. By default, operations involving multiple files will show a progressbar.", -) -@click.option( - "--overwrite/--no-overwrite", - "-o/-O", - default=True, - help="Whether existing files should be overwritten. Default: true", -) -@click.option( - "--format", - "-f", - required=False, - multiple=False, - help="The format used for output. Possible values: gda (default), json, yaml", -) -def decompile( - input=None, output=None, recursive=False, quiet=False, format=None, overwrite=None -): - """ - Decompiles the GDS script(s) at INPUT into a human-readable text format at OUTPUT. - - INPUT can be a single file or a directory (which obviously has to exist). In the latter case subfiles with the correct file ending will be processed. - If unset, defaults to the current working directory. - - The meaning of OUTPUT may depend on INPUT: - - If INPUT is a file, then OUTPUT is expected to be a file, unless it explicitly ends with a slash indicating a directory. - In this case, if unset OUTPUT will default to the INPUT filename with `.json` exchanged/appended. - - Otherwise OUTPUT has to be a directory as well (or an error will be shown). - In this case, if unset OUTPUT will default to the INPUT directory (which may itself default to the current working directory). - - In the file-to-file case, the paths are explicitly used as they are. Otherwise, if multiple input files were collected, or OUTPUT is a directory, - an output path is inferred for each input file by exchanging the `.gds` file ending for `.json`, or otherwise appending the `.json` file ending. - """ - out_ending = "" - if format == "gda" or format is None: - out_ending = ".gda" - elif format == "json": - out_ending = ".json" - elif format in ["yaml", "yml"]: - out_ending = ".yml" - else: - raise Exception(f"Unsupported output format: '{format}'") - - def process(input, output): - input = open(input, "rb").read() - gds = GDS.from_gds(input) - - nonlocal format - if format is None: - if output.lower().endswith(".gda"): - format = "gda" - elif output.lower().endswith(".json"): - format = "json" - elif output.lower().endswith(".yml") or output.lower().endswith(".yaml"): - format = "yaml" - else: - print( - f"WARNING: output format couldn't be inferred from filename '{output}'; using default (gda). To remove this warning, please explicitly specify a format.", - file=sys.stderr, - ) - - with open(output, "w", encoding="utf-8") as output: - if format == "gda": - output.write(gds.to_gda()) - elif format == "json": - output.write(gds.to_json()) - elif format in ["yaml", "yml"]: - output.write(gds.to_yaml()) - - pairs = cli_file_pairs( - input, output, in_endings=[".gds"], out_ending=out_ending, recursive=recursive - ) - if not overwrite: - new_pairs = [] - existing = [] - for ip, op in pairs: - if os.path.exists(op): - existing.append(op) - else: - new_pairs.append((ip, op)) - - if not quiet: - print(f"Skipping {len(existing)} existing output files.") - - pairs = new_pairs - foreach_file_pair(pairs, process, quiet=quiet) diff --git a/formats/gds/test_gda.py b/formats/gds/test_gda.py index fe9a932..4d46ad0 100644 --- a/formats/gds/test_gda.py +++ b/formats/gds/test_gda.py @@ -6,9 +6,9 @@ import tqdm from .gds import read_gds, write_gds -import formats.gds.gda as gda +from .gda import read_gda, write_gda, parse_element, CommentContext, format_comment from .patch import patch -from .value import * +from .value import GDSIntType, GDSIntValue from utils import RESOURCES @@ -18,13 +18,13 @@ def test_file(gdspath: str, base: str): gdsb = gdsf.read() gdsb = patch(gdsb, gdspath) prog = read_gds(gdsb, gdspath) - gdas = gda.write_gda(prog, gdspath, base) + gdas = write_gda(prog, gdspath, base) gdapath = gdspath.replace(".gds", ".gda") with open(os.path.join(base, gdapath), "w", encoding="utf8") as gdaf: gdaf.write(gdas) try: - reread = gda.read_gda(gdas, gdapath) + reread = read_gda(gdas, gdapath) except Exception as e: print(f"ERR: {gdapath}: could not read decompiled script: {e}") return @@ -64,9 +64,9 @@ def test_comment(): $(/data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt) """ print( - gda.format_comment( + format_comment( EXAMPLE, - gda.CommentContext( + CommentContext( args=[GDSIntValue(7, GDSIntType())], filename="/data/script/event/e6.gds", workdir=BASE, @@ -76,25 +76,72 @@ def test_comment(): def test_parsers(): - print(gda.parse_element.parse("if not 0x49:{ # test \n0x49 # test2\n#test3\n}")) + print(parse_element.parse("if not 0x49:{ # test \n0x49 # test2\n#test3\n}")) -def test_readfile(gdapath: str, base: str): - with open(os.path.join(base, gdapath), "r", encoding="utf8") as gdaf: - gdas = gdaf.read() - - prog = gda.read_gda(gdas, gdapath) - compiled = write_gds(prog) - - gdspath = gdapath.replace(".gda", ".gds") - with open(os.path.join(base, gdspath), "rb") as gdsf: - gdsb = gdsf.read() - gdsb = patch(gdsb, gdspath) - - assert gdsb == compiled - +class TestGDSVanillaFiles(unittest.TestCase): + """ + A basic parameterized test case, checking if the decompiler is able to decompile and recompile the + original game scripts and produce identical data. + + For this suite to be possible, the original files from the LT1 EU version .nds ROM need to be + extracted/symlinked into `{flora root dir}/data/game_root/lt1_eu/`, such that the folder contains the + subfolders `data`, `dwc` and `ftc`. Eventually Flora should support this, but for now you can use Tinke to do it. + """ + def test_all_files(self): + """ + Checks if the decompiler is able to disassemble and reassemble each of the unmodified original game scripts. + This baseline sanity check ensures that merely running `decompile` and then `compile` does not corrupt + game files. + """ + base = BASE + # sourcery skip: no-conditionals-in-tests + if not os.path.exists(base) or not os.path.isdir(base): + self.skipTest( + 'Could not find vanilla game files under "data/game_root/lt1_eu". Please make sure ' + "to extract and either copy or symlink the entire contents of a valid LT1 EU version .nds ROM " + "into that location." + ) + # sourcery skip: no-loop-in-tests + for root, _subdirs, files in os.walk(os.path.join(base, "data/script")): + for f in files: + if not f.endswith(".gds"): + continue + path = os.path.relpath(os.path.join(root, f), base) + with self.subTest(msg=f"Check file {path}"): + self.single_file(path, base) + + def single_file(self, gdspath: str, base: str): + with open(os.path.join(base, gdspath), "rb") as f: + gdsb = f.read() + # For now we require these known exceptions + gdsb = patch(gdsb, gdspath) + try: + prog = read_gds(gdsb, gdspath) + except Exception as e: + self.fail(f"Decompile: {e}") + raise e + try: + gdas = write_gda(prog, gdspath, base) + except Exception as e: + self.fail(f"Write GDA: {e}") + raise e + + gdapath = gdspath.replace(".gds", ".gda") + try: + prog2 = read_gda(gdas, gdapath) + except Exception as e: + self.fail(f"Read GDA: {e}") + raise e + try: + gdsb2 = write_gds(prog2) + except Exception as e: + self.fail(f"Recompile: {e}") + raise e + + self.assertEqual(gdsb, gdsb2, "Recompiled result is not identical") if __name__ == "__main__": - # test_file("data/script/event/e49.gds", BASE) + # test_file("data/script/rooms/room4_param.gds", BASE) test_all(BASE) # test_parsers() diff --git a/formats/gds/test_gds.py b/formats/gds/test_gds.py index 3c6c812..19b36cf 100644 --- a/formats/gds/test_gds.py +++ b/formats/gds/test_gds.py @@ -7,6 +7,7 @@ from .patch import patch from utils import RESOURCES + def print_hexdump(data: bytes): res = "" padstart = 4 @@ -16,14 +17,14 @@ def print_hexdump(data: bytes): elif i % 4 == 0: res += " " res += "## " - + i = padstart for b in data: if i % 16 == 0: res += "\n" elif i % 4 == 0: res += " " - + if b < 16: res += "0" res += hex(b)[2:] @@ -70,27 +71,31 @@ def test_all(base: str): path = os.path.relpath(os.path.join(root, f), base) test_file(path, base) + # test_all(BASE) # test_file("data/script/qscript/q4_param.gds", BASE) + class TestGDSVanillaFiles(unittest.TestCase): def test_all_files(self): """ Checks if the decompiler is able to disassemble and reassemble each of the unmodified original game scripts. This baseline sanity check ensures that merely running `decompile` and then `compile` does not corrupt game files. - + Note that for this test to be possible, original files from the LT1 EU version .nds ROM need to be extracted/symlinked into `{flora root dir}/data/game_root/lt1_eu/`, such that the folder contains the subfolders `data`, `dwc` and `ftc`. Eventually Flora should support this, but for now you can use Tinke to do it. """ base = BASE -# sourcery skip: no-conditionals-in-tests + # sourcery skip: no-conditionals-in-tests if not os.path.exists(base) or not os.path.isdir(base): - self.skipTest("Could not find vanilla game files under \"data/game_root/lt1_eu\". Please make sure " - "to extract and either copy or symlink the entire contents of a valid LT1 EU version .nds ROM " - "into that location.") -# sourcery skip: no-loop-in-tests + self.skipTest( + 'Could not find vanilla game files under "data/game_root/lt1_eu". Please make sure ' + "to extract and either copy or symlink the entire contents of a valid LT1 EU version .nds ROM " + "into that location." + ) + # sourcery skip: no-loop-in-tests for root, _subdirs, files in os.walk(os.path.join(base, "data/script")): for f in files: if not f.endswith(".gds"): @@ -98,7 +103,7 @@ def test_all_files(self): path = os.path.relpath(os.path.join(root, f), base) with self.subTest(msg=f"Check file {path}"): self.single_file(path, base) - + def single_file(self, filepath: str, base: str): with open(os.path.join(base, filepath), "rb") as f: data = f.read() @@ -113,7 +118,7 @@ def single_file(self, filepath: str, base: str): except Exception as e: self.fail(f"Recompile: {e}") raise e - + self.assertEqual(data, data2, "Recompiled result is not identical") # if data != data2: # print( @@ -123,4 +128,3 @@ def single_file(self, filepath: str, base: str): # f.write(data2) # elif os.path.exists(os.path.join(base, filepath + "2")): # os.remove(os.path.join(base, filepath + "2")) - diff --git a/formats/gds/value.py b/formats/gds/value.py index 1ea5077..3e55305 100644 --- a/formats/gds/value.py +++ b/formats/gds/value.py @@ -18,7 +18,7 @@ def parse_type(descriptor: str) -> Optional["GDSValueType"]: """ Given a type descriptor (used internally in the GDS command definition files), creates a corresponding type object, or returns None if the descriptor is invalid. - + See the individual type classes registered in `TYPES` for information on valid descriptor formats """ @@ -34,6 +34,7 @@ class GDSValueType(ABC): A GDS Value Type. Defines how a value of this type is created from a GDS token or a string parsed in a GDA file. """ + @classmethod @abstractmethod def parse_type(cls, descriptor: str) -> Optional["GDSValueType"]: @@ -43,6 +44,12 @@ def parse_type(cls, descriptor: str) -> Optional["GDSValueType"]: otherwise returns None to signal the next candidate type should be tried. """ + @abstractmethod + def describe(self) -> str: + """ + Returns a descriptor for this variant of the type. The inverse of `parse_type`, in a sense. + """ + @abstractmethod def from_token(self, tok: gds.GDSTokenValue) -> "GDSValue": """ @@ -64,6 +71,7 @@ class GDSValue(ABC): is printed, either in Python code or in a GDA script, and conversion to other value types / GDS tokens. """ + type: GDSValueType value: Any @@ -98,6 +106,7 @@ class GDSIntType(GDSValueType): An integer type, with a fixed byte length, and a marker whether it is intended to be read as an unsigned integer in the game. """ + bytelen: int = 4 unsigned: bool = True @@ -107,12 +116,12 @@ def parse_type(cls, descriptor: str) -> "GDSIntType": Valid descriptors are: int(n): n bytes, signed uint(n): n bytes, unsigned - + or alternatively the shorthands: int / uint: 4 bytes, optionally unsigned short / ushort: 2 bytes, optionally unsigned byte / ubyte: 1 byte, optionally unsigned - + Note that all int values are still written as 4 little endian bytes in GDS files, but only a subset of these values is actually read by the game. Note also that this hard limits the byte length to 4 at most. @@ -137,11 +146,14 @@ def parse_type(cls, descriptor: str) -> "GDSIntType": return None return GDSIntType(bytelen=bytelen, unsigned=unsigned) + def describe(self) -> str: + return ("uint" if self.unsigned else "int") + f"({self.bytelen})" + def from_token(self, tok: gds.GDSTokenValue) -> "GDSIntValue": if tok not in gds.GDSTokenValue.int: return None val = tok() - if not self.unsigned and val >= (256**self.bytelen)/2: + if not self.unsigned and val >= (256**self.bytelen) / 2: val -= 256**self.bytelen return GDSIntValue(val, self) @@ -151,6 +163,7 @@ def parser(self) -> Parser: base 16/2 (prefixed by 0x or 0b respectively). Note that if a dot is found at the end of the literal, parsing will fail, as this designates the value as a float instead. """ + def parser_map( val: int, lit_fmt: Literal["hex", "bin", "dec"] = "dec" ) -> "GDSIntValue": @@ -201,6 +214,7 @@ class GDSIntValue(GDSValue): An integer value. If written in a non-decimal base in a GDA script, this information is preserved here (but not after being written to a GDS binary). """ + value: int lit_fmt: Literal["hex", "bin", "dec"] = "dec" @@ -231,10 +245,14 @@ class GDSFloatType(GDSValueType): A float type, of which there are no variations. It matches the IEEE-754 32-bit float specification, though it is stored in little-endian byte order in GDS files. """ + @classmethod def parse_type(cls, descriptor: str) -> "GDSFloatType": return GDSFloatType() if descriptor == "float" else None + def describe(self) -> str: + return "float" + def from_token(self, tok: gds.GDSTokenValue) -> "GDSFloatValue": return GDSFloatValue(tok(), self) if tok in gds.GDSTokenValue.float else None @@ -244,7 +262,7 @@ def parser(self) -> Parser: as a nonempty int literal to either side of it. Omitting the other will denote it as 0, therefore `123.` is a convenient way to write integers explicitly as floats. Literals may also include an exponent. - + TODO: allow writing integers here? Since we already know where the program expects a float due to the command list, we could make a convenient conversion here... """ @@ -263,6 +281,7 @@ class GDSFloatValue(GDSValue): we correct this by attempting to round to the float with the least decimal digits which still produces the same value as the original. """ + value: float def __init__(self, value: float, ty: GDSFloatType): @@ -283,6 +302,7 @@ class GDSStringType(GDSValueType): a "long string". The main difference between the two is that a long string is stored in a different buffer, meaning the two types can NOT be substituted for each other! """ + maxlen: int = 63 longstr: bool = False @@ -314,6 +334,9 @@ def parse_type(cls, descriptor: str) -> "GDSStringType": return None return GDSStringType(maxlen=maxlen, longstr=longstr) + def describe(self) -> str: + return ("longstr" if self.longstr else "string") + f"({self.maxlen})" + def from_token(self, tok: gds.GDSTokenValue) -> "GDSStringValue": # TODO: this should allow me to make things more flexible later if self.longstr: @@ -330,9 +353,10 @@ def parser(self) -> Parser: Valid literals can be delimited by double quotes or single quotes; the chosed quote type will need to be escaped if it appears in the literal. Generally most Python string escape sequences should be supported. - + Long strings are denoted by prepending a single "l" to the front of the quoted literal. """ + def parser_map(args: List[Any]) -> Parser: is_longstr: bool = args[0] text: str = args[1] @@ -363,7 +387,9 @@ def parser_map(args: List[Any]) -> Parser: ) quoted = ( string('"') >> (string_part_dq | string_esc).many().concat() << string('"') - ) | (string("'") >> (string_part_sq | string_esc).many().concat() << string("'")) + ) | ( + string("'") >> (string_part_sq | string_esc).many().concat() << string("'") + ) return seq(regex("l?").map(lambda s: s != ""), quoted).bind(parser_map) @@ -378,6 +404,7 @@ class GDSStringValue(GDSValue): whether a long string is expected; however, in the future interoperability of the types might be possible. """ + value: str longstr: bool @@ -402,6 +429,7 @@ class GDSBoolType(GDSValueType): file format; however it is a useful abstraction for writing GDA scripts, and can be easily converted into the GDS format (though this process may be lossy). """ + force_rep: Optional[Literal["int", "str"]] = None @classmethod @@ -426,6 +454,13 @@ def parse_type(cls, descriptor: str) -> "GDSBoolType": else: return None + def describe(self) -> str: # sourcery skip: assign-if-exp, reintroduce-else + if self.force_rep == "int": + return "bool|int" + if self.force_rep == "str": + return "bool|string" + return "bool" + def from_token(self, tok: gds.GDSTokenValue) -> "GDSBoolValue": if (tok in gds.GDSTokenValue.int and self.force_rep != "str") or ( tok in gds.GDSTokenValue.str and self.force_rep != "int" @@ -438,7 +473,7 @@ def parser(self) -> Any: `true` and `false` are always valid literals, and will be converted to the required backing token type (default is int). Otherwise, if the backing type allows it, either a string or an int literal can be used instead. - + As all values other than 0/1 or "true"/"false" are flattened into a truth value by the game, this is only possible to preserve binary parity between decompiled-then-recompiled original scripts (which sometimes show a preference for either type despite the command @@ -462,6 +497,7 @@ class GDSBoolValue(GDSValue): here is encoded as the opposite backing type, the value will be converted in a lossy manner (this doesn't matter for the game though.) """ + value: Union[bool, int, str] def __init__(self, value: bool, ty: GDSBoolType): diff --git a/main.py b/main.py index 29f50e6..e70382c 100644 --- a/main.py +++ b/main.py @@ -10,7 +10,7 @@ def cli(): pass -cli.add_command(formats.gds.old.cli, "gds") +cli.add_command(formats.gds.cli, "gds") cli.add_command(formats.bg.cli, "bg") cli.add_command(formats.pcm.cli, "pcm") cli.add_command(formats.puzzle.cli, "puzzle") diff --git a/utils.py b/utils.py index 973d3ef..446e2cf 100644 --- a/utils.py +++ b/utils.py @@ -105,18 +105,18 @@ def default_filter_infer(ipath, force_accept=False): rel_pairs = [(ip, op) for (ip, op) in rel_pairs if op is not None] if opaths is None: - output = input_dir + opaths = input_dir if ( os.path.isfile(ipaths) - and not os.path.isdir(output) - and os.path.split(output)[1] != "" + and not os.path.isdir(opaths) + and os.path.split(opaths)[1] != "" ): - return [(ipaths, output)] + return [(ipaths, opaths)] - if os.path.isfile(output): - raise OSError(f"Output path exists but is not a directory: '{output}'") - output_dir = output + if os.path.isfile(opaths): + raise OSError(f"Output path exists but is not a directory: '{opaths}'") + output_dir = opaths pairs = [ (os.path.join(input_dir, ip), os.path.join(output_dir, op)) From d597db4dcc38b887d26a353e4211c740550a675c Mon Sep 17 00:00:00 2001 From: ilona Date: Mon, 7 Oct 2024 16:24:08 +0200 Subject: [PATCH 15/17] doc(gds): added capabilities and requirements to the readme --- README.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b8d0085..a4f2ac2 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,12 @@ + Warning: the only version with guaranteed compatibility is EU/AUS. Please specify the region of your ROM when reporting a bug or opening an issue. ### Current utilities -* GDS script extracting and repacking to a custom readable format. - + Currently, the parameters supported are those of types 1 (int), 2 (double int) and 3 (string). Since some scripts use other parameter types, this will be fixed in the future. +* GDS script decompilation and recompilation into a custom human-readable scripting language. + + Supports control flow instructions with conditionals + + An effort to catalogue all command IDs with speaking names and parameter lists is ongoing + + All vanilla files from the EU version of CV decompile* and recompile into identical bytecode. + (* Some scripts are structurally incorrect in ways that make parsing difficult, so a set of baseline + patches is applied first to make them functional) * Exporting BG ARC files to PNG, and creating them from PNGs. + Currently, the PNG's color mode must be set to indexed. This will be changed in future versions. + The image must have at most 256 different colors. This is a limitation of the format. @@ -35,10 +39,13 @@ You can also extract the match puzzle tutorial (from puzzle #10) using "match_tu Some unused variants of puzzles can't be extracted yet through Flora, as well as some Japanese-exclusive weekly puzzles, however, this will be fixed in a future update. ### Requirements -* Python 3 +* Python 3.10 * [click](https://pypi.org/project/click/) * [pillow](https://pypi.org/project/pillow/) * [ndspy](https://pypi.org/project/ndspy/) +* [pyyaml](https://pypi.org/project/pyyaml) +* [dacite](https://pypi.org/project/dacite) +* [parsy](https://pypi.org/project/parsy) * Optional: [tqdm](https://pypi.org/project/tqdm) (to display a progress bar for long-running processes) To install all modules, use: `pip install -r requirements.txt` From 0cf5c5e1ff966f1270806607679f6f399871c381 Mon Sep 17 00:00:00 2001 From: ilonachan Date: Wed, 9 Oct 2024 07:08:02 +0200 Subject: [PATCH 16/17] WIP: going through all commands, analyzing the meaning of some I previously didn't understand, as well as actual parameter lengths --- data/gds_commands/common.yml | 4 - data/gds_commands/event.yml | 1 + data/gds_commands/fx.yml | 7 +- data/gds_commands/minigames/dog.yml | 17 +- data/gds_commands/minigames/hotel.yml | 60 +- data/gds_commands/puzzle/buttonlist.yml | 4 +- data/gds_commands/puzzle/coin.yml | 13 +- data/gds_commands/puzzle/common.yml | 81 +- data/gds_commands/puzzle/connect.yml | 29 +- data/gds_commands/puzzle/cup.yml | 19 +- data/gds_commands/puzzle/cut.yml | 1 + data/gds_commands/puzzle/drag.yml | 1 + data/gds_commands/puzzle/drawinput.yml | 3 + data/gds_commands/puzzle/match.yml | 11 +- data/gds_commands/puzzle/queen.yml | 7 +- data/gds_commands/puzzle/rivercross.yml | 1 + data/gds_commands/puzzle/scale.yml | 23 +- data/gds_commands/puzzle/shape.yml | 3 +- data/gds_commands/puzzle/slide.yml | 1 + data/gds_commands/puzzle/tile.yml | 7 +- data/gds_commands/puzzle/trace.yml | 23 +- data/gds_commands/room.yml | 34 +- doc/sample_file.sav.yml | 2438 +++++++++++++++++++++++ formats/gds/preview.py | 3 +- formats/gds/value.py | 1 + 25 files changed, 2663 insertions(+), 129 deletions(-) create mode 100644 doc/sample_file.sav.yml diff --git a/data/gds_commands/common.yml b/data/gds_commands/common.yml index 14a4427..d20acd5 100644 --- a/data/gds_commands/common.yml +++ b/data/gds_commands/common.yml @@ -3,7 +3,6 @@ # By default the prefix is generated from the directory structure and package file name, # we reset it explicitly here to get these functions into the common namespace. prefix: null - # Commands that we know the meaning of, reasonably well at least. commands: playSfx: @@ -120,9 +119,6 @@ commands: params: unk_1: type: int - 0x1c: - params: - - type: int 0x4a: condition: true desc: checks if some global flag is false diff --git a/data/gds_commands/event.yml b/data/gds_commands/event.yml index 1ac6fc9..81c8700 100644 --- a/data/gds_commands/event.yml +++ b/data/gds_commands/event.yml @@ -1,3 +1,4 @@ +context: "event" commands: setCurrent: id: 0x60 diff --git a/data/gds_commands/fx.yml b/data/gds_commands/fx.yml index bfd4e86..a6276ab 100644 --- a/data/gds_commands/fx.yml +++ b/data/gds_commands/fx.yml @@ -25,13 +25,15 @@ commands: desc: Shows a fade out effect on the bottom screen. displayImgBottom: id: 0x0b + context: "puzzle.drawinput2" desc: Shows the specified image path on the bottom screen params: bgxName: type: string desc: The name of the image file, with bgx file ending (will look for a .arc file with that name under /bg) unk_2: - type: int + # TODO: datatype int[0,4] + type: byte desc: Seems to always be 2 (for now) displayImgTop: id: 0x0c @@ -41,5 +43,6 @@ commands: type: string desc: The name of the image file, with bgx file ending (will look for a .arc file with that name under /bg) unk_2: - type: int + # TODO: datatype int[0,4] + type: byte desc: Seems to always be 3 (for now) diff --git a/data/gds_commands/minigames/dog.yml b/data/gds_commands/minigames/dog.yml index 822dcae..da877bb 100644 --- a/data/gds_commands/minigames/dog.yml +++ b/data/gds_commands/minigames/dog.yml @@ -1,10 +1,15 @@ commands: partdef: id: 0xc2 + desc: | + Defines a gizmo/dog part for use in the minigame. This definition consists of the display position of the part + sprite once placed, and a separate "slide target" point where the gizmo sprite will linearly slide toward when + clicked, before vanishing and being replaced with the realized part. Note that the sprite ID for the gizmo in the + tray is not assumed to be the same ID as the part itself, though in practice it turned out that way. params: - - int - - int - - int - - int - - int - - int \ No newline at end of file + partId: int + xPos: int + yPos: int + gizmoSpriteId: int + slideTargetX: int + slideTargetY: int \ No newline at end of file diff --git a/data/gds_commands/minigames/hotel.yml b/data/gds_commands/minigames/hotel.yml index 89758e3..6f9d9d6 100644 --- a/data/gds_commands/minigames/hotel.yml +++ b/data/gds_commands/minigames/hotel.yml @@ -13,12 +13,27 @@ commands: itemId: int xPos: int yPos: int - unk_4: int - unk_5: int - unk_6: int - unk_7: int - unk_8: int - unk_9: int + zIndex: + type: int + uncertain: true + desc: Defines the z-order of the item when displayed in the room. Required since some configurations definitely overlap. + blockerId: + type: int + desc: > + The ID of another item in the room, which if present, implies that this item's preferred spot can't/shouldn't be used. + Instead the position is calculated relative to the blocker item. + blockerOffsetX: int + blockerOffsetY: int + blockerZ: + type: int + uncertain: true + desc: Definitely the same kind of value as zIndex, and if -1 will reuse the above. But if the blocker is present, this value is used instead. + overlayId: + type: int + desc: > + Some of the furniture items have a completely different sprite when their blocker is present, namely the flowers in the vase and the books on the shelf. + If that is the case, this value will be the ID of the sprite for that case. If the item doesn't have a blocker, this value is ignored; set it to -1 + to mean there is no alternate sprite. itemComment1Layton: id: 0xc6 desc: > @@ -39,15 +54,30 @@ commands: itemLuke: id: 0xbd params: - - int - - int - - int - - int - - int - - int - - int - - int - - int + itemId: int + xPos: int + yPos: int + zIndex: + type: int + uncertain: true + desc: Defines the z-order of the item when displayed in the room. Required since some configurations definitely overlap. + blockerId: + type: int + desc: > + The ID of another item in the room, which if present, implies that this item's preferred spot can't/shouldn't be used. + Instead the position is calculated relative to the blocker item. + blockerOffsetX: int + blockerOffsetY: int + blockerZ: + type: int + uncertain: true + desc: Definitely the same kind of value as zIndex, and if -1 will reuse the above. But if the blocker is present, this value is used instead. + overlayId: + type: int + desc: > + Some of the furniture items have a completely different sprite when their blocker is present, namely the flowers in the vase and the books on the shelf. + If that is the case, this value will be the ID of the sprite for that case. If the item doesn't have a blocker, this value is ignored; set it to -1 + to mean there is no alternate sprite. itemComment1Luke: id: 0xc8 desc: > diff --git a/data/gds_commands/puzzle/buttonlist.yml b/data/gds_commands/puzzle/buttonlist.yml index 9e2d536..3373590 100644 --- a/data/gds_commands/puzzle/buttonlist.yml +++ b/data/gds_commands/puzzle/buttonlist.yml @@ -1,10 +1,10 @@ +context: "puzzle.buttonlist" commands: count: id: 0x1d desc: Defines how many buttons there should be in the list. Note that the engine entirely controls button placement. params: - count: - type: int + count: int correct: id: 0x1e desc: Defines which of the buttons is the correct one. diff --git a/data/gds_commands/puzzle/coin.yml b/data/gds_commands/puzzle/coin.yml index 931dfbf..e422834 100644 --- a/data/gds_commands/puzzle/coin.yml +++ b/data/gds_commands/puzzle/coin.yml @@ -1,9 +1,14 @@ +context: "puzzle.coin" commands: - 0x25: + coin: + id: 0x25 + desc: Places a coin on the puzzle field, and defines its reset slot params: - - type: int - - type: int - 0x26: + xPos: int + yPos: int + target: + id: 0x26 + desc: Defines one of the target locations where a coin needs to be placed for the puzzle to solve params: - type: int - type: int diff --git a/data/gds_commands/puzzle/common.yml b/data/gds_commands/puzzle/common.yml index 041b948..9071ef3 100644 --- a/data/gds_commands/puzzle/common.yml +++ b/data/gds_commands/puzzle/common.yml @@ -1,4 +1,5 @@ prefix: "puzzle." +context: "puzzle" commands: setCurrent: id: 0x48 @@ -6,8 +7,7 @@ commands: Sets the current puzzle ID in the global game state. If at the end of script execution the global "current script type" is set to "question", the puzzle with this ID will be started. params: - puzzleId: - type: int + puzzleId: int isCurrentNotSolved: id: 0x4c condition: true @@ -34,6 +34,8 @@ commands: engine: id: 0x1b + # TECHNICALLY. But nobody would use this outside of puzzle scripts, because the grading and everything would be missing. + context: "all" desc: Loads a puzzle engine with the specified name, and continues the script in that context. params: engineName: @@ -41,6 +43,7 @@ commands: desc: the name of the engine to load imgWin: id: 0x1f + context: "all" desc: Selects the image to be displayed when the puzzle is done, probably. uncertain: true params: @@ -52,18 +55,19 @@ commands: desc: seems to always be 0 slideItemGoalTile: id: 0x38 + context: ["puzzle.shape", "puzzle.slide", "puzzle.drag"] desc: For puzzles that involve sliding objects into correct locations (slide, drag, shape (unused)) define the location that would be an object's goal. params: - xPos: - type: int - yPos: - type: int + xPos: int + yPos: int drawLineColor: id: 0x42 + context: "all" desc: | Many puzzle types give the user the ability to draw lines freehand or between points; this command sets the RGB color of those lines (with 5bit channel precision). - A full list of those puzzle types: trace, traceButton, drawInput2, cut, drag. + A full list of the puzzle types that use this: [trace, traceButton, drawInput2, cut, drag] + However, the function is available independent of engine. params: r: type: int @@ -159,32 +163,53 @@ commands: type: string desc: There is no fixed list of options, but the length may not exceed 64 characters. - - 0x20: + 0x1c: + desc: > + Effectively calls 0x20 times with parameter 0, though this value is hardcoded to 3 in the release. + Therefore, fills that 3-element list with 0 (all puzzles in the game do this at the start) params: - type: int - 0x27: - desc: Used by the match, scale, coin and cut puzzle types. + desc: gets overwritten by 3, so unused + 0x20: + desc: Pushes the specified byte to a list with at most 3 elements. + unused: true params: - - type: int - 0x2b: - desc: Used by the match and tileRotate puzzle types, so it's likely a way to define the initial positions for movable and rotatable pieces. + - type: byte + + moveLimit: + id: 0x27 + context: ["puzzle.coin", "puzzle.match", "puzzle.scale", "puzzle.cut"] + desc: Sets the move limit for this puzzle. Note that the max for match and coin puzzles is 255. params: - - type: int - - type: int - - type: float - # ok so apparently it can happen that an instruction checks if the next token - # is of the type it expects, and if not it uses a default value instead. - - type: float - optional: true - default: 16.0 - - type: float - optional: true - default: 30.0 - 0x39: - desc: For puzzles that involve tiles in a grid (tile, tileRotate, shape (unused)) do ??? + # TODO: types dependent on conditions? Maybe allow defining the same command multiple times for different contexts??? + limit: int + + targetPosRot: + id: 0x2b + context: ["puzzle.match", "puzzle.tilerotate"] + desc: Defines a target position for movable and rotatable objects. params: - unk_1: + xPos: int + yPos: int + angle: float + posTolerance: + type: float + optional: true + uncertain: true + desc: (Probably) the radius of pixels that the player may be off for a match to still be counted + angleTolerance: + type: float + optional: true + uncertain: true + desc: (Probably) the angle in degrees that the player may be off for a match to still be counted + + + targetAngle: + id: 0x39 + context: ["puzzle.tile", "puzzle.tilerotate", "puzzle.shape"] + desc: Sets the target angle of the previously specified object. The object must be rotated to that exact angle for the puzzle to solve. + params: + angle: type: float 0x49: diff --git a/data/gds_commands/puzzle/connect.yml b/data/gds_commands/puzzle/connect.yml index d17db3c..bb1d10d 100644 --- a/data/gds_commands/puzzle/connect.yml +++ b/data/gds_commands/puzzle/connect.yml @@ -1,12 +1,23 @@ +context: "puzzle.connect" commands: - 0x28: + barrier: + id: 0x28 + desc: > + Fills the specified area in the grid with "barrier" tiles, i.e. tiles that the line cannot cross into. + By default the entire grid is accessible. params: - - type: int - - type: int - - type: int - - type: int - 0x29: + # TODO: range [0, 19) + xPos: int + yPos: int + width: int + height: int + goal: + id: 0x29 + desc: > + Sets the specified cell in the grid to be a goal with the given ID. Two goals of the same ID must be matched up for the puzzle to solve. params: - - type: int - - type: int - - type: int \ No newline at end of file + xPos: int + yPos: int + goalId: + type: int + desc: Note that this cannot be zero, as that would be treated like no goal at all. \ No newline at end of file diff --git a/data/gds_commands/puzzle/cup.yml b/data/gds_commands/puzzle/cup.yml index aa18b1c..52b41df 100644 --- a/data/gds_commands/puzzle/cup.yml +++ b/data/gds_commands/puzzle/cup.yml @@ -1,14 +1,19 @@ +context: "puzzle.cup" commands: cup: id: 0x3a - desc: Adds a new cup object. + desc: Adds a new cup object. The position is is relative to the bottom-center of the sprite, as well as the top left of the litre count display. params: - - type: string - - type: int - - type: int - - type: int - - type: int - - type: int + spriteset: string + xPos: short + yPos: short + spriteId: int + capacity: + type: int + desc: Note this is x10 + target: + type: int + desc: The amount that should be in the cup for the puzzle to solve. Note this is x10 color: id: 0xbf desc: Sets the color of the liquid diff --git a/data/gds_commands/puzzle/cut.yml b/data/gds_commands/puzzle/cut.yml index 66c55a0..6fbc029 100644 --- a/data/gds_commands/puzzle/cut.yml +++ b/data/gds_commands/puzzle/cut.yml @@ -1,3 +1,4 @@ +context: "puzzle.cut" commands: gridFillType: id: 0x9f diff --git a/data/gds_commands/puzzle/drag.yml b/data/gds_commands/puzzle/drag.yml index f04e4a7..93f3493 100644 --- a/data/gds_commands/puzzle/drag.yml +++ b/data/gds_commands/puzzle/drag.yml @@ -1,3 +1,4 @@ +context: "puzzle.drag" commands: 0xf2: params: [float] \ No newline at end of file diff --git a/data/gds_commands/puzzle/drawinput.yml b/data/gds_commands/puzzle/drawinput.yml index c7ebea6..c6be12b 100644 --- a/data/gds_commands/puzzle/drawinput.yml +++ b/data/gds_commands/puzzle/drawinput.yml @@ -1,3 +1,5 @@ +context: ["puzzle.drawinput", "puzzle.drawinput2"] + commands: setup: id: 0xa8 @@ -7,6 +9,7 @@ commands: numInputs: int 0x24: + context: "puzzle.drawinput" params: - type: int - type: int diff --git a/data/gds_commands/puzzle/match.yml b/data/gds_commands/puzzle/match.yml index 462d35e..a939c19 100644 --- a/data/gds_commands/puzzle/match.yml +++ b/data/gds_commands/puzzle/match.yml @@ -1,6 +1,9 @@ +context: "puzzle.match" commands: - 0x2a: + match: + id: 0x2a + desc: Defines a new match on the playfield, with position and rotation. params: - - type: int - - type: int - - type: float \ No newline at end of file + xPos: short + yPos: short + angle: float \ No newline at end of file diff --git a/data/gds_commands/puzzle/queen.yml b/data/gds_commands/puzzle/queen.yml index 1c82e4c..ac5c71b 100644 --- a/data/gds_commands/puzzle/queen.yml +++ b/data/gds_commands/puzzle/queen.yml @@ -1,5 +1,6 @@ +context: "puzzle.queen" commands: - defineBoard: + board: id: 0x3b desc: Defines base parameters for the overall board of the Queen puzzle. params: @@ -18,7 +19,7 @@ commands: params: count: type: int - fixedQueen: + fixed: id: 0x3d desc: Places an immovable (golden) queen on the board in the specified tile location params: @@ -26,7 +27,7 @@ commands: type: int tileY: type: int - partialMode: + partial: id: 0x3e desc: "Determines the \"partial mode\" setting, which is used in 'Too Many Queens 3': there are less queens than tile width, but they still need to block the entire grid." params: diff --git a/data/gds_commands/puzzle/rivercross.yml b/data/gds_commands/puzzle/rivercross.yml index 720badf..107081e 100644 --- a/data/gds_commands/puzzle/rivercross.yml +++ b/data/gds_commands/puzzle/rivercross.yml @@ -1,3 +1,4 @@ +context: "puzzle.rivercross" commands: chicken: id: 0x31 diff --git a/data/gds_commands/puzzle/scale.yml b/data/gds_commands/puzzle/scale.yml index a103013..e13c9c7 100644 --- a/data/gds_commands/puzzle/scale.yml +++ b/data/gds_commands/puzzle/scale.yml @@ -1,11 +1,16 @@ +context: "puzzle.scale" commands: - 0x2d: + count: + id: 0x2d + desc: Defines how many weights there should be, and creates the objects. The maximum is 12. params: - - type: int - 0x2e: - params: [] - 0x2f: - params: [] - 0x30: - params: [] - \ No newline at end of file + count: int + light: + id: 0x2e + desc: Sets a random weight to be lighter than the others, and also the correct solution + heavy: + id: 0x2f + desc: Sets a random weight to be heavier than the others, and also the correct solution + random: + id: 0x30 + desc: Sets a random weight to be randomly either lighter or heavier than the others, and also the correct solution diff --git a/data/gds_commands/puzzle/shape.yml b/data/gds_commands/puzzle/shape.yml index f0d27fd..3e4af0c 100644 --- a/data/gds_commands/puzzle/shape.yml +++ b/data/gds_commands/puzzle/shape.yml @@ -1,3 +1,4 @@ +context: "puzzle.shape" # This puzzle type seems to be unused, which means I can't infer anything from usage. commands: shape: @@ -26,4 +27,4 @@ commands: - type: int 0x37: params: - - type: float \ No newline at end of file + angle: float \ No newline at end of file diff --git a/data/gds_commands/puzzle/slide.yml b/data/gds_commands/puzzle/slide.yml index 304a690..01e98b5 100644 --- a/data/gds_commands/puzzle/slide.yml +++ b/data/gds_commands/puzzle/slide.yml @@ -1,3 +1,4 @@ +context: "puzzle.slide" commands: item: id: 0xa6 diff --git a/data/gds_commands/puzzle/tile.yml b/data/gds_commands/puzzle/tile.yml index a396f5c..07dd7d3 100644 --- a/data/gds_commands/puzzle/tile.yml +++ b/data/gds_commands/puzzle/tile.yml @@ -1,3 +1,4 @@ +context: ["puzzle.tile", "puzzle.tilerotate"] commands: tile: id: 0x73 @@ -14,7 +15,7 @@ commands: params: xPos: int yPos: int - assign: + targetSlot: id: 0x75 desc: > Assigns the given tile ID (in declaration order) to the given slot ID. To solve the puzzle @@ -22,7 +23,9 @@ commands: multiple, which will result in the puzzle having multiple solutions. params: tileId: int - slotId: int + slotId: + type: int + desc: Note this value is negative rotButton: id: 0xb5 desc: > diff --git a/data/gds_commands/puzzle/trace.yml b/data/gds_commands/puzzle/trace.yml index 0e04077..0e17239 100644 --- a/data/gds_commands/puzzle/trace.yml +++ b/data/gds_commands/puzzle/trace.yml @@ -1,32 +1,27 @@ +context: "puzzle.trace" commands: - traceCenter: + center: id: 0x3f desc: Places the centerpoint of the shape to be traced. I'm not yet sure what the significance of that is, because it seems very lax. params: - xPos: - type: int - yPos: - type: int + xPos: int + yPos: int inPoint: id: 0x40 desc: Adds an Inpoint, a point that needs to be inside the traced shape for the puzzle to complete. params: - xPos: - type: int - yPos: - type: int + xPos: int + yPos: int outPoint: id: 0x41 desc: Adds an Outpoint, a point that mustn't be inside the traced shape for the puzzle to complete. params: - xPos: - type: int - yPos: - type: int - + xPos: int + yPos: int circleAnswer: id: 0xd4 + context: puzzle.tracebutton" desc: In the TraceButton (ie Circle Answer) puzzles, used to define a possible circle target params: xPos: int diff --git a/data/gds_commands/room.yml b/data/gds_commands/room.yml index 492135f..d538b60 100644 --- a/data/gds_commands/room.yml +++ b/data/gds_commands/room.yml @@ -1,11 +1,13 @@ +context: "room" commands: isCurrent: id: 0x13 + context: "all" condition: true desc: Returns true if the engine's current room is equal to the provided ID. Can be used anywhere. params: roomId: - type: int + type: ubyte desc: the room ID to be tested against setCurrent: @@ -19,28 +21,24 @@ commands: exit: id: 0x22 - desc: Defines an exit the user can click on to change rooms. + desc: > + Defines an exit the user can click on to change rooms, by its bounding box, target room, and + (TODO: very uncertain) the movement of the lil layton sprite on the top screen map??? params: exitType: type: int desc: Value from 0-5. Not sure what zero does, but 5 is a door in the scene, and 1-4 stands for the screen edges starting left ccw. - xPos: - type: int - desc: x position of the sprite - yPos: - type: int - desc: y position of the sprite - unk_4: - type: int - unk_5: - type: int + xPos: int + yPos: int + width: int + height: int exitRoom: - type: int - desc: Value from 0-255. Sets the room the player will move to when they take this exit + type: ubyte + desc: Sets the room the player will move to when they take this exit unk_7: - type: int + type: short unk_8: - type: int + type: short exitSound: id: 0x5b desc: Defines which footsteps sound is used on room transition @@ -80,7 +78,7 @@ commands: desc: Defines a Text Object, i.e. an area of the screen where clicking on it opens a brief flavor text box from one of the characters. params: speaker: - type: int + type: ubyte desc: > 1 for Layton, 2 for Luke. Probably corresponds to the frames in ani/room_tobjp.spr, which also includes the hint coin, but doesn't include anything else; my assumption is @@ -231,7 +229,7 @@ commands: 0x2c: - desc: sets some kind of flag at the given index + desc: sets some kind of flag at the given index to true params: - type: int diff --git a/doc/sample_file.sav.yml b/doc/sample_file.sav.yml new file mode 100644 index 0000000..752d9eb --- /dev/null +++ b/doc/sample_file.sav.yml @@ -0,0 +1,2438 @@ +files: +- filename: ilona + cur_location: Tower Top Floor + playtime: 43131 + playtime_simple: + - 11 + - 58 + puzzles_discovered: 135 + puzzles_solved: 135 + game_cleared: true + cur_room_id: 78 + cur_puzzle_id: 139 + cur_puzzle_solved: 1 + cur_puzzle_retry: 0 + cur_puzzle_aborted: 0 + cur_script_type: 3 + return_script_type: 30 + puzzle_win_img: 0 + puzzle_flags: + 0: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 1: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 2: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 3: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 4: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 5: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 6: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 7: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 8: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 9: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 10: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 11: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 12: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 13: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 14: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 15: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 16: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 17: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 18: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 19: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 20: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 21: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 22: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 23: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 24: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 25: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 26: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 27: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 28: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 29: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 30: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 31: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 32: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 33: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 34: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 35: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 36: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 37: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 38: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 39: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 40: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 41: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 42: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 43: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 44: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 45: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 46: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 47: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 48: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 49: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 50: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 51: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 52: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 53: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 54: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 55: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 56: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 57: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 58: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 59: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 60: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 61: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 62: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 63: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 64: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 65: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 66: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 67: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 68: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 69: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 70: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 71: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 72: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 73: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 74: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 75: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 76: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 77: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 78: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 79: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 80: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 81: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 82: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 83: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 84: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 85: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 86: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 87: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 88: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 89: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 90: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 91: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 92: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 93: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 94: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 95: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 96: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 97: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 98: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 99: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 100: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 101: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 102: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 103: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 104: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 3 + 105: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 106: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 107: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 108: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 109: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 110: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 111: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 112: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 113: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 114: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 115: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 116: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 117: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 118: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 119: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 120: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 121: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 122: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 123: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 124: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 125: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 126: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 127: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 128: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 129: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 130: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 131: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 132: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 133: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 134: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 135: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 136: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 137: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 138: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 139: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 140: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 141: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 142: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 143: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 144: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 145: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 146: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 147: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 148: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 149: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 150: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 3 + 151: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 152: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 153: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 3 + 154: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 155: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 156: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 157: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 2 + 158: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 1 + 159: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 160: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 161: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 162: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 163: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 164: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 165: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 166: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 167: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 168: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 169: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 170: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 171: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 172: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 173: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 174: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 175: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 176: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 177: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 178: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 179: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 180: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 181: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 182: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 183: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 184: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 185: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 186: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 187: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 188: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 189: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 190: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 191: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 192: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 193: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 194: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 195: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 196: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 197: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 198: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 199: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 200: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 201: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 202: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 203: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 204: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 205: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 206: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 207: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 208: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 209: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 210: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 211: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 212: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 213: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 214: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 215: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 216: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 217: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 218: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 219: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 220: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 221: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 222: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 223: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 224: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 225: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 226: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 227: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 228: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 229: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 230: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 231: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 232: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 233: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 234: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 235: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 236: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 237: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 238: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 239: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 240: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 241: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 242: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 243: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 244: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 245: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 246: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 247: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 248: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 249: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 250: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 251: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 252: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 253: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 254: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 255: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + cur_event: 259 + event_flags: + 0: false + 1: true + 2: true + 3: true + 4: true + 5: true + 6: true + 7: true + 8: true + 9: true + 10: true + 11: true + 12: true + 13: true + 14: true + 15: true + 16: true + 17: true + 18: true + 19: true + 20: true + 21: true + 22: true + 23: true + 24: true + 25: true + 26: true + 27: true + 28: true + 29: true + 30: true + 31: true + 32: true + 33: true + 34: false + 35: true + 36: true + 37: true + 38: false + 39: true + 40: false + 41: true + 42: true + 43: true + 44: true + 45: true + 46: true + 47: true + 48: true + 49: true + 50: true + 51: false + 52: true + 53: true + 54: true + 55: true + 56: true + 57: false + 58: true + 59: true + 60: true + 61: false + 62: true + 63: false + 64: true + 65: true + 66: true + 67: true + 68: true + 69: true + 70: true + 71: true + 72: false + 73: true + 74: true + 75: true + 76: true + 77: true + 78: true + 79: true + 80: true + 81: true + 82: true + 83: true + 84: true + 85: true + 86: true + 87: true + 88: true + 89: false + 90: true + 91: true + 92: true + 93: true + 94: true + 95: true + 96: true + 97: true + 98: true + 99: true + 100: true + 101: true + 102: true + 103: true + 104: true + 105: true + 106: true + 107: false + 108: true + 109: true + 110: true + 111: true + 112: true + 113: true + 114: true + 115: true + 116: true + 117: true + 118: true + 119: true + 120: true + 121: true + 122: true + 123: true + 124: true + 125: true + 126: true + 127: true + 128: true + 129: true + 130: true + 131: true + 132: true + 133: true + 134: true + 135: true + 136: true + 137: true + 138: true + 139: true + 140: false + 141: true + 142: false + 143: true + 144: true + 145: false + 146: true + 147: false + 148: false + 149: true + 150: true + 151: true + 152: true + 153: true + 154: false + 155: true + 156: true + 157: true + 158: true + 159: true + 160: true + 161: true + 162: true + 163: false + 164: true + 165: true + 166: true + 167: false + 168: true + 169: false + 170: false + 171: true + 172: true + 173: true + 174: true + 175: true + 176: true + 177: true + 178: true + 179: true + 180: true + 181: true + 182: true + 183: true + 184: false + 185: true + 186: true + 187: true + 188: true + 189: true + 190: true + 191: false + 192: true + 193: true + 194: true + 195: true + 196: true + 197: true + 198: false + 199: true + 200: true + 201: true + 202: true + 203: true + 204: true + 205: true + 206: false + 207: false + 208: true + 209: true + 210: true + 211: true + 212: true + 213: true + 214: true + 215: true + 216: true + 217: true + 218: true + 219: true + 220: true + 221: true + 222: true + 223: true + 224: true + 225: true + 226: true + 227: true + 228: true + 229: true + 230: true + 231: true + 232: true + 233: true + 234: true + 235: true + 236: true + 237: true + 238: true + 239: true + 240: true + 241: true + 242: true + 243: true + 244: false + 245: true + 246: true + 247: true + 248: true + 249: true + 250: true + 251: true + 252: true + 253: true + 254: true + 255: false + 256: true + 257: true + 258: true + 259: true + 260: true + 261: true + 262: true + 263: true + 264: true + 265: true + 266: true + 267: false + 268: false + 269: true + 270: true + 271: true + 272: true + 273: true + 274: true + 275: true + 276: true + 277: false + 278: true + 279: false + 280: false + 281: true + 282: true + 283: true + 284: false + 285: false + 286: false + 287: false + 288: true + 289: false + 290: true + 291: false + 292: false + 293: false + 294: false + 295: true + 296: false + 297: false + 298: false + 299: false + 300: true + 301: true + 302: false + 303: false + 304: false + 305: false + 306: false + 307: false + 308: false + 309: false + 310: false + 311: true + 312: true + 313: true + 314: false + 315: false + 316: true + 317: true + 318: true + 319: true + 320: true + 321: true + 322: true + 323: true + 324: true + 325: true + 326: true + 327: true + 328: true + 329: true + 330: true + 331: true + 332: true + 333: true + 334: true + 335: true + 336: true + 337: true + 338: true + 339: true + 340: true + 341: true + 342: true + 343: true + 344: true + 345: true + 346: false + 347: false + 348: false + 349: false + 350: true + 351: true + 352: true + 353: true + 354: true + 355: true + 356: true + 357: true + 358: true + 359: true + 360: true + 361: true + 362: true + 363: true + 364: true + 365: false + 366: false + 367: false + 368: false + 369: false + 370: false + 371: false + 372: false + 373: false + 374: false + 375: false + 376: false + 377: false + 378: false + 379: false + 380: false + 381: false + 382: false + 383: false + 384: false + 385: false + 386: false + 387: false + 388: false + 389: false + 390: false + 391: false + 392: false + 393: false + 394: false + 395: false + 396: false + 397: false + 398: false + 399: false + 400: false + 401: false + 402: false + 403: false + 404: false + 405: false + 406: false + 407: false + 408: false + 409: false + 410: false + 411: false + 412: false + 413: false + 414: false + 415: false + 416: false + 417: false + 418: false + 419: false + 420: false + 421: false + 422: false + 423: false + 424: false + 425: false + 426: false + 427: false + 428: false + 429: false + 430: false + 431: false + 432: false + 433: false + 434: false + 435: false + 436: false + 437: false + 438: false + 439: false + 440: false + 441: false + 442: false + 443: false + 444: false + 445: false + 446: false + 447: false + 448: false + 449: false + 450: false + 451: false + 452: false + 453: false + 454: false + 455: false + 456: false + 457: false + 458: false + 459: false + 460: false + 461: false + 462: false + 463: false + 464: false + 465: false + 466: false + 467: false + 468: false + 469: false + 470: false + 471: false + 472: false + 473: false + 474: false + 475: false + 476: false + 477: false + 478: false + 479: false + hint_coins: 93 + inventory: + - 24 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + story_flags: + dog_unlocked: true + painting_unlocked: true + hotelroom_unlocked: true + puzzle_at_nazobaba: {} + cur_objective: 45 + dog_parts_backlog: + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + dog_parts_placed: + 0: true + 1: true + 2: true + 3: true + 4: true + 5: true + 6: true + 7: true + 8: true + 9: true + 10: true + 11: true + 12: true + 13: true + 14: true + 15: true + 16: true + 17: true + 18: true + 19: true + 20: false + 21: false + 22: false + 23: false + 24: false + 25: false + 26: false + 27: false + 28: false + 29: false + 30: false + 31: false + puzzle_pieces_obtained: + 0: true + 1: true + 2: true + 3: true + 4: true + 5: true + 6: true + 7: true + 8: true + 9: true + 10: true + 11: true + 12: true + 13: true + 14: true + 15: true + 16: true + 17: true + 18: true + 19: true + puzzle_pieces_location: {} + hotelroom_items: + - 1 + - 2 + - 2 + - 1 + - 1 + - 1 + - 2 + - 2 + - 2 + - 1 + - 1 + - 1 + - 1 + - 1 + - 2 + - 2 + - 1 + - 1 + - null + - null + - 0 + - 1 + - 0 + - 1 + - 1 + - 0 + - 1 + - 1 + - 0 + - 0 + - 0 + - 0 + news_journal: 1 + news_dog: 0 + news_hotelroom: 0 + news_painting: 0 + news_mysteries: 0 + journal_entries_unread: + 0: false + 1: true + 2: true + 3: true + 4: true + 5: true + 6: true + 7: true + 8: true + 9: true + 10: true + 11: true + 12: true + 13: true + 14: true + 15: true + 16: true + 17: true + 18: true + 19: true + 20: true + 21: true + 22: true + 23: true + 24: true + 25: true + 26: true + 27: true + 28: true + 29: true + 30: true + 31: true + 32: true + 33: true + 34: true + 35: true + 36: true + 37: true + 38: true + 39: true + 40: true + 41: true + 42: true + 43: true + 44: true + 45: true + 46: false + 47: false + 48: false + 49: false + 50: false + 51: false + 52: false + 53: false + 54: false + 55: false + 56: false + 57: false + 58: false + 59: false + 60: false + 61: false + 62: false + 63: false + dog_name: pochi + unk1: 0 + unk2: 0 + unk3: 0 + unk4: 0 + unk5: 1 + unk6: 12 + unk7: + - 0 + - 0 + - 0 + - 1 + - 1 + - 0 + - 1 + - 1 + - 0 + - 1 + - 0 + - 0 + - 0 + - 0 + - 0 + - 1 + - 1 + - 1 + - 1 + - 0 + - 0 + - 1 + - 1 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + unk8: + - 192 + - 231 + - 95 + - 199 + - 136 + - 252 + - 106 + - 252 + - 17 + - 193 + - 0 + - 0 + - 48 + - 129 + - 10 + - 231 + - 166 + - 36 + - 140 + - 211 + - 194 + - 141 + - 117 + - 104 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + unk9: 5302 + unk10: + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + unk_bitfield2: + - false + - false + - false + - false + - false + - false + - false + - false + - false + - false + - false + - false + - false + - false + - false + - false + unk11: 299 + unk12: 22 +- null +- null +cur_file: 0 +bonus_data: {} diff --git a/formats/gds/preview.py b/formats/gds/preview.py index 8ac33d7..44557b7 100644 --- a/formats/gds/preview.py +++ b/formats/gds/preview.py @@ -3,7 +3,8 @@ explorable representation of room layouts, a structured view of dialogue events, and an overview of the script's logical control flow. -TODO: WIP, and no priority at the moment +TODO: WIP, and no priority at the moment. Though given Flora's main motivation statement is to make +puzzle editing a one-step process, maybe the preview at least for simple puzzle types should be prioritized. """ from .model import GDSProgram diff --git a/formats/gds/value.py b/formats/gds/value.py index 3e55305..7a2198c 100644 --- a/formats/gds/value.py +++ b/formats/gds/value.py @@ -100,6 +100,7 @@ def write(self) -> str: return format(self) +# TODO: int value with a specified range (e.g. "int[0,3)" or just "int[0,2]") @dataclass class GDSIntType(GDSValueType): """ From 221fc66fa4e71fc4d8ddb8c1894d84ca50a755e2 Mon Sep 17 00:00:00 2001 From: ilonachan Date: Wed, 9 Oct 2024 22:11:42 +0200 Subject: [PATCH 17/17] WIP: continue with 0x69 (nice) --- data/gds_commands/common.yml | 18 ++---- data/gds_commands/event.yml | 3 + data/gds_commands/puzzle/common.yml | 90 +++++++++++++++++++---------- data/gds_commands/room.yml | 52 +++++++---------- 4 files changed, 91 insertions(+), 72 deletions(-) diff --git a/data/gds_commands/common.yml b/data/gds_commands/common.yml index d20acd5..e256913 100644 --- a/data/gds_commands/common.yml +++ b/data/gds_commands/common.yml @@ -25,6 +25,7 @@ commands: soundId: int playSfx700: id: 0x59 + unused: true desc: I'm not sure what's so special about SFX 700, and I can't even find it in the files! (also this command is unused.) setScriptType: @@ -47,7 +48,7 @@ commands: `Taiken`: ??? `Challenge Mode`: Might have something to do with Layton's Challenges `Staff Roll` - setResumeScriptType: + setReturnScriptType: id: 0x52 desc: > Some script types aren't actually "scripts" but technical events; they cannot decide how @@ -119,32 +120,23 @@ commands: params: unk_1: type: int - 0x4a: - condition: true - desc: checks if some global flag is false - 0x4b: - condition: true - desc: checks if some global flag (same as 0x4a) is true 0x5f: params: [] 0x62: desc: Sets some global variable. params: value: - type: int + type: short desc: the value to set 0x63: desc: Checks the global variable set by `0x62` (could this be global story progress?) condition: true params: value: - type: int + type: short desc: the value to check against 0x64: - desc: sets some global flag to true - 0x66: - desc: sets some global flag to false - 0x67: + unused: true desc: sets some global flag to true 0x69: params: diff --git a/data/gds_commands/event.yml b/data/gds_commands/event.yml index 81c8700..2dabc52 100644 --- a/data/gds_commands/event.yml +++ b/data/gds_commands/event.yml @@ -2,6 +2,7 @@ context: "event" commands: setCurrent: id: 0x60 + context: "all" desc: > Sets the current event in global state. Once the current script is completed, and the script type is set to "Event", that script will be executed. @@ -144,6 +145,8 @@ commands: setFlag2: id: 0x55 + context: "all" + unused: true desc: > Every event has a byte's worth of story flags to access, and this instruction sets byte 2 of that to true. Maybe it's a simple unified way to tell if an event has already been seen diff --git a/data/gds_commands/puzzle/common.yml b/data/gds_commands/puzzle/common.yml index 9071ef3..b4c3089 100644 --- a/data/gds_commands/puzzle/common.yml +++ b/data/gds_commands/puzzle/common.yml @@ -3,6 +3,7 @@ context: "puzzle" commands: setCurrent: id: 0x48 + context: "all" desc: | Sets the current puzzle ID in the global game state. If at the end of script execution the global "current script type" is set to "question", the puzzle with this ID will be started. @@ -10,19 +11,42 @@ commands: puzzleId: int isCurrentNotSolved: id: 0x4c + context: "all" + condition: true + desc: Checks if the currently relevant puzzle has NOT been solved + + isCurrentDiscovered: + id: 0x49 + context: "all" condition: true - desc: Checks if the most recently (perhaps currently) attempted puzzle has NOT been solved + desc: Checks if the currently relevant puzzle has been discovered/attempted before. + isDiscovered: + id: 0x58 + context: "all" + condition: true + desc: checks if the specified puzzle has been discovered. + params: + puzzleId: bool + isCurrentSolved: id: 0x4d + context: "all" condition: true - desc: Checks if the most recently (perhaps currently) attempted puzzle has been solved - + desc: Checks if the currently relevant puzzle has been solved isSolved: id: 0x54 + context: "all" condition: true desc: Checks if the specified puzzle ID has been solved (puzzle flag 2) params: puzzleId: int + + isSolvedInStory: + id: 0x4e + context: "all" + condition: true + desc: Checks if the currently relevant puzzle has been solved in its correct location, i.e. the event surrounding it has completed. + countEnoughSolved: id: 0x77 condition: true @@ -32,6 +56,27 @@ commands: params: threshold: int + eventAfterPuzzle: + id: 0x4b + context: "all" + condition: true + desc: Checks a special flag set if the event script is run right after a puzzle was ended (solved or given up). + eventNotAfterPuzzle: + id: 0x4a + context: "all" + condition: true + desc: Checks a special flag set if the event script is run right after a puzzle was ended, and returns true if that is not the case. + + solverLayton: + id: 0x66 + context: "all" + desc: Sets the next puzzle to be "solved by" Layton (i.e. one of his sequences will play during grading) + solverLuke: + id: 0x77 + context: "all" + desc: Sets the next puzzle to be "solved by" Luke (i.e. one of his sequences will play during grading) + + engine: id: 0x1b # TECHNICALLY. But nobody would use this outside of puzzle scripts, because the grading and everything would be missing. @@ -78,12 +123,11 @@ commands: button: id: 0x5d - desc: For the puzzle types OnOff and FreeButton, this adds a button with the specified sprite texture at the given location. + context: ["puzzle.onoff", "puzzle.freebutton"] + desc: Adds a button with the specified sprite texture at the given location. params: - xPos: - type: int - yPos: - type: int + xPos: short + yPos: short spriteName: type: string desc: The sprite used for the button. As with all button sprites, it should define a depressed state in the animation "shadow". @@ -95,16 +139,16 @@ commands: placeTarget: id: 0x5e - desc: The sole defining command for the PlaceTarget puzzle type. + context: "puzzle.placetarget" + desc: > + The sole defining command for the PlaceTarget puzzle type. It seems there can be multiple targets defined this way, + all of which need to be hit in order for the puzzle to solve; but this is never used + (maybe it felt too cumbersome during playtesting, because you can't reset?) params: - unk_1: - type: int - unk_2: - type: int - unk_3: - type: string - unk_4: - type: float + xPos: int + yPos: int + spriteName: string + radius: float unk_5: type: int @@ -211,18 +255,6 @@ commands: params: angle: type: float - - 0x49: - condition: true - desc: checks some kind of flag for the current puzzle (puzzle flags mask 1) - 0x58: - condition: true - desc: checks some kind of flag for the given puzzle (puzzle flags mask 1); for regular puzzles at least. - params: - - int - 0x4e: - condition: true - desc: checks some kind of flag for the current puzzle (puzzle flags mask 4) 0xb7: desc: sets all the flags for the given puzzle ID to true (1, 4, and 2 ie "solved") params: diff --git a/data/gds_commands/room.yml b/data/gds_commands/room.yml index d538b60..f9769e1 100644 --- a/data/gds_commands/room.yml +++ b/data/gds_commands/room.yml @@ -12,6 +12,7 @@ commands: setCurrent: id: 0x53 + context: "all" desc: > Sets the engine's current room ID to the specified value. This takes effect after the script completes, the next time the script type is set to 'Event'. @@ -44,10 +45,10 @@ commands: desc: Defines which footsteps sound is used on room transition params: exitId: - type: int + type: byte desc: the ID of the exit (in declaration order) soundId: - type: int + type: byte desc: Can be any number, but only 1 (echoy footsteps) and 2 (creaky door) have special handling sprite: @@ -65,10 +66,8 @@ commands: id: 0x5c desc: Adds a non-interactive background sprite to the room, for animated room background effects. params: - xPos: - type: int - yPos: - type: int + xPos: byte + yPos: byte spriteName: type: string desc: path to the sprite to be used. @@ -107,20 +106,16 @@ commands: type: int desc: Appears unused in code, and seems to always be 1 - npcObject: + eventObject: id: 0x50 desc: > Adds an interactable object to the room. This is used for all NPCs, and some other items such as the crumpled notes and newspapers. params: - xPos: - type: int - yPos: - type: int - width: - type: int - height: - type: int + xPos: short + yPos: short + width: int + height: int spriteId: type: int desc: The sprite that will be placed at the given location. It only becomes interactable within the defined bounding box. Sourced from ani/obj_%i.spr with default animation. @@ -160,14 +155,10 @@ commands: coinId: type: int desc: Every hint coin has a unique ID, presumably to allow saving which ones you've collected. - xPos: - type: int - yPos: - type: int - width: - type: int - height: - type: int + xPos: short + yPos: short + width: short + height: short unk_6: type: int desc: appears to be unused. @@ -175,9 +166,11 @@ commands: mapInfo: id: 0x5a + context: ["room", "event"] desc: > - As far as I can tell, this is used to setup info on the top screen (& probably for the save - process). Can also be used from events to briefly change scenes. + Sets the current room ID and the map ID, which determines the name of the location. + Also sets the coordinates for the little Layton sprite map marker on the top screen. + Can be used in events to enqueue a room change as well. params: roomId: type: int @@ -185,12 +178,11 @@ commands: mapId: type: int desc: the ID of the top screen map image to be used - unk_3: - type: int - unk_4: - type: int - unk_5: + xPos: short + yPos: short + markerSpriteId: type: int + desc: It seems there were multiple possible map marker icons at some point, but only ID 1 has survived. nazobabaBottle: