diff --git a/README.md b/README.md index b8d0085..a4f2ac2 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,12 @@ + Warning: the only version with guaranteed compatibility is EU/AUS. Please specify the region of your ROM when reporting a bug or opening an issue. ### Current utilities -* GDS script extracting and repacking to a custom readable format. - + Currently, the parameters supported are those of types 1 (int), 2 (double int) and 3 (string). Since some scripts use other parameter types, this will be fixed in the future. +* GDS script decompilation and recompilation into a custom human-readable scripting language. + + Supports control flow instructions with conditionals + + An effort to catalogue all command IDs with speaking names and parameter lists is ongoing + + All vanilla files from the EU version of CV decompile* and recompile into identical bytecode. + (* Some scripts are structurally incorrect in ways that make parsing difficult, so a set of baseline + patches is applied first to make them functional) * Exporting BG ARC files to PNG, and creating them from PNGs. + Currently, the PNG's color mode must be set to indexed. This will be changed in future versions. + The image must have at most 256 different colors. This is a limitation of the format. @@ -35,10 +39,13 @@ You can also extract the match puzzle tutorial (from puzzle #10) using "match_tu Some unused variants of puzzles can't be extracted yet through Flora, as well as some Japanese-exclusive weekly puzzles, however, this will be fixed in a future update. ### Requirements -* Python 3 +* Python 3.10 * [click](https://pypi.org/project/click/) * [pillow](https://pypi.org/project/pillow/) * [ndspy](https://pypi.org/project/ndspy/) +* [pyyaml](https://pypi.org/project/pyyaml) +* [dacite](https://pypi.org/project/dacite) +* [parsy](https://pypi.org/project/parsy) * Optional: [tqdm](https://pypi.org/project/tqdm) (to display a progress bar for long-running processes) To install all modules, use: `pip install -r requirements.txt` diff --git a/data/.gitignore b/data/.gitignore new file mode 100644 index 0000000..d5440b6 --- /dev/null +++ b/data/.gitignore @@ -0,0 +1 @@ +game_root/** \ No newline at end of file diff --git a/data/commands.json b/data/commands.json index c88bdc1..857e390 100644 --- a/data/commands.json +++ b/data/commands.json @@ -1,10 +1,245 @@ { - "engine": 27, - "img_win": 31, - "puzzle_title": 186, - "puzzle_picarats": 195, - "circle_answer": 212, - "puzzle_info": 220, - "input_answer": 251, - "input_bg": 252 -} \ No newline at end of file + "engine": { + "id": 27, + "desc": "Called at the start of a puzzle script. Determines which puzzle engine to use.", + "params": [ + { "type": "string", "desc": "Engine name (from a predefined list)" } + ] + }, + "img_win": { + "id": 31, + "desc": "Seemingly denotes when a special top-screen image should be shown on the puzzle completion screen.", + "params": [{ "type": "int" }, { "type": "int" }] + }, + "puzzle_title": { + "id": 186, + "desc": "Defines the title of a puzzle. Called centrally as a list in `puzzletitle/qtitle.gds`.", + "params": [ + { "type": "int", "desc": "Puzzle ID" }, + { "type": "string", "desc": "Puzzle Title" } + ] + }, + "puzzle_picarats": { + "id": 195, + "desc": "Defines how much picarat a puzzle awards. Called centrally in `puzzletitle/pscript.gds`.", + "params": [ + { "type": "int", "desc": "Puzzle ID" }, + { "type": "int", "desc": "Value on first attempt" }, + { "type": "int", "desc": "Value on second attempt" }, + { "type": "int", "desc": "Value on all further attempts" } + ] + }, + "puzzle_info": { + "id": 220, + "desc": "Defines additional information about the puzzle displayed in the puzzle index. Some puzzle IDs ", + "params": [ + { "type": "int", "desc": "Puzzle ID" }, + { + "type": "string", + "desc": "Puzzle type (e.g. \"Write Answer\" or \"Draw Line\", but can be anything)" + }, + { + "type": "string", + "desc": "Puzzle original location (should be a recognizable map name, but can technically be anything, such as \"Layton's Challenges\")" + } + ] + }, + "circle_answer": { "id": 212 }, + "input_answer": { "id": 251 }, + "input_bg": { "id": 252 }, + "dog_unknown": { + "id": 194, + "desc": "Used in the Gizmo minigame, presumably to manage the placement of the dog parts", + "params": [ + { "type": "int", "desc": "Between 1-20, probably the part index?" }, + { "type": "int" }, + { "type": "int" }, + { "type": "int", "desc": "Always the same as param 1" }, + { "type": "int" }, + { "type": "int" } + ] + }, + "inn_hint_layton": { + "id": 202, + "desc": "Used in the Inn minigame to display the Professor's assessment of the room layout.", + "params": [ + { + "type": "string", + "desc": "The text to be displayed" + }, + { + "type": "int", + "desc": "Increments with the quality of the room (unsure about effect)" + }, + { + "type": "int", + "desc": "Always 0" + } + ] + }, + "inn_hint_luke": { + "id": 203, + "desc": "Used in the Inn minigame to display Luke's assessment of the room layout.", + "params": [ + { + "type": "string", + "desc": "The text to be displayed" + }, + { + "type": "int", + "desc": "Increments with the quality of the room (unsure about effect)" + }, + { + "type": "int", + "desc": "Always 0" + } + ] + }, + "inn_item": { + "id": 207, + "desc": "Used in the hotelscript to define the available Inn items. All used at the end for some reason.", + "params": [ + { "type": "int", "desc": "item ID" }, + { "type": "string", "desc": "Name of the item" } + ] + }, + "inn_item_layton": { + "id": 188, + "desc": "Used in the hotelscript, always followed by `inn_item_layton_comment` and `inn_item_layton_comment2`, likely to define Inn item placements in Layton's room.", + "params": [ + { "type": "int", "desc": "item ID" }, + { "type": "int" }, + { "type": "int" }, + { "type": "int" }, + { "type": "int" }, + { "type": "int" }, + { "type": "int" }, + { "type": "int" }, + { "type": "int" } + ] + }, + "inn_item_layton_comment": { + "id": 198, + "desc": "Used in the hotelscript after `inn_item_layton`. Defines the Professor's comment about an Inn minigame item.", + "params": [{ "type": "string", "desc": "The Professor's comment" }] + }, + "inn_item_layton_comment2": { + "id": 199, + "desc": "Used in the hotelscript after `inn_item_layton`. Defines the comment made by the professor if a second item of the same type is being added, along with the ID of the item that is the 'same type'.", + "params": [ + { "type": "string", "desc": "The Professor's comment" }, + { + "type": "int", + "desc": "The ID of the item that's considered a duplicate of this one's type, or -1 if there is no such item. (not sure which of the two comments is used in that case)" + } + ] + }, + "inn_item_luke": { + "id": 189, + "desc": "Used in the hotelscript, always followed by `inn_item_luke_comment` and `inn_item_luke_comment2`, likely to define Inn item placements in Luke's room." + }, + "inn_item_luke_comment": { + "id": 200, + "desc": "Used in the hotelscript with `inn_item_luke`. Defines Luke's comment about an Inn minigame item.", + "params": [{ "type": "string", "desc": "Luke's comment" }] + }, + "inn_item_luke_comment2": { + "id": 201, + "desc": "Used in the hotelscript with `inn_item_luke`. Defines the comment made by Luke if a second item of the same type is being added, along with the ID of the item that is the 'same type'.", + "params": [ + { "type": "string", "desc": "Luke's comment" }, + { + "type": "int", + "desc": "The ID of the item that's considered a duplicate of this one's type, or -1 if there is no such item. (not sure which of the two comments is used in that case)" + } + ] + }, + "nazobaba": { + "id": 229, + "desc": "Used in the Granny Riddleton script. One of these commands is used for every puzzle, possibly to define after which story progression point a puzzle gets sent to the cottage. All but one of the three unknown params are always -1, and sometimes all are (presumably for puzzles that never get sent to the cottage). Which value is present may determine the bottle color, and the value may determine after which progression point the puzzle shows up.", + "params": [ + { "type": "int", "desc": "Puzzle ID" }, + { "type": "int" }, + { "type": "int" }, + { "type": "int" } + ] + }, + + "slide2_grid_dimensions": { + "id": 164, + "desc": "Defines the grid dimensions in a 'Slide Puzzle 2'.", + "params": [ + { "type": "int", "desc": "grid width" }, + { "type": "int", "desc": "grid height" } + ] + }, + "slide2_grid_offset": { + "id": 163, + "desc": "Determines the offset of the grid's visual placement relative to the touch screen's top left corner (0,0). Used to align the puzzle itself with the background.", + "params": [ + { "type": "int", "desc": "x offset in px" }, + { "type": "int", "desc": "y offset in px" } + ] + }, + "slide2_grid_cellsize": { + "id": 165, + "desc": "Defines the visual scale of the grid by the width and height of the cells. This puzzle engine only allows square cells.", + "params": [ + { + "type": "int", + "desc": "width & height of the square grid cells in px" + } + ] + }, + "slide2_item": { + "id": 166, + "desc": "Place an item in the grid of a 'Slide Puzzle 2'", + "params": [ + { "type": "int", "desc": "x coordinate" }, + { "type": "int", "desc": "y coordinate" }, + { + "type": "int", + "desc": "unknown, but likely to do with properties such as draggability & obstacle behavior" + }, + { + "type": "string", + "desc": "Item sprite name, from `ani/slidepuzzle2.arc`. 'x' in particular has no image associated, and will show the background through (used for walls)." + } + ] + }, + "slide2_goalid": { + "id": 177, + "desc": "Defines a grid cell as the goal for a specific item type index. These indices seem to be attached to objects adhoc, but an object with each goal's specified index must be placed on all goal cells defined like this in order for the puzzle to solve.", + "params": [ + { "type": "int", "desc": "x coordinate" }, + { "type": "int", "desc": "y coordinate" }, + { "type": "int", "desc": "Item type index" } + ] + }, + "slide2_item_goalid": { + "id": 178, + "desc": "Following a `slide2_item` definition, attaches the given item type index to that object. Used to determine if the puzzle has been solved.", + "params": [ + { + "type": "int", + "desc": "Item type index (can seemingly be chosen adhoc to be anything)" + } + ] + }, + "slide2_item_goalcell": { + "id": 56, + "desc": "Following a `slide2_item` definition, sets a target location at which the cell needs to be placed for the puzzle to be solved." + }, + + "placetarget": { + "id": 94, + "desc": "Main definition of a 'Place Target' puzzle", + "params": [ + {"type": "int", "desc": "Target x position"}, + {"type": "int", "desc": "Target y position"}, + {"type": "string", "desc": "Sprite to be displayed at the touched location"}, + {"type": "unknown-2"}, + {"type": "int"} + ] + } +} diff --git a/data/gds_commands/common.yml b/data/gds_commands/common.yml new file mode 100644 index 0000000..e256913 --- /dev/null +++ b/data/gds_commands/common.yml @@ -0,0 +1,292 @@ +# a prefix prepended to each command name with a dot. +# ex: a command "hi" in a package with prefix "test" will be decompiled to "test.hi" +# By default the prefix is generated from the directory structure and package file name, +# we reset it explicitly here to get these functions into the common namespace. +prefix: null +# Commands that we know the meaning of, reasonably well at least. +commands: + playSfx: + id: 0x10 + desc: Only possible candidate instruction for playing the jingle on the level5 logo screen. Uses a lookup table internally, + the sound IDs are irregular and look nonsensical. Needs more research + # This probably won't be queried programmatically, but it's important to admit our limitations. + uncertain: true + # This can either be an unnamed (or empty) list of parameters, or an ordered dict with parameter names marked. + # If omitted or null, the command takes no arguments. + params: + soundId: + type: int + desc: the ID of the sfx to be played + playSfxAny: + id: 0xf3 + uncertain: true + desc: Can play any sound from the list without indirection, I think. + params: + soundId: int + playSfx700: + id: 0x59 + unused: true + desc: I'm not sure what's so special about SFX 700, and I can't even find it in the files! (also this command is unused.) + + setScriptType: + id: 0x51 + desc: > + Sets the "current" script type to the specified name. The currently running script will + continue, but once it completes this value hints to the engine what should happen next. + params: + scriptType: + # TODO: enum types? + type: string + desc: | + One of the following: + `Question`: Puzzle + `Room` + `Event` + `Movie` + `Event Movie` (TODO: what's the difference?) + `Reset`: Possibly returns to the title screen? + `Taiken`: ??? + `Challenge Mode`: Might have something to do with Layton's Challenges + `Staff Roll` + setReturnScriptType: + id: 0x52 + desc: > + Some script types aren't actually "scripts" but technical events; they cannot decide how + flow should continue once they're done. That's why for these events (things like puzzles, + event movies etc) you can additionally set a script type that should be set after that + event is done. + params: + scriptType: + type: string + desc: | + Supports only the following values (see `0x51 setScriptType`): + + `Question` + `Room` + `Event` + `Challenge Mode` + + spriteTop: + id: 0x7e + desc: Displays a sprite on the top screen; ordinarily all sprite/bg/ui operations apply to the touchscreen. + params: + xPos: int + yPos: int + spriteName: string + animName: string + setSpriteTopAnim: + id: 0x7f + params: + spriteId: int + animName: string + setSpriteTopPos: + id: 0x80 + params: + spriteId: int + xPos: int + yPos: int + spriteTopReveal: + id: 0x81 + params: + spriteId: int + spriteTopHide: + id: 0x82 + params: + spriteId: int + + spriteTopMoveMode: + id: 0x94 + uncertain: true + desc: + params: + actorId: int + moveMode: + type: string + desc: either "Normal" or "Move To Target" + + # Commands that we don't understand well enough to name yet. Their format is similar to known commands, except the ID is the name. + 0x09: + params: [] + 0x0a: + params: [] + 0x0d: + params: + - type: int + 0x0e: + params: [] + 0x0f: + params: [] + 0x11: + params: + unk_1: + type: int + 0x5f: + params: [] + 0x62: + desc: Sets some global variable. + params: + value: + type: short + desc: the value to set + 0x63: + desc: Checks the global variable set by `0x62` (could this be global story progress?) + condition: true + params: + value: + type: short + desc: the value to check against + 0x64: + unused: true + desc: sets some global flag to true + 0x69: + params: + - type: int + 0x6a: + params: + - type: int + 0x6b: + params: + - type: int + 0x7a: + desc: Seems to be a wait instruction, that perhaps advances by n frames + params: [int] + 0x7b: + params: + - int + - int + - int + - int + 0x7c: + params: + - int + - int + - int + - int + 0x7d: + params: [] + 0x85: + desc: pushes a nonzero byte value to a (very short) list. + params: [int] + 0x86: + desc: checks if the list from `0x85` contains the given byte value + condition: true + params: [int] + 0x87: + params: [int] + 0x88: + params: [int] + 0x8d: + desc: checks if some global flag at the given location (out of 1024) is set + condition: true + params: + flagId: int + 0x8e: + desc: sets some global flag at the given location (out of 1024) + condition: true + params: + flagId: int + value: + type: bool + desc: true if either a nonzero number or the string "true", otherwise false + 0x93: + desc: sets some property on a top sprite + params: + spriteId: int + unk_2: int + 0x95: + desc: sets some point on a top sprite + params: + spriteId: int + xPos: int + yPos: int + 0x96: + desc: sets some property on a top sprite + params: + spriteId: int + unk_2: int + 0x9e: + params: [int] + 0xae: + params: + - string + - string + - int + - int + - int + - int + + storyscript: + id: 0xb6 + desc: > + In the debug storyscript, registers a starting point from which (presumably) playtesters + could jump into the game without having to replay large earlier parts of it. + params: + desc: + type: string + desc: The description of the starting point + scriptPath: + type: string + desc: The path to the script executed to setup the game state to the specified starting point + + 0xc0: + params: [int] + 0xc1: + condition: true + params: [] + 0xce: + params: [int] + + 0xdd: + desc: Selects a movie (of the emb%i pool) and sets script type to MOVIE2 + params: + movieId: int + + 0xde: + params: [] + 0xdf: + desc: noop + params: [] + 0xe0: + desc: noop + params: [] + 0xe1: + desc: noop + params: [] + 0xe2: + desc: noop + params: [] + 0xe3: + desc: noop + params: [] + 0xe9: + desc: sets a global value + params: [int] + 0xea: + desc: reads the global value set by `0xe9`, and if it's a valid event ID, set the curScriptType and curEvent appropriately (if it's -1, set to ROOM) + params: [] + 0xeb: + condition: true + # You have entered GStruct1 Zone + 0xec: + params: [int, int] + 0xed: + params: [int, int] + 0xee: + params: [int] + # + 0xef: + params: [int, int, int, int] + 0xf0: + params: [int, int, int] + 0xf1: + params: [int, int] + 0xf4: + params: [int] + 0xf6: + params: [] + 0xf7: + params: [] + 0xf8: + params: [] + 0xf9: + params: [] \ No newline at end of file diff --git a/data/gds_commands/event.yml b/data/gds_commands/event.yml new file mode 100644 index 0000000..2dabc52 --- /dev/null +++ b/data/gds_commands/event.yml @@ -0,0 +1,383 @@ +context: "event" +commands: + setCurrent: + id: 0x60 + context: "all" + desc: > + Sets the current event in global state. Once the current script is completed, + and the script type is set to "Event", that script will be executed. + params: + eventId: + type: int + + + textbox: + id: 0x98 + desc: > + Displays a textbox for the specified on-screen actor, pointing in the + direction indicated by that actor's sprite location. + comment: | + /data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt: + + $(/data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt) + params: + textId: + type: int + desc: > + the text to display, in textbox format. This is sourced from + etext/?/e???/e_t.txt. + actorId: + type: int + desc: the id (in initialization order) of the actor sprite saying the text. + textboxRight: + id: 0x9c + desc: > + Displays a textbox for the specified on-screen actor, pointing right. + comment: | + /data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt: + + $(/data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt) + params: + textId: + type: int + desc: > + the text to display, in textbox format. This is sourced from + etext/?/e???/e_t.txt. + actorId: + type: int + desc: the id (in initialization order) of the actor sprite saying the text. + textboxLeft: + id: 0x9d + desc: > + Displays a textbox for the specified on-screen actor, pointing left. + comment: | + /data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt: + + $(/data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt) + params: + textId: + type: int + desc: > + the text to display, in textbox format. This is sourced from + etext/?/e???/e_t.txt. + actorId: + type: int + desc: the id (in initialization order) of the actor sprite saying the text. + textboxMiddle: + id: 0xcd + desc: Displays a text box for the specified on-screen actor, with no arrow pointing. + comment: | + /data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt: + + $(/data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt) + params: + textId: + type: int + desc: > + the text to display, in textbox format. This is sourced from + etext/?/e???/e_t.txt. + actorId: + type: int + desc: the id (in initialization order) of the actor sprite saying the text. + + textboxNoSpeaker: + id: 0x9b + desc: Displays a textbox without a speaker on screen. + comment: | + /data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt: + + $(/data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt) + params: + textId: + type: int + desc: > + the text to display, in textbox format. This is sourced from + etext/?/e???/e_t.txt. + textboxNoSpeaker2: + id: 0x44 + desc: Displays a textbox without a speaker on screen. (I'm not sure what the difference between these two functions is) + comment: | + /data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt: + + $(/data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt) + params: + textId: + type: int + desc: > + the text to display, in textbox format. This is sourced from + etext/?/e???/e_t.txt. + textboxNoSpeakerRight: + id: 0x99 + desc: Displays a textbox pointing right, but without a speaker on screen. + comment: | + /data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt: + + $(/data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt) + params: + textId: + type: int + desc: > + the text to display, in textbox format. This is sourced from + etext/?/e???/e_t.txt. + textboxNoSpeakerLeft: + id: 0x9a + desc: Displays a textbox pointing left, but without a speaker on screen. + comment: | + /data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt: + + $(/data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt) + params: + textId: + type: int + desc: > + the text to display, in textbox format. This is sourced from + etext/?/e???/e_t.txt. + + textboxBgRight: + id: 0x45 + desc: Displays the background of a textbox pointing right, but doesn't print anything. + textboxBgLeft: + id: 0x46 + desc: Displays the background of a textbox pointing left, but doesn't print anything. + textboxBgNoSpeaker: + id: 0x65 + desc: Displays the background of a textbox pointing in no direction. + + setFlag2: + id: 0x55 + context: "all" + unused: true + desc: > + Every event has a byte's worth of story flags to access, and this instruction sets byte 2 of + that to true. Maybe it's a simple unified way to tell if an event has already been seen + by the player. + + pickupItem: + id: 0x56 + desc: Shows an item pickup display for the given item ID, and adds it to the inventory. + params: + itemId: + type: int + desc: > + The item ID. It's likely the only valid ones are 1 ("some fish bones") and 3 ("a wristwatch"), + but more can *potentially* be added by providing stext/?/stext.pcm//p_%i.txt and a sprite + frame in ani/prize.spr + pickupGizmo: + id: 0xc4 + desc: Shows a pickup display for the given Gizmo ID + params: + gizmoId: int + pickupPainting: + id: 0xc5 + desc: Shows a pickup display for the given painting scrap ID + params: + scrapId: int + pickupHotel: + id: 0xcc + desc: Shows a pickup display for the given hotel item ID + params: + itemId: int + + + puzzleIndexPopup: + id: 0x57 + desc: Shows the notification that " has been added to the puzzle index". + + mysteryPopup: + id: 0xe8 + desc: Shows the notification that " has been added to your mysteries". + params: + mysteryId: + type: int + desc: this is actually bounds-checked to be between 1-10. The names are source from storytext.pcm//it_%i.txt + + promptSave: + id: 0x61 + desc: Shows a popup allowing the player to save. + + actorSprite: + id: 0x6c + desc: > + Initializes an actor sprite in the current engine. I'm not sure what exactly makes them special, + but they're used mostly in events to display the speakers (and their name tags); however they + also appear in the Drag Puzzle type. + params: + # For parameters that you seriously don't have anything else to say about, this is a nice shortcut + xPos: int + yPos: int + spriteName: + type: string + unk_4: + type: string + + actorSpriteHead: + id: 0x6d + desc: > + Many of the actors have their facial expression separated from their body, probably to save on + space since otherwise all permutations would have to be baked in separately. + params: + xPos: int + yPos: int + spriteName: string + animName: string + actorId: + type: int + desc: The ID (in declaration order) of the actor that this head belongs to. + + setActorAnim: + id: 0x6e + desc: Sets the specified actor's (body) animation to the specified name + params: + actorId: int + animName: string + setActorAnimHead: + id: 0x6f + desc: Sets the specified actor's head animation to the specified name + params: + actorId: int + animName: string + + setActorPos: + id: 0x70 + desc: Sets the specified actor's (body) position to the specified point + params: + # TODO: could the compiler be smart here and emit warnings depending on logic? + actorId: int + xPos: int + yPos: int + + revealActor: + id: 0x71 + uncertain: true + desc: Sets the specified actor to be visible(?) + params: + actorId: int + hideActor: + id: 0x72 + uncertain: true + desc: Sets the specified actor to be visible(?) + params: + actorId: int + + actorMoveMode: + id: 0x90 + uncertain: true + desc: + params: + actorId: int + moveMode: + type: string + desc: either "Normal" or "Move To Target" + + + currentMovie: + id: 0xd3 + desc: Sets the current movie ID. Once the current script completes, if the current script state is set to "Movie", that movie will play. + params: + movieId: int + + + choice3: + id: 0xd5 + desc: Displays a three-option choice, like the Chelmey/DonPaolo deduction minigame. + choice2: + id: 0xdb + desc: Displays a two-option choice, like the endgame continue prompt. Might actually have logic specific to that. + choiceQuestion: + id: 0xd9 + desc: The question to be displayed for this choice + params: + question: + type: string(64) + desc: Unlike most strings in this game, this one is hardcoded in the script, and therefore NOT localized! + choiceAnswer1: + id: 0xd6 + desc: Answer 1 of this choice + params: + answer: + type: string(64) + desc: Just like the question, not localizable. + unk_2: int + choiceAnswer2: + id: 0xd7 + desc: Answer 2 of this choice + params: + answer: + type: string(64) + desc: Just like the question, not localizable. + unk_2: int + choiceAnswer3: + id: 0xd8 + desc: Answer 3 of this choice + params: + answer: + type: string(64) + desc: Just like the question, not localizable. + unk_2: int + choiceAnswerIs: + id: 0xda + condition: true + desc: Checks if the given answer ID (from 1-3) was chosen by the user + params: + answer: int + + + movieSub: + id: 0xfd + desc: Adds a subtitle to the upcoming movie sequence + params: + textId: int + startTime: float + duration: float + moviePlay: + id: 0xfe + desc: Plays the specified movie clip + params: + movieId: int + + 0x47: + params: + - type: int + 0x78: + desc: sets some float value for the specified actor and its head (rotation??) + params: + actorId: int + unk_2: float + 0x79: + desc: sets some float value for the specified actor and its head (rotation??) + params: + actorId: int + unk_2: float + 0x8f: + params: + actorId: int + unk_2: int + + 0x91: + params: + actorId: int + unk_2: int + 0x92: + params: + actorId: int + unk_2: int + 0x97: + params: + actorId: int + unk_2: int + + 0xaf: + params: [int] + 0xb0: + params: [int] + 0xb3: + params: [int] + 0xb4: + params: + actorId: int + unk_2: bool + + 0xb8: + desc: sets the eventFlag with the given ID to 1 + params: + eventFlagId: int \ No newline at end of file diff --git a/data/gds_commands/fx.yml b/data/gds_commands/fx.yml new file mode 100644 index 0000000..a6276ab --- /dev/null +++ b/data/gds_commands/fx.yml @@ -0,0 +1,48 @@ +commands: + fadeIn: + id: 0x01 + desc: Shows a fast fade in effect on both screens. + fadeIn2: + id: 0x02 + desc: Shows a slow fade in effect on both screens. + fadeInTop: + id: 0x03 + desc: Shows a fade in effect on the top screen. + fadeInBottom: + id: 0x04 + desc: Shows a fade in effect on the bottom screen. + fadeOut: + id: 0x05 + desc: Shows a fast fade out effect on both screens. + fadeOut2: + id: 0x06 + desc: Shows a slow fade out effect on both screens. + fadeOutTop: + id: 0x07 + desc: Shows a fade out effect on the top screen. + fadeOutBottom: + id: 0x08 + desc: Shows a fade out effect on the bottom screen. + displayImgBottom: + id: 0x0b + context: "puzzle.drawinput2" + desc: Shows the specified image path on the bottom screen + params: + bgxName: + type: string + desc: The name of the image file, with bgx file ending (will look for a .arc file with that name under /bg) + unk_2: + # TODO: datatype int[0,4] + type: byte + desc: Seems to always be 2 (for now) + displayImgTop: + id: 0x0c + desc: Shows the specified image path on the top screen + params: + bgxName: + type: string + desc: The name of the image file, with bgx file ending (will look for a .arc file with that name under /bg) + unk_2: + # TODO: datatype int[0,4] + type: byte + desc: Seems to always be 3 (for now) diff --git a/data/gds_commands/logic.yml b/data/gds_commands/logic.yml new file mode 100644 index 0000000..8be922c --- /dev/null +++ b/data/gds_commands/logic.yml @@ -0,0 +1,46 @@ +prefix: null +commands: + if: + id: 0x12 + aliases: + - jumpUnless + desc: Executes a boolean expression consisting of instructions and logic flags, and jumps away if the condition is false (which is the default if no condition is provided). + complex: true + elif: + id: 0x16 + aliases: + - jumpUnlessTargetAnd + desc: If reached through regular program flow, always jumps. If reached from a `jump` instruction, executes the logic of `jump`. + complex: true + else: + id: 0x17 + aliases: + - jumpUnlessTarget + desc: If reached through regular program flow, always jumps. If reached from a `jump` instruction, simply continues. + complex: true + repeatN: + id: 0x14 + desc: Takes an loop count (int) and a jump target (addr), and loops the instructions up to that target location that many times. + complex: true + while: + id: 0x15 + desc: Takes a boolean expression like `if`, as well as a jump target (addr), and loops the instructions up to that target location until the condition evaluates to false or a break token is encountered. + complex: true + + "true": + id: 0x19 + condition: true + desc: Always returns true + "false": + id: 0x1a + condition: false + desc: Always returns false + + return: + id: 0x4f + desc: Aborts the script by setting the instruction pointer past its end. + + 0x18: + condition: true + desc: Might have been a debug condition, because the logic to determine its value is just a function that returns false. Probably got compiled out. + \ No newline at end of file diff --git a/data/gds_commands/minigames/dog.yml b/data/gds_commands/minigames/dog.yml new file mode 100644 index 0000000..da877bb --- /dev/null +++ b/data/gds_commands/minigames/dog.yml @@ -0,0 +1,15 @@ +commands: + partdef: + id: 0xc2 + desc: | + Defines a gizmo/dog part for use in the minigame. This definition consists of the display position of the part + sprite once placed, and a separate "slide target" point where the gizmo sprite will linearly slide toward when + clicked, before vanishing and being replaced with the realized part. Note that the sprite ID for the gizmo in the + tray is not assumed to be the same ID as the part itself, though in practice it turned out that way. + params: + partId: int + xPos: int + yPos: int + gizmoSpriteId: int + slideTargetX: int + slideTargetY: int \ No newline at end of file diff --git a/data/gds_commands/minigames/hotel.yml b/data/gds_commands/minigames/hotel.yml new file mode 100644 index 0000000..6f9d9d6 --- /dev/null +++ b/data/gds_commands/minigames/hotel.yml @@ -0,0 +1,116 @@ +commands: + + item: + id: 0xcf + desc: Defines an inn item's name + params: + itemId: int + name: string + + itemLayton: + id: 0xbc + params: + itemId: int + xPos: int + yPos: int + zIndex: + type: int + uncertain: true + desc: Defines the z-order of the item when displayed in the room. Required since some configurations definitely overlap. + blockerId: + type: int + desc: > + The ID of another item in the room, which if present, implies that this item's preferred spot can't/shouldn't be used. + Instead the position is calculated relative to the blocker item. + blockerOffsetX: int + blockerOffsetY: int + blockerZ: + type: int + uncertain: true + desc: Definitely the same kind of value as zIndex, and if -1 will reuse the above. But if the blocker is present, this value is used instead. + overlayId: + type: int + desc: > + Some of the furniture items have a completely different sprite when their blocker is present, namely the flowers in the vase and the books on the shelf. + If that is the case, this value will be the ID of the sprite for that case. If the item doesn't have a blocker, this value is ignored; set it to -1 + to mean there is no alternate sprite. + itemComment1Layton: + id: 0xc6 + desc: > + The comment from Layton when placing the given item in his room. Must be called right after an + `0xbc itemLayton` command, and refers to that item. + params: + comment: string(82) + itemComment2Layton: + id: 0xc7 + desc: > + The comment from Layton when placing the given item in his room, given that the specified item ID + is already in there. Used to point out duplications, as well as two things going together well. + Like `0xc6 itemComment1Layton`, has to follow after a `0xbc itemLayton` call. + params: + comment: string(82) + otherItem: int + + itemLuke: + id: 0xbd + params: + itemId: int + xPos: int + yPos: int + zIndex: + type: int + uncertain: true + desc: Defines the z-order of the item when displayed in the room. Required since some configurations definitely overlap. + blockerId: + type: int + desc: > + The ID of another item in the room, which if present, implies that this item's preferred spot can't/shouldn't be used. + Instead the position is calculated relative to the blocker item. + blockerOffsetX: int + blockerOffsetY: int + blockerZ: + type: int + uncertain: true + desc: Definitely the same kind of value as zIndex, and if -1 will reuse the above. But if the blocker is present, this value is used instead. + overlayId: + type: int + desc: > + Some of the furniture items have a completely different sprite when their blocker is present, namely the flowers in the vase and the books on the shelf. + If that is the case, this value will be the ID of the sprite for that case. If the item doesn't have a blocker, this value is ignored; set it to -1 + to mean there is no alternate sprite. + itemComment1Luke: + id: 0xc8 + desc: > + The comment from Luke when placing the given item in his room. Must be called right after an + `0xbd itemLuke` command, and refers to that item. + params: + comment: string(82) + itemComment2Luke: + id: 0xc9 + desc: > + The comment from Luke when placing the given item in his room, given that the specified item ID + is already in there. Used to point out duplications, as well as two things going together well. + Like `0xc8 itemComment1Like`, has to follow after a `0xbd itemLuke` call. + params: + comment: string(82) + otherItem: int + + + roomCommentLayton: + id: 0xca + desc: The comment from Layton, expressing his overall satisfaction with the room layout. + params: + comment: string(75) + level: int + unused: + type: int + desc: always 0 + roomCommentLuke: + id: 0xcb + desc: The comment from Luke, expressing his overall satisfaction with the room layout. + params: + comment: string(75) + level: int + unused: + type: int + desc: always 0 \ No newline at end of file diff --git a/data/gds_commands/puzzle/buttonlist.yml b/data/gds_commands/puzzle/buttonlist.yml new file mode 100644 index 0000000..3373590 --- /dev/null +++ b/data/gds_commands/puzzle/buttonlist.yml @@ -0,0 +1,14 @@ +context: "puzzle.buttonlist" +commands: + count: + id: 0x1d + desc: Defines how many buttons there should be in the list. Note that the engine entirely controls button placement. + params: + count: int + correct: + id: 0x1e + desc: Defines which of the buttons is the correct one. + params: + btnId: + type: int + desc: The index of the button that should be the correct answer. \ No newline at end of file diff --git a/data/gds_commands/puzzle/coin.yml b/data/gds_commands/puzzle/coin.yml new file mode 100644 index 0000000..e422834 --- /dev/null +++ b/data/gds_commands/puzzle/coin.yml @@ -0,0 +1,24 @@ +context: "puzzle.coin" +commands: + coin: + id: 0x25 + desc: Places a coin on the puzzle field, and defines its reset slot + params: + xPos: int + yPos: int + target: + id: 0x26 + desc: Defines one of the target locations where a coin needs to be placed for the puzzle to solve + params: + - type: int + - type: int + 0x83: + params: + - int + - int + - int + 0x84: + params: + - int + - int + - int \ No newline at end of file diff --git a/data/gds_commands/puzzle/common.yml b/data/gds_commands/puzzle/common.yml new file mode 100644 index 0000000..b4c3089 --- /dev/null +++ b/data/gds_commands/puzzle/common.yml @@ -0,0 +1,271 @@ +prefix: "puzzle." +context: "puzzle" +commands: + setCurrent: + id: 0x48 + context: "all" + desc: | + Sets the current puzzle ID in the global game state. If at the end of script execution the + global "current script type" is set to "question", the puzzle with this ID will be started. + params: + puzzleId: int + isCurrentNotSolved: + id: 0x4c + context: "all" + condition: true + desc: Checks if the currently relevant puzzle has NOT been solved + + isCurrentDiscovered: + id: 0x49 + context: "all" + condition: true + desc: Checks if the currently relevant puzzle has been discovered/attempted before. + isDiscovered: + id: 0x58 + context: "all" + condition: true + desc: checks if the specified puzzle has been discovered. + params: + puzzleId: bool + + isCurrentSolved: + id: 0x4d + context: "all" + condition: true + desc: Checks if the currently relevant puzzle has been solved + isSolved: + id: 0x54 + context: "all" + condition: true + desc: Checks if the specified puzzle ID has been solved (puzzle flag 2) + params: + puzzleId: int + + isSolvedInStory: + id: 0x4e + context: "all" + condition: true + desc: Checks if the currently relevant puzzle has been solved in its correct location, i.e. the event surrounding it has completed. + + countEnoughSolved: + id: 0x77 + condition: true + desc: > + Counts how many puzzles the player has solved, and returns true if that number is greater + than or equal to the given threshold. + params: + threshold: int + + eventAfterPuzzle: + id: 0x4b + context: "all" + condition: true + desc: Checks a special flag set if the event script is run right after a puzzle was ended (solved or given up). + eventNotAfterPuzzle: + id: 0x4a + context: "all" + condition: true + desc: Checks a special flag set if the event script is run right after a puzzle was ended, and returns true if that is not the case. + + solverLayton: + id: 0x66 + context: "all" + desc: Sets the next puzzle to be "solved by" Layton (i.e. one of his sequences will play during grading) + solverLuke: + id: 0x77 + context: "all" + desc: Sets the next puzzle to be "solved by" Luke (i.e. one of his sequences will play during grading) + + + engine: + id: 0x1b + # TECHNICALLY. But nobody would use this outside of puzzle scripts, because the grading and everything would be missing. + context: "all" + desc: Loads a puzzle engine with the specified name, and continues the script in that context. + params: + engineName: + type: string + desc: the name of the engine to load + imgWin: + id: 0x1f + context: "all" + desc: Selects the image to be displayed when the puzzle is done, probably. + uncertain: true + params: + id: + type: int + desc: seems to always be the current puzzle ID + unk_2: + type: int + desc: seems to always be 0 + slideItemGoalTile: + id: 0x38 + context: ["puzzle.shape", "puzzle.slide", "puzzle.drag"] + desc: For puzzles that involve sliding objects into correct locations (slide, drag, shape (unused)) define the location that would be an object's goal. + params: + xPos: int + yPos: int + drawLineColor: + id: 0x42 + context: "all" + desc: | + Many puzzle types give the user the ability to draw lines freehand or between points; this command sets the RGB color of those lines (with 5bit channel precision). + + A full list of the puzzle types that use this: [trace, traceButton, drawInput2, cut, drag] + However, the function is available independent of engine. + params: + r: + type: int + g: + type: int + b: + type: int + + button: + id: 0x5d + context: ["puzzle.onoff", "puzzle.freebutton"] + desc: Adds a button with the specified sprite texture at the given location. + params: + xPos: short + yPos: short + spriteName: + type: string + desc: The sprite used for the button. As with all button sprites, it should define a depressed state in the animation "shadow". + correct: + type: bool|int + desc: Whether this button should be (de)pressed in order for the puzzle to solve. + unk_5: + type: int + + placeTarget: + id: 0x5e + context: "puzzle.placetarget" + desc: > + The sole defining command for the PlaceTarget puzzle type. It seems there can be multiple targets defined this way, + all of which need to be hit in order for the puzzle to solve; but this is never used + (maybe it felt too cumbersome during playtesting, because you can't reset?) + params: + xPos: int + yPos: int + spriteName: string + radius: float + unk_5: + type: int + + gridOffset: + id: 0xa3 + desc: For grid-based puzzles, determines the offset in pixels that grid should have on the screen. + params: + xPos: int + yPos: int + gridDimensions: + id: 0xa4 + desc: For grid-based puzzles, determines the dimensions of the grid in tiles. This also initializes the grid. + params: + width: int + height: int + gridTileSize: + id: 0xa5 + desc: For grid-based puzzles, determines the size in pixels that each grid cell should have. + params: + sizePx: int + + + title: + id: 0xba + desc: > + Used in the technical script/puzzletitle/?/qtitle.gds, to determine the localized name + of all puzzles at once. This script is read, for example, by the Puzzle Index and Wifi Puzzle list. + params: + puzzleId: + type: int + desc: the ID of the puzzle to set the name for + name: string + picarat: + id: 0xc3 + desc: Defines how many picarat are awarded for each attempt. Stored in script/pcarot/pscript.gds + params: + puzzleId: int + first: + type: int + desc: The amount awarded solving on first try + second: + type: int + desc: The amount awarded after one error + other: + type: int + desc: The amount awarded after the second failed attempt, on all subsequent tries + indexInfo: + id: 0xdc + desc: Loads information for the puzzle index + params: + puzzleId: int + type: + type: string + desc: There is no fixed list of options, but the length may not exceed 64 characters. + location: + type: string + desc: There is no fixed list of options, but the length may not exceed 64 characters. + + 0x1c: + desc: > + Effectively calls 0x20 times with parameter 0, though this value is hardcoded to 3 in the release. + Therefore, fills that 3-element list with 0 (all puzzles in the game do this at the start) + params: + - type: int + desc: gets overwritten by 3, so unused + 0x20: + desc: Pushes the specified byte to a list with at most 3 elements. + unused: true + params: + - type: byte + + moveLimit: + id: 0x27 + context: ["puzzle.coin", "puzzle.match", "puzzle.scale", "puzzle.cut"] + desc: Sets the move limit for this puzzle. Note that the max for match and coin puzzles is 255. + params: + # TODO: types dependent on conditions? Maybe allow defining the same command multiple times for different contexts??? + limit: int + + targetPosRot: + id: 0x2b + context: ["puzzle.match", "puzzle.tilerotate"] + desc: Defines a target position for movable and rotatable objects. + params: + xPos: int + yPos: int + angle: float + posTolerance: + type: float + optional: true + uncertain: true + desc: (Probably) the radius of pixels that the player may be off for a match to still be counted + angleTolerance: + type: float + optional: true + uncertain: true + desc: (Probably) the angle in degrees that the player may be off for a match to still be counted + + + targetAngle: + id: 0x39 + context: ["puzzle.tile", "puzzle.tilerotate", "puzzle.shape"] + desc: Sets the target angle of the previously specified object. The object must be rotated to that exact angle for the puzzle to solve. + params: + angle: + type: float + 0xb7: + desc: sets all the flags for the given puzzle ID to true (1, 4, and 2 ie "solved") + params: + puzzleId: int + + 0xaa: + desc: used by all the tile-based puzzles + params: [int] + + + 0x00: + desc: DO NOT USE! The function pointer slot for this command is never populated, trying to call it will probably crash! + 0xff: + desc: DO NOT USE! The function pointer slot for this command is never populated, trying to call it will probably crash! \ No newline at end of file diff --git a/data/gds_commands/puzzle/connect.yml b/data/gds_commands/puzzle/connect.yml new file mode 100644 index 0000000..bb1d10d --- /dev/null +++ b/data/gds_commands/puzzle/connect.yml @@ -0,0 +1,23 @@ +context: "puzzle.connect" +commands: + barrier: + id: 0x28 + desc: > + Fills the specified area in the grid with "barrier" tiles, i.e. tiles that the line cannot cross into. + By default the entire grid is accessible. + params: + # TODO: range [0, 19) + xPos: int + yPos: int + width: int + height: int + goal: + id: 0x29 + desc: > + Sets the specified cell in the grid to be a goal with the given ID. Two goals of the same ID must be matched up for the puzzle to solve. + params: + xPos: int + yPos: int + goalId: + type: int + desc: Note that this cannot be zero, as that would be treated like no goal at all. \ No newline at end of file diff --git a/data/gds_commands/puzzle/cup.yml b/data/gds_commands/puzzle/cup.yml new file mode 100644 index 0000000..52b41df --- /dev/null +++ b/data/gds_commands/puzzle/cup.yml @@ -0,0 +1,23 @@ +context: "puzzle.cup" +commands: + cup: + id: 0x3a + desc: Adds a new cup object. The position is is relative to the bottom-center of the sprite, as well as the top left of the litre count display. + params: + spriteset: string + xPos: short + yPos: short + spriteId: int + capacity: + type: int + desc: Note this is x10 + target: + type: int + desc: The amount that should be in the cup for the puzzle to solve. Note this is x10 + color: + id: 0xbf + desc: Sets the color of the liquid + params: + r: int + g: int + b: int \ No newline at end of file diff --git a/data/gds_commands/puzzle/cut.yml b/data/gds_commands/puzzle/cut.yml new file mode 100644 index 0000000..6fbc029 --- /dev/null +++ b/data/gds_commands/puzzle/cut.yml @@ -0,0 +1,41 @@ +context: "puzzle.cut" +commands: + gridFillType: + id: 0x9f + params: + xPos: int + yPos: int + width: int + height: int + value: int + gridSetType3: + id: 0xa0 + desc: Sets the type of the given cell in the grid to 3 (whatever that means). In the CutPuzzle this + means a source/target for cut lines. + params: + xPos: int + yPos: int + solutionCut: + id: 0xa1 + desc: Adds a cut between two grid(!) points to the solution. + params: + xPos1: int + yPos1: int + xPos2: int + yPos2: int + + 0xa2: + 0xac: + desc: sets the color of the cut line once it's done + uncertain: true + params: + - int + - int + - int + 0xad: + desc: sets the color of the cut line while it's still being drawn + uncertain: true + params: + - int + - int + - int \ No newline at end of file diff --git a/data/gds_commands/puzzle/drag.yml b/data/gds_commands/puzzle/drag.yml new file mode 100644 index 0000000..93f3493 --- /dev/null +++ b/data/gds_commands/puzzle/drag.yml @@ -0,0 +1,4 @@ +context: "puzzle.drag" +commands: + 0xf2: + params: [float] \ No newline at end of file diff --git a/data/gds_commands/puzzle/drawinput.yml b/data/gds_commands/puzzle/drawinput.yml new file mode 100644 index 0000000..c6be12b --- /dev/null +++ b/data/gds_commands/puzzle/drawinput.yml @@ -0,0 +1,41 @@ +context: ["puzzle.drawinput", "puzzle.drawinput2"] + +commands: + setup: + id: 0xa8 + desc: Sets up the type and amount of draw inputs there should be + params: + inputType: int + numInputs: int + + 0x24: + context: "puzzle.drawinput" + params: + - type: int + - type: int + - type: int + 0xa7: + params: + - int + - string + 0xa9: + params: + - int + - string + 0xb9: + params: + inputId: int + unk_2: + type: int + desc: possibly the target value of the puzzle solution + 0xf5: + desc: seems to reset the inputs? + 0xfa: + desc: only in type2 + params: [int, int, int, int] + 0xfb: + desc: only in type2 + params: [int, string] + 0xfc: + desc: setInputBg? only in type2 + params: [string] \ No newline at end of file diff --git a/data/gds_commands/puzzle/match.yml b/data/gds_commands/puzzle/match.yml new file mode 100644 index 0000000..a939c19 --- /dev/null +++ b/data/gds_commands/puzzle/match.yml @@ -0,0 +1,9 @@ +context: "puzzle.match" +commands: + match: + id: 0x2a + desc: Defines a new match on the playfield, with position and rotation. + params: + xPos: short + yPos: short + angle: float \ No newline at end of file diff --git a/data/gds_commands/puzzle/queen.yml b/data/gds_commands/puzzle/queen.yml new file mode 100644 index 0000000..ac5c71b --- /dev/null +++ b/data/gds_commands/puzzle/queen.yml @@ -0,0 +1,37 @@ +context: "puzzle.queen" +commands: + board: + id: 0x3b + desc: Defines base parameters for the overall board of the Queen puzzle. + params: + screenX: + type: int + desc: The pixel offset of the top left corner of the grid from the left edge of the screen + screenY: + type: int + desc: The pixel offset of the top left corner of the grid from the top edge of the screen + boardSize: + type: int + desc: The size of the board in cells + queenCount: + id: 0x3c + desc: Sets the number of queens the player needs to place + params: + count: + type: int + fixed: + id: 0x3d + desc: Places an immovable (golden) queen on the board in the specified tile location + params: + tileX: + type: int + tileY: + type: int + partial: + id: 0x3e + desc: "Determines the \"partial mode\" setting, which is used in 'Too Many Queens 3': there are less queens than tile width, but they still need to block the entire grid." + params: + enable: + # bool is one special datatype that must be converted into another type before saving, and is just separate for convenience in GDA scripts. + type: bool|int + desc: Set to true in order to make a puzzle with the ruleset of 'Too Many Queens 3' diff --git a/data/gds_commands/puzzle/rivercross.yml b/data/gds_commands/puzzle/rivercross.yml new file mode 100644 index 0000000..107081e --- /dev/null +++ b/data/gds_commands/puzzle/rivercross.yml @@ -0,0 +1,35 @@ +context: "puzzle.rivercross" +commands: + chicken: + id: 0x31 + desc: Adds a chicken to the puzzle's play field. + params: + xPos: int + yPos: int + wolf: + id: 0x32 + desc: Adds a wolf to the puzzle's play field. + params: + xPos: int + yPos: int + man: + id: 0x89 + desc: Adds a man to the puzzle's play field. + params: + xPos: int + yPos: int + cabbage: + id: 0x8a + desc: Adds a man to the puzzle's play field. + params: + xPos: int + yPos: int + sheep: + id: 0x8b + desc: Adds a man to the puzzle's play field. + params: + xPos: int + yPos: int + + 0x8c: + params: [int] \ No newline at end of file diff --git a/data/gds_commands/puzzle/scale.yml b/data/gds_commands/puzzle/scale.yml new file mode 100644 index 0000000..e13c9c7 --- /dev/null +++ b/data/gds_commands/puzzle/scale.yml @@ -0,0 +1,16 @@ +context: "puzzle.scale" +commands: + count: + id: 0x2d + desc: Defines how many weights there should be, and creates the objects. The maximum is 12. + params: + count: int + light: + id: 0x2e + desc: Sets a random weight to be lighter than the others, and also the correct solution + heavy: + id: 0x2f + desc: Sets a random weight to be heavier than the others, and also the correct solution + random: + id: 0x30 + desc: Sets a random weight to be randomly either lighter or heavier than the others, and also the correct solution diff --git a/data/gds_commands/puzzle/shape.yml b/data/gds_commands/puzzle/shape.yml new file mode 100644 index 0000000..3e4af0c --- /dev/null +++ b/data/gds_commands/puzzle/shape.yml @@ -0,0 +1,30 @@ +context: "puzzle.shape" +# This puzzle type seems to be unused, which means I can't infer anything from usage. +commands: + shape: + id: 0x33 + desc: Adds a shape object (max 8) + point: + id: 0x34 + desc: Adds a point to the last created shape object (max 16) + params: + xPos: + type: int + desc: x-position of the point + yPos: + type: int + desc: y-position of the point (orientation unclear) + + + 0x35: + params: + - type: int + - type: int + - type: int + 0x36: + params: + - type: int + - type: int + 0x37: + params: + angle: float \ No newline at end of file diff --git a/data/gds_commands/puzzle/slide.yml b/data/gds_commands/puzzle/slide.yml new file mode 100644 index 0000000..01e98b5 --- /dev/null +++ b/data/gds_commands/puzzle/slide.yml @@ -0,0 +1,22 @@ +context: "puzzle.slide" +commands: + item: + id: 0xa6 + desc: Adds a slidable item into the grid + params: + xPos: int + yPos: int + unk_3: int + imgName: + type: string + desc: The "animation" name representing this item's image. Sourced from either ani/slidepuzzle.spr or ani/slidepuzzle2.spr + tileGroup: + id: 0xb1 + params: + xPos: int + yPos: int + group: int + itemGroup: + id: 0xb2 + params: + group: int \ No newline at end of file diff --git a/data/gds_commands/puzzle/tile.yml b/data/gds_commands/puzzle/tile.yml new file mode 100644 index 0000000..07dd7d3 --- /dev/null +++ b/data/gds_commands/puzzle/tile.yml @@ -0,0 +1,49 @@ +context: ["puzzle.tile", "puzzle.tilerotate"] +commands: + tile: + id: 0x73 + desc: Adds a tile to the puzzle field. A reset-linked slot is automatically created. + params: + xPos: int + yPos: int + spriteName: string + animId: + type: int + slot: + id: 0x74 + desc: Adds a slot to the puzzle field; tiles can only be dropped in slots. + params: + xPos: int + yPos: int + targetSlot: + id: 0x75 + desc: > + Assigns the given tile ID (in declaration order) to the given slot ID. To solve the puzzle + it is checked whether one of these assignments is satisfied; note particularly that you can add + multiple, which will result in the puzzle having multiple solutions. + params: + tileId: int + slotId: + type: int + desc: Note this value is negative + rotButton: + id: 0xb5 + desc: > + For the non-rotate tile puzzles, there may be buttons on the side to rotate the pieces + (there's actually only one example of this in the entire game, Puzzle #116p112) + params: + xPos: int + yPos: int + width: int + height: int + + + 0x76: + desc: Most likely sets the solution count for a tile puzzle. + params: + - int + 0xbb: + desc: pushes a truth value into a list + params: [bool] + 0xbe: + desc: sets some field to 0 \ No newline at end of file diff --git a/data/gds_commands/puzzle/trace.yml b/data/gds_commands/puzzle/trace.yml new file mode 100644 index 0000000..0e17239 --- /dev/null +++ b/data/gds_commands/puzzle/trace.yml @@ -0,0 +1,37 @@ +context: "puzzle.trace" +commands: + center: + id: 0x3f + desc: Places the centerpoint of the shape to be traced. I'm not yet sure what the significance of that is, because it seems very lax. + params: + xPos: int + yPos: int + inPoint: + id: 0x40 + desc: Adds an Inpoint, a point that needs to be inside the traced shape for the puzzle to complete. + params: + xPos: int + yPos: int + outPoint: + id: 0x41 + desc: Adds an Outpoint, a point that mustn't be inside the traced shape for the puzzle to complete. + params: + xPos: int + yPos: int + + circleAnswer: + id: 0xd4 + context: puzzle.tracebutton" + desc: In the TraceButton (ie Circle Answer) puzzles, used to define a possible circle target + params: + xPos: int + yPos: int + radius?: float + unk_4: bool + + 0xd0: + params: + - int + - int + - int + - int \ No newline at end of file diff --git a/data/gds_commands/room.yml b/data/gds_commands/room.yml new file mode 100644 index 0000000..f9769e1 --- /dev/null +++ b/data/gds_commands/room.yml @@ -0,0 +1,247 @@ +context: "room" +commands: + isCurrent: + id: 0x13 + context: "all" + condition: true + desc: Returns true if the engine's current room is equal to the provided ID. Can be used anywhere. + params: + roomId: + type: ubyte + desc: the room ID to be tested against + + setCurrent: + id: 0x53 + context: "all" + desc: > + Sets the engine's current room ID to the specified value. This takes effect after the script + completes, the next time the script type is set to 'Event'. + params: + roomId: + type: int + + exit: + id: 0x22 + desc: > + Defines an exit the user can click on to change rooms, by its bounding box, target room, and + (TODO: very uncertain) the movement of the lil layton sprite on the top screen map??? + params: + exitType: + type: int + desc: Value from 0-5. Not sure what zero does, but 5 is a door in the scene, and 1-4 stands for the screen edges starting left ccw. + xPos: int + yPos: int + width: int + height: int + exitRoom: + type: ubyte + desc: Sets the room the player will move to when they take this exit + unk_7: + type: short + unk_8: + type: short + exitSound: + id: 0x5b + desc: Defines which footsteps sound is used on room transition + params: + exitId: + type: byte + desc: the ID of the exit (in declaration order) + soundId: + type: byte + desc: Can be any number, but only 1 (echoy footsteps) and 2 (creaky door) have special handling + + sprite: + id: 0x23 + desc: > + Defines a tiny sprite used in the room, from the "room_%i_%i.arc" files. Never actually used, + but the sprites look like sweet cut content. + uncertain: true + params: + xPos: + type: int + yPos: + type: int + bgsprite: + id: 0x5c + desc: Adds a non-interactive background sprite to the room, for animated room background effects. + params: + xPos: byte + yPos: byte + spriteName: + type: string + desc: path to the sprite to be used. + + textObj: + id: 0x43 + desc: Defines a Text Object, i.e. an area of the screen where clicking on it opens a brief flavor text box from one of the characters. + params: + speaker: + type: ubyte + desc: > + 1 for Layton, 2 for Luke. Probably corresponds to the frames in ani/room_tobjp.spr, + which also includes the hint coin, but doesn't include anything else; my assumption is + that using a number outside of these two (and maybe 3) will break something. It's worth + testing how adding frames to the sprite file will affect things... + # TODO: something really cool here would be compound datatypes + # ex: + # params: + # ... + # bbox: + # type: BBox(int xPos, int yPos, int width, int height) + # + # And in a GDA script this could be written as `BBox(1,2,3,4)` to be more readable. + xPos: + type: int + yPos: + type: int + width: + type: int + height: + type: int + textId: + type: int + desc: The id of the text to be displayed. They are sourced from room/tobj/?/tobj.pcm//t_%i.txt + unk_7: + type: int + desc: Appears unused in code, and seems to always be 1 + + eventObject: + id: 0x50 + desc: > + Adds an interactable object to the room. This is used for all NPCs, and some other + items such as the crumpled notes and newspapers. + params: + xPos: short + yPos: short + width: int + height: int + spriteId: + type: int + desc: The sprite that will be placed at the given location. It only becomes interactable within the defined bounding box. Sourced from ani/obj_%i.spr with default animation. + eventId: + type: int + desc: The event that will be triggered when the player taps on the object + + eventTrigger: + id: 0x21 + desc: > + Unused command that does something similar to `0x87 npcObject`, but without registering a sprite. + Internally this method is called by that one, because it is responsible for determining the popup + icon on tap (though this is hardcoded now to always be the exclamation point). + params: + xPos: + type: int + yPos: + type: int + width: + type: int + height: + type: int + spriteName: + type: string + desc: the name of the sprite container holding the sprite that should popup when the trigger is tapped + animName: + type: string + desc: Within the sprite container, the name of the animation to be displayed. + eventId: + type: int + desc: The event that will be started when the trigger is tapped + + hintcoin: + id: 0x68 + desc: Adds a hint coin to the room. + params: + coinId: + type: int + desc: Every hint coin has a unique ID, presumably to allow saving which ones you've collected. + xPos: short + yPos: short + width: short + height: short + unk_6: + type: int + desc: appears to be unused. + + + mapInfo: + id: 0x5a + context: ["room", "event"] + desc: > + Sets the current room ID and the map ID, which determines the name of the location. + Also sets the coordinates for the little Layton sprite map marker on the top screen. + Can be used in events to enqueue a room change as well. + params: + roomId: + type: int + desc: the current room. Mostly useful when called from event scripts. + mapId: + type: int + desc: the ID of the top screen map image to be used + xPos: short + yPos: short + markerSpriteId: + type: int + desc: It seems there were multiple possible map marker icons at some point, but only ID 1 has survived. + + + nazobabaBottle: + id: 0xe4 + desc: Only used in the Nazobaba (Granny Riddleton) house. Almost certainly registers the bottles that puzzles may appear in. + params: + xPos: int + yPos: int + width: int + height: int + unused: int + nazobabaAwards: + id: 0xe5 + desc: > + Registers a puzzle in the puzzle index, and awards a puzzle's assigned item, when solved from Nazobaba house + (since there is no story event to give you that item normally). Value -1 means no item given in that category. All + these commands are placed in script/nazobaba/babascript.gds in a list, but only the command corresponding to the + current puzzle ID will actually execute. (Runs in Event context only) + uncertain: true + params: + puzzleId: int + hotelId: int + gizmoId: int + paintingId: int + nazobabaGiveAward: + id: 0xe6 + desc: Responsible for calling babascript.gds (see `0xe5 nazobabaAwards`) + nazobabaSend: + id: 0xe7 + uncertain: true + desc: | + Sends the specified puzzle to the Nazobaba house, if it wasn't already solved. Also writes all those puzzle names somewhere, but not their IDs. + It seems these are called in bulk in certain otherwise empty scripts, which in turn are probably called in spots where puzzles would vanish. + params: + puzzleId: int + + + 0x2c: + desc: sets some kind of flag at the given index to true + params: + - type: int + + 0xab: + params: + - int + + 0xd1: + params: + - int + - int + - int + - bool + - int + - int + 0xd2: + params: + - int + - int + - int + - bool + - int + - int \ No newline at end of file diff --git a/doc/gds/gda.md b/doc/gds/gda.md new file mode 100644 index 0000000..1bc6c2c --- /dev/null +++ b/doc/gds/gda.md @@ -0,0 +1,144 @@ +# GDA: a human-friendly decompilation of GDS + +With how significant [GDS](./gds.md) is for controlling this game, obviously modders would be interested in modifying or writing these script files themselves. But that's pretty difficult to do with a binary format that has location-sensitive jump markers... of course we could build a visual editor, but that's extremely +out of scope for Flora. + +Incidentally, this problem is identical to the one of writing actual real-world executable programs in machine code. And the easiest solution people came up with was **Assembly**: a slightly more human-readable format, where instructions weren't just numbers but had understandable (or at least decipherable) names, and crucially, label and jump locations would be assembled automatically. While most of us write programs in high-level, sometimes even interpreted languages, Assembly is also still widely used! + +And so it only makes sense that, in order to make both the reading and the creation of GDS scripts easier, Flora should include a (dis)assembler. The special dialect we use is called **GDA**, which may stand for **G**ame **D**ata **A**ssembly. This doc describes our specification of the format's syntax. + +## Structure + +The GDA specification and compiler were both written by the Flora devs, based on knowledge from the decompilation of the game and its scripts. It's a hopefully good attempt at a readable scripting language matching the feature set of GDS. + +A GDA script consists of a few elements: + +- An (optional but recommended) version comment at the beginning +- Instructions with parameters +- Jump labels referred to by the jump instructions +- Optional comments + +### Comments + +At any point in the file, in a new line or after an instruction in the same line, you may use the `#` character to have GDA ignore the entire rest of the line. This is used purely for your own documentation purposes. Unfortunately when re-decompiling a GDS file, the compiler doesn't yet have a way to keep these comments in place. + +A special comment of the form `#!version 1.0` can be placed as the first non-empty line of a GDA file, to mark the version of the language that the script was written in. This is useful for the compiler, because it allows disambiguating commands whose names have changed across Flora/GDA versions (due to improved understanding and research). + +### Instructions + +An instruction is referred to by its name, which comes from a predefined list of commands. Since the instructions in GDS are named neither in the data files nor in the assembly, we took our best guess to name them descriptively and accurately. But as our understanding evolves (or mods add new capabilities to GDS), these names may change over time. + +You can ask Flora to print help for a certain command (by name or hex id), or list all known name (and parameter) assignments. + +### Parameters + +There are four types of parameter values usable in a script: +- Integers, which are just written as regular numbers. Negative numbers are allowed, and sometimes used as sentinel values (markers that denote a special case and are not meant to be interpreted as a number, usually -1). Remember that the value of these numbers needs to fit into 4 bytes, which means it must be in the (inclusive) range from -2^31 to 2^31-1 (alternatively, if you know the number is treated as unsigned positive, you can go from 0 to 2^32-1). + > TODO: many commands have a far smaller number range, this should be denoted in their type! + + You are also allowed to write them in different bases, by using the prefixes `0x` for hex, and `0b` for binary. We don't support writing numbers in octal, what the hell. (This means you can add leading zeroes without worrying.) +- Floats, which are also written as numbers, but which contain a decimal point or are written in scientific notation. If you want to force an integer to be a float, just write it like `1234.` (the decimal part is implied zero). Likewise you can omit the integer part for numbers like `.5`. Scientific notation works like it does in most languages, by separating the mantisse and exponent with an `e`. +- Strings, which may contain up to 63 standard ASCII characters, and are enclosed in either single or double quotes. Common escape sequences like line breaks (`\n`), arbitrary ASCII (`\x12`) or escaping the quotes (`\"`) or escape character (`\\`) are supported; but keep in mind that this is separate from the escape mechanisms some instructions use internally (for example, "break page" or "change expression" commands inlined in dialogue text). Backslash escaping is understood and processed only by the GDA compiler. + > Technical note: If you want to know why the max length isn't a power of two when 64 is right there: that's because the buffer itself is 64 bytes long, but must keep room for a single null byte at the end of the string. GDA automatically takes care of this for you. +- Long Strings, which work exactly like regular Strings, but may be (practically) as long as you wish! ...they're also completely unused in the game, and you can't use them in places where a regular string is required (maybe someone could make a mod that makes more or even all GDS commands accept them in place of strings). The only reason they even exist seems to be because `0x21 eventTrigger`'s filename seems to have been very long at some point, but it is now simply a format string derived in code (and also completely unused). If you still want to use one, just prepend `l` to the front of a regular string literal `l'like so'`. + > TODO: Check other language versions of this game. Perhaps the Asian versions needed more string length since their character set is larger. + +### Jump Labels + +A jump label denotes a location in the GDS binary file, right before the next instruction after it. They are used for jumps, and physically represented in code as a number pointing to the location that jumps to them (there is only one at all times). + +You can define one (or more) jump labels by name in a location in GDA code, by beginning the line with an `@` symbol and writing space-separated label names: +``` +@JumpLoc_1 JumpLoc2 +print "this is the next instruction after the jump" +``` + +Referencing a label is equally simple, by just writing `@` before its label name. These references can be used as another type of function parameter: +``` +jump @Label +print "This gets skipped" +@Label +print "This gets printed" +``` + +The GDA compiler will take care of converting these names into their actual binary file locations for you. + +In the degenerate case that a jump address doesn't correspond to a physical label at its target location (which seems possible, since that is technically completely unnecessary), this syntax wouldn't produce identical results as its input... So when defining a jump label, you can prepend `!` to (each of) the names, indicating that no label should be created in the binary file. This is mostly for completeness, and you shouldn't ever need to do this in your own scripts, so it's best if you don't. + +### `if` statements + +In practice there are only a handful of statements which accept an address as a parameter, and all of them do it via special handling: `if`, `elif`, `else`, `repeatN` and `while`. If handled naïvely as described above, the if statement would look roughly like this: + +``` +if @else_1 + +@else_1 +elif @else_2 + +@else_2 +else @endif_1 + +@endif + +``` + +But that's unnecessarily cluttered, so instead there's special syntax for if blocks that more resembles how other languages do it: + +``` +if { + +} elif { + +} else { + +} + +``` + +In detail, the behavior of the commands is as follows: +- `if` will evaluate a condition, and jump if it is `false` (if there is no condition, that's the default). If after jumping it encounters an `elif` as the next instruction, it'll evaluate that condition too and again jump if it is false. If it encounters an `else` as the next instruction, it will always jump to the location it specifies. +- `elif` and `else`, when encountered outside the context of an `if` instruction being executed, will simply discard all tokens before the jump address and always jump there. + +Because of the way these functions play together, none of them can always be used as a pure jump instruction without side effects; but the compiler may make an effort to let you know when that did happen. + +It is important to note that while all scripts in the base game have well-structured `if` blocks, this may not necessarily be the case for third-party scripts. The main problems arise when non-condition instructions, in particular other jumps, are placed inside a condition block: +- An `if` statement may jump out of the condition block, which is technically fine, but can't be represented with the nice block syntax. In that case we fall back to the basic syntax. +- Similarly, a `loop` may have its end outside of the condition block, and while this will almost certainly crash the game, the basic syntax of the loop section below can be used as fallback. + +> TODO: A big problem happens when the condition of an `elif` statement contains a jump: when reached from `if` the condition will be evaluated and the next address read, but when encountered regularly the first address (of the inner if) will be used instead! This actually can't be represented at all, I think... We could solve this by simply refusing to (de)compile files that are so badly malformed, but we need to detect it. + +### loop statements + +Similarly, loops in their simplest form would be written like this: +``` +while : @endwhile_1 +# or +#repeatN : @endwhile_1 + +@endwhile_1 + +``` + +And likewise, in regular cases this might be simplified to: +``` +while { +# or +#repeatN { + +} + +``` + +It's possible that there isn't actually a degenerate case here, since the loop statement seems to only really be usable for its intended purpose. The only possible edge case is when the loop doesn't end with a physical jump label, which shouldn't matter and is addressed by using `@!endwhile_1` as described above. + +## Compiler + +The Compiler and Decompiler are part of Flora, and executable using `flora gds (de)compile` respectively. Its goal is to produce a GDA script that, when reassembled into GDS, produces an identical binary. (The opposite is certainly impossible, since comments are lost, but the logic should obviously be identical.) + +If a text file doesn't seem to be a valid GDA script, the compiler will report (hopefully helpful) errors. It will *not* output a faulty binary file, or overwrite/delete existing GDS files with invalid ones. + +If a `.gds` file doesn't seem to be a valid script binary, the decompiler will report so. All GDS files in the base game are valid scripts, so if you encounter this while unpacking a fresh ROM, it may be corrupted. This is not guaranteed for third-party files, but all scripts compiled by Flora should be valid as well; if that is not the case, this is a bug you should report to us. + +### Versioning + +The current version of the GDA language and compiler can be found using `flora gds -v`. This version mostly refers to the command list, of which instruction and parameter names may change in the future; these changes will be documented in the resource files, to ensure the compiler still understands what to do when encountering a script of an older version. It can only recognize this by the script's version comment, which is why you should always include it and match it to your current compiler version. (The decompiler does this automatically.) diff --git a/doc/gds/gds.md b/doc/gds/gds.md new file mode 100644 index 0000000..9845b58 --- /dev/null +++ b/doc/gds/gds.md @@ -0,0 +1,119 @@ +# GDS: The Scripting Language of Professor Layton + +Most complex events in the Professor Layton game(s) are driven by **GDS** (which may stand for **G**ame **D**ata **S**cript), +a binary scripting language that can trigger events in the currently running +script engine. This is used, among other things, to: + +- choose a puzzle engine, and initialize details of the puzzle specific to that engine +- define room layouts, i.e. entities, text objects, hint coins, exits,... +- write out dialogue sequences, including all NPC interactions +- provide localized data for certain minigames and use cases, where it is easier to read a single + script containing all the definitions into an existing engine, than to open a specific text file +- setup subtitles for the event movies +- It also defines what the logo sequence on bootup looks like. Yes, that is not hardcoded, + you can change the logo sequence! + +While far too many things are still hard-defined in code (such as Puzzle IDs), this makes +the Layton engine fairly data-driven and moddable! It's therefore extremely important to gain +an understanding of this scripting format, to take control of the engine in custom scenarios. + +This page describes the basic structure of GDS scripts, according to what we know now. This knowledge +was gained mostly from common-sense guesses about commands that were only used in specific scripts, +as well as a disassembly effort by [@ilonachan](https://github.com/ilonachan). The information here +*may* be limited to the European version of the game, but it seems unlikely the scripting language +would have been changed significantly between regional releases. + +This knowledge gives important insights about how **GDA** (which may stand for **G**ame **D**ata **A**ssembly), +a custom language designed to be an easy-to-read decompilation target for GDS, should be structured. + +## Structure + +In general, all scripts (after their 4-byte length header) are made up of *tokens* of specified type. +The type is defined in two bytes, though it never comes close to using that range: +all possible values (as far as we know so far) are: + +- `0` for executing commands, +- `1,2,3,4` for data parameters, +- `6,7` for jump addressing, +- `8,9,10` for logic operators, and +- `11` and `12` for loop behavior, +- `12` to end the script. + +`5` is technically handled in code, but never used and contains no data. All type values higher than 12 will cause the read function to fail and the program to behave unpredictably (including infinite loops). + +### Basic Commands + +A GDS script consists mostly of these: a type 0 token contains the command ID, and may be followed by multiple +type 1-4 parameter tokens; how many are read is determined by the command's logic itself. + +Nearly all of these commands have a static parameter list with fixed meaning, and these commands and their parameters are +defined for programmatic purposes by the YAML files in this directory (split into categories of relevance, because many +commands are only implemented in certain engine contexts). + +Certain commands have a more variable parameter list or parsing behavior than the regular ones: these commands are documented in YAML, but their parameter list is specially handled in [the gds parser module](../../formats/gds.py). Specific ones used for control flow are described further down. + +Commands may also modify a game-internal conditional flag, which will be described further below. + +### Data parameters + +The following are all the data type parameters handled: + +- `1`: a 4-byte integer +- `2`: a 4-byte floating point number +- `3`: a string, beginning with a 2-byte length, and then that many 1-byte "ASCII" characters (plus null terminator) +- `4`: unknown, but also read as a string + +Each command decides which types of data it will read; it will *not* check if the read parameter actually corresponds to that +type, so specifying incorrect types will cause unpredictable error behavior. + +### Conditional Jump Instructions + +At the core these are also implemented as type-0 instructions, but their "parameter" fetching logic differs significantly. +The most common example is the instruction ensemble `0x12 jumpUnless`, `0x17 jumpIfNotArrived`, and `0x16 jumpChainUnless`. +These would probably be more aptly described as `if`, `else` and `else if` (I haven't 100% confirmed that, but their behavior and use imply this). + +To define jump targets, the following tokens are used: + +- `6`: a 4-byte jump location within the binary script file. Always seems to point to the beginning of a type-7 token. +- `7`: a 4-byte jump source address within the binary file. Seems to always be the target of a single jump instruction, + and holds the address of its source. While this doesn't seem strictly required by the jump routine, + it makes reading and debugging scripts easier, and that's likely why they never bothered to remove it. + +`if` and `else if` are obviously not very useful without any conditions to check; for this purpose, a list of type-0 commands (with arguments) are executed right after them, and the conditional flag they returned is incorporated into the boolean expression (note that not all instructions modify the conditional flag, the ones that do are marked in our analysis files). +The conditional chain ends once a type-6 token with a jump address is encountered. + +To define complex conditional chains, the following flag tokens can be interspersed with instruction executions: + +- `8`: All following commands' condition output is treated as negated. + Note that while you'd think this flag would only apply to the next command, or be reset by toggling it, + this is not the case! Once set the flag persists, and there's no way to unset it (within the same conditional). + They probably never bothered to check this, because I only see this used to negate single conditions. +- `9`: Changes the mode of how multiple commands' flags are combined: ANDs their results. +- `10`: Changes the mode of how multiple commands' flags are combined: ORs their results. + +When writing your own scripts, note the following pitfalls: +- If no conditions are provided, `0x12` will always jump (because the default condition value is false) +- If neither flag 9 or 10 are specified, the result of a new instruction's condition will always overwrite all previous ones, + until either one is specified. +- The 8 flag may seem like a one-shot or toggle, but internally it actually can't be reset! Once used, all future instruction + results will be negated, and there is no easy way around this. Plan your conditionals around this fact, or nest if blocks! +- Using a command that does not produce a conditional output will simply leave the previous one untouched. This is fairly useless + because there is no short-circuiting, and contitional instructions normally don't modify state to prevent a command from running before the `if` block entirely. + But if you choose to do this, keep in mind that the previously negated condition will be negated again! In DISCARD mode this would undo the negate flag's effect (but that's pretty useless because ideally you're only in DISCARD mode in the first instruction), and in AND/OR modes it fixes the value to `false`/`true` respectively (which is also very useless). Therefore, ideally only condition instructions should be used in conditions. + +### Loop Instructions + +The instructions `0x14 repeatN` and `0x15 while` are assumed to be loops (hence why I named them that). `repeatN` takes a single number parameter for the amount of times to loop, and then simply ignores all tokens until it finds a jump address; `while` consumes a conditional in the same way as `if`, and re-executes it on each loop. + +The loop will execute commands until: +1. the GDS instruction pointer exceeds the loop exit address, in which case it will be moved back to the start of the loop (checking conditions) +2. it encounters a type-`11` token (a flag with no data), in which case it will break the loop by jumping to its end (in hex this type is recognizable as `0xb` for **b**reak) +3. it encounters a type-`12` token (a flag with no data), in which case it will abort the loop but leave the program counter as + it is. This is not equivalent to the `continue` of most programming languages, rather saying "we don't want to loop anymore, + but finish this final iteration". + This token is also always found at the end of a script, and I'm still looking for instances outside of that, meaning there is a chance this is simply a failsafe to make sure the loop doesn't continue beyond the script buffer. However, if this behavior seems useful, it doesn't look like using it would have any negative side effects. +4. the next instruction would be `0x4f return`, in which case the loop manually handles to respect the early return. + +## Future Work: A Turing Machine in Professor Layton 1 + +Most script engines only provide a limited number of flags to freely set and access, but maybe this is enough to do a fun proof-of-concept turing machine design. \ No newline at end of file diff --git a/doc/sample_file.sav.yml b/doc/sample_file.sav.yml new file mode 100644 index 0000000..752d9eb --- /dev/null +++ b/doc/sample_file.sav.yml @@ -0,0 +1,2438 @@ +files: +- filename: ilona + cur_location: Tower Top Floor + playtime: 43131 + playtime_simple: + - 11 + - 58 + puzzles_discovered: 135 + puzzles_solved: 135 + game_cleared: true + cur_room_id: 78 + cur_puzzle_id: 139 + cur_puzzle_solved: 1 + cur_puzzle_retry: 0 + cur_puzzle_aborted: 0 + cur_script_type: 3 + return_script_type: 30 + puzzle_win_img: 0 + puzzle_flags: + 0: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 1: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 2: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 3: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 4: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 5: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 6: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 7: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 8: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 9: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 10: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 11: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 12: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 13: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 14: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 15: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 16: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 17: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 18: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 19: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 20: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 21: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 22: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 23: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 24: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 25: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 26: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 27: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 28: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 29: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 30: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 31: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 32: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 33: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 34: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 35: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 36: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 37: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 38: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 39: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 40: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 41: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 42: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 43: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 44: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 45: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 46: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 47: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 48: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 49: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 50: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 51: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 52: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 53: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 54: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 55: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 56: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 57: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 58: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 59: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 60: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 61: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 62: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 63: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 64: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 65: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 66: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 67: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 68: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 69: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 70: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 71: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 72: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 73: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 74: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 75: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 76: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 77: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 78: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 79: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 80: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 81: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 82: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 83: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 84: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 85: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 86: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 87: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 88: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 89: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 90: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 91: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 92: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 93: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 94: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 95: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 96: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 97: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 98: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 99: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 100: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 101: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 102: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 103: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 104: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 3 + 105: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 106: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 107: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 108: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 109: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 110: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 111: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 112: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 113: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 114: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 115: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 116: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 117: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 118: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 119: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 120: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 121: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 122: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 123: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 124: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 125: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 126: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 127: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 128: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 129: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 130: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 131: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 132: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 133: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 134: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 135: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 136: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 137: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 138: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 139: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 140: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 141: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 142: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 143: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 144: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 145: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 146: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 147: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 148: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 149: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 150: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 3 + 151: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 152: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 153: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 3 + 154: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 155: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 156: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 157: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 2 + 158: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 1 + 159: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 160: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 161: + discovered: true + solved: true + unk: true + hints_unlocked: 0 + fail_count: 0 + 162: + discovered: true + solved: true + unk: false + hints_unlocked: 0 + fail_count: 0 + 163: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 164: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 165: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 166: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 167: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 168: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 169: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 170: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 171: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 172: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 173: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 174: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 175: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 176: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 177: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 178: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 179: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 180: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 181: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 182: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 183: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 184: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 185: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 186: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 187: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 188: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 189: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 190: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 191: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 192: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 193: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 194: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 195: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 196: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 197: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 198: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 199: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 200: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 201: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 202: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 203: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 204: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 205: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 206: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 207: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 208: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 209: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 210: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 211: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 212: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 213: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 214: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 215: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 216: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 217: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 218: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 219: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 220: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 221: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 222: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 223: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 224: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 225: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 226: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 227: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 228: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 229: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 230: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 231: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 232: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 233: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 234: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 235: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 236: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 237: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 238: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 239: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 240: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 241: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 242: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 243: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 244: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 245: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 246: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 247: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 248: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 249: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 250: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 251: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 252: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 253: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 254: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + 255: + discovered: false + solved: false + unk: false + hints_unlocked: 0 + fail_count: 0 + cur_event: 259 + event_flags: + 0: false + 1: true + 2: true + 3: true + 4: true + 5: true + 6: true + 7: true + 8: true + 9: true + 10: true + 11: true + 12: true + 13: true + 14: true + 15: true + 16: true + 17: true + 18: true + 19: true + 20: true + 21: true + 22: true + 23: true + 24: true + 25: true + 26: true + 27: true + 28: true + 29: true + 30: true + 31: true + 32: true + 33: true + 34: false + 35: true + 36: true + 37: true + 38: false + 39: true + 40: false + 41: true + 42: true + 43: true + 44: true + 45: true + 46: true + 47: true + 48: true + 49: true + 50: true + 51: false + 52: true + 53: true + 54: true + 55: true + 56: true + 57: false + 58: true + 59: true + 60: true + 61: false + 62: true + 63: false + 64: true + 65: true + 66: true + 67: true + 68: true + 69: true + 70: true + 71: true + 72: false + 73: true + 74: true + 75: true + 76: true + 77: true + 78: true + 79: true + 80: true + 81: true + 82: true + 83: true + 84: true + 85: true + 86: true + 87: true + 88: true + 89: false + 90: true + 91: true + 92: true + 93: true + 94: true + 95: true + 96: true + 97: true + 98: true + 99: true + 100: true + 101: true + 102: true + 103: true + 104: true + 105: true + 106: true + 107: false + 108: true + 109: true + 110: true + 111: true + 112: true + 113: true + 114: true + 115: true + 116: true + 117: true + 118: true + 119: true + 120: true + 121: true + 122: true + 123: true + 124: true + 125: true + 126: true + 127: true + 128: true + 129: true + 130: true + 131: true + 132: true + 133: true + 134: true + 135: true + 136: true + 137: true + 138: true + 139: true + 140: false + 141: true + 142: false + 143: true + 144: true + 145: false + 146: true + 147: false + 148: false + 149: true + 150: true + 151: true + 152: true + 153: true + 154: false + 155: true + 156: true + 157: true + 158: true + 159: true + 160: true + 161: true + 162: true + 163: false + 164: true + 165: true + 166: true + 167: false + 168: true + 169: false + 170: false + 171: true + 172: true + 173: true + 174: true + 175: true + 176: true + 177: true + 178: true + 179: true + 180: true + 181: true + 182: true + 183: true + 184: false + 185: true + 186: true + 187: true + 188: true + 189: true + 190: true + 191: false + 192: true + 193: true + 194: true + 195: true + 196: true + 197: true + 198: false + 199: true + 200: true + 201: true + 202: true + 203: true + 204: true + 205: true + 206: false + 207: false + 208: true + 209: true + 210: true + 211: true + 212: true + 213: true + 214: true + 215: true + 216: true + 217: true + 218: true + 219: true + 220: true + 221: true + 222: true + 223: true + 224: true + 225: true + 226: true + 227: true + 228: true + 229: true + 230: true + 231: true + 232: true + 233: true + 234: true + 235: true + 236: true + 237: true + 238: true + 239: true + 240: true + 241: true + 242: true + 243: true + 244: false + 245: true + 246: true + 247: true + 248: true + 249: true + 250: true + 251: true + 252: true + 253: true + 254: true + 255: false + 256: true + 257: true + 258: true + 259: true + 260: true + 261: true + 262: true + 263: true + 264: true + 265: true + 266: true + 267: false + 268: false + 269: true + 270: true + 271: true + 272: true + 273: true + 274: true + 275: true + 276: true + 277: false + 278: true + 279: false + 280: false + 281: true + 282: true + 283: true + 284: false + 285: false + 286: false + 287: false + 288: true + 289: false + 290: true + 291: false + 292: false + 293: false + 294: false + 295: true + 296: false + 297: false + 298: false + 299: false + 300: true + 301: true + 302: false + 303: false + 304: false + 305: false + 306: false + 307: false + 308: false + 309: false + 310: false + 311: true + 312: true + 313: true + 314: false + 315: false + 316: true + 317: true + 318: true + 319: true + 320: true + 321: true + 322: true + 323: true + 324: true + 325: true + 326: true + 327: true + 328: true + 329: true + 330: true + 331: true + 332: true + 333: true + 334: true + 335: true + 336: true + 337: true + 338: true + 339: true + 340: true + 341: true + 342: true + 343: true + 344: true + 345: true + 346: false + 347: false + 348: false + 349: false + 350: true + 351: true + 352: true + 353: true + 354: true + 355: true + 356: true + 357: true + 358: true + 359: true + 360: true + 361: true + 362: true + 363: true + 364: true + 365: false + 366: false + 367: false + 368: false + 369: false + 370: false + 371: false + 372: false + 373: false + 374: false + 375: false + 376: false + 377: false + 378: false + 379: false + 380: false + 381: false + 382: false + 383: false + 384: false + 385: false + 386: false + 387: false + 388: false + 389: false + 390: false + 391: false + 392: false + 393: false + 394: false + 395: false + 396: false + 397: false + 398: false + 399: false + 400: false + 401: false + 402: false + 403: false + 404: false + 405: false + 406: false + 407: false + 408: false + 409: false + 410: false + 411: false + 412: false + 413: false + 414: false + 415: false + 416: false + 417: false + 418: false + 419: false + 420: false + 421: false + 422: false + 423: false + 424: false + 425: false + 426: false + 427: false + 428: false + 429: false + 430: false + 431: false + 432: false + 433: false + 434: false + 435: false + 436: false + 437: false + 438: false + 439: false + 440: false + 441: false + 442: false + 443: false + 444: false + 445: false + 446: false + 447: false + 448: false + 449: false + 450: false + 451: false + 452: false + 453: false + 454: false + 455: false + 456: false + 457: false + 458: false + 459: false + 460: false + 461: false + 462: false + 463: false + 464: false + 465: false + 466: false + 467: false + 468: false + 469: false + 470: false + 471: false + 472: false + 473: false + 474: false + 475: false + 476: false + 477: false + 478: false + 479: false + hint_coins: 93 + inventory: + - 24 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + story_flags: + dog_unlocked: true + painting_unlocked: true + hotelroom_unlocked: true + puzzle_at_nazobaba: {} + cur_objective: 45 + dog_parts_backlog: + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + dog_parts_placed: + 0: true + 1: true + 2: true + 3: true + 4: true + 5: true + 6: true + 7: true + 8: true + 9: true + 10: true + 11: true + 12: true + 13: true + 14: true + 15: true + 16: true + 17: true + 18: true + 19: true + 20: false + 21: false + 22: false + 23: false + 24: false + 25: false + 26: false + 27: false + 28: false + 29: false + 30: false + 31: false + puzzle_pieces_obtained: + 0: true + 1: true + 2: true + 3: true + 4: true + 5: true + 6: true + 7: true + 8: true + 9: true + 10: true + 11: true + 12: true + 13: true + 14: true + 15: true + 16: true + 17: true + 18: true + 19: true + puzzle_pieces_location: {} + hotelroom_items: + - 1 + - 2 + - 2 + - 1 + - 1 + - 1 + - 2 + - 2 + - 2 + - 1 + - 1 + - 1 + - 1 + - 1 + - 2 + - 2 + - 1 + - 1 + - null + - null + - 0 + - 1 + - 0 + - 1 + - 1 + - 0 + - 1 + - 1 + - 0 + - 0 + - 0 + - 0 + news_journal: 1 + news_dog: 0 + news_hotelroom: 0 + news_painting: 0 + news_mysteries: 0 + journal_entries_unread: + 0: false + 1: true + 2: true + 3: true + 4: true + 5: true + 6: true + 7: true + 8: true + 9: true + 10: true + 11: true + 12: true + 13: true + 14: true + 15: true + 16: true + 17: true + 18: true + 19: true + 20: true + 21: true + 22: true + 23: true + 24: true + 25: true + 26: true + 27: true + 28: true + 29: true + 30: true + 31: true + 32: true + 33: true + 34: true + 35: true + 36: true + 37: true + 38: true + 39: true + 40: true + 41: true + 42: true + 43: true + 44: true + 45: true + 46: false + 47: false + 48: false + 49: false + 50: false + 51: false + 52: false + 53: false + 54: false + 55: false + 56: false + 57: false + 58: false + 59: false + 60: false + 61: false + 62: false + 63: false + dog_name: pochi + unk1: 0 + unk2: 0 + unk3: 0 + unk4: 0 + unk5: 1 + unk6: 12 + unk7: + - 0 + - 0 + - 0 + - 1 + - 1 + - 0 + - 1 + - 1 + - 0 + - 1 + - 0 + - 0 + - 0 + - 0 + - 0 + - 1 + - 1 + - 1 + - 1 + - 0 + - 0 + - 1 + - 1 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + unk8: + - 192 + - 231 + - 95 + - 199 + - 136 + - 252 + - 106 + - 252 + - 17 + - 193 + - 0 + - 0 + - 48 + - 129 + - 10 + - 231 + - 166 + - 36 + - 140 + - 211 + - 194 + - 141 + - 117 + - 104 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + unk9: 5302 + unk10: + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + - 0 + unk_bitfield2: + - false + - false + - false + - false + - false + - false + - false + - false + - false + - false + - false + - false + - false + - false + - false + - false + unk11: 299 + unk12: 22 +- null +- null +cur_file: 0 +bonus_data: {} diff --git a/formats/gds.py b/formats/gds.py deleted file mode 100644 index 9a6ac9a..0000000 --- a/formats/gds.py +++ /dev/null @@ -1,245 +0,0 @@ -import click -import json -import os - -import parse -from utils import cli_file_pairs, foreach_file_pair -from version import v - -@click.group(help="Script-like format, also used to store puzzle parameters.",options_metavar='') -def cli(): - pass - -dir_path = "/".join(os.path.dirname(os.path.realpath(__file__).replace("\\", "/")).split("/")[:-1]) -commands = json.load(open(f"{dir_path}/data/commands.json", encoding="utf-8")) -commands_i = {val: key for key, val in commands.items()} # Inverted version of commands - -class GDSModeException (Exception): - def __init__(self, mode): - self.mode = mode - super().__init__( - f"'{mode}' is not a valid mode for GDS.__init__(): must be one of 'bin', 'json', 'gda'") - -class GDS: - def __init__(self, file, mode="bin"): #modes: "bin"/"b", "json"/"j", "gda"/"a" - if mode == "bin" or mode == "b": - self.from_gds(file) - elif mode == "json" or mode == "j": - self.from_json(file) - elif mode == "gda" or mode == "a": - self.from_old(file) - else: - raise GDSModeException(mode) - - def from_gds(self, file): - length = int.from_bytes(file[0:4], "little") - if file[4:6] == b"\x0c\x00": - self.cmds = [] - return - cmd_data = file[6:length+4] - cmds = [] - - cmd = None - params = [] - c = 0 - while True: - if c >= length: - raise Exception("GDS file error: End of file reached with no 0xC command!") - if cmd == None: - cmd = int.from_bytes(cmd_data[c:c+2], "little") - if cmd in commands_i: - cmd = commands_i[cmd] - c += 2 - continue - - p_type = int.from_bytes(cmd_data[c:c+2], "little") - - if p_type == 0: - cmds.append({"command":cmd, "parameters":params}) - cmd = None - params = [] - c += 2 - elif p_type == 1: - params.append({"type": "int", "data": int.from_bytes(cmd_data[c+2:c+6], "little")}) - c += 6 - elif p_type == 2: - params.append({"type": "unknown-2", "data": int.from_bytes(cmd_data[c+2:c+6], "little")}) - c += 6 - elif p_type == 3: - str_len = int.from_bytes(cmd_data[c+2:c+4], "little") - params.append({"type": "string", "data": cmd_data[c+4:c+4+str_len].decode("ascii").rstrip("\x00")}) #TODO: JP/KO compatibility - c += str_len+4 - elif p_type == 6: - params.append({"type": "unknown-6", "data": int.from_bytes(cmd_data[c+2:c+6], "little")}) - c += 6 - elif p_type == 7: - params.append({"type": "unknown-7", "data": int.from_bytes(cmd_data[c+2:c+6], "little")}) - c += 6 - elif p_type == 8: - params.append({"type": "unknown-8"}) - c += 2 - elif p_type == 9: - params.append({"type": "unknown-9"}) - c += 2 - elif p_type == 0xb: - params.append({"type": "unknown-b"}) - c += 2 - elif p_type == 0xc: - #cmd = hex(cmd) - cmds.append({"command":cmd, "parameters":params}) - break - else: - raise Exception(f"GDS file error: Invalid or unsupported parameter type {hex(p_type)}!") - - self.cmds = cmds - - def from_json (self, file): - self.cmds = json.loads(file)["data"] - #TODO: reject non-compatible json files - - def from_old (self, file): #TODO: make this, so gds_old can be completely removed - cmds = [] - - for line in file.split("\n"): - data = {} - if line.startswith("#"): - continue - - line, strings = parse.remove_strings(line) - line = line.rstrip().split(" ") - cmd = line[0] - - if cmd == "engine": - cmd = "0x1b" - elif cmd == "img_win": - cmd = "0x1f" - - cmd = int(cmd[2:], base=16) - params = [] - - for param in line[1:]: - if param.isdigit(): - params.append({"type":"int", "data":int(param)}) - elif param.startswith("0x"): - params.append({"type":"unknown-2", "data":int(param[2:], 16)}) - elif param.startswith('"') and param.endswith('"'): - param = strings[int(param[1:-1])] - params.append({"type":"string", "data":param}) - else: - raise Exception(f"Invalid GDA parameter: {param}") - - cmds.append({"command":cmd, "parameters":params}) - - self.cmds = cmds - - def __getitem__ (self, index): - index = int(index) - return self.cmds[index] - - def to_json (self): - return json.dumps({"version": v, "data": self.cmds}, indent=4) - - def to_gds (self): - out = b"\x00" * 2 - for command in self.cmds: - if type(command["command"]) == int: - out += command["command"].to_bytes(2, "little") - else: - out += commands[command["command"]].to_bytes(2, "little") - for param in command["parameters"]: - if param["type"] == "int": - out += b"\x01\x00" - out += param["data"].to_bytes(4, "little") - elif param["type"] == "unknown-2": - out += b"\x02\x00" - out += param["data"].to_bytes(4, "little") - elif param["type"] == "string": - out += b"\x03\x00" - out += (len(param["data"])+1).to_bytes(2, "little") - out += param["data"].encode("ASCII") + b"\x00" #TODO: JP/KO compatibility - elif param["type"] == "unknown-6": - out += b"\x06\x00" - out += param["data"].to_bytes(4, "little") - elif param["type"] == "unknown-7": - out += b"\x07\x00" - out += param["data"].to_bytes(4, "little") - elif param["type"] == "unknown-8": - out += b"\x08\x00" - elif param["type"] == "unknown-9": - out += b"\x09\x00" - elif param["type"] == "unknown-b": - out += b"\x0b\x00" - else: - raise Exception(f"GDS JSON error: Invalid or unsupported parameter type '{param['type']}'!") - out += b"\x00\x00" - out = out[:-2] + b"\x0c\x00" - - return len(out).to_bytes(4, "little") + out - - def to_bin (self): #alias - return self.to_gds() - -@cli.command( - name="extract", - no_args_is_help = False - ) -@click.argument("input", required=False, type=click.Path(exists=True)) -@click.argument("output", required=False, type=click.Path(exists=False)) -@click.option("--recursive", "-r", is_flag=True, help="Recurse into subdirectories of the input directory to find more applicable files.") -@click.option("--quiet", "-q", is_flag=True, help="Suppress all output. By default, operations involving multiple files will show a progressbar.") -def unpack_json(input = None, output = None, recursive = False, quiet = False): - """ - Converts the GDS script(s) at INPUT to JSON files at OUTPUT. - - INPUT can be a single file or a directory (which obviously has to exist). In the latter case subfiles with the correct file ending will be processed. - If unset, defaults to the current working directory. - - The meaning of OUTPUT may depend on INPUT: - - If INPUT is a file, then OUTPUT is expected to be a file, unless it explicitly ends with a slash indicating a directory. - In this case, if unset OUTPUT will default to the INPUT filename with `.json` exchanged/appended. - - Otherwise OUTPUT has to be a directory as well (or an error will be shown). - In this case, if unset OUTPUT will default to the INPUT directory (which may itself default to the current working directory). - - In the file-to-file case, the paths are explicitly used as they are. Otherwise, if multiple input files were collected, or OUTPUT is a directory, - an output path is inferred for each input file by exchanging the `.gds` file ending for `.json`, or otherwise appending the `.json` file ending. - """ - def process(input, output): - input = open(input, "rb").read() - output = open(output, "w", encoding="utf-8") - gds = GDS(input) - output.write(gds.to_json()) - output.close() - - pairs = cli_file_pairs(input, output, in_ending=".gds", out_ending=".json", recursive=recursive) - foreach_file_pair(pairs, process, quiet=quiet) - - -@cli.command( - name="create", - help="Converts a JSON to GDS.", - no_args_is_help = True - ) -@click.argument("input") -@click.argument("output") -def create_json(input, output): - input = open(input, encoding="utf-8").read() - output = open(output, "wb") - - gds = GDS(input, "json") - output.write(gds.to_bin()) - output.close() - -@cli.command( - name="gdaimport", - help="Creates a GDS JSON file from the old GDA format.", - no_args_is_help = True - ) -@click.argument("input") -@click.argument("output") -def create_from_gda(input, output): - input = open(input, encoding="utf-8").read() - output = open(output, "w", encoding="utf-8") - - gds = GDS(input, "gda") - output.write(gds.to_json()) - output.close() diff --git a/formats/gds/__init__.py b/formats/gds/__init__.py new file mode 100644 index 0000000..b355049 --- /dev/null +++ b/formats/gds/__init__.py @@ -0,0 +1,204 @@ +import click +import contextlib +import collections +import sys +import os + +import formats.gds.cmddef as cmddef +import formats.gds.gds as gds +import formats.gds.gda as gda +import formats.gds.patch as patch +from utils import cli_file_pairs, foreach_file_pair + +cmddef.init_commands() + + +@click.group( + help="Script-like format, also used to store puzzle parameters.", options_metavar="" +) +def cli(): + pass + + +@cli.command(name="compile", no_args_is_help=True) +@click.argument("input", required=False, type=click.Path(exists=True)) +@click.argument("output", required=False, type=click.Path(exists=False)) +@click.option( + "--recursive", + "-r", + is_flag=True, + help="Recurse into subdirectories of the input directory to find more applicable files.", +) +@click.option( + "--quiet", + "-q", + is_flag=True, + help="Suppress all output. By default, operations involving multiple files will show a progressbar.", +) +@click.option( + "--overwrite/--no-overwrite", + "-o/-O", + default=True, + help="Whether existing files should be overwritten. Default: true", +) +@click.option( + "--workspace", + "-w", + type=click.Path(dir_okay=True, file_okay=False, exists=True), + help="The path to the workspace directory, i.e. the root folder into which the NDS ROM was extracted. " + "Paths in scripts are relative to this. If unset, defaults to the current working directory (usually not what you want).", +) +def gds_compile(input=None, output=None, recursive=False, quiet=False, overwrite=None, workspace=None): + """ + Compiles the human-readable script(s) at INPUT into the game's binary script files at OUTPUT. + + INPUT can be a single file or a directory (which obviously has to exist). In the latter case subfiles with the correct file ending will be processed. + If unset, defaults to the current working directory. + + The meaning of OUTPUT may depend on INPUT: + - If INPUT is a file, then OUTPUT is expected to be a file, unless it explicitly ends with a slash indicating a directory. + In this case, if unset OUTPUT will default to the INPUT filename with `.gds` exchanged/appended. + - Otherwise OUTPUT has to be a directory as well (or an error will be shown). + In this case, if unset OUTPUT will default to the INPUT directory (which may itself default to the current working directory). + + In the file-to-file case, the paths are explicitly used as they are. Otherwise, if multiple input files were collected, or OUTPUT is a directory, + an output path is inferred for each input file by exchanging the input format's file ending for, or otherwise appending the `.gds` file ending. + + In the case where INPUT is a directory, if no format is specified, this command will collect all files with ending `.gda`. + """ + workdir = workspace or os.getcwd() + + def process(inpath, outpath): + with open(inpath, "r", encoding="utf-8") as inf: + filename = os.path.relpath(inpath, workdir) + prog = gda.read_gda(inf.read(), filename) + + with open(outpath, "wb") as outf: + outf.write(gds.write_gds(prog)) + + pairs = cli_file_pairs( + input, output, in_endings=[".gda"], out_ending=".gds", recursive=recursive + ) + + duplicates = collections.defaultdict(list) + for ip, op in pairs: + duplicates[op].append(ip) + duplicates = {k: v for k, v in duplicates.items() if len(v) > 1} + if duplicates: + print( + f"ERROR: {len(duplicates)} {'files have' if len(duplicates) > 1 else 'file has'} multiple conflicting source files; please explicitly specify a format to determine which should be used.", + file=sys.stderr, + ) + for op, ips in duplicates.items(): + pathlist = ", ".join(f"'{ip}'" for ip in ips) + print(f"'{op}' could be compiled from {pathlist}", file=sys.stderr) + sys.exit(-1) + + if not overwrite: + new_pairs = [] + existing = [] + for ip, op in pairs: + if os.path.exists(op): + existing.append(op) + else: + new_pairs.append((ip, op)) + + if not quiet: + print(f"Skipping {len(existing)} existing output files.") + + pairs = new_pairs + + foreach_file_pair(pairs, process, quiet=quiet) + + +@cli.command(name="decompile", no_args_is_help=True) +@click.argument("input", required=False, type=click.Path(exists=True)) +@click.argument("output", required=False, type=click.Path(exists=False)) +@click.option( + "--recursive", + "-r", + is_flag=True, + help="Recurse into subdirectories of the input directory to find more applicable files.", +) +@click.option( + "--quiet", + "-q", + is_flag=True, + help="Suppress all output. By default, operations involving multiple files will show a progressbar.", +) +@click.option( + "--overwrite/--no-overwrite", + "-o/-O", + default=True, + help="Whether existing files should be overwritten. Default: true", +) +@click.option( + "--workspace", + "-w", + type=click.Path(dir_okay=True, file_okay=False, exists=True), + help="The path to the workspace directory, i.e. the root folder into which the NDS ROM was extracted. " + "Paths in scripts are relative to this. If unset, defaults to the current working directory (usually not what you want).", +) +@click.option( + "--patches/--no-patches", + "-p/-P", + default=True, + help="Whether to apply 'baseline patches' that fix known errors in the vanilla scripts. Which patch to apply is determined " + "by the filename relative to the workspace directory. Default: true; note that some of the vanilla scripts WILL NOT PARSE without " + "these patches, so when in doubt, keep it enabled." +) +def gds_decompile( + input=None, output=None, recursive=False, quiet=False, overwrite=None, workspace=None, patches=None +): + """ + Decompiles the GDS script(s) at INPUT into the human-readable GDA script format at OUTPUT. + + INPUT can be a single file or a directory (which obviously has to exist). In the latter case subfiles with the correct file ending will be processed. + If unset, defaults to the current working directory. + + The meaning of OUTPUT may depend on INPUT: + - If INPUT is a file, then OUTPUT is expected to be a file, unless it explicitly ends with a slash indicating a directory. + In this case, if unset OUTPUT will default to the INPUT filename with `.gda` exchanged/appended. + - Otherwise OUTPUT has to be a directory as well (or an error will be shown). + In this case, if unset OUTPUT will default to the INPUT directory (which may itself default to the current working directory). + + In the file-to-file case, the paths are explicitly used as they are. Otherwise, if multiple input files were collected, or OUTPUT is a directory, + an output path is inferred for each input file by exchanging the `.gds` file ending for `.gda`, or otherwise appending the `.gda` file ending. + """ + # TODO: once/if workspaces are a thing, use auto-discovery to place the file in the workspace correctly + workdir = workspace or os.getcwd() + + def process(inpath, outpath): + try: + filepath = os.path.relpath(inpath, workdir) + with open(inpath, "rb") as inf: + indata = inf.read() + if patches: + indata = patch.patch(indata, filepath) + prog = gds.read_gds(indata, inpath) + + with open(outpath, "w", encoding="utf-8") as outf: + # TODO: determine the working directory? somehow?? + outf.write(gda.write_gda(prog, filepath, workdir)) + except KeyboardInterrupt: + raise + except Exception as e: + print(f"ERR: {inpath}: could not decompile: {e}") + + pairs = cli_file_pairs( + input, output, in_endings=[".gds"], out_ending=".gda", recursive=recursive + ) + if not overwrite: + new_pairs = [] + existing = [] + for ip, op in pairs: + if os.path.exists(op): + existing.append(op) + else: + new_pairs.append((ip, op)) + + if not quiet: + print(f"Skipping {len(existing)} existing output files.") + + pairs = new_pairs + foreach_file_pair(pairs, process, quiet=quiet) diff --git a/formats/gds/_test/sub/subby.yaml b/formats/gds/_test/sub/subby.yaml new file mode 100644 index 0000000..3db6426 --- /dev/null +++ b/formats/gds/_test/sub/subby.yaml @@ -0,0 +1,7 @@ +version: 0.1 +prefix: "nope." + +commands: + hi: + id: 0x24 + desc: hi \ No newline at end of file diff --git a/formats/gds/_test/test.yml b/formats/gds/_test/test.yml new file mode 100644 index 0000000..e511dfb --- /dev/null +++ b/formats/gds/_test/test.yml @@ -0,0 +1,39 @@ +version: 0.1 +prefix: null +context: all + +commands: + 0x01: + name: test + desc: Test Command + condition: false + uncertain: false + params: + - int + - type: int + name: testparam + desc: some number + uncertain: false + test2: + id: 0x02 + desc: Test Command 2 + context: ["event", "puzzle"] + aliases: + - tc2 + condition: true + uncertain: true + params: + unk_1: string + unk_2: + type: int + uncertain: true + optional: true + +groups: + ouchie: + commands: + nice: + id: 0x69 + desc: nice + 0x74: + 0x75: \ No newline at end of file diff --git a/formats/gds/cmddef.py b/formats/gds/cmddef.py new file mode 100755 index 0000000..eba6736 --- /dev/null +++ b/formats/gds/cmddef.py @@ -0,0 +1,405 @@ +#!/usr/bin/env python3 + +from typing import Optional, List, Union, Set, Any, Mapping +import os +from dataclasses import dataclass, field + +import logging + +import yaml +from dacite import from_dict + +from utils import RESOURCES +import formats.gds.value as value + + +logger = logging.getLogger("flora.debug.gds.cmddef") + + +@dataclass +class GDSCommandParam: + cmd: int + """The command for which this is a parameter""" + idx: int + """The index of the parameter in order""" + type: "value.GDSValueType" + """ + The data type of the command. Real builtin GDS datatypes are: + int (1) + (also usable as uint, short, ushort, byte, ubyte; TODO: document which parameters use which) + float (2) + string (3) + longstr (4, never used) + + Our GDA framework also defines some auxiliary datatypes: + bool: backed by either an int (true = nonzero) or a string (true = "true") + bool|int: backed by an int (true = nonzero) + bool|string: backed by a string (true = "true") + + TODO: Compound datatypes may make the code more understandable, by combining semantically + related parameters into a single one: + pt(x: int, y: int) + rect(x: int, y: int, w: int, h: int) + ... + """ + + name: Optional[str] = None + """A descriptive name for the parameter. Can optionally be written in the script.""" + desc: Optional[str] = None + """ + Description of what the parameter means. + """ + uncertain: bool = False + """ + Purely informative: whether research is conclusive about the meaning of the parameter. + The fact that the parameter exists is NOT uncertain, but its meaning may not be clear yet. + Parameters without a name are normally uncertain, even when they don't mark it explicitly. + """ + optional: bool = False + """ + A small number of commands have optional parameters at the end of their argument list: + they check if the next token (see general info about GDS) has the expected type, and if + not stops reading the argument list. + """ + + +@dataclass +class GDSCommand: + id: int + """ID of the command, as it appears in binary scripts""" + + name: Optional[str] = None + """Human-readable name, as it would appear in the decompiled assembly""" + aliases: list[str] = field(default_factory=lambda: []) + """ + Other names this command may be recognized by when compiling scripts, + for example a naming convention from a previous decompiler version + """ + + desc: Optional[str] = None + """ + Description of what the command does, where it's used etc. + """ + uncertain: bool = False + """ + Purely informative: whether the research is conclusive about the meaning, purpose, functionality + etc of this command. Note that the structural properties below are obvious to read off from + disassembly, and are therefore not subject to this; it purely qualifies our understanding + of the command's meaning and usage. + """ + comment: Optional[str] = None + """ + A format string for a comment that should be placed above every execution of this command in + decompiled scripts. Useful to give the reader additional context, such as the path to a file + referenced by an abstract numeric parameter, or even the contents of said file! + """ + + condition: bool = False + """ + True if the command sets the condition flag. Those commands usually don't have side effects. + """ + context: Set[str] = field(default_factory=lambda: {"all"}) + """ + (list of) context name(s) where the command is defined. The game contains many different + script engines called in different contexts, and while they all run the same type of script, + most commands are noops in engines they weren't meant to be used in. + + The special string "all" denotes that a command is implemented everywhere (for example, + if it doesn't proxy through the script engine itself but directly to the sound system etc.) + """ + + params: list[GDSCommandParam] = field(default_factory=lambda: []) + """ + List of parameters the command accepts, if it's a simple command with a fixed parameter list. + """ + complex: bool = False + """ + Set to true for special structural commands that don't have a fixed parameter list, + but instead need to be parsed in a more complex way. The logic for that will be looked + up in Python code. + """ + + file: Optional[str] = None + """ + The file path where the command is defined; for debug purposes + """ + + +CONTEXTS = ["all", "event", "room", "puzzle"] + + +def load_group( + data: dict, + group_name: str = None, + parent_prefix: str = None, + parent_context: Union[str, List[str]] = None, + filename: str = None, +) -> List[GDSCommand]: # sourcery skip: remove-pass-body + """ + Reads a single group object from the command definition YAML files. That group might be the top-level file itself. + """ + if "prefix" in data: + prefix = data["prefix"] + else: + prefix = parent_prefix or "" + if group_name is not None: + prefix += f"{group_name}." + + context = data["context"] if "context" in data else parent_context + commands = data["commands"] + if isinstance(commands, dict): + + def command_kv(k, v): + if v is None: + v = {} + cid = v.get("id") + name = v.get("name") + if isinstance(k, int): + if cid is not None: + if k != cid: + raise ValueError( + "id field specified in a GDSCommand listed by id, and the two values didn't match" + ) + logger.warning( + "id field specified in a GDSCommand listed by id is redundant" + ) + cid = k + elif isinstance(k, str): + if name is not None: + if k != name: + raise ValueError( + "name field specified in a GDSCommand listed by name, and the two values didn't match" + ) + logger.warning( + "name field specified in a GDSCommand listed by name is redundant" + ) + # TODO: handle hex ID in string? + name = k + else: + raise ValueError( + "GDSCommands can only be listed by their name or numeric id" + ) + v["id"] = cid + v["name"] = name + return v + + commands = [command_kv(k, v) for k, v in commands.items()] + + def process_command(c): + if c.get("id") is None: + raise ValueError("GDSCommand must define an id") + + if "context" not in c: + c["context"] = context or "all" + if isinstance(c["context"], str): + c["context"] = [c["context"]] + c["context"] = set(c["context"]) + if "prefix" not in c: + c["prefix"] = prefix + if c["prefix"] is not None and c["name"] is not None: + c["name"] = c["prefix"] + c["name"] + + params = c.get("params") or [] + if isinstance(params, dict): + + def param_kv(k, v): + # have to do that here already, otherwise I don't have any object to put the name into + if isinstance(v, str): + v = {"type": v} + if isinstance(k, int): + # specifying a number here means we don't want a name + pass + else: + if "name" in v: + # pylint: disable=logging-not-lazy + logger.warning( + "name field specified in a parameter that was listed by name" + + (", and did not match" if k != v["name"] else "") + ) + + v["name"] = k + return v + + params = [param_kv(k, v) for k, v in params.items()] + + def process_param(idx, p): + if isinstance(p, str): + p = {"type": p} + p["cmd"] = c["id"] + p["idx"] = idx + + p["type"] = value.parse_type(p["type"]) + + return from_dict(data_class=GDSCommandParam, data=p) + + c["params"] = [process_param(i, p) for i, p in enumerate(params)] + + c["file"] = filename + + return from_dict(data_class=GDSCommand, data=c) + + commands = [process_command(v) for v in commands] + + groups = data.get("groups") + if isinstance(groups, dict): + for k, v in groups.items(): + commands.extend(load_group(v, k, prefix, context, filename)) + elif groups is not None: + logger.warning( + "groups key has to be a dictionary mapping a group name to a command list" + ) + + return commands + + +def load_file(filename: str, root: str) -> list[GDSCommand]: + """ + Reads a single command definition YAML file. Default prefix is the directory structure relative to the root, if specified. + """ + with open(filename, encoding="utf8") as f: + data = yaml.safe_load(f) + + # version = data.get("version") + + if root is not None: + cur = os.path.splitext(os.path.relpath(filename, root))[0] + cur, group_name = os.path.split(cur) + parent_prefix = "" + while cur != "": + cur, seg = os.path.split(cur) + parent_prefix = f"{seg}.{parent_prefix}" + else: + parent_prefix = None + group_name = None + + return load_group(data, group_name, parent_prefix, None, filename) + + +def load_cmdrepo(root: str) -> list[GDSCommand]: + """ + Reads all the command definitions from the YAML files in the specified directory, and returns them as a list of data objects. + Only very basic sanity checking internal to each command definition is performed by this point. + """ + commands = [] + for r, _dirs, files in os.walk(root): + for f in files: + if f.lower().endswith("yml") or f.lower().endswith("yaml"): + path = os.path.join(r, f) + commands.extend(load_file(path, root)) + return commands + + +def build_maps( + cmds: list[GDSCommand], +) -> tuple[dict[int, GDSCommand], dict[str, GDSCommand]]: + """ + Structures a list of commands into a bidirectional lookup table by command ID and human-readable name. + """ + by_id = {} + by_name = {} + + for cmd in cmds: + if cmd.id in by_id: + raise ValueError(f"Command id {hex(cmd.id)} defined twice") + by_id[cmd.id] = cmd + + if cmd.name in by_name: + existing = by_name[cmd.name] + if not isinstance(existing, GDSCommand): + other = '", "'.join(c.name for c in existing["ALIAS_CONFLICT"]) + logger.warning( + 'Command name "%s" is already used as alias by commands "%s". ' + "The former will take precedence. Please make sure to update any GDA files decompiled " + "with a previous version of Flora.", + cmd.name, + other, + ) + elif existing.name != cmd.name: + logger.warning( + 'Command "%s" has alias "%s", which is also a command name. ' + "The latter will take precedence. Please make sure to update any GDA files decompiled " + "with a previous version of Flora.", + existing.name, + cmd.name, + ) + else: + raise ValueError(f'Command name "{cmd.name}" defined twice') + + if cmd.name is None: + continue + by_name[cmd.name] = cmd + for alias in cmd.aliases: + if alias in by_name: + existing = by_name[alias] + if not isinstance(existing, GDSCommand): + existing["ALIAS_CONFLICT"].append(cmd) + elif existing.name == alias: + logger.warning( + 'Command "%s" has alias "%s", which is also a command name. ' + "The latter will take precedence. Please make sure to update any GDA files decompiled " + "with a previous version of Flora.", + cmd.name, + alias, + ) + else: + by_name[alias] = {"ALIAS_CONFLICT": [existing, cmd]} + else: + by_name[alias] = cmd + + for k, v in by_name.items(): + if not isinstance(v, GDSCommand): + other = '", "'.join(c.name for c in existing["ALIAS_CONFLICT"]) + logger.warning( + 'Commands "{other}" all define alias "%s"; none of them will be registered.', + k, + ) + del by_name[k] + + return (by_id, by_name) + + +COMMANDS_BYNAME = {} +COMMANDS_BYID = {} + +CMDDEF_ROOT = os.path.join(RESOURCES, "gds_commands") + + +def init_commands(root: str = None): + global COMMANDS_BYID, COMMANDS_BYNAME # pylint: disable=global-statement + if root is None: + root = CMDDEF_ROOT + try: + commands = load_cmdrepo(root) + COMMANDS_BYID, COMMANDS_BYNAME = build_maps(commands) + except ValueError as e: + logger.error(e) + print( + "An error occured when reading command definitions from metadata. Your installation of Flora might be corrupted. If you're sure it's not, " + "you can file an issue on GitHub: https://github.com/patataofcourse/Flora" + ) + return + + if undefined_commands := [i for i in range(0x100) if i not in COMMANDS_BYID]: + logger.warning( + "The definition for the following commands is missing:\n%s\nThis may cause significant soundness issues in the (de)compiler!" + "Please make sure to correctly identify and define the parameter counts for these commands.", + ", ".join(hex(i) for i in undefined_commands), + ) + print( + "An error occured when reading command definitions from metadata. Your installation of Flora might be corrupted. If you're sure it's not, " + "you can file an issue on GitHub: https://github.com/patataofcourse/Flora" + ) + + +# sourcery skip: hoist-statement-from-if +if __name__ == "__main__": + logging.basicConfig() + + # TODO: unit tests just to make sure this function's understanding of syntax is still correct + # init_commands("formats/gds/_test") + init_commands() + + import pprint + + pprint.pp(COMMANDS_BYID) + pprint.pp(COMMANDS_BYNAME) diff --git a/formats/gds/dump.py b/formats/gds/dump.py new file mode 100644 index 0000000..0735cce --- /dev/null +++ b/formats/gds/dump.py @@ -0,0 +1,78 @@ +""" +Legacy support for writing a read GDS program to JSON or YAML. + +This doesn't seem very useful now, since the format and internal +representation have changed drastically. I still wanted to add +it just in case. +""" + +import json +from typing import List, Dict + +import version +import formats.gds.model as model + + +def dump_command(cmd: model.GDSInvocation) -> Dict: + # sourcery skip: remove-redundant-pass + obj = { + "command": cmd.command.name or cmd.command.id, + } + + if isinstance(cmd, (model.GDSIfInvocation, model.GDSLoopInvocation)): + if cmd.condition is not None: + if isinstance(cmd.condition, int): + obj["loop_cnt"] = cmd.condition + else: + cond = [] + + for c in cmd.condition: + if c in model.GDSConditionToken.AND: + cond.append({"flag": "and"}) + elif c in model.GDSConditionToken.OR: + cond.append({"flag": "or"}) + elif c in model.GDSConditionToken.NOT: + cond.append({"flag": "not"}) + elif c in model.GDSConditionToken.command: + cond.append(dump_command(c())) + else: + # unreachable + pass + + obj["condition"] = cond + if cmd.block is not None: + obj["block"] = dump_block(cmd.block) + + else: + obj["args"] = [ + {"type": arg.type.describe(), "value": arg.value} for arg in cmd.args + ] + + return obj + + +def dump_block(block: List[model.GDSElement]) -> List: + els = [] + + for el in block: + if el in model.GDSElement.command: + els.append(dump_command(el())) + elif el in model.GDSElement.BREAK: + els.append({"flag": "break"}) + elif el in model.GDSElement.label: + lbl = {"name": el().name, "present": el().present} + if el().loc is not None: + lbl["loc"] = el().loc + els.append({"label": lbl}) + + return els + + +def dump_json(prog: model.GDSProgram) -> str: + return json.dumps( + {"version": version.__version__, "data": dump_block(prog.elements)} + ) + + +def load_json(json: str) -> model.GDSProgram: + pass diff --git a/formats/gds/gda.py b/formats/gds/gda.py new file mode 100644 index 0000000..042c8e2 --- /dev/null +++ b/formats/gds/gda.py @@ -0,0 +1,618 @@ +""" +Provides the `read_gda` and `write_gda` methods to read/write a `GDSProgram` (see the `model` module) from/to a a human-readable GDA text file (buffer). + +Requires the command definitions provided by `cmddef` for type checking and parameter data, as well as (possibly in the future) documentation features. +""" + +import contextlib +from typing import Any, Optional, List, Mapping, Union +from dataclasses import dataclass, field + +from parsy import ( + forward_declaration, + regex, + seq, + string, + generate, + eof, + Parser, + success, + fail, + peek, + Result as ParseResult, +) + +import formats.gds.value as value +import formats.gds.cmddef as cmddef +from .model import ( + GDSJumpAddress, + GDSProgram, + GDSElement, + GDSLabel, + GDSInvocation, + GDSIfInvocation, + GDSLoopInvocation, + GDSConditionToken, +) + +# === PARSER === + + +@Parser +def peek_parser_state(stream, index): + print(stream[index:]) + return ParseResult.success(index, None) + + +ws = regex(r"[^\S\r\n]+") +nl = regex(r"\n\s*") | eof.result("\n") +comment = string("#") >> regex(r".*") << nl + + +def has_nl(res: str) -> Parser: + return success(res) if "\n" in res else fail("newline required") + + +# TODO: make at least one nl required here +ws_nl = ( + ((regex(r"\n\s*") | ws | comment.map(lambda v: v + "\n")).many()) + .concat() + .bind(has_nl) +) +lexeme = lambda p: p << ws.optional() + +lparen = lexeme(string("(")) +rparen = lexeme(string(")")) +lbrace = lexeme(string("{")) +rbrace = lexeme(string("}")) +colon = lexeme(string(":")) +comma = lexeme(string(",")) +eqsign = lexeme(string("=")) + +BREAK = lexeme(string("break")) +NOT = lexeme(string("not")) +AND = lexeme(string("and")) +OR = lexeme(string("or")) + +meta_comment = ( + lexeme(string("#!")) + >> ( + string("version") + >> ws + >> lexeme(regex(r"[\w\.]+")).map(lambda v: ("version", v)) + ) + | regex(r".*").map(lambda v: (None, v)) << nl +) + + +command_name = lexeme( + regex(r"[a-zA-Z_][\w\.]*") | regex(r"0x[0-9a-fA-F]{1,2}").map(lambda i: int(i, 16)) +) + + +@generate +def parse_condition(): + conds = [] + while True: + kw = yield ( + NOT.result(GDSConditionToken.NOT()) + | AND.result(GDSConditionToken.AND()) + | OR.result(GDSConditionToken.OR()) + ).optional() + if kw is not None: + conds.append(kw) + continue + cmd = yield parse_command.map(GDSConditionToken.command).optional() + if cmd is None: + break + conds.append(cmd) + return conds + + +@generate +def parse_label(): + yield string("@") + present = yield string("!").result(False).optional(True) + name = yield lexeme(regex("[\w]+")) + loc = None + if (yield lparen.optional()) is not None: + loc = yield lexeme( + regex(r"0x[0-9a-fA-F]+").map(lambda v: int(v, 16)) | regex(r"\d+").map(int) + ).optional() + yield rparen + + return GDSLabel(name=name, present=present, loc=loc) + + +@generate +def parse_addr(): + yield string("@") + primary = yield string("!").result(False).optional(True) + label = yield lexeme(regex("[\w]+")) + return GDSJumpAddress(label=label, primary=primary) + + +@generate +def parse_block(): + yield lbrace + yield ws_nl.optional() + block = [] + while True: + el = yield parse_element | peek(rbrace).result(None) | eof.result(None) + if el is None: + break + block.append(el) + yield ws_nl + yield rbrace + return block + + +def parse_simple(cmdobj: cmddef.GDSCommand) -> Parser: + # pylint: disable=redefined-outer-name + @generate + def parse_simple(): + if next((p for p in cmdobj.params if not p.optional), None) is None: + if (yield lparen.result(False).optional(True)): + return GDSInvocation(command=cmdobj, args=[]) + + yield lparen + + args = [] + for param in cmdobj.params: + p = param.type.parser() + if param.optional: + p = p.optional() + value = yield lexeme(p) + if value is None and not param.optional: + raise ValueError(f"Could not read parameter {param.name}") + args.append(value) + + yield rparen + + return GDSInvocation(command=cmdobj, args=args) + + return parse_simple + + +def parse_if(cmdobj: cmddef.GDSCommand) -> Parser: + # pylint: disable=redefined-outer-name + @generate + def parse_if(): + condition = yield parse_condition + + yield colon + + target = yield parse_addr.optional() + block = None + if target is None: + block = yield parse_block + + return GDSIfInvocation( + command=cmdobj, + args=[], + condition=condition, + target=target, + block=block, + elseif=cmdobj.id == 0x16, + elze=False, + ) + + return parse_if + + +def parse_else(cmdobj: cmddef.GDSCommand) -> Parser: + # pylint: disable=redefined-outer-name + @generate + def parse_else(): + yield colon + + target = yield parse_addr.optional() + block = None + if target is None: + block = yield parse_block + + return GDSIfInvocation( + command=cmdobj, + args=[], + condition=None, + target=target, + block=block, + elseif=False, + elze=True, + ) + + return parse_else + + +def parse_repeatN(cmdobj: cmddef.GDSCommand) -> Parser: + # pylint: disable=redefined-outer-name + @generate + def parse_repeatN(): + condition = yield value.GDSIntType(4, True).parser() + + yield colon + + target = yield parse_addr.optional() + block = None + if target is None: + block = yield parse_block + + return GDSLoopInvocation( + command=cmdobj, + args=[], + condition=condition, + target=target, + block=block, + ) + + return parse_repeatN + + +def parse_while(cmdobj: cmddef.GDSCommand) -> Parser: + # pylint: disable=redefined-outer-name + @generate + def parse_while(): + condition = yield parse_condition + + yield colon + + target = yield parse_addr.optional() + block = None + if target is None: + block = yield parse_block + + return GDSLoopInvocation( + command=cmdobj, + args=[], + condition=condition, + target=target, + block=block, + ) + + return parse_while + + +@generate +def parse_command(): + name = yield command_name + cmdobj = cmddef.COMMANDS_BYNAME.get(name) or cmddef.COMMANDS_BYID.get(name) + if cmdobj is None: + raise ValueError(f"Command {name} not defined") + + if cmdobj.complex: + cmd = yield PARSE_COMPLEX.get(cmdobj.name)(cmdobj) or PARSE_COMPLEX.get( + cmdobj.id + )(cmdobj) + else: + cmd = yield parse_simple(cmdobj) + + return cmd + + +PARSE_COMPLEX = { + "if": parse_if, + "elif": parse_if, + "else": parse_else, + "repeatN": parse_repeatN, + "while": parse_while, +} + + +@generate +def parse_element(): + kw = yield BREAK.result(GDSElement.BREAK()).optional() + if kw is not None: + return kw + label = yield parse_label.optional() + if label is not None: + return GDSElement.label(label) + cmd = yield parse_command + return GDSElement.command(cmd) + + +@generate +def parse_gda_source(): + version = yield meta_comment.map( + lambda v: v[1] if v[0] == "version" else None + ).optional() + + elements = [] + while True: + yield ws_nl.many() + yield ws.optional() + # yield peek_parser_state + el = yield parse_element | eof.result(None) + if el is None: + break + elements.append(el) + yield ws_nl | eof + return (version, elements) + + +def read_gda(data: str, path: Optional[str] = None) -> GDSProgram: + version, elements = parse_gda_source.parse(data) + + def collect_labels(elements: List[GDSElement], labels: Mapping[str, List[Union[GDSLabel, GDSJumpAddress]]] = None): + if labels is None: + labels = {} + for el in elements: + if el in GDSElement.label: + name = el().name + if name not in labels: + labels[name] = [] + labels[name].append(el()) + elif el in GDSElement.command and isinstance(el(), (GDSIfInvocation, GDSLoopInvocation)): + block_cmd = el() + if block_cmd.target is not None: + name = block_cmd.target.label + if name not in labels: + labels[name] = [] + labels[name].append(block_cmd.target) + + collect_labels(block_cmd.block, labels) + return labels + + labels = collect_labels(elements) + + return GDSProgram(path=path, elements=elements, labels=labels) + + +# === WRITER === + + +@dataclass(kw_only=True) +class WriterContext: + filename: Optional[str] = None + workdir: Optional[str] = None + result: str = "" + indent: int = 0 + + def write(self, s: str): + self.result += s + + def nl(self): + self.result += "\n" + " " * self.indent + + # pylint: disable=redefined-outer-name + def insert_comment(self, comment: str): + self.result, latest_line = self.result.rsplit("\n", 1) + for line in comment.splitlines(): + self.nl() + self.write(f"# {line}") + self.write("\n" + latest_line) + + +def write_gda( + prog: GDSProgram, filename: Optional[str] = None, workdir: Optional[str] = None +) -> str: + ctx = WriterContext(filename=filename, workdir=workdir) + version = "0.1" + ctx.write(f"#!version {version}\n") + ctx.write(f"# {filename}\n") + + for el in prog.elements: + ctx.nl() + write_element(ctx, el) + return ctx.result + + +def write_element(ctx: WriterContext, el: GDSElement): + if el in GDSElement.BREAK: + ctx.write("break") + elif el in GDSElement.label: + lbl: GDSLabel = el() + ctx.write(f'{"@" if lbl.present else "@!"}{lbl.name}') + if lbl.loc is not None: + ctx.write(f"({lbl.loc})") + elif el in GDSElement.command: + cmd: GDSInvocation = el() + ctx.write(cmd.command.name or hex(cmd.command.id)) + if cmd.command.complex: + (WRITE_COMPLEX.get(cmd.command.name) or WRITE_COMPLEX.get(cmd.command.id))( + ctx, cmd + ) + else: + write_simple(ctx, cmd) + + +def write_simple(ctx: WriterContext, cmd: GDSInvocation): + if len(cmd.args) == 0: + return + ctx.write(" (") + ctx.write(" ".join(arg.write() for arg in cmd.args if arg is not None)) + ctx.write(")") + + if cmd.command.comment is not None: + ctx.insert_comment( + format_comment( + cmd.command.comment, + CommentContext( + args=cmd.args, filename=ctx.filename, workdir=ctx.workdir + ), + ) + ) + + +def write_condition(ctx: WriterContext, cond: List[GDSConditionToken]): + first = True + for tok in cond: + if first: + first = False + else: + ctx.write(" ") + if tok in GDSConditionToken.NOT: + ctx.write("not") + elif tok in GDSConditionToken.AND: + ctx.write("and") + elif tok in GDSConditionToken.OR: + ctx.write("or") + elif tok in GDSConditionToken.command: + cmd: GDSInvocation = tok() + ctx.write(cmd.command.name or hex(cmd.command.id)) + if cmd.command.complex: + ( + WRITE_COMPLEX.get(cmd.command.name) + or WRITE_COMPLEX.get(cmd.command.id) + )(ctx, cmd) + else: + write_simple(ctx, cmd) + + +def write_block(ctx: WriterContext, cond: List[GDSElement]): + ctx.write(" {") + ctx.indent += 2 + for el in cond: + ctx.nl() + write_element(ctx, el) + ctx.indent -= 2 + ctx.nl() + ctx.write("}") + + +def write_if(ctx: WriterContext, cmd: GDSIfInvocation): + ctx.write(" ") + write_condition(ctx, cmd.condition) + ctx.write(":") + if cmd.target is not None: + ctx.write(f' {"@" if cmd.target.primary else "@!"}{cmd.target.label}') + if cmd.block is not None: + write_block(ctx, cmd.block) + + +def write_else(ctx: WriterContext, cmd: GDSIfInvocation): + ctx.write(":") + if cmd.target is not None: + ctx.write(f' {"@" if cmd.target.primary else "@!"}{cmd.target.label}') + if cmd.block is not None: + write_block(ctx, cmd.block) + + +def write_repeatN(ctx: WriterContext, cmd: GDSLoopInvocation): + count: int = cmd.condition + ctx.write(f" {count}:") + if cmd.target is not None: + ctx.write(f' {"@" if cmd.target.primary else "@!"}{cmd.target.label}') + if cmd.block is not None: + write_block(ctx, cmd.block) + + +def write_while(ctx: WriterContext, cmd: GDSLoopInvocation): + ctx.write(" ") + write_condition(ctx, cmd.condition) + ctx.write(":") + if cmd.target is not None: + ctx.write(f' {"@" if cmd.target.primary else "@!"}{cmd.target.label}') + if cmd.block is not None: + write_block(ctx, cmd.block) + + +WRITE_COMPLEX = { + "if": write_if, + "elif": write_if, + "else": write_else, + "repeatN": write_repeatN, + "while": write_while, +} + +import os + +@dataclass +class CommentContext: + filename: Optional[str] = None + workdir: Optional[str] = None + args: List[value.GDSValue] = field(default_factory=list) + omit_file_contents: bool = False + lang: Optional[str] = "en" + + +# pylint: disable=redefined-outer-name +def format_comment(comment: str, ctx: CommentContext): + if ctx.filename is not None and os.path.isabs(ctx.filename): + ctx.filename = os.path.relpath(ctx.filename, "/") + # sourcery skip: assign-if-exp + import re + + def get_var(name: str) -> Any: + if name.lower() == "lang": + return ctx.lang or "??" + if name.lower() == "eventid": + if ctx.filename is None: + return "?" + match: re.Match = re.match(r"data/script/event/e(\d+).gd[as]", ctx.filename) + if match: + return int(match.group(1)) + return "?" + with contextlib.suppress(ValueError): + arg_id = int(name) - 1 + if arg_id < 0 or arg_id >= len(ctx.args): + return "?" + + arg: value.GDSValue = ctx.args[arg_id] + return arg.value + + return "?" + + def format_value(val: Any, modifiers: List[str]) -> str: + for m in modifiers: + if not m.startswith("r"): + continue + step, max_, *_ = m[1:].split("<=") + [None] + if not isinstance(val, int): + val = "?" + break + step = int(step) + val = (val // step) * step + if max_ is not None: + val = min(val, int(max_)) + + for m in modifiers: + if not m.startswith("0"): + continue + l = int(m[1:]) + if not isinstance(val, int): + val = "?" * l + break + val = str(val) + val = "0" * max(0, l - len(val)) + val + + return str(val) + + def readfile(path: str) -> str: + if path is None or ctx.workdir is None or ctx.omit_file_contents: + return "<...>" + if os.path.isabs(path): + path = os.path.relpath(path, "/") + realpath = os.path.join(ctx.workdir, path) + try: + with open(realpath, encoding="utf8") as f: + return f.read() + except FileNotFoundError: + return "" + + str_part = regex(r"[^$]") + str_esc = string(r"$$") + # str_part_fvar = regex(r"[^$}:]") + str_part_expr = regex(r"[^$)]") + # fvar_esc = string(r"}}") | string(r"::") + expr_esc = string(r"))") + + format_args = regex(r"0\d+") | regex(r"r\d+(<=\d+)?") + fvar = forward_declaration() + expr = forward_declaration() + str_var = string("$") >> ( + string("{") >> fvar << string("}") | string("(") >> expr << string(")") + ) + + varname = regex(r"\w+") + + # fvar_expr = (str_part_fvar | fvar_esc | str_var).many().concat() + fvar.become( + seq(varname.map(get_var), (string(":") >> format_args).many()).map( + lambda a: format_value(a[0], a[1]) + ) + ) + expr.become((str_part_expr | expr_esc | str_var).many().concat().map(readfile)) + + str_ = (str_part | str_esc | str_var).many().concat() + + return str_.parse(comment) diff --git a/formats/gds/gds.py b/formats/gds/gds.py new file mode 100644 index 0000000..c488181 --- /dev/null +++ b/formats/gds/gds.py @@ -0,0 +1,772 @@ +""" +Provides the `read_gds` and `write_gds` methods to read/write a `GDSProgram` (see the `model` module) from/to a binary GDS file (buffer). + +Requires the command definitions provided by `cmddef` to know parameter counts and behavior +""" + +from typing import Tuple, List, Optional, Callable, TypeVar, Mapping, Union +from dataclasses import dataclass, field +import struct +import functools + +# pylint: disable=unused-wildcard-import,wildcard-import +from utils import tagged_union, TU, match, nested_break + +import formats.gds.model as model +import formats.gds.cmddef as cmddef +import formats.gds.value as value + + +@tagged_union +class GDSTokenValue: + """ + Raw token + """ + + command: TU[int] + int: TU[int] + float: TU[float] + str: TU[str] + longstr: TU[str] + unused5: TU[None] + # the source address is the one pointing to the target + saddr: TU[model.GDSAddress] + taddr: TU[model.GDSAddress] + NOT: TU[None] + AND: TU[None] + OR: TU[None] + BREAK: TU[None] + fileend: TU[None] + + +@dataclass(kw_only=True) +class BinaryLocalized: + loc: int + + +@dataclass(kw_only=True) +class GDSToken(BinaryLocalized): + val: GDSTokenValue + + +@dataclass(kw_only=True) +class LabelUse(BinaryLocalized): + loc: int + addr: int + use: Optional[str] = None + primary: bool = False + + +@dataclass(kw_only=True) +class LabelToken(BinaryLocalized): + loc: int + addr: Optional[int] + pointsto: Optional[LabelUse] = None + + +@dataclass(kw_only=True) +class BinaryCommand(model.GDSInvocation, BinaryLocalized): + loc: int = None + + +@dataclass(kw_only=True) +class BinaryIfCommand(model.GDSIfInvocation, BinaryLocalized): + loc: int = None + + +@dataclass(kw_only=True) +class BinaryLoopCommand(model.GDSLoopInvocation, BinaryLocalized): + loc: int = None + + +F = TypeVar("F", bound=Callable) + + +def prevcursor(fn: F) -> F: + """ + For the CompilerState methods, records the cursor before the function call + so it can be restored for peeking functionality + """ + + @functools.wraps(fn) + def wrapper(self, *args, **kwargs): + prev = self.cursor + val = fn(self, *args, **kwargs) + self.prev_cursor = prev + return val + + return wrapper + + +@dataclass(kw_only=True) +class DecompilerState: + data: bytes + len: int + cursor: int + prev_cursor: int + elements: List[model.GDSElement] = field(default_factory=lambda: []) + labels: Mapping[int, List[Union[LabelUse, LabelToken]]] = field( + default_factory=dict + ) + context: model.GDSContext = field(default_factory=model.GDSContext) + + def __init__(self, data: bytes): + self.data = data + self.len = int.from_bytes(data[:4], "little") + self.cursor = 4 + self.prev_cursor = 4 + self.elements = [] + self.labels = {} + self.context = model.GDSContext() + + @prevcursor + def read_bytes(self, l: int) -> bytes: + val = self.data[self.cursor : self.cursor + l] + self.cursor += l + return val + + @prevcursor + def read_int(self, l: int) -> int: + return int.from_bytes(self.read_bytes(l), "little") + + @prevcursor + def read_token(self) -> GDSToken: + loc = self.cursor + p_type = self.read_int(2) + val = None + if p_type == 0: + cmd = self.read_int(2) + val = GDSTokenValue.command(cmd) + elif p_type == 1: + val = self.read_int(4) + val = GDSTokenValue.int(val) + elif p_type == 2: + val = struct.unpack(" 64: + # print( + # "WARN: string literal is too long (max is 64 bytes); this may lead to errors in the game." + # ) + val = ( + self.read_bytes(str_len) + .decode("ascii") # TODO: JP/KO compatibility + .rstrip("\x00") + ) + val = GDSTokenValue.str(val) + elif p_type == 4: + str_len = self.read_int(2) + val = ( + self.read_bytes(str_len) + .decode("ascii") # TODO: JP/KO compatibility + .rstrip("\x00") + ) + val = GDSTokenValue.longstr(val) + elif p_type == 6: + addr = self.read_int(4) + val = GDSTokenValue.saddr(addr) + elif p_type == 7: + addr = self.read_int(4) + val = GDSTokenValue.taddr(addr) + elif p_type == 8: + val = GDSTokenValue.NOT() + elif p_type == 9: + val = GDSTokenValue.AND() + elif p_type == 10: + val = GDSTokenValue.OR() + elif p_type == 11: + val = GDSTokenValue.BREAK() + elif p_type == 12: + val = GDSTokenValue.fileend() + elif p_type == 5: + val = GDSTokenValue.unused5() + else: + raise ValueError("Invalid GDS token type") + return GDSToken(loc=loc, val=val) + + def read_label(self, token: GDSToken) -> LabelToken: + if token.val not in GDSTokenValue.taddr: + raise ValueError("Not a target label") + ltoken = LabelToken(loc=token.loc, addr=model.GDSAddress(token.val())) + + if (ltoken.loc + 2) not in self.labels: + self.labels[ltoken.loc + 2] = [] + self.labels[ltoken.loc + 2].append(ltoken) + self.elements.append(model.GDSElement.label(ltoken)) + + return ltoken + + def read_address(self, token: GDSToken, use: str = "") -> LabelUse: + if token.val not in GDSTokenValue.saddr: + raise ValueError("Not a jump address") + luse = LabelUse(loc=token.loc, addr=model.GDSAddress(token.val()), use=use) + + if luse.addr not in self.labels: + self.labels[luse.addr] = [] + self.labels[luse.addr].append(luse) + + return luse + + def name_labels(self): + label_names = {} + label_prefix_counter: Mapping[str, int] = {} + + for addr, labels in self.labels.items(): + target = None + sources = [] + use = None + for l in labels: + if isinstance(l, LabelToken): + target = l + else: + sources.append(l) + use = l.use if use is None or use == l.use else "" + if use is None: + use = "" + if target is None: + i = 0 + for x in self.elements: + if x.loc >= addr + 4: + break + i += 1 + target = LabelToken(loc=addr - 2, addr=None) + labels.insert(target) + self.elements.insert(i, target) + + # name the label according to its use + prefix = f"{use}_" if use != "" else "" + if prefix not in label_prefix_counter: + label_prefix_counter[prefix] = 1 + counter = label_prefix_counter[prefix] + label_prefix_counter[prefix] += 1 + name = prefix + str(counter) + label_names[addr] = name + + # set which address is the primary one, if any + for s in sources: + if s.loc + 2 == target.addr: + s.primary = True + target.pointsto = s + break + + # # Fold if/elif/else and loop blocks if possible + # # i.e. the corresponding label is only used by the one ifelseloop jump, + # # and it jumps forward so a block even makes sense + # # TODO: I need to combine all the labels before a statement together + # if len(sources) == 1 and sources[0].addr >= sources[0].loc+6: + # pass + return label_names + + def collapse_blocks(self, block: List[model.GDSElement]) -> List[model.GDSElement]: + sid = 0 + while sid < len(block): + with nested_break() as continue_outer: + el = block[sid] + if el not in model.GDSElement.command or not isinstance( + el(), (model.GDSIfInvocation, model.GDSLoopInvocation) + ): + raise continue_outer # also very convenient for preserving a cleanup block in a continue + + block_cmd: Union[model.GDSIfInvocation, model.GDSLoopInvocation] = el() + + source: LabelUse = block_cmd.target + if source is None: + raise continue_outer + target: LabelToken = None + for label in self.labels[source.addr]: + if isinstance(label, LabelUse) and label is not source: + # more than one thing jumps here, + # that's not pro-level + raise continue_outer + if isinstance(label, LabelToken): + target = label + if target.loc <= source.loc: + # backwards jumps are VERY not pro-level + raise continue_outer + + # Cut out the frame between these two tokens and put it in a block in the command right before. + # Also remove the labels from the map + tid = sid + 1 + while tid < len(block) and not ( + block[tid] in model.GDSElement.label and block[tid]() is target + ): + tid += 1 + if tid == len(block): + # I believe the only way this can happen at this point, + # is if the jump target is outside of the current block. Which is perfect! + raise continue_outer + + # excludes the command in front and the target label + sub_block = block[sid + 1 : tid] + block = block[: sid + 1] + block[tid + 1 :] + block_cmd.target = None + block_cmd.block = sub_block + + block_cmd.block = self.collapse_blocks(sub_block) + sid += 1 + return block + + def finalize_labels( + self, block: List[model.GDSElement], label_names: Mapping[int, str], labels=None + ) -> Mapping[str, Union[model.GDSLabel, model.GDSAddress]]: + if labels is None: + labels = {} + + oldlabel_indices = [ + (i, el()) for i, el in enumerate(block) if el in model.GDSElement.label + ] + olduse_indices = [ + (el, el().target) + for el in block + if el in model.GDSElement.command + and isinstance(el(), (BinaryIfCommand, BinaryLoopCommand)) + and isinstance(el().target, LabelUse) + ] + + for addr, name in label_names.items(): + if name not in labels: + labels[name] = [] + for i in self.labels[addr]: + if isinstance(i, LabelToken): + newlabel = model.GDSLabel( + name=name, + present=i.addr is not None, + loc=i.addr if i.pointsto is None else None, + ) + found = False + idx = None + for idx, el in oldlabel_indices: + if el == i: + found = True + break + if found: + block[idx] = model.GDSElement.label(newlabel) + labels[name].append(newlabel) + elif isinstance(i, LabelUse): + newuse = model.GDSJumpAddress(label=name, primary=i.primary) + found = False + el = None + for el, olduse in olduse_indices: + if olduse == i: + found = True + break + if found: + el().target = newuse + labels[name].append(newuse) + else: + raise TypeError() + + for el in block: + if el not in model.GDSElement.command or not isinstance( + el(), (BinaryIfCommand, BinaryLoopCommand) + ): + continue + block_cmd = el() + if block_cmd.block is None: + continue + self.finalize_labels(block_cmd.block, label_names, labels) + + return labels + + +def read_gds(data: bytes, path: str = None) -> model.GDSProgram: + ctx = DecompilerState(data) + + cur_token = ctx.read_token() + # TODO + while cur_token.val not in GDSTokenValue.fileend: + if cur_token.val in GDSTokenValue.command: + ctx.elements.append(model.GDSElement.command(read_command(ctx, cur_token))) + elif cur_token.val in GDSTokenValue.taddr: + ctx.read_label(cur_token) + elif cur_token.val in GDSTokenValue.BREAK: + b = model.GDSElement.BREAK() + b.loc: int = cur_token.loc + ctx.elements.append(b) + else: + # The game technically allows this, but it's meaningless nonsense so for now we forbid it + raise ValueError("Unexpected token type") + + cur_token = ctx.read_token() + + label_names = ctx.name_labels() + + ctx.elements = ctx.collapse_blocks(ctx.elements) + + labels = ctx.finalize_labels(ctx.elements, label_names) + + return model.GDSProgram( + context=ctx.context, path=path, elements=ctx.elements, labels=labels + ) + + +def read_condition( + ctx: DecompilerState, use: str = "" +) -> Tuple[List[model.GDSConditionToken], LabelUse]: + cur_token = ctx.read_token() + cond = [] + while cur_token.val not in GDSTokenValue.saddr: + if cur_token.val in GDSTokenValue.NOT: + cond.append(model.GDSConditionToken.NOT()) + elif cur_token.val in GDSTokenValue.AND: + cond.append(model.GDSConditionToken.AND()) + elif cur_token.val in GDSTokenValue.OR: + cond.append(model.GDSConditionToken.OR()) + elif cur_token.val in GDSTokenValue.command: + cond.append(model.GDSConditionToken.command(read_command(ctx, cur_token))) + else: + raise ValueError("Unexpected token type") + cur_token = ctx.read_token() + + addr = ctx.read_address(cur_token, use) + + return cond, addr + + +def read_command(ctx: DecompilerState, token: GDSToken) -> BinaryCommand: + if token.val not in GDSTokenValue.command: + raise ValueError("Expected instruction") + + cmdid = token.val() + cmdobj = cmddef.COMMANDS_BYID.get(cmdid) + if cmdobj is None: + raise ValueError(f"Command {cmdid} not defined") + + if cmdobj.complex: + return (READ_COMPLEX.get(cmdobj.name) or READ_COMPLEX.get(cmdobj.id))( + ctx, cmdobj + ) + + cmd = read_simple(ctx, cmdobj) + cmd.loc = token.loc + return cmd + + +def read_simple(ctx: DecompilerState, cmdobj: "cmddef.GDSCommand") -> BinaryCommand: + args = [] + for param in cmdobj.params: + + arg = ctx.read_token() + val = param.type.from_token(arg.val) + if val is None: + if not param.optional: + raise ValueError( + f"Unexpected parameter token type: should have been {param.type}, token was {arg}" + ) + val = None + ctx.cursor = ctx.prev_cursor + + args.append(val) + return BinaryCommand(command=cmdobj, args=args) + + +def read_if(ctx: DecompilerState, cmdobj: "cmddef.GDSCommand") -> BinaryIfCommand: + cond, addr = read_condition(ctx, "if") + + # this diagnostic is literally just objectively wrong + # pylint: disable=unexpected-keyword-arg + return BinaryIfCommand( + command=cmdobj, + args=[], + condition=cond, + block=None, + elze=False, + elseif=False, + target=addr, + ) + + +def read_elif(ctx: DecompilerState, cmdobj: "cmddef.GDSCommand") -> BinaryIfCommand: + cond, addr = read_condition(ctx, "elif") + + # this diagnostic is literally just objectively wrong + # pylint: disable=unexpected-keyword-arg + return BinaryIfCommand( + command=cmdobj, + args=[], + condition=cond, + block=None, + elze=False, + elseif=True, + target=addr, + ) + + +def read_else(ctx: DecompilerState, cmdobj: "cmddef.GDSCommand") -> BinaryIfCommand: + cur_token = ctx.read_token() + while cur_token.val not in GDSTokenValue.saddr: + cur_token = ctx.read_token() + addr = ctx.read_address(cur_token, "else") + + # this diagnostic is literally just objectively wrong + # pylint: disable=unexpected-keyword-arg + return BinaryIfCommand( + command=cmdobj, + args=[], + condition=None, + block=None, + elze=True, + elseif=False, + target=addr, + loc=None, + ) + + +def read_repeatN(ctx: DecompilerState, cmdobj: "cmddef.GDSCommand") -> BinaryLoopCommand: + cntt = ctx.read_token() + if cntt.val not in GDSTokenValue.int: + raise ValueError( + f"Unexpected parameter token type: should have been int, token was {cntt}" + ) + cnt = cntt.val() + + saddrt = ctx.read_token() + while saddrt.val not in GDSTokenValue.saddr: + if saddrt.val in GDSTokenValue.fileend: + raise ValueError("repeatN: encountered EOF looking for jump address") + saddrt = ctx.read_token() + addr = ctx.read_address(saddrt, "loop") + + # this diagnostic is literally just objectively wrong + # pylint: disable=unexpected-keyword-arg + return BinaryLoopCommand( + command=cmdobj, condition=cnt, target=addr, args=[], block=[] + ) + + +def read_while(ctx: DecompilerState, cmdobj: "cmddef.GDSCommand") -> BinaryLoopCommand: + cond, addr = read_condition(ctx, "loop") + + # this diagnostic is literally just objectively wrong + # pylint: disable=unexpected-keyword-arg + return BinaryLoopCommand( + command=cmdobj, condition=cond, target=addr, args=[], block=[] + ) + + +READ_COMPLEX = { + "if": read_if, + "elif": read_elif, + "else": read_else, + "repeatN": read_repeatN, + "while": read_while, +} + + +@dataclass(kw_only=True) +class CompilerState: + data: bytearray + elements: List[model.GDSElement] + labels: Mapping[str, List[Union[model.GDSLabel, model.GDSJumpAddress]]] + label_locs: Mapping[str, Tuple[int, model.GDSLabel]] + use_locs: Mapping[str, List[Tuple[int, model.GDSJumpAddress]]] + + def __init__(self, prog: model.GDSProgram): + self.data = bytearray() + # self.context = prog.context + self.elements = prog.elements + self.labels = prog.labels + self.label_locs = {} + self.use_locs = {} + + def write_bytes(self, b: bytes): + self.data += b + + def write_token(self, token: GDSTokenValue): + els = match( + token, + { + GDSTokenValue.command: lambda val: ( + (0).to_bytes(2, "little"), + val.to_bytes(2, "little"), + ), + GDSTokenValue.int: lambda val: ( + (1).to_bytes(2, "little"), + val.to_bytes(4, "little"), + ), + GDSTokenValue.float: lambda val: ( + (2).to_bytes(2, "little"), + struct.pack(" bytes: + ctx = CompilerState(prog) + + for el in ctx.elements: + if el in model.GDSElement.BREAK: + ctx.write_token(GDSTokenValue.BREAK()) + elif el in model.GDSElement.command: + ctx.write_command(el()) + elif el in model.GDSElement.label: + ctx.write_label(el()) + + ctx.write_token(GDSTokenValue.fileend()) + return len(ctx.data).to_bytes(4, "little") + bytes(ctx.data) + + +def write_simple(ctx: CompilerState, cmd: model.GDSInvocation): + args = cmd.args + if len(args) < len(cmd.command.params): + args += [None] * len(cmd.command.params) - len(args) + for arg, param in zip(args, cmd.command.params): + if arg is None: + if param.optional: + continue + else: + raise ValueError("Too few parameters") + + # pylint: disable=not-callable + tok = arg.as_token() + if tok is None: + raise ValueError( + f"Unexpected parameter type: should have been {param.type}, value was {arg}" + ) + + ctx.write_token(tok) + + +def write_condition(ctx: CompilerState, cond: List[model.GDSConditionToken]): + for tok in cond: + if tok in model.GDSConditionToken.NOT: + ctx.write_token(GDSTokenValue.NOT()) + elif tok in model.GDSConditionToken.AND: + ctx.write_token(GDSTokenValue.AND()) + elif tok in model.GDSConditionToken.OR: + ctx.write_token(GDSTokenValue.OR()) + elif tok in model.GDSConditionToken.command: + ctx.write_command(tok()) + else: + raise ValueError("Unexpected token type") + + +def write_if(ctx: CompilerState, cmd: model.GDSIfInvocation): + write_condition(ctx, cmd.condition) + if cmd.target is not None: + ctx.write_addr(cmd.target) + elif cmd.block is not None: + ctx.write_block(cmd.block) + + +def write_else(ctx: CompilerState, cmd: model.GDSIfInvocation): + # write_condition(ctx, cmd.condition) + if cmd.target is not None: + ctx.write_addr(cmd.target) + elif cmd.block is not None: + ctx.write_block(cmd.block) + + +def write_repeatN(ctx: CompilerState, cmd: model.GDSLoopInvocation): + ctx.write_token(GDSTokenValue.int(cmd.condition)) + if cmd.target is not None: + ctx.write_addr(cmd.target) + elif cmd.block is not None: + ctx.write_block(cmd.block) + + +def write_while(ctx: CompilerState, cmd: model.GDSLoopInvocation): + write_condition(ctx, cmd.condition) + if cmd.target is not None: + ctx.write_addr(cmd.target) + elif cmd.block is not None: + ctx.write_block(cmd.block) + + +WRITE_COMPLEX = { + "if": write_if, + "elif": write_if, + "else": write_else, + "repeatN": write_repeatN, + "while": write_while, +} diff --git a/formats/gds/model.py b/formats/gds/model.py new file mode 100644 index 0000000..0f9f009 --- /dev/null +++ b/formats/gds/model.py @@ -0,0 +1,197 @@ +from typing import List, Optional, NewType, Union, Set, Mapping +from dataclasses import dataclass, field +# pylint: disable=unused-wildcard-import,wildcard-import + +from utils import tagged_union, TU +import formats.gds.value as value +import formats.gds.cmddef as cmddef + +GDSAddress = NewType('GDSAddress', int) + +@dataclass(kw_only=True) +class GDSInvocation: + """ + The specific invocation of a GDS commmand, with the given parameter values. + """ + command: "cmddef.GDSCommand" + args: List["value.GDSValue"] + + +@tagged_union +class GDSConditionToken: + """ + A token that appears in the condition for a flow statement. + """ + command: TU[GDSInvocation] + NOT: TU[None] + AND: TU[None] + OR: TU[None] + + +@dataclass(kw_only=True) +class GDSJumpAddress: + """ + A jump address used by flow instructions in GDS scripts. + """ + label: str + """ + The name of the label to be jumped to. + """ + primary: bool = True + """ + Whether the label this address points to actually points back to this address + instance. Only one of the pointers to a single label can have this flag set + (but normally, there is also a 1:1 correspondence between addresses and labels) + """ + + +@dataclass(kw_only=True) +class GDSIfInvocation(GDSInvocation): + condition: List[GDSConditionToken] + target: GDSJumpAddress + block: Optional[List["GDSElement"]] = None + elseif: bool = False + elze: bool = False + + +@dataclass(kw_only=True) +class GDSLoopInvocation(GDSInvocation): + condition: Union[List[GDSConditionToken], int] + block: Optional[List["GDSElement"]] = None + target: GDSJumpAddress + + +@dataclass(kw_only=True) +class GDSLabel: + """ + A target label from a GDS script file. + """ + name: str + """ + The name of the label + """ + present: bool = True + """ + Whether the label actually physically exists, or was inserted as the target + of a jump that didn't point to a target label + """ + loc: Optional[int] = None + """ + The location encoded in the physically stored label word. + Only set if it differs from what would be expected, i.e. the location + of a jump address pointing to the label. + """ + + +@tagged_union +class GDSElement: + """ + An entry in a GDS script file. + """ + command: TU[GDSInvocation] + label: TU[GDSLabel] + BREAK: TU[None] + + +@dataclass(kw_only=True) +class GDSContext: + """ + The context that was determined for a GDS script, either manually or by context narrowing. + + TODO: find the best way to represent conflicts + """ + manual_name: Optional[str] = None + """ + A manual context set on construction, which will never change on narrowing + """ + candidates: Set[str] = field(default_factory=lambda: ["all"]) + """ + A list of candidate contexts, if any are still applicable. This list will be empty if narrowing caused a conflict. + """ + conflicts: List["cmddef.GDSCommand"] = field(default_factory=list) + """ + Whenever an instruction does not match any of the candidate contexts, the offending command is recorded here, and its + context candidates added to the global candidate list. + """ + + def narrow(self, cmd: "cmddef.GDSCommand") -> bool: + """ + Checks if the given command is compatible with the currently assumed candidates, and narrows the list if so. + Otherwise, records the conflict and returns false. + """ + intersection = GDSContext.intersection([self.manual_name] if self.manual_name else self.candidates, cmd.context) + if not intersection: + self.conflicts.append(cmd) + if not self.manual_name: + self.candidates = GDSContext.union(self.candidates, cmd.context) + return False + if not self.manual_name: + self.candidates = intersection + return True + + @staticmethod + def intersection(n1: Union[str, Set[str]], n2: Union[str, Set[str]]) -> Set[str]: + if isinstance(n1, list): + res = set() + for n in n1: + res.update(GDSContext.intersection(n, n2)) + return res + if isinstance(n2, list): + res = set() + for n in n2: + res.update(GDSContext.intersection(n1, n)) + return res + + # sourcery skip: assign-if-exp, reintroduce-else + if n1 == n2: + return {n1} + if n2.startswith(f"{n1}."): + return {n2} + if n1.startswith(f"{n2}."): + return {n1} + return set() + + @staticmethod + def union(n1: Union[str, Set[str]], n2: Union[str, Set[str]]) -> Set[str]: + if isinstance(n1, list): + res = set() + for n in n1: + res.update(GDSContext.intersection(n, n2)) + return res + if isinstance(n2, list): + res = set() + for n in n2: + res.update(GDSContext.intersection(n1, n)) + return res + + # sourcery skip: assign-if-exp, reintroduce-else + if n1 == n2: + return {n1} + if n2.startswith(f"{n1}."): + return {n1} + if n1.startswith(f"{n2}."): + return {n2} + return set() + + +@dataclass(kw_only=True) +class GDSProgram: + """ + The program contained in a GDS script file. + """ + context: Optional[GDSContext] = None + """ + The execution context of the script, or any candidates for it. + """ + path: Optional[str] = None + """ + The file path this script was loaded from, if any. + """ + elements: List[GDSElement] = field(default_factory=list) + """ + The instructions or other flow elements in the script. + """ + labels: Mapping[str, List[Union[GDSLabel, GDSJumpAddress]]] = field(default_factory=list) + """ + A list of all the labels present in the script. May not technically be necessary. + """ diff --git a/formats/gds/patch.py b/formats/gds/patch.py new file mode 100644 index 0000000..4f21cb6 --- /dev/null +++ b/formats/gds/patch.py @@ -0,0 +1,76 @@ +# TODO: what to do with these? +# Do we just provide a baseline patch internally? Or do I expand the GDA language +# to accommodate for these mistakes? +PATCHES = { + # These scripts use `0x12 if` in a condition, in places where that doesn't seem legal. + # The game would likely crash if this inner if's condition was ever true. The intention + # seems to have been to write `if ... and if ...`, which hints at the nature of the scripting + # language used in the development process! But is still incorrect. + "data/script/rooms/room4_param.gds": [(0x2B1, b"\0\0\x12\0", b"\x09\0\x09\0")], + "data/script/rooms/room13_in.gds": [(0x5C, b"\0\0\x12\0", b"\x09\0\x09\0")], + # Similarly someone might forget that "elseif" is its own command, and write "else if" instead. + # It's actually possible that this doesn't cause an error though! + "data/script/rooms/room12_in.gds": [ + (0x127, b"\0\0\x17\0\0\0\x12\0", b"\0\0\x16\0\x09\0\x09\0") + ], + # I have no idea about these. Maybe stuff got cut? + "data/script/rooms/room23_in.gds": [ + (0x18, b"\0\0\x12\0\0\0\x8d\0", b"\0\0\xdf\0\0\0\xdf\0") + ], + "data/script/rooms/room24_in.gds": [ + (0x18, b"\0\0\x12\0\0\0\x8d\0", b"\0\0\xdf\0\0\0\xdf\0") + ], + # The code says that the second parameter of 0x78 is a float, but it's called with an + # int here... Apparently in their language it was very easy to accidentally write an integer + # instead of a float, and there were no compile-time checks for that. + "data/script/event/e49.gds": [ + (0x24D, b"\x01\0\xfa\xff\xff\xff", b"\x02\0\xc0\0\xc0\0"), + (0x25D, b"\x01\0\xfa\xff\xff\xff", b"\x02\0\xc0\0\xc0\0"), + ], + "data/script/event/e126.gds": [(0x398, b"\x01", b"\x02")], + "data/script/event/e276.gds": [(0x1B4, b"\x01", b"\x02")], + "data/script/event/e233.gds": [(0x1F8, b"\x01", b"\x02")], + "data/script/event/e42.gds": [(0x1C3, b"\x01", b"\x02")], +} +""" +A list of patches to the vanilla scripts, correcting some common mistakes(?). + +For some of these it's more obvious than for others that they are incorrect; an example being +command arguments that should be floats but are integers (likely because of a forgotten decimal point). +There are also cases of extremely nonstandard overlapping use of multiple if statements, which would +crash the game in certain cases, but look like they are obviously meant to mean something correct. +""" + + +def patch(data: bytes, filepath: str) -> bytes: + if filepath not in PATCHES: + return data + + data = bytearray(data) + + for start, old, new in PATCHES[filepath]: + if data[start : start + len(old)] != old: + print( + f"WARN: {filepath}: patch at {hex(start)} incorrect: expected {old}, was {data[start:start+len(old)]}" + ) + continue + data[start : start + len(old)] = new + + return bytes(data) + + +def unpatch(data: bytes, filepath: str) -> bytes: + if filepath not in PATCHES: + return data + + data = bytearray(data) + + for start, old, new in PATCHES[filepath]: + if data[start : start + len(new)] != new: + print( + f"WARN: {filepath}: patch at {hex(start)} incorrect: expected {new}, was {data[start:start+len(new)]}" + ) + continue + data[start : start + len(new)] = old + + return bytes(data) \ No newline at end of file diff --git a/formats/gds/preview.py b/formats/gds/preview.py new file mode 100644 index 0000000..44557b7 --- /dev/null +++ b/formats/gds/preview.py @@ -0,0 +1,70 @@ +""" +This module (eventually) contains a small GDS simulator, suitable for piecing together a visually +explorable representation of room layouts, a structured view of dialogue events, and an overview +of the script's logical control flow. + +TODO: WIP, and no priority at the moment. Though given Flora's main motivation statement is to make +puzzle editing a one-step process, maybe the preview at least for simple puzzle types should be prioritized. +""" + +from .model import GDSProgram + +def preview_room(prog: GDSProgram): + """ + Executes the provided room script, by building the room in a small view window + like the game itself would, but with many introspection tools. + + The program should consist of + 1. a main view in which the content visible on the bottom screen is assembled. + All items placed here should be represented by their actual animated sprites in game, + and bounding boxes should be shown. Clicking on an item should show technical details, + such as exactly what type of item it is, or in the case of triggers, what event they cause. + This information also includes the conditions under which the object is included in the scene. + 2. a topscreen view displaying what map information is being set by the room script, and possibly + other adhoc sprites like the objectives arrows. Again, bounding boxes where relevant. + 3. a sidebar that lists all items placed in the room by category, also split by what conditions need + to be required to make the item appear. + + This information could all just be an informational display, but optionally, certain properties like the + positions, names and events of items could be editable. A "save" functionality should write the result of + these edits to a new GDA script. + """ + pass + +def preview_event(prog: GDSProgram): + """ + Executes the provided event script, by playing through the sequence in a small view window + like the game itself would, but with many introspection tools. + + The program should consist of + 1. a main view in which actors and text bubbles are displayed like in the game. + Items placed here should be represented by their actual animated sprites in game, + and clickable bounding boxes shown to display technical details of placement and animation logic. + 2. (optionally, depending on what the event can do) a topscreen view displaying the results of special + logic displaying information on the top screen. + 3. a sidebar listing all events in the dialogue tree in order, highlighting the currently displayed step, + allowing the user to click on previous ones to rewind. Should also display the logic instructions + determining which parts of the event are played, but allow the user to override them and see + all parts of the dialogue. + + This information could all just be an informational display, but optionally, certain properties like the + positions, count and animations of the actors, as well as the dialogue text, could be editable. A "save" + functionality should write the result of these edits to a new GDA script. + """ + pass + +def preview_puzzle(prog: GDSProgram): + """ + Least likely to actually get done: executes the provided puzzle script. These seldom have much interactivity, + but they're extremely contextual on the engine used. + + The program should consist of + 1. a main view displaying the assembled touchscreen view a player would be presented with in-game, + with animations and bounding boxes for important elements (and grid line hints where applicable). + 2. a "topscreen view" which may not need to be completely accurate, instead listing all crucial information + about the puzzle such as title, description, picarat counts, even the hints in a separate tab. + 3. a sidebar with debug information about the internal state of the puzzle engine, and the ability + to show/hide technical objects used for puzzle grading that would otherwise clutter the screen. + + Of course this view is highly dependent on the puzzle engine, which makes it difficult to conceptualize at this point. + """ \ No newline at end of file diff --git a/formats/gds/test_cmddef.py b/formats/gds/test_cmddef.py new file mode 100644 index 0000000..e69de29 diff --git a/formats/gds/test_gda.py b/formats/gds/test_gda.py new file mode 100644 index 0000000..4d46ad0 --- /dev/null +++ b/formats/gds/test_gda.py @@ -0,0 +1,147 @@ +import os +import pprint + +import unittest + +import tqdm + +from .gds import read_gds, write_gds +from .gda import read_gda, write_gda, parse_element, CommentContext, format_comment +from .patch import patch +from .value import GDSIntType, GDSIntValue + +from utils import RESOURCES + + +def test_file(gdspath: str, base: str): + with open(os.path.join(base, gdspath), "rb") as gdsf: + gdsb = gdsf.read() + gdsb = patch(gdsb, gdspath) + prog = read_gds(gdsb, gdspath) + gdas = write_gda(prog, gdspath, base) + gdapath = gdspath.replace(".gds", ".gda") + with open(os.path.join(base, gdapath), "w", encoding="utf8") as gdaf: + gdaf.write(gdas) + + try: + reread = read_gda(gdas, gdapath) + except Exception as e: + print(f"ERR: {gdapath}: could not read decompiled script: {e}") + return + try: + recompiled = write_gds(reread) + except Exception as e: + print(f"ERR: {gdapath}: could not recompile script: {e}") + return + + if recompiled != gdsb: + print( + f"ERR: {gdspath}: Recompiled contents not identical. Result written as {gdspath}2 for comparison." + ) + with open(os.path.join(base, f"{gdspath}2"), "wb") as gdsf: + gdsf.write(recompiled) + + +BASE = os.path.join(RESOURCES, "game_root/lt1_eu") + + +def test_all(base: str): + for f in tqdm.tqdm( + os.path.join(root, f) + for root, _subdirs, files in os.walk(os.path.join(base, "data/script")) + for f in files + ): + if not f.endswith(".gds"): + continue + path = os.path.relpath(f, base) + test_file(path, base) + + +def test_comment(): + EXAMPLE = """ + /data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt: + + $(/data/etext/${lang}/e${eventid:r100<=200:03}{pcm}/e${eventid}_t${1}.txt) + """ + print( + format_comment( + EXAMPLE, + CommentContext( + args=[GDSIntValue(7, GDSIntType())], + filename="/data/script/event/e6.gds", + workdir=BASE, + ), + ) + ) + + +def test_parsers(): + print(parse_element.parse("if not 0x49:{ # test \n0x49 # test2\n#test3\n}")) + + +class TestGDSVanillaFiles(unittest.TestCase): + """ + A basic parameterized test case, checking if the decompiler is able to decompile and recompile the + original game scripts and produce identical data. + + For this suite to be possible, the original files from the LT1 EU version .nds ROM need to be + extracted/symlinked into `{flora root dir}/data/game_root/lt1_eu/`, such that the folder contains the + subfolders `data`, `dwc` and `ftc`. Eventually Flora should support this, but for now you can use Tinke to do it. + """ + def test_all_files(self): + """ + Checks if the decompiler is able to disassemble and reassemble each of the unmodified original game scripts. + This baseline sanity check ensures that merely running `decompile` and then `compile` does not corrupt + game files. + """ + base = BASE + # sourcery skip: no-conditionals-in-tests + if not os.path.exists(base) or not os.path.isdir(base): + self.skipTest( + 'Could not find vanilla game files under "data/game_root/lt1_eu". Please make sure ' + "to extract and either copy or symlink the entire contents of a valid LT1 EU version .nds ROM " + "into that location." + ) + # sourcery skip: no-loop-in-tests + for root, _subdirs, files in os.walk(os.path.join(base, "data/script")): + for f in files: + if not f.endswith(".gds"): + continue + path = os.path.relpath(os.path.join(root, f), base) + with self.subTest(msg=f"Check file {path}"): + self.single_file(path, base) + + def single_file(self, gdspath: str, base: str): + with open(os.path.join(base, gdspath), "rb") as f: + gdsb = f.read() + # For now we require these known exceptions + gdsb = patch(gdsb, gdspath) + try: + prog = read_gds(gdsb, gdspath) + except Exception as e: + self.fail(f"Decompile: {e}") + raise e + try: + gdas = write_gda(prog, gdspath, base) + except Exception as e: + self.fail(f"Write GDA: {e}") + raise e + + gdapath = gdspath.replace(".gds", ".gda") + try: + prog2 = read_gda(gdas, gdapath) + except Exception as e: + self.fail(f"Read GDA: {e}") + raise e + try: + gdsb2 = write_gds(prog2) + except Exception as e: + self.fail(f"Recompile: {e}") + raise e + + self.assertEqual(gdsb, gdsb2, "Recompiled result is not identical") + +if __name__ == "__main__": + # test_file("data/script/rooms/room4_param.gds", BASE) + test_all(BASE) + # test_parsers() diff --git a/formats/gds/test_gds.py b/formats/gds/test_gds.py new file mode 100644 index 0000000..19b36cf --- /dev/null +++ b/formats/gds/test_gds.py @@ -0,0 +1,130 @@ +import os +import pprint + +import unittest + +from .gds import read_gds, write_gds +from .patch import patch +from utils import RESOURCES + + +def print_hexdump(data: bytes): + res = "" + padstart = 4 + for i in range(padstart): + if i % 16 == 0: + res += "\n" + elif i % 4 == 0: + res += " " + res += "## " + + i = padstart + for b in data: + if i % 16 == 0: + res += "\n" + elif i % 4 == 0: + res += " " + + if b < 16: + res += "0" + res += hex(b)[2:] + res += " " + i += 1 + return res + + +BASE = os.path.join(RESOURCES, "game_root/lt1_eu") + + +def test_file(filepath: str, base: str): + # sourcery skip: use-fstring-for-concatenation + + with open(os.path.join(base, filepath), "rb") as f: + data = f.read() + data = patch(data, filepath) + try: + prog = read_gds(data, filepath) + except Exception as e: + print(f"ERR: {filepath}: could not decompile: {e}") + return + try: + data2 = write_gds(prog) + except Exception as e: + print(f"ERR: {filepath}: could not recompile: {e}") + return + if data != data2: + print( + f"ERR: {filepath}: Recompiled contents not identical. Result written as {filepath}2 for comparison." + ) + with open(os.path.join(base, filepath + "2"), "wb") as f: + f.write(data2) + elif os.path.exists(os.path.join(base, filepath + "2")): + os.remove(os.path.join(base, filepath + "2")) + + +# TODO: make this a unit test! +def test_all(base: str): + for root, _subdirs, files in os.walk(os.path.join(base, "data/script")): + for f in files: + if not f.endswith(".gds"): + continue + path = os.path.relpath(os.path.join(root, f), base) + test_file(path, base) + + +# test_all(BASE) +# test_file("data/script/qscript/q4_param.gds", BASE) + + +class TestGDSVanillaFiles(unittest.TestCase): + def test_all_files(self): + """ + Checks if the decompiler is able to disassemble and reassemble each of the unmodified original game scripts. + This baseline sanity check ensures that merely running `decompile` and then `compile` does not corrupt + game files. + + Note that for this test to be possible, original files from the LT1 EU version .nds ROM need to be + extracted/symlinked into `{flora root dir}/data/game_root/lt1_eu/`, such that the folder contains the + subfolders `data`, `dwc` and `ftc`. Eventually Flora should support this, but for now you can use Tinke to do it. + """ + base = BASE + # sourcery skip: no-conditionals-in-tests + if not os.path.exists(base) or not os.path.isdir(base): + self.skipTest( + 'Could not find vanilla game files under "data/game_root/lt1_eu". Please make sure ' + "to extract and either copy or symlink the entire contents of a valid LT1 EU version .nds ROM " + "into that location." + ) + # sourcery skip: no-loop-in-tests + for root, _subdirs, files in os.walk(os.path.join(base, "data/script")): + for f in files: + if not f.endswith(".gds"): + continue + path = os.path.relpath(os.path.join(root, f), base) + with self.subTest(msg=f"Check file {path}"): + self.single_file(path, base) + + def single_file(self, filepath: str, base: str): + with open(os.path.join(base, filepath), "rb") as f: + data = f.read() + data = patch(data, filepath) + try: + prog = read_gds(data, filepath) + except Exception as e: + self.fail(f"Decompile: {e}") + raise e + try: + data2 = write_gds(prog) + except Exception as e: + self.fail(f"Recompile: {e}") + raise e + + self.assertEqual(data, data2, "Recompiled result is not identical") + # if data != data2: + # print( + # f"ERR: {filepath}: Recompiled contents not identical. Result written as {filepath}2 for comparison." + # ) + # with open(os.path.join(base, filepath + "2"), "wb") as f: + # f.write(data2) + # elif os.path.exists(os.path.join(base, filepath + "2")): + # os.remove(os.path.join(base, filepath + "2")) diff --git a/formats/gds/value.py b/formats/gds/value.py new file mode 100644 index 0000000..7a2198c --- /dev/null +++ b/formats/gds/value.py @@ -0,0 +1,549 @@ +""" +Defines the way values and their types are defined and handled in GDS and GDA. +""" + +from abc import ABC, abstractmethod +from typing import Any, Union, Optional, Literal, List +from dataclasses import dataclass + +from parsy import regex, string, Parser, seq, success, fail, peek + +import formats.gds.gds as gds +from utils import round_perfect + +TYPES: List["GDSValueType"] = [] + + +def parse_type(descriptor: str) -> Optional["GDSValueType"]: + """ + Given a type descriptor (used internally in the GDS command definition files), + creates a corresponding type object, or returns None if the descriptor is invalid. + + See the individual type classes registered in `TYPES` for information on valid + descriptor formats + """ + for t in TYPES: + ty = t.parse_type(descriptor) + if ty is not None: + return ty + return None + + +class GDSValueType(ABC): + """ + A GDS Value Type. Defines how a value of this type is created from + a GDS token or a string parsed in a GDA file. + """ + + @classmethod + @abstractmethod + def parse_type(cls, descriptor: str) -> Optional["GDSValueType"]: + """ + Parses the descriptor and returns the corresponding type object + if the descriptor corresponds to a variant of this type, + otherwise returns None to signal the next candidate type should be tried. + """ + + @abstractmethod + def describe(self) -> str: + """ + Returns a descriptor for this variant of the type. The inverse of `parse_type`, in a sense. + """ + + @abstractmethod + def from_token(self, tok: gds.GDSTokenValue) -> "GDSValue": + """ + Parses a value of this type from a GDSToken(Value), or returns + None if there is no way to correspond that token to a value of this type. + """ + + @abstractmethod + def parser(self) -> Parser: + """ + Returns a `parsy` parser that attempts to read a value of this type from a GDA script, + and if so returns a value instance of this type. + """ + + +class GDSValue(ABC): + """ + A GDS Value, of a specific `GDSValueType`. Defines how a value + is printed, either in Python code or in a GDA script, + and conversion to other value types / GDS tokens. + """ + + type: GDSValueType + value: Any + + @abstractmethod + def as_token(self) -> gds.GDSTokenValue: + """ + Converts the value into a GDSToken(Value). There should + rarely be a reason for this to fail, + but if it does this function should return None. + """ + + @abstractmethod + def __format__(self, format_spec: Optional[str] = None) -> str: + pass + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({format(self)})" + + def write(self) -> str: + """ + Returns a string representation of this value, as it would be written in a GDA script. + Separate from Python's format and repr mechanisms, though it corresponds more closely + to the latter; however some representations in GDA would not be valid repr strings in Python + (specifically long strings). + """ + return format(self) + + +# TODO: int value with a specified range (e.g. "int[0,3)" or just "int[0,2]") +@dataclass +class GDSIntType(GDSValueType): + """ + An integer type, with a fixed byte length, and a marker whether it is intended to be + read as an unsigned integer in the game. + """ + + bytelen: int = 4 + unsigned: bool = True + + @classmethod + def parse_type(cls, descriptor: str) -> "GDSIntType": + """ + Valid descriptors are: + int(n): n bytes, signed + uint(n): n bytes, unsigned + + or alternatively the shorthands: + int / uint: 4 bytes, optionally unsigned + short / ushort: 2 bytes, optionally unsigned + byte / ubyte: 1 byte, optionally unsigned + + Note that all int values are still written as 4 little endian bytes in GDS files, + but only a subset of these values is actually read by the game. Note also that + this hard limits the byte length to 4 at most. + """ + unsigned = False + if descriptor.startswith("u"): + unsigned = True + descriptor = descriptor[1:] + + if descriptor.startswith("int"): + if descriptor == "int": + bytelen = 4 + elif descriptor[3] == "(": + bytelen = int(descriptor[4:-1]) + else: + return None + elif descriptor == "short": + bytelen = 2 + elif descriptor == "byte": + bytelen = 1 + else: + return None + return GDSIntType(bytelen=bytelen, unsigned=unsigned) + + def describe(self) -> str: + return ("uint" if self.unsigned else "int") + f"({self.bytelen})" + + def from_token(self, tok: gds.GDSTokenValue) -> "GDSIntValue": + if tok not in gds.GDSTokenValue.int: + return None + val = tok() + if not self.unsigned and val >= (256**self.bytelen) / 2: + val -= 256**self.bytelen + return GDSIntValue(val, self) + + def parser(self) -> Parser: + """ + Valid integer literals can include a sign (+-), and can be written in base 10 (default) or + base 16/2 (prefixed by 0x or 0b respectively). Note that if a dot is found at the end of + the literal, parsing will fail, as this designates the value as a float instead. + """ + + def parser_map( + val: int, lit_fmt: Literal["hex", "bin", "dec"] = "dec" + ) -> "GDSIntValue": + if self.unsigned: + if val < 0: + new_val = (256**self.bytelen) + val + if new_val >= 0: + print( + f"WARN: assigned negative value {val} to uint; this will be converted " + f"to the two's complement in {self.bytelen} bytes ({new_val})" + ) + val = new_val + else: + print( + f"WARN: assigned negative value {val} to uint; this value is out of " + f"bounds for a {self.bytelen}-byte int, and cannot be represented in " + "two's complement. This WILL lead to errors!" + ) + if val >= 256**self.bytelen: + print( + f"WARN: value {val} is out of bounds for {self.bytelen}-byte uint; " + "the value will be stored in full (truncated to 4 bytes due to " + f"technical limitations), but only the least significant {self.bytelen} " + "bytes will be read." + ) + elif val * 2 < -(256**self.bytelen) or val * 2 >= 256**self.bytelen: + print( + f"WARN: value {val} is out of bounds for {self.bytelen}-byte int; " + "the value will be stored in full (truncated to 4 bytes due to " + f"technical limitations), but only the least significant {self.bytelen} " + "bytes will be read." + ) + + return GDSIntValue(val, self, lit_fmt=lit_fmt) + + return ( + regex(r"[+-]?0x[0-9a-fA-F]+").map(lambda i: parser_map(int(i, 16), "hex")) + | regex(r"[+-]?0b[01]+").map(lambda i: parser_map(int(i, 2), "bin")) + | regex(r"[+-]?[0-9]+").map(lambda i: parser_map(int(i), "dec")) + ) << peek(regex(r"[^\.]?")) + + +TYPES.append(GDSIntType) + + +class GDSIntValue(GDSValue): + """ + An integer value. If written in a non-decimal base in a GDA script, this information is + preserved here (but not after being written to a GDS binary). + """ + + value: int + lit_fmt: Literal["hex", "bin", "dec"] = "dec" + + def __init__( + self, value: int, ty: GDSIntType, lit_fmt: Literal["hex", "bin", "dec"] = "dec" + ): + self.type = ty + self.value = value + self.lit_fmt = lit_fmt + + def as_token(self) -> gds.GDSToken: + val = self.value + if val < 0: + val += 256**self.type.bytelen + return gds.GDSTokenValue.int(val) + + def __format__(self, format_spec: Optional[str] = None) -> str: + # sourcery skip: assign-if-exp, reintroduce-else + if self.lit_fmt == "hex": + return hex(self.value) + if self.lit_fmt == "bin": + return bin(self.value) + return str(self.value) + + +class GDSFloatType(GDSValueType): + """ + A float type, of which there are no variations. It matches the IEEE-754 32-bit float + specification, though it is stored in little-endian byte order in GDS files. + """ + + @classmethod + def parse_type(cls, descriptor: str) -> "GDSFloatType": + return GDSFloatType() if descriptor == "float" else None + + def describe(self) -> str: + return "float" + + def from_token(self, tok: gds.GDSTokenValue) -> "GDSFloatValue": + return GDSFloatValue(tok(), self) if tok in gds.GDSTokenValue.float else None + + def parser(self) -> Parser: + """ + Valid float literals can include a sign (+-), and MUST include a period as well + as a nonempty int literal to either side of it. Omitting the other will denote it as 0, + therefore `123.` is a convenient way to write integers explicitly as floats. + Literals may also include an exponent. + + TODO: allow writing integers here? Since we already know where the program expects + a float due to the command list, we could make a convenient conversion here... + """ + return regex(r"[+-]?([0-9]+\.|[0-9]*\.[0-9])[0-9]*([eE][+-]?[0-9]+)?").map( + lambda f: GDSFloatValue(float(f), self) + ) + + +TYPES.append(GDSFloatType) + + +class GDSFloatValue(GDSValue): + """ + A float value. The only interesting aspect is the way it's printed in GDA files: + since IEEE-754 introduces nasty rounding errors even for simple float values, + we correct this by attempting to round to the float with the least decimal digits + which still produces the same value as the original. + """ + + value: float + + def __init__(self, value: float, ty: GDSFloatType): + self.type = ty + self.value = value + + def as_token(self) -> gds.GDSToken: + return gds.GDSTokenValue.float(self.value) + + def __format__(self, format_spec: Optional[str] = None) -> str: + return str(round_perfect(self.value)) + + +@dataclass +class GDSStringType(GDSValueType): + """ + A string type, with a maximum buffer length, and a flag indicating whether the game expects + a "long string". The main difference between the two is that a long string is stored in a + different buffer, meaning the two types can NOT be substituted for each other! + """ + + maxlen: int = 63 + longstr: bool = False + + @classmethod + def parse_type(cls, descriptor: str) -> "GDSStringType": + """ + Valid descriptors are: str, string, longstr (for long strings) + optionally followed by the buffer length in bytes / characters. + """ + # NOTE: if the game uses a 64-char buffer internally, remember the null terminator + # also needs to fit! The length of the corresponding GDS type should then be 63. + # TODO: I haven't yet determined the game's actual buffer size for both string types... + if descriptor.startswith("string"): + longstr = False + descriptor = descriptor[6:] + elif descriptor.startswith("str"): + longstr = False + descriptor = descriptor[3:] + elif descriptor.startswith("longstr"): + longstr = True + descriptor = descriptor[7:] + else: + return None + if descriptor == "": + maxlen = 63 + elif descriptor[0] == "(": + maxlen = int(descriptor[1:-1]) + else: + return None + return GDSStringType(maxlen=maxlen, longstr=longstr) + + def describe(self) -> str: + return ("longstr" if self.longstr else "string") + f"({self.maxlen})" + + def from_token(self, tok: gds.GDSTokenValue) -> "GDSStringValue": + # TODO: this should allow me to make things more flexible later + if self.longstr: + return ( + GDSStringValue(tok(), self) + if tok in gds.GDSTokenValue.longstr + else None + ) + else: + return GDSStringValue(tok(), self) if tok in gds.GDSTokenValue.str else None + + def parser(self) -> Parser: + """ + Valid literals can be delimited by double quotes or single quotes; + the chosed quote type will need to be escaped if it appears in the literal. + Generally most Python string escape sequences should be supported. + + Long strings are denoted by prepending a single "l" to the front of the quoted literal. + """ + + def parser_map(args: List[Any]) -> Parser: + is_longstr: bool = args[0] + text: str = args[1] + if is_longstr != self.longstr: + return fail( + f'A {"long" if is_longstr else "regular"} string cannot be ' + f'used in place of a {"long" if self.longstr else "regular"} string' + ) + if len(text) > self.maxlen: + print( + f"WARN: String {repr(text)} is too long ({len(text)}, max is {self.maxlen})" + ) + return success(GDSStringValue(text, self)) + + string_part_sq = regex(r"[^'\\]+") + string_part_dq = regex(r'[^"\\]+') + string_esc = string("\\") >> ( + string("\\") + | string("/") + | string('"') + | string("'") + | string("b").result("\b") + | string("f").result("\f") + | string("n").result("\n") + | string("r").result("\r") + | string("t").result("\t") + | regex(r"u[0-9a-fA-F]{4}").map(lambda s: chr(int(s[1:], 16))) + ) + quoted = ( + string('"') >> (string_part_dq | string_esc).many().concat() << string('"') + ) | ( + string("'") >> (string_part_sq | string_esc).many().concat() << string("'") + ) + + return seq(regex("l?").map(lambda s: s != ""), quoted).bind(parser_map) + + +TYPES.append(GDSStringType) + + +class GDSStringValue(GDSValue): + """ + A string value, also denoting whether it was defined as a long string. + This is currently redundant, since the GDSStringType already determines + whether a long string is expected; however, in the future interoperability + of the types might be possible. + """ + + value: str + longstr: bool + + def __init__(self, value: str, ty: GDSStringType): + self.type = ty + self.value = value + self.longstr = ty.longstr + + def as_token(self) -> gds.GDSToken: + if self.longstr: + return gds.GDSTokenValue.longstr(self.value) + return gds.GDSTokenValue.str(self.value) + + def __format__(self, format_spec: Optional[str] = None) -> str: + return ("l" if self.longstr else "") + repr(self.value) + + +@dataclass +class GDSBoolType(GDSValueType): + """ + A boolean type. This type is virtual, meaning it doesn't actually exist in the GDS + file format; however it is a useful abstraction for writing GDA scripts, and can be easily + converted into the GDS format (though this process may be lossy). + """ + + force_rep: Optional[Literal["int", "str"]] = None + + @classmethod + def parse_type(cls, descriptor: str) -> "GDSBoolType": + """ + Valid descriptors are: + bool|int : a bool backed by an integer token. Considered true if the value is != 0. + bool|string : a bool backed by a string token. Considered true exactly if the value + is the string "true" (WARNING: this is very specific and in particular, + case-sensitive. Especially for Python developers, accidentally writing + "True" and having the game recognize it as false is a serious pitfall! + For this reason, we discourage use of this type in its string form.) + bool : a bool that can be backed by either an integer or a string token; + the game will check both and uses whichever is applicable! + """ + if descriptor == "bool": + return GDSBoolType() + elif descriptor == "bool|int": + return GDSBoolType(force_rep="int") + elif descriptor == "bool|string": + return GDSBoolType(force_rep="str") + else: + return None + + def describe(self) -> str: # sourcery skip: assign-if-exp, reintroduce-else + if self.force_rep == "int": + return "bool|int" + if self.force_rep == "str": + return "bool|string" + return "bool" + + def from_token(self, tok: gds.GDSTokenValue) -> "GDSBoolValue": + if (tok in gds.GDSTokenValue.int and self.force_rep != "str") or ( + tok in gds.GDSTokenValue.str and self.force_rep != "int" + ): + return GDSBoolValue(tok(), self) + return None + + def parser(self) -> Any: + """ + `true` and `false` are always valid literals, and will be converted to the required + backing token type (default is int). Otherwise, if the backing type allows it, + either a string or an int literal can be used instead. + + As all values other than 0/1 or "true"/"false" are flattened into a truth value by + the game, this is only possible to preserve binary parity between decompiled-then-recompiled + original scripts (which sometimes show a preference for either type despite the command + supporting both). + """ + p = regex(r"true|false").map(lambda b: GDSBoolValue(b == "true", self)) + if self.force_rep != "int": + p |= GDSStringType().parser().map(lambda v: GDSBoolValue(v.value, self)) + if self.force_rep != "str": + p |= GDSIntType().parser().map(lambda v: GDSBoolValue(v.value, self)) + return p + + +TYPES.append(GDSBoolType) + + +class GDSBoolValue(GDSValue): + """ + A boolean value, backed either by an actual boolean, or an int or string. + Note that if the GDSBoolType declares a forced backing type, but the value + here is encoded as the opposite backing type, the value will be converted + in a lossy manner (this doesn't matter for the game though.) + """ + + value: Union[bool, int, str] + + def __init__(self, value: bool, ty: GDSBoolType): + self.type = ty + self.value = value + + def as_token(self) -> gds.GDSToken: + preferred_rep = self.type.force_rep or None + value = self.value + if isinstance(value, int): + if preferred_rep == "str": + if value not in ["true", "false"]: + print( + f"WARN: assigning string value {repr(value)} (false) to int-backed bool; " + "this may lose information, but this information would be ignored " + "by the game anyway." + ) + value = value != 0 + else: + preferred_rep = "int" + elif isinstance(value, str): + if preferred_rep == "int": + if value not in [0, 1]: + print( + f"WARN: assigning int value {value} (true) to string-backed bool; " + "this may lose information, but this information would be ignored " + "by the game anyway." + ) + value = value == "true" + else: + preferred_rep = "str" + + if preferred_rep is None: + preferred_rep = "int" + + if preferred_rep == "int": + if isinstance(value, bool): + value = 1 if value else 0 + return gds.GDSTokenValue.int(value) + elif preferred_rep == "str": + if isinstance(value, bool): + value = "true" if value else "false" + return gds.GDSTokenValue.str(value) + # unreachable + return None + + def __format__(self, format_spec: Optional[str] = None) -> str: + return repr(self.value) diff --git a/main.py b/main.py index 2c52fa0..e70382c 100644 --- a/main.py +++ b/main.py @@ -1,12 +1,12 @@ import click import formats -from version import v +import version CONTEXT_SETTINGS = dict(help_option_names = ['--help', '-h', '-?']) @click.group(name="flora", context_settings=CONTEXT_SETTINGS) -@click.version_option(v, '--version', '-v', prog_name="flora", message=f"Flora v{v} by patataofcourse") +@click.version_option(version.__version__, '--version', '-v', prog_name="flora", message=f"Flora v{version.__version__} by patataofcourse") def cli(): pass diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..e69de29 diff --git a/requirements-optional.txt b/requirements-optional.txt new file mode 100644 index 0000000..fa9cf06 --- /dev/null +++ b/requirements-optional.txt @@ -0,0 +1 @@ +tqdm \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 7389c82..5a305e6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,6 @@ click pillow ndspy +parsy +pyyaml +dacite \ No newline at end of file diff --git a/utils.py b/utils.py index 646e0f3..446e2cf 100644 --- a/utils.py +++ b/utils.py @@ -1,6 +1,29 @@ import os +from contextlib import contextmanager, suppress +from typing import ( + Any, + Callable, + Generic, + Mapping, + TypeVar, + Union, + Optional, + get_origin, + get_type_hints, + List, +) +import struct -def cli_file_pairs(input = None, output = None, *, in_ending = None, out_ending = None, recursive = False): + +def cli_file_pairs( + ipaths: Optional[str] = None, + opaths: Optional[str] = None, + *, + in_endings: Optional[List[str]] = None, + out_ending: Optional[str] = None, + filter_infer: Callable[[str, bool], bool] = None, + recursive: bool = False, +): """ Given the file path inputs to the various CLI commands, determines which input files should be operated on and mapped to which output files. @@ -9,28 +32,28 @@ def cli_file_pairs(input = None, output = None, *, in_ending = None, out_ending - If the input is a directory, all files in this directory (with the file ending `in_ending`) will be separately treated as input paths. - If there is no input (which also means there's no output), the current working directory is used as both input and output (which should be fine since most commands produce files of a different ending). - + Assuming the input is a file, then if the output is the same this pair is used as-is. Otherwise, if the output isn't specified (or a directory) he output path is *inferred* like this: - If the input ends with the expected file ending, that is stripped and replaced with the output file ending. It's expected that if a user wants more control over this file extension, they should provide an output manually for each input file. - If the input has a different ending (or none at all), the output file ending is simply appended to the full name. - + This same inference is used if the input is a directory: here we have multiple input paths, for which the targets can not have been specified, and so the inference is applied to each of them separately. The output must be a directory (or the input directory itself if not specified) and all the output paths are calculated such that input paths relative to the input directory become output paths relative to the output directory (i.e. `in/a/b/c.input` becomes `out/a/b/c.output`). If the output exists but isn't a directory, the CLI exits with an error explaining how this doesn't make sense. """ - if input is None: - input = "." - - if not os.path.exists(input): - raise FileNotFoundError(input) - + if ipaths is None: + ipaths = "." + + if not os.path.exists(ipaths): + raise FileNotFoundError(ipaths) + def listfiles(path): if recursive: - for (dp, _, fn) in os.walk(path, topdown=True): + for dp, _, fn in os.walk(path, topdown=True): for f in fn: yield os.path.join(dp, f) else: @@ -38,56 +61,192 @@ def listfiles(path): if not os.path.isfile(os.path.join(path, f)): continue yield os.path.join(path, f) - - def filter_infer(input): - if in_ending is not None and not input.lower().endswith(in_ending): + + def default_filter_infer(ipath, force_accept=False): + if ( + in_endings is not None + and not any(ipath.lower().endswith(ie) for ie in in_endings) + and not force_accept + ): return None - if out_ending is not None and input.lower().endswith(out_ending): + if ( + out_ending is not None + and ipath.lower().endswith(out_ending) + and not force_accept + ): return None - - output = input - if in_ending is not None and input.lower().endswith(in_ending): - output = input[:-len(in_ending)] + + output = ipath + if in_endings is not None: + endings = [ie for ie in in_endings if ipath.lower().endswith(ie)] + if endings: + output = ipath[: -len(endings[0])] if out_ending is None: - raise ValueError("Can't infer output file names without a target file ending specified") + raise ValueError( + "Can't infer output file names without a target file ending specified" + ) output += out_ending return output - + + if filter_infer is None: + filter_infer = default_filter_infer + input_dir = "" input_paths = [] - if os.path.isfile(input): - input_dir, ip = os.path.split(input) + rel_pairs = None + if os.path.isfile(ipaths): + input_dir, ip = os.path.split(ipaths) input_paths = [ip] + rel_pairs = [(ip, filter_infer(ip, True)) for ip in input_paths] else: - input_dir = input - input_paths = [os.path.relpath(f, input_dir) for f in listfiles(input)] - rel_pairs = [(ip, filter_infer(ip)) for ip in input_paths] + input_dir = ipaths + input_paths = [os.path.relpath(f, input_dir) for f in listfiles(ipaths)] + rel_pairs = [(ip, filter_infer(ip, False)) for ip in input_paths] rel_pairs = [(ip, op) for (ip, op) in rel_pairs if op is not None] - - if output is None: - output = input_dir - - if os.path.isfile(input) and not os.path.isdir(output) and os.path.split(output)[1] != '': - return [(input, output)] - - if os.path.isfile(output): - raise OSError(f"Output path exists but is not a directory: '{output}'") - output_dir = output - - pairs = [(os.path.join(input_dir, ip), os.path.join(output_dir, op)) for (ip, op) in rel_pairs] + + if opaths is None: + opaths = input_dir + + if ( + os.path.isfile(ipaths) + and not os.path.isdir(opaths) + and os.path.split(opaths)[1] != "" + ): + return [(ipaths, opaths)] + + if os.path.isfile(opaths): + raise OSError(f"Output path exists but is not a directory: '{opaths}'") + output_dir = opaths + + pairs = [ + (os.path.join(input_dir, ip), os.path.join(output_dir, op)) + for (ip, op) in rel_pairs + ] return pairs -def foreach_file_pair(pairs, fn, quiet = False): - try: + +def foreach_file_pair(pairs, fn, quiet=False): + # If TQDM isn't installed, continue as if --quiet was specified + with suppress(ImportError): from tqdm import tqdm - if not quiet: + + if not quiet and len(pairs) > 5: progress = tqdm(pairs) - for (input, output) in progress: - progress.set_description(input) - fn(input, output) + for ipath, opath in progress: + progress.set_description(ipath) + fn(ipath, opath) return - except ImportError: - # TQDM isn't installed; just don't show a progress bar. + for ipath, opath in pairs: + fn(ipath, opath) + + +T = TypeVar("T") + + +class TU(Generic[T]): + union: type + name: str + + def __init__(self, union: type, name: str): + self.union = union + self.name = name + + def __call__(self, val=None) -> "TUI[T]": + # assert isinstance(val, T) + return TUI(self, val) + + def __contains__(self, other): + return other.variant == self if isinstance(other, TUI) else False + + # def __eq__(self, other): + # return self.union == other.union and self.name == other.name + + def __hash__(self): + return hash((self.union, self.name)) + + def __repr__(self): + return f"{self.union.__name__}.{self.name}" + + +class TUI(Generic[T]): + variant: TU[T] + value: T + + def __init__(self, variant: TU[T], value: T): + self.variant = variant + self.value = value + + def __call__(self) -> T: + return self.value + + def __eq__(self, other): + return self.variant == other.variant and self.value == other.value + + def __hash__(self): + return hash((self.variant, self.value)) + + def __repr__(self): + return f"{repr(self.variant)}({repr(self.value)})" + + +def tagged_union(cls: type): + hints = get_type_hints(cls) + members = [(k, v) for (k, v) in hints.items() if get_origin(v) == TU] + + for name, t in members: + setattr(cls, name, t(cls, name)) + + return cls + + +class Test: + a: TU[int] + b: TU[str] + + +R = TypeVar("R") + + +def match(union: TUI, fns: Mapping[Union[TU, type(...)], Callable[[Any], R]]) -> R: + ellipsis_fn = None + for k, v in fns.items(): + if k is Ellipsis: + ellipsis_fn = v + continue + if union in k: + return v(union()) + if ellipsis_fn is not None: + return ellipsis_fn() + + +# TODO: use importlib.resources instead +RESOURCES = os.path.join(os.path.dirname(__file__), "data") + + +@contextmanager +def nested_break(): + """ + Raise the returned value to instantly bail out of the with block. + Replaces loop labels for breaking out of nested iterations. + """ + + class NestedBreakException(Exception): pass - for (input, output) in pairs: - fn(input, output) \ No newline at end of file + + with suppress(NestedBreakException): + yield NestedBreakException + + +def round_places(x: float, places: int = 0) -> float: + return round(x * (10**places)) / (10**places) + + +def round_perfect(x: float) -> float: + for i in range(1, 8): + y = round_places(x, i) + # If the rounded form is equivalent in bits to the full version, + # just use this simpler rounded version. + if struct.unpack("