From b55980b904cdca44f70cf7de19a0a366572145a0 Mon Sep 17 00:00:00 2001 From: Sandy Carter Date: Mon, 30 Sep 2024 08:54:16 -0400 Subject: [PATCH] headers: Generate a globals.h header with global variables --- extracted_reversing_data_bw_141.json | 98 +++++++++--- scripts/headers/analyze_headers.py | 67 +++++--- scripts/headers/bw1_decomp_gen/functions.py | 15 +- .../bw1_decomp_gen/generate_headers.py | 45 +++--- scripts/headers/bw1_decomp_gen/header.py | 144 ++++++++++++------ .../bw1_decomp_gen/src/reversing_utils.h | 2 + scripts/headers/bw1_decomp_gen/structs.py | 76 +++++---- scripts/headers/bw1_decomp_gen/vftable.py | 33 ++-- scripts/headers/tests/header_test.py | 75 +++++++-- scripts/headers/tests/struct_test.py | 2 +- 10 files changed, 373 insertions(+), 184 deletions(-) diff --git a/extracted_reversing_data_bw_141.json b/extracted_reversing_data_bw_141.json index 0a6cfb8..7dad4a3 100644 --- a/extracted_reversing_data_bw_141.json +++ b/extracted_reversing_data_bw_141.json @@ -110068,6 +110068,34 @@ ], "signature": "unsigned int (struct SetupList *, int, int, int, int, int, int, int) __attribute__((stdcall))" }, + { + "kind": "FUNCTIONPROTO", + "type": "SetupList__ListBoxDraw", + "size": 4, + "call_type": "__stdcall", + "result": "uint32_t", + "args": [ + "struct SetupList *", + "int", + "int", + "int", + "int", + "int", + "int", + "int" + ], + "arg_names": [ + "", + "", + "", + "", + "", + "", + "", + "" + ], + "signature": "uint32_t (struct SetupList *, int, int, int, int, int, int, int) __attribute__((stdcall))" + }, { "kind": "STRUCT_DECL", "type": "struct SetupList", @@ -110140,7 +110168,7 @@ }, { "name": "ListBoxDraw", - "type": "SetupList__ListBoxDraw_t **", + "type": "SetupList__ListBoxDraw *", "offset": 620 }, { @@ -125838,6 +125866,40 @@ "offset": 592 } ] + }, + { + "kind": "FUNCTIONPROTO", + "type": "globals_funcptr__DAT_00eb9a98_t", + "size": 4, + "call_type": "__fastcall", + "result": "void", + "args": [ + "void *", + "const void *", + "struct MobileWallHug *" + ], + "arg_names": [ + "this", + "edx", + "param_3" + ], + "signature": "void (void *, const void *, struct MobileWallHug *) __attribute__((fastcall))" + }, + { + "kind": "FUNCTIONPROTO", + "type": "globals_funcptr__PTR_00fa51d0_t", + "size": 4, + "call_type": "__cdecl", + "result": "void", + "args": [ + "int", + "int" + ], + "arg_names": [ + "", + "" + ], + "signature": "void (int, int)" } ], "functions": [ @@ -146238,27 +146300,27 @@ { "name": "FLOAT_00bf33f0", "type": "float", - "address": 12534472 + "address": 12530672 }, { "name": "DAT_00bf42c8", "type": "int", - "address": 12534476 + "address": 12534472 }, { "name": "DAT_00bf42cc", "type": "int", - "address": 12576908 + "address": 12534476 }, { "name": "DAT_00bfe88c", "type": "int", - "address": 12577044 + "address": 12576908 }, { "name": "DAT_00bfe914", "type": "int", - "address": 12530672 + "address": 12577044 }, { "name": "LH3DColor_ARRAY_00bff0b8", @@ -146538,7 +146600,7 @@ { "name": "FLOAT_ARRAY_00d1a280", "type": "float[8]", - "address": 8 + "address": 13738624 }, { "name": "Point2D_00d3ee60", @@ -146608,7 +146670,7 @@ { "name": "CHAR_ARRAY_00d414d8", "type": "char[128]", - "address": 128 + "address": 13898968 }, { "name": "ARGS_NEWGAME", @@ -146718,12 +146780,12 @@ { "name": "CHAR_ARRAY_00d99580", "type": "char[200]", - "address": 200 + "address": 14259584 }, { "name": "CHAR_ARRAY_00d99648", "type": "char[128]", - "address": 128 + "address": 14259784 }, { "name": "PTR_00d99724", @@ -146843,32 +146905,32 @@ { "name": "LH3DMesh_g_current_list_matrix", "type": "struct LHMatrix[256]", - "address": 15320420 + "address": 15322664 }, { "name": "LH3DMesh_g_pack", "type": "struct g_pack_t *", - "address": 15321444 + "address": 15334964 }, { "name": "LH3DMesh_g_current_matrix", "type": "struct LHMatrix *", - "address": 15322664 + "address": 15334984 }, { "name": "LH3DMesh_g_b_dont_care_about_texture", "type": "_Bool", - "address": 15334964 + "address": 15334996 }, { "name": "g_ptr_blocks__10LH3DIsland", "type": "struct LandBlock *[256]", - "address": 15334984 + "address": 15320420 }, { "name": "g_index_block__10LH3DIsland", "type": "uint8_t[32][32]", - "address": 15334996 + "address": 15321444 }, { "name": "LH3DObject", @@ -146972,7 +147034,7 @@ }, { "name": "DAT_00eb9a98", - "type": "void (*)(void *, const void *, struct MobileWallHug *) __attribute__((fastcall))", + "type": "globals_funcptr__DAT_00eb9a98_t*", "address": 15440536 }, { @@ -147092,7 +147154,7 @@ }, { "name": "PTR_00fa51d0", - "type": "void (*)(int, int)", + "type": "globals_funcptr__PTR_00fa51d0_t*", "address": 16404944 }, { diff --git a/scripts/headers/analyze_headers.py b/scripts/headers/analyze_headers.py index e544ad9..c736399 100644 --- a/scripts/headers/analyze_headers.py +++ b/scripts/headers/analyze_headers.py @@ -7,7 +7,7 @@ from pathlib import Path from typing import List, Tuple, Set, Optional, Dict, Union -from clang.cindex import TranslationUnit, Diagnostic, Config, TranslationUnitLoadError +from clang.cindex import TranslationUnit, Diagnostic, Config, Token, TranslationUnitLoadError @dataclass @@ -133,22 +133,26 @@ def parse_source(path: Optional[Path] = None, source: Optional[str] = None) -> T return translation_unit -def extract_function_pointers(struct_type) -> Dict[str, TypeInfo]: +def extract_function_pointers(struct_type) -> Dict[str, Tuple[TypeInfo, int]]: function_map: Dict[str, TypeInfo] = {} for t in struct_type.get_fields(): - if t.type.kind.name != "POINTER": - continue - p = t.type.get_pointee() - if p.kind.name != "FUNCTIONPROTO": + underlying_type = t.type + pointer_depth = 0 + while underlying_type.kind.name == "POINTER": + pointer_depth += 1 + underlying_type = underlying_type.get_pointee() + + if pointer_depth == 0 or underlying_type.kind.name != "FUNCTIONPROTO": continue - result = p.get_result().spelling - args = list(c.spelling for c in p.argument_types()) + + result = underlying_type.get_result().spelling + args = list(c.spelling for c in underlying_type.argument_types()) arg_names = list(c.spelling for c in t.get_definition().get_children() if c.kind.name == 'PARM_DECL') if len(arg_names) == 0: arg_names = [''] * len(args) assert(len(args) == len(arg_names)) - function_map[t.spelling] = TypeInfo(kind_name=p.kind.name, size=t.type.get_size(), - location=Path(t.location.file.name), children=(result, list(zip(args, arg_names)), p.spelling)) + function_map[t.spelling] = (TypeInfo(kind_name=underlying_type.kind.name, size=t.type.get_size(), + location=Path(t.location.file.name), children=(result, list(zip(args, arg_names)), underlying_type.spelling)), pointer_depth) return function_map @@ -168,7 +172,7 @@ def extract_type_info(tu: TranslationUnit) -> Tuple[bool, Dict[str, TypeInfo]]: children: List[Tuple[str, int] | Tuple[str, str, int]] | str | Tuple[str, List[str], str] = [] if c.kind.name == "STRUCT_DECL": extra_types = extract_function_pointers(struct_type) - types.update((f"{name.removeprefix("struct ")}__{k}", v) for k, v in extra_types.items()) + types.update((f"{name.removeprefix("struct ")}__{k}", v[0]) for k, v in extra_types.items()) children = [] for f in struct_type.get_fields(): child_name = f.spelling @@ -249,23 +253,38 @@ def extract_globals_in_header_info(tu: TranslationUnit) -> Tuple[bool, Dict[str, globals_cursor = next(c for c in tu.cursor.get_children() if c.kind.name == "VAR_DECL" and c.spelling == "globals") globals_decl_cursor = globals_cursor.type.get_fields() globals_init_list = list(next(i for i in globals_cursor.get_children() if i.kind.name == 'INIT_LIST_EXPR').get_children()) - globals_init_cursor = (next(next(j.get_tokens()) for j in i.walk_preorder() if j.kind.name == 'INTEGER_LITERAL') for i in globals_init_list) + literal_map: dict[str, Token] = {} + for i in globals_init_list: + for j in i.walk_preorder(): + if j.kind.name == 'MEMBER_REF': + key = j.spelling + elif j.kind.name == 'INTEGER_LITERAL': + value = next(j.get_tokens()) + literal_map[key] = value + + types: Dict[str, TypeInfo] = {} + extra_types = extract_function_pointers(globals_cursor.type.get_declaration().type) + types.update((f"globals_funcptr__{k}_t", v[0]) for k, v in extra_types.items()) found_issues = False global_addresses: Dict[str, GlobalInfo] = {} - for identifier, literal in zip(globals_decl_cursor, globals_init_cursor): + for identifier in globals_decl_cursor: if identifier.spelling in GLOBALS_TO_IGNORE: continue - var_type = identifier.type.get_pointee().spelling var_name = identifier.spelling + if var_name in extra_types: + var_type = f"globals_funcptr__{var_name}_t" + (extra_types[var_name][1] - 1) * "*" + else: + var_type = identifier.type.get_pointee().spelling + literal_spelling = literal_map[var_name].spelling try: - literal_value: int = ast.literal_eval(literal.spelling) + literal_value: int = ast.literal_eval(literal_spelling) except ValueError as e: found_issues = True - sys.stderr.write(f"global declaration \"{identifier.spelling}\" can't parse: \"{literal.spelling}\"\n") + sys.stderr.write(f"global declaration \"{identifier.spelling}\" can't parse: \"{literal_spelling}\"\n") continue global_addresses[var_name] = GlobalInfo(var_type, literal_value) - return found_issues, global_addresses + return found_issues, global_addresses, types def extract_function_info(tu: TranslationUnit, known_types: Set[str], decorated_names: Set[str], @@ -344,7 +363,7 @@ def extract_function_info(tu: TranslationUnit, known_types: Set[str], decorated_ def main(out_path) -> bool: - paths: List[Path] = list(itertools.chain([Path("rtti.h")], *(p.glob("*.h") for p in PATHS))) + paths: List[Path] = list(filter(lambda x: x != Path("src/globals.h"), itertools.chain([Path("rtti.h")], *(p.glob("*.h") for p in PATHS)))) include_all_headers_src = '\n'.join(f'#include "{p}"' for p in paths) found_issues, types = extract_type_info(parse_source(source=include_all_headers_src)) @@ -359,15 +378,14 @@ def main(out_path) -> bool: decorated_names: Set[str] = set() function_metadata: List[FunctionMetadata] = [] for path in paths: - found_issues |= extract_function_info(parse_source(path=path), set(types.keys()), decorated_names, - function_metadata) + found_issues |= extract_function_info(parse_source(path=path), set(types.keys()), decorated_names, function_metadata) - new_issues, global_values = extract_globals_info(parse_source(source=include_all_headers_src), - set(types.keys())) + new_issues, global_values = extract_globals_info(parse_source(source=include_all_headers_src), set(types.keys())) found_issues |= new_issues - new_issues, globals_in_header_values = extract_globals_in_header_info(parse_source(path=Path("src/globals.h")), set(types.keys())) + new_issues, globals_in_header_values, globals_extra_types = extract_globals_in_header_info(parse_source(path=Path("globals.h"))) global_values.update(globals_in_header_values) + types |= globals_extra_types found_issues |= new_issues # for global_name, (global_type, global_value) in global_values.items(): @@ -418,8 +436,7 @@ def main(out_path) -> bool: f.consolidate_thiscall() result_functions.append(asdict(f)) - result_globals: List[Dict[str, str | int]] = [dict(name=global_name, type=g.type_name, value=g.address) for - global_name, g in global_values.items()] + result_globals: List[Dict[str, str | int]] = [dict(name=global_name, type=g.type_name, address=g.address) for global_name, g in global_values.items()] result = dict(types=result_types, functions=result_functions, globals=result_globals) diff --git a/scripts/headers/bw1_decomp_gen/functions.py b/scripts/headers/bw1_decomp_gen/functions.py index 969247f..aaeccbd 100644 --- a/scripts/headers/bw1_decomp_gen/functions.py +++ b/scripts/headers/bw1_decomp_gen/functions.py @@ -25,16 +25,18 @@ def clean_up_type(typename): class CSnakeFuncPtr(csnake.FuncPtr): - __slots__ = ("return_type", "arguments", "calling_convention") + __slots__ = ("return_type", "arguments", "calling_convention", "indirection_level") def __init__( self, return_type: str, arguments: Optional[Iterable] = None, calling_convention: Optional[str] = None, + indirection_level = 1, ) -> None: super().__init__(return_type, arguments) self.calling_convention = calling_convention + self.indirection_level = indirection_level def get_declaration( self, @@ -46,13 +48,14 @@ def get_declaration( arg.generate_declaration() for arg in self.arguments ) - retval = "{rt} ({conv}* {qual}{name}{arr})({arguments})".format( + retval = "{rt} ({conv}{indir}{qual}{name}{arr})({arguments})".format( rt=self.return_type, qual=qualifiers if qualifiers else "", name=name, arguments=jointargs if self.arguments else "", arr=array if array else "", - conv=self.calling_convention if self.calling_convention else "" + conv=self.calling_convention if self.calling_convention else "", + indir="*" * self.indirection_level + (" " if qualifiers or name or array else ""), ) return retval @@ -87,14 +90,16 @@ class FuncPtr: args: list[str] arg_labels: list[str] decorated_name: str + indirection_level: int - def __init__(self, name: str, call_type: str, result: str, args: list[str], arg_labels: list[str]): + def __init__(self, name: str, call_type: str, result: str, args: list[str], arg_labels: list[str], indirection_level: int = 1): self.name = name self.call_type = CALL_TYPE_SUBSTITUTIONS.get(call_type, call_type) self.result = clean_up_type(result) self.args = list(map(clean_up_type, args)) self.arg_labels = arg_labels self.decorated_name = name + self.indirection_level = indirection_level def get_types(self) -> set[str]: result = {self.result} @@ -120,7 +125,7 @@ def to_csnake(self) -> CSnakeFuncPtr: params.insert(1, ["edx", "const void*"]) conv = "__fastcall" - return CSnakeFuncPtr(self.result, params or [("", " void")], conv) + return CSnakeFuncPtr(self.result, params or [("", " void")], conv, self.indirection_level) def to_code(self, cw: csnake.CodeWriter): fptr = self.to_csnake() diff --git a/scripts/headers/bw1_decomp_gen/generate_headers.py b/scripts/headers/bw1_decomp_gen/generate_headers.py index 3d64a7c..b44c27a 100644 --- a/scripts/headers/bw1_decomp_gen/generate_headers.py +++ b/scripts/headers/bw1_decomp_gen/generate_headers.py @@ -8,7 +8,7 @@ from pathlib import Path import csnake -from header import Header, C_STDLIB_HEADER_IMPORT_MAP +from header import Header, GlobalsHeader, C_STDLIB_HEADER_IMPORT_MAP from structs import Struct, Union, Enum, RTTIClass from typedef import Typedef from functions import FuncPtr, DefinedFunctionPrototype, CSnakeFuncPtr @@ -38,27 +38,6 @@ def find_methods(function_db: list[dict]) -> tuple[dict[str, DefinedFunctionProt return thiscall_map, static_method_map, remainder -def is_globals_helper_struct(name: str) -> bool: - return name in [ - "globals_t", - "SetupThingDraw_t", - "SetupBox_t", - "LH3DObject_namespace", - "LH3DComplexObject_namespace", - "LH3DMist_namespace", - "LH3DObject_region", - "LH3DMem_t", - "GameThing_t", - "ape_hair_t", - "custom_footpath_display_control_t", - "g_pack_t", - "g_anim_pack_t", - "GlobalLH3DTextures", - "SetRenderModeData", - "ModAddedGlobals_t", - "globals_Script", - "SetRenderModeData__callback" - ] primitive_look_up = { 'STRUCT_DECL': Struct, 'UNION_DECL': Union, @@ -128,6 +107,14 @@ def batched_arg_to_csnake(type_decls): type_decls[int(i)].args[int(j)] = type_ +def generate_globals_header(globals_decl: dict, function_proto_map: dict[str, FuncPtr], local_header_import_map: dict[str, Path]) -> GlobalsHeader: + members = [Struct.Member(g["name"], g["type"], g["address"]) for g in globals_decl] + globals_t = Struct("globals_t", None, members) + header = GlobalsHeader(Path("globals.h"), includes=[], structs=[globals_t], function_proto_map=function_proto_map) + header.build_include_list(local_header_import_map) + return header + + # TODO: For each global and their types, create inspector entires: webserver or imgui inspector window if __name__ == "__main__": tic = time.perf_counter() @@ -218,10 +205,10 @@ def is_ignore_struct(data_type) -> bool: # Do some selecting ( - globals_t, vftables, bases, vftable_function_prototypes, + global_function_ptr_prototypes, header_structs, header_enums, remainder_enums, @@ -233,10 +220,10 @@ def is_ignore_struct(data_type) -> bool: remainder_primitives, remainder, ) = partition([ - lambda x: is_globals_helper_struct(x.name), lambda x: type(x) is Struct and (x.name.endswith('Vftable') or x.name.startswith('vt_')), lambda x: type(x) is Union and x.name.endswith('Base'), lambda x: type(x) is FuncPtr and ('Vftable__' in x.name or x.name.startswith('vt_')), + lambda x: type(x) is FuncPtr and x.name.startswith('globals_funcptr__'), is_header_struct, lambda x: type(x) is Enum and get_enum_header_name_key(x) is not None, lambda x: type(x) is Enum, @@ -249,8 +236,10 @@ def is_ignore_struct(data_type) -> bool: ], primitives) batched_arg_to_csnake(vftable_function_prototypes) + batched_arg_to_csnake(global_function_ptr_prototypes) vftable_function_proto_map = {i.name: i for i in vftable_function_prototypes} + global_function_ptr_proto_map = {i.name: i for i in global_function_ptr_prototypes} lh_linked_list_pointer_structs = {"struct " + i.name.removeprefix("LHLinkedList__p_").removeprefix("LHLinkedNode__p_") for i in lh_linked_pointer_lists} lh_linked_list_structs = {"struct " + i.name.removeprefix("LHLinkedList__").removeprefix("LHLinkedNode__") for i in lh_linked_lists} @@ -283,7 +272,7 @@ def get_path(name): raise RuntimeError(f"Need to add guessed path for {name} in vanilla_filepaths.py") return Path(project) / f"{stem}.h" - local_header_import_map: dict[str, str] = {} + local_header_import_map: dict[str, Path] = {} header_map: dict[Path, Header] = {} for e in header_enums: @@ -355,6 +344,9 @@ def get_path(name): headers: list[Header] = list(header_map.values()) + # Create header for globals_t with actual addresses + headers.append(generate_globals_header(remainder_globals, global_function_ptr_proto_map, local_header_import_map)) + if remainder_functions: header = Header(Path("TodoRemainderFunctions.h"), [], remainder_functions) header.build_include_list(local_header_import_map) @@ -372,9 +364,6 @@ def get_path(name): header.build_include_list(local_header_import_map) headers.append(header) - # TODO: create header for globals_t with actual addresses and remove from ignored count - to_ignore += globals_t - output_path = Path("generated_headers_output") if output_path.exists(): shutil.rmtree(output_path) diff --git a/scripts/headers/bw1_decomp_gen/header.py b/scripts/headers/bw1_decomp_gen/header.py index b761463..94e58e6 100644 --- a/scripts/headers/bw1_decomp_gen/header.py +++ b/scripts/headers/bw1_decomp_gen/header.py @@ -2,6 +2,7 @@ import re import os +from typing import Union from dataclasses import dataclass from inflection import underscore from pathlib import Path @@ -34,21 +35,6 @@ } -C_STDLIB_TYPEDEFS = { - "bool", - "int8_t", - "int16_t", - "int32_t", - "int64_t", - "uint8_t", - "uint16_t", - "uint32_t", - "uint64_t", - "uintptr_t", - "char16_t", -} - - C_STDLIB_HEADER_IMPORT_MAP = { "static_assert": "assert.h", "bool": "stdbool.h", @@ -62,10 +48,14 @@ "uint64_t": "stdint.h", "uintptr_t": "stdint.h", "char16_t": "uchar.h", - "D3DTLVERTEX": "d3dtypes.h" + "IDirect3DDevice7": "d3d.h", + "D3DMATRIX": "d3dtypes.h", + "D3DTLVERTEX": "d3dtypes.h", } + UTILITY_HEADER_IMPORT_MAP = { + "RTL_CRITICAL_SECTION": Path("reversing_utils.h"), "bool32_t": Path("reversing_utils.h"), "struct vec2u16": Path("reversing_utils.h"), "DECLARE_LH_LINKED_LIST": Path("lionhead/lhlib/LHLinkedList.h"), @@ -86,7 +76,7 @@ def strip_pointers_arrays_and_modifiers(c_type): return c_type c_type = re.sub(r'\*', '', c_type) c_type = strip_arrays_and_modifiers(c_type) - return c_type + return c_type.strip() @dataclass @@ -161,15 +151,27 @@ def get_types(self) -> set[str]: def get_includes(self) -> list[str]: return sorted(list(self.includes.values())) + @staticmethod + def is_forward_declarable_pointer(typename: Union[str, csnake.FuncPtr]): + if isinstance(typename, csnake.FuncPtr): + return True + if '*' in typename: + # array pointer + if '(*' in typename and typename.endswith("]"): + return False + else: + return strip_pointers_arrays_and_modifiers(typename) not in (C_STDLIB_HEADER_IMPORT_MAP.keys() | UTILITY_HEADER_IMPORT_MAP.keys()) + return False + def get_direct_dependencies(self) -> set[str]: result = self.get_types() - pointers = set(filter(lambda x: isinstance(x, csnake.FuncPtr) or ('*' in x and strip_pointers_arrays_and_modifiers(x) not in C_STDLIB_TYPEDEFS), result)) + pointers = set(filter(self.is_forward_declarable_pointer, result)) result.difference_update(pointers) lh_lists = {i for i in result if i.startswith("struct LHListHead__") or i.startswith("struct LHLinkedList__")} lh_lists_underlying_type = {"struct " + i.removeprefix("struct ").removeprefix("LHListHead__").removeprefix("LHLinkedList__p_").removeprefix("LHLinkedList__") for i in lh_lists} result.difference_update(lh_lists) result.update(lh_lists_underlying_type) - if self.structs: + if any(hasattr(i, "size") and i.size is not None for i in self.structs): result.add("static_assert") result = result - {f"struct {s.name}" for s in self.structs} - C_FUNDAMENTAL_TYPES if self.linked_lists_pointered: @@ -178,7 +180,7 @@ def get_direct_dependencies(self) -> set[str]: result.add("DECLARE_LH_LINKED_LIST") if self.lists_heads: result.add("DECLARE_LH_LIST_HEAD") - result = set(map(strip_arrays_and_modifiers, result)) + result = set(map(strip_pointers_arrays_and_modifiers, result)) return result def get_forward_declare_types(self) -> set[str]: @@ -200,41 +202,85 @@ def get_forward_declare_types(self) -> set[str]: result = {i for i in result if not i.startswith("struct LHListHead__") and not i.startswith("struct LHLinkedList__")} return result - def to_code(self, cw: csnake.CodeWriter): + def to_code_includes(self, cw: csnake.CodeWriter): + if not self.includes: + return + include_categories = partition([lambda x: x.system], self.get_includes()) + for c in include_categories: + c = list(c) + if not c: + continue + for i in c: + cw.include(('<' if i.system else '"') + i.header_path.as_posix() + ('>' if i.system else '"'), + ("For " + ", ".join(sorted(i.dependencies)) if i.dependencies else None)) + cw.add_line() + + def to_code_forward_declares(self, cw: csnake.CodeWriter): fwd = self.get_forward_declare_types() - cw.start_if_def(HEADER_GUARD_TEMPLATE % str.upper(underscore(self.path.stem)), invert=True) - cw.add_define(HEADER_GUARD_TEMPLATE % str.upper(underscore(self.path.stem))) + if not fwd: + return + cw.add_line("// Forward Declares") + cw.add_line() + for f in sorted(fwd): + cw.add_line(f"{f};") cw.add_line() - if self.includes: - include_categories = partition([lambda x: x.system], self.get_includes()) - for c in include_categories: - c = list(c) - if not c: - continue - for i in c: - cw.include(('<' if i.system else '"') + i.header_path.as_posix() + ('>' if i.system else '"'), - ("For " + ", ".join(sorted(i.dependencies)) if i.dependencies else None)) + def to_code_struct_decl(self, cw: csnake.CodeWriter): + if not self.structs: + return + for s in self.structs: + s.to_code(cw) + if s.decorated_name in self.linked_lists_pointered | self.linked_lists | self.lists_heads: + if s.decorated_name in self.linked_lists: + cw.add_line(f"DECLARE_LH_LINKED_LIST({s.name});") + if s.decorated_name in self.linked_lists_pointered: + cw.add_line(f"DECLARE_P_LH_LINKED_LIST({s.name});") + if s.decorated_name in self.lists_heads: + cw.add_line(f"DECLARE_LH_LIST_HEAD({s.name});") cw.add_line() - if fwd: - cw.add_line("// Forward Declares") - cw.add_line() - for f in sorted(fwd): - cw.add_line(f"{f};") - cw.add_line() + def to_code_inner(self, cw: csnake.CodeWriter): + self.to_code_includes(cw) + self.to_code_forward_declares(cw) + self.to_code_struct_decl(cw) + + def to_code(self, cw: csnake.CodeWriter): + cw.start_if_def(HEADER_GUARD_TEMPLATE % str.upper(underscore(self.path.stem)), invert=True) + cw.add_define(HEADER_GUARD_TEMPLATE % str.upper(underscore(self.path.stem))) + cw.add_line() - if self.structs: - for s in self.structs: - s.to_code(cw) - if s.decorated_name in self.linked_lists_pointered | self.linked_lists | self.lists_heads: - if s.decorated_name in self.linked_lists: - cw.add_line(f"DECLARE_LH_LINKED_LIST({s.name});") - if s.decorated_name in self.linked_lists_pointered: - cw.add_line(f"DECLARE_P_LH_LINKED_LIST({s.name});") - if s.decorated_name in self.lists_heads: - cw.add_line(f"DECLARE_LH_LIST_HEAD({s.name});") - cw.add_line() + self.to_code_inner(cw) cw.end_if_def() cw.add_line() + + +class GlobalsHeader(Header): + def __init__(self, path: Path, includes: list[Header.Include], structs: list[Composite], function_proto_map: dict[str, FuncPtr]): + super().__init__(path, includes, structs) + self.globals_struct = next(filter(lambda s: s.name == "globals_t", self.structs)) + self.globals_struct.print_offset_at_each = 1 # TODO: also print range based on member size: e.g. struct GGlobal* global; /* 00cd3b20-00d01020 */ + # For each type, deref it + for m in self.globals_struct.members: + m.data_type = function_proto_map.get(m.data_type.rstrip("*"), m.data_type) + if isinstance(m.data_type, FuncPtr): + m.data_type.indirection_level += 1 + m.data_type = m.data_type.to_csnake() + elif "(*" in m.data_type and m.data_type.endswith("]"): + m.data_type = m.data_type.replace("(*", "(**") + elif "[" in m.data_type: + m.data_type = m.data_type.replace("[", " (*)[", 1) + else: + m.data_type += "*" + + def to_code_inner(self, cw: csnake.CodeWriter): + super().to_code_inner(cw) + + cw.add_line(f"volatile static struct {self.globals_struct.name} globals = {{") + for m in sorted(self.globals_struct.members): + data_type = m.data_type + if isinstance(data_type, csnake.FuncPtr): + data_type = data_type.get_declaration("") + cw.add_line(f" .{m.name} = ({data_type})0x{m.offset:08x},") + cw.add_line("};") + cw.add_line() diff --git a/scripts/headers/bw1_decomp_gen/src/reversing_utils.h b/scripts/headers/bw1_decomp_gen/src/reversing_utils.h index a9958bb..17185f8 100644 --- a/scripts/headers/bw1_decomp_gen/src/reversing_utils.h +++ b/scripts/headers/bw1_decomp_gen/src/reversing_utils.h @@ -3,6 +3,8 @@ #include /* For uint32_t */ +typedef struct _RTL_CRITICAL_SECTION RTL_CRITICAL_SECTION; + typedef uint32_t bool32_t; struct vec2u16 diff --git a/scripts/headers/bw1_decomp_gen/structs.py b/scripts/headers/bw1_decomp_gen/structs.py index 3d717ce..66ce32d 100644 --- a/scripts/headers/bw1_decomp_gen/structs.py +++ b/scripts/headers/bw1_decomp_gen/structs.py @@ -1,7 +1,7 @@ import csnake from dataclasses import dataclass -from typing import Optional +from typing import Optional, Union from functions import DefinedFunctionPrototype from utils import partition, extract_type_name @@ -12,43 +12,51 @@ class Composite: @dataclass class Member: name: str - type: str + data_type: Union[str, csnake.FuncPtr] offset: int - def __init__(self, name: str, type: str, offset: int): + def __init__(self, name: str, data_type: str, offset: int): self.name = name - self.type = type.replace(" *", "*") + self.data_type = data_type + if type(self.data_type) is str: + self.data_type = self.data_type.replace(" *", "*") self.offset = offset + def __lt__(self, other: "Composite.Member") -> bool: + return self.offset < other.offset + def get_types(self) -> set[str]: - result = {self.type} + result = {self.data_type} return result def to_csnake(self) -> csnake.Variable: # Check if it's a pointer to an array (e.g., int(*)[2] or int (*)[2]) - if self.type.endswith('[]'): - self.type = self.type.rstrip('[]') - self.name += '[]' - if '(*' in self.type and '[' in self.type: - base_type, array_part = self.type.split('(*') - formatted_name = f"(*{self.name})" - array_part = array_part.lstrip('(*)') + if type(self.data_type) is str: + if self.data_type.endswith('[]'): + self.data_type = self.data_type.rstrip('[]') + self.name += '[]' + if '(*' in self.data_type and '[' in self.data_type: + base_type, array_part = self.data_type.split('(*') + formatted_name = f"(*{self.name})" + array_part = array_part.lstrip('(*)') + else: + # Handle regular arrays and non-pointer arrays + base_type, array_part = self.data_type.split( + '[', 1) if '[' in self.data_type else (self.data_type, '') + formatted_name = self.name + # Extract dimensions for arrays (e.g., "int[2][3]" -> [2, 3]) + dimensions = [dim for dim in array_part.replace(']', '').split('[') if dim] + base_type = base_type.strip() else: - # Handle regular arrays and non-pointer arrays - base_type, array_part = self.type.split( - '[', 1) if '[' in self.type else (self.type, '') - + dimensions = None + base_type = self.data_type formatted_name = self.name - # Extract dimensions for arrays (e.g., "int[2][3]" -> [2, 3]) - dimensions = [int(dim) for dim in array_part.replace( - ']', '').split('[') if dim] - # Create the csnake variable with either pointer/array or just array - return csnake.Variable(formatted_name, base_type.strip(), array=dimensions) + return csnake.Variable(formatted_name, base_type, array=dimensions) name: str - size: int + size: Optional[int] members: list[Member] @property @@ -57,6 +65,8 @@ def decorated_name(self): class Struct(Composite): + print_offset_at_each: Optional[int] = None + @property def decorated_name(self): return f"struct {self.name}" @@ -65,7 +75,7 @@ def decorated_name(self): def from_json(cls, decl: dict) -> "Struct": name = extract_type_name(decl['type']) size = decl['size'] - members = [cls.Member(**m) for m in decl["members"]] + members = [cls.Member(**{"data_type": m.pop("type"), **m}) for m in decl["members"]] return cls(name, size, members) def get_types(self) -> set[str]: @@ -76,14 +86,26 @@ def get_types(self) -> set[str]: def to_csnake(self) -> csnake.Struct: result = csnake.Struct(self.name, typedef=False) - variables = map(self.Member.to_csnake, self.members) - for v in variables: + # Check if sortable + if all(map(lambda x: x.__class__.__lt__ != object.__lt__, self.members)): + sorted_members = sorted(self.members) + else: + sorted_members = self.members + variables = map(self.Member.to_csnake, sorted_members) + last_offset: int = -1 + for v, m in zip(variables, sorted_members): + if self.print_offset_at_each: + if last_offset < 0 or m.offset - last_offset >= self.print_offset_at_each: + hoffset = f"{m.offset:08x}" if m.offset > 0x400000 else f"{m.offset:x}" + v.comment = None if hoffset in m.name else "0x" + hoffset + last_offset = m.offset result.add_variable(v) return result def to_code(self, cw: csnake.CodeWriter): cw.add_struct(self.to_csnake()) - cw.add_line(f'static_assert(sizeof({self.decorated_name}) == 0x{self.size:x}, "Data type is of wrong size");') + if self.size is not None: + cw.add_line(f'static_assert(sizeof({self.decorated_name}) == 0x{self.size:x}, "Data type is of wrong size");') cw.add_line() @@ -117,7 +139,7 @@ def decorated_name(self): def from_json(cls, decl: dict) -> "Union": name = extract_type_name(decl['type']) size = decl['size'] - members = [cls.Member(**m, offset=0) for m in decl["aliases"]] + members = [cls.Member(**{"data_type": m.pop("type"), **m}, offset=0) for m in decl["aliases"]] return cls(name, size, members) def get_types(self) -> set[str]: diff --git a/scripts/headers/bw1_decomp_gen/vftable.py b/scripts/headers/bw1_decomp_gen/vftable.py index c8363a7..81ced7e 100644 --- a/scripts/headers/bw1_decomp_gen/vftable.py +++ b/scripts/headers/bw1_decomp_gen/vftable.py @@ -8,39 +8,36 @@ @dataclass class Vftable(Struct): + skip_size_assert: bool @dataclass class Member: name: str - type: Union[FuncPtr, str] - comment: Optional[str] + data_type: Union[FuncPtr, str] + offset: int def to_csnake(self) -> csnake.Variable: - if type(self.type) is FuncPtr: - return csnake.Variable(self.name, self.type.to_csnake(), comment=self.comment) + if type(self.data_type) is FuncPtr: + return csnake.Variable(self.name, self.data_type.to_csnake()) else: - return csnake.Variable(self.name, self.type, comment=self.comment) + return csnake.Variable(self.name, self.data_type) def get_types(self): - if type(self.type) is FuncPtr: - return set(self.type.args).union((self.type.result, )) - if type(self.type) is str: - return set({self.type}) - raise NotImplementedError(f"Don't know what to do with {self.type}") + if type(self.data_type) is FuncPtr: + return set(self.data_type.args).union((self.data_type.result, )) + if type(self.data_type) is str: + return set({self.data_type}) + raise NotImplementedError(f"Don't know what to do with {self.data_type}") def __init__(self, struct: Struct, function_proto_map: dict[str, FuncPtr]): self.name = struct.name self.size = struct.size + self.print_offset_at_each = 0x10 substitutions = { "__dt__": "__dt", } self.members = [] - last_offset = 0 - for m in struct.members: + for i, m in enumerate(struct.members): name = substitutions.get(m.name, m.name) - type_ = function_proto_map.get(m.type.removesuffix("*"), m.type) - comment = None - if m.offset >= last_offset + 0x10: - comment = hex(m.offset) - last_offset = m.offset & ~0xF - self.members.append(self.Member(name, type_, comment)) + data_type = function_proto_map.get(m.data_type.rstrip("*"), m.data_type) + self.members.append(self.Member(name, data_type, offset=i*4)) diff --git a/scripts/headers/tests/header_test.py b/scripts/headers/tests/header_test.py index 8664000..e6dc6f6 100644 --- a/scripts/headers/tests/header_test.py +++ b/scripts/headers/tests/header_test.py @@ -8,7 +8,7 @@ from pathlib import Path from functions import FuncPtr, DefinedFunctionPrototype -from header import Header +from header import Header, GlobalsHeader from structs import Struct, RTTIClass from vftable import Vftable @@ -54,8 +54,8 @@ def test_direct_dependencies(self): Struct.Member("field_0x4", "enum TestEnum", 0x4), Struct.Member("field_0x8", "union TestUnion", 0x8), Struct.Member("field_0xc", "struct TestStruct3*", 0xc), - Struct.Member("field_0x10", "struct TestStruct3(*)[2]", 0x10), - Struct.Member("field_0x14", "struct TestStruct3*[2]", 0x14), + Struct.Member("field_0x10", "struct TestStruct4(*)[2]", 0x10), + Struct.Member("field_0x14", "struct TestStruct5*[2]", 0x14), Struct.Member("field_0x1c", "int", 0x1c), Struct.Member("field_0x20", "struct TestStruct1", 0x20), Struct.Member("field_0x24", "struct TestStruct1*", 0x24), @@ -65,6 +65,7 @@ def test_direct_dependencies(self): self.assertSetEqual(h.get_direct_dependencies(), { "static_assert", "struct TestStruct1", + "struct TestStruct4", "enum TestEnum", "union TestUnion", }) @@ -82,8 +83,8 @@ def test_forward_declare_list(self): Struct.Member("field_0x4", "enum TestEnum*", 0x4), Struct.Member("field_0x8", "union TestUnion*", 0x8), Struct.Member("field_0xc", "struct TestStruct3*", 0xc), - Struct.Member("field_0x10", "struct TestStruct3(*)[2]", 0x10), - Struct.Member("field_0x14", "struct TestStruct3*[2]", 0x14), + Struct.Member("field_0x10", "struct TestStruct5(*)[2]", 0x10), + Struct.Member("field_0x14", "struct TestStruct6*[2]", 0x14), Struct.Member("field_0x1c", "int", 0x1c), Struct.Member("field_0x1c", "int*", 0x20), Struct.Member("field_0x1c", "uint32_t*", 0x24), @@ -98,6 +99,7 @@ def test_forward_declare_list(self): "union TestUnion", "struct TestStruct3", "struct TestStruct4", + "struct TestStruct6", }) @@ -471,8 +473,8 @@ def test_structs_with_forward_declare(self): Struct.Member("field_0x4", "enum TestEnum*", 0x4), Struct.Member("field_0x8", "union TestUnion*", 0x8), Struct.Member("field_0xc", "struct TestStruct3*", 0xc), - Struct.Member("field_0x10", "struct TestStruct3(*)[2]", 0x10), - Struct.Member("field_0x14", "struct TestStruct3*[2]", 0x14), + Struct.Member("field_0x10", "uint32_t(*)[2]", 0x10), + Struct.Member("field_0x14", "struct TestStruct6*[2]", 0x14), Struct.Member("field_0x1c", "int", 0x1c), Struct.Member("field_0x20", "struct TestStruct1", 0x20), Struct.Member("field_0x24", "struct TestStruct1*", 0x24), @@ -488,6 +490,7 @@ def test_structs_with_forward_declare(self): #define BW1_DECOMP_TEST_HEADER_INCLUDED_H #include /* For static_assert */ +#include /* For uint32_t */ // Forward Declares @@ -495,11 +498,12 @@ def test_structs_with_forward_declare(self): struct TestStruct1; struct TestStruct3; struct TestStruct4; +struct TestStruct6; union TestUnion; struct TestStruct1Vftable { - char* (__fastcall* Foo)(struct TestStruct1* this, const void* edx, int param_1, struct TestStruct4* param_2); + char* (__fastcall* Foo)(struct TestStruct1* this, const void* edx, int param_1, struct TestStruct4* param_2); /* 0x0 */ void (__fastcall* Bar)(struct TestStruct1* this, const void* edx, float param_1); }; static_assert(sizeof(struct TestStruct1Vftable) == 0x8, "Data type is of wrong size"); @@ -516,8 +520,8 @@ def test_structs_with_forward_declare(self): enum TestEnum* field_0x4; union TestUnion* field_0x8; struct TestStruct3* field_0xc; - struct TestStruct3 (*field_0x10)[2]; - struct TestStruct3* field_0x14[2]; + uint32_t (*field_0x10)[2]; + struct TestStruct6* field_0x14[2]; int field_0x1c; struct TestStruct1 field_0x20; struct TestStruct1* field_0x24; @@ -575,7 +579,7 @@ def test_structs_with_functions(self): struct TestStructVftable { - char* (__fastcall* Foo)(struct TestStruct* this, const void* edx, int param_1); + char* (__fastcall* Foo)(struct TestStruct* this, const void* edx, int param_1); /* 0x0 */ void (__fastcall* Bar)(struct TestStruct* this); }; static_assert(sizeof(struct TestStructVftable) == 0x8, "Data type is of wrong size"); @@ -607,7 +611,7 @@ def test_structs_with_functions(self): struct TestChildStructVftable { - struct TestStructVftable super; + struct TestStructVftable super; /* 0x0 */ char* (__fastcall* Qux)(struct TestChildStruct* this, const void* edx, int test); }; static_assert(sizeof(struct TestChildStructVftable) == 0xc, "Data type is of wrong size"); @@ -651,11 +655,12 @@ def test_class_with_callback_in_vftable(self): #include /* For static_assert */ // Forward Declares + struct TestStruct; struct TestStructVftable { - void (__fastcall* Foo)(struct TestStruct* this, const void* edx, void (*foo)(int param_1, float param_2, int param_3)); + void (__fastcall* Foo)(struct TestStruct* this, const void* edx, void (*foo)(int param_1, float param_2, int param_3)); /* 0x0 */ }; static_assert(sizeof(struct TestStructVftable) == 0x4, "Data type is of wrong size"); @@ -665,5 +670,49 @@ def test_class_with_callback_in_vftable(self): }; static_assert(sizeof(struct TestStruct) == 0x4, "Data type is of wrong size"); +#endif /* BW1_DECOMP_TEST_HEADER_INCLUDED_H */ +""") + + def test_globals_header(self): + + test_funcptr_global_t = FuncPtr("test_funcptr_global_t", "__fastcall", "void", ["struct TestStruct*", "const void*", "int8_t"], ["this", "edx", "param_1"]) + + structs: list[Struct] = [ + Struct("globals_t", None, [ + Struct.Member("test_int_global", "int", 0xdeadbee7), + Struct.Member("test_struct_global", "struct TestStruct", 0xbaadc0de), + Struct.Member("test_uint_array_global", "uint32_t[0x800][0x900]", 0xdeadbeef), + Struct.Member("test_funcptr_global", "test_funcptr_global_t*", 0xdeadc0de), + ]), + ] + + header = GlobalsHeader(self.path, includes=[], structs=structs, function_proto_map={"test_funcptr_global_t": test_funcptr_global_t}) + header.build_include_list({}) + header.to_code(self.cw) + + self.assertEqual(self.cw.code, + """\ +#ifndef BW1_DECOMP_TEST_HEADER_INCLUDED_H +#define BW1_DECOMP_TEST_HEADER_INCLUDED_H + +// Forward Declares + +struct TestStruct; + +struct globals_t +{ + struct TestStruct* test_struct_global; /* 0xbaadc0de */ + int* test_int_global; /* 0xdeadbee7 */ + uint32_t (*test_uint_array_global)[0x800][0x900]; /* 0xdeadbeef */ + void (__fastcall** test_funcptr_global)(struct TestStruct* this, const void* edx, int8_t param_1); /* 0xdeadc0de */ +}; + +volatile static struct globals_t globals = { + .test_struct_global = (struct TestStruct*)0xbaadc0de, + .test_int_global = (int*)0xdeadbee7, + .test_uint_array_global = (uint32_t (*)[0x800][0x900])0xdeadbeef, + .test_funcptr_global = (void (__fastcall**)(struct TestStruct* this, const void* edx, int8_t param_1))0xdeadc0de, +}; + #endif /* BW1_DECOMP_TEST_HEADER_INCLUDED_H */ """) diff --git a/scripts/headers/tests/struct_test.py b/scripts/headers/tests/struct_test.py index 12dde16..4e5baac 100644 --- a/scripts/headers/tests/struct_test.py +++ b/scripts/headers/tests/struct_test.py @@ -40,5 +40,5 @@ def test_func_ptr_func_ptr_arg_to_code(self): self.assertEqual(csnake_obj.declaration.code, """\ struct TestStructVftable { - void (__fastcall* Foo)(struct TestStruct* this, const void* edx, void (*foo)(int param_1, float param_2, int param_3)); + void (__fastcall* Foo)(struct TestStruct* this, const void* edx, void (*foo)(int param_1, float param_2, int param_3)); /* 0x0 */ };""")