From 0d294c98a4df236f22edcffecf145330828472db Mon Sep 17 00:00:00 2001 From: Grant Date: Thu, 12 Sep 2024 15:50:52 +0100 Subject: [PATCH 1/7] start adding bitcode support for android --- devkit.py | 167 +++++++++++++++++++++++++++++++++++++++++++++++++ env_android.py | 4 +- env_generic.py | 4 +- 3 files changed, 171 insertions(+), 4 deletions(-) diff --git a/devkit.py b/devkit.py index ba638560..e06ee970 100644 --- a/devkit.py +++ b/devkit.py @@ -263,8 +263,22 @@ def _do_generate_library_unix(self, library_paths): objcopy = meson_config.get("objcopy", None) if objcopy is not None: + llvm_dis = [objcopy[0].replace("-objcopy", "-dis")] + llvm_as = [llvm_dis[0].replace("-dis", "-as")] + print("Generating symbol mappings...") thirdparty_symbol_mappings = get_thirdparty_symbol_mappings(output_path, meson_config) + """ + If there is llvmbc then we must patch the bitcode too. + 1. Check for the llvmbc section + 2. Extract the IR from the .o + 3. Patch the IR + 4. Re-assemble the .o + """ + new_archive = rename_symbols_in_library(str(output_path), thirdparty_symbol_mappings, objcopy, llvm_dis, llvm_as) + if new_archive: + output_path = new_archive + renames = "\n".join([f"{original} {renamed}" for original, renamed in thirdparty_symbol_mappings]) + "\n" with tempfile.NamedTemporaryFile() as renames_file: renames_file.write(renames.encode("utf-8")) @@ -533,3 +547,156 @@ def tweak_flags(cflags, ldflags): def deduplicate(items): return list(OrderedDict.fromkeys(items)) + + +def identify_bitcode_files(extract_dir): + """Identify bitcode-enabled object files.""" + bitcode_files = [] + for obj_file in os.listdir(extract_dir): + obj_path = os.path.join(extract_dir, obj_file) + # Check if the file is LLVM bitcode + result = subprocess.run(['readelf', '-S', obj_path], capture_output=True, text=True) + if '.llvmbc' in result.stdout: + bitcode_files.append(obj_path) + return bitcode_files + + +def disassemble_bitcode(llvm_dis, bitcode_file): + """Disassemble a bitcode object file into LLVM IR (.ll).""" + ll_file = bitcode_file.replace('.bc', '.ll') + subprocess.run(llvm_dis + [bitcode_file, '-o', ll_file], check=True) + return ll_file + + +def rename_symbols_in_ll(ll_file, rename_pairs): + """Rename symbols in the LLVM IR file using sed.""" + print(f"[+] Renaming symbols in {ll_file}") + for old_symbol, new_symbol in rename_pairs: + with open(ll_file, 'r') as file: + lines = file.readlines() + + # Regular expression to match function declarations, definitions, and calls for the specific symbol + # Prepare a list to hold modified lines + modified_lines = [] + cautious_symbol = "file" == old_symbol or "load" == old_symbol or "store" == old_symbol or "free" == old_symbol or "alloca" == old_symbol + if cautious_symbol: + print(f"Cautious symbol: {old_symbol}") + + pattern = rf'\B\@{old_symbol}\b' + + for line in lines: + if cautious_symbol: + if 'define ' not in line and 'declare ' not in line and 'call ' not in line: + modified_lines.append(line) + continue + + # modified_line = line.replace(f"@{old_symbol}", f"@{new_symbol}") + modified_line = re.sub(pattern, f"@{new_symbol}", line) + modified_lines.append(modified_line) + + # Write the modified lines back to the file + with open(ll_file, 'w') as file: + file.writelines(modified_lines) + + +def reassemble_bitcode(llvm_as, ll_file): + """Reassemble the modified LLVM IR file back to a bitcode object file.""" + bitcode_file = ll_file.replace('.ll', '.bc') + subprocess.run(llvm_as + [ll_file, '-o', bitcode_file], check=True) + return bitcode_file + + +def rebuild_library(extract_dir, output_library_path): + """Rebuild the static library from modified object files.""" + object_files = [f for f in os.listdir(extract_dir) if f.endswith('.o')] + subprocess.run(['ar', 'rcs', output_library_path] + object_files, cwd=extract_dir, check=True) + + +def extract_bitcode(objcopy, obj_file, bitcode_path): + """Extract the .llvmbc section from an object file.""" + subprocess.run(objcopy + ['--dump-section=.llvmbc=' + bitcode_path, obj_file], check=True) + + +def embed_bitcode(objcopy, obj_file, bitcode_path): + """Re-embed the modified bitcode back into the object file.""" + subprocess.run(objcopy + ['--update-section', f'.llvmbc={bitcode_path}', obj_file], check=True) + + +def extract_objects_from_archive(archive_path): + """Extract object files from a static library.""" + extract_dir = tempfile.TemporaryDirectory() + if not os.path.exists(extract_dir.name): + os.makedirs(extract_dir.name) + if os.path.exists(extract_dir.name): + print(f"Path does exist: {extract_dir.name}") + subprocess.run(['ar', 'x', archive_path], cwd=extract_dir.name, check=True) + return extract_dir + + +def process_object_file(object_file, rename_pairs, objcopy, llvm_dis, llvm_as): + # print(f"Processing {object_file}") + bitcode_path = object_file.replace('.o', '.bc') + # print(f"1. extracting bitcode from {object_file}: {bitcode_path}") + extract_bitcode(objcopy, object_file, bitcode_path) + + # Disassemble, rename symbols, and reassemble the bitcode + # print(f"2. disassembling bitcode: {bitcode_path}") + ll_file = disassemble_bitcode(llvm_dis, bitcode_path) + # print(f"3. renaming... {ll_file}") + # should_copy = False + # if "backend-libdwarf_gumsymbolutil-libdwarf.c." in ll_file: + # import shutil + # print("Copying backend-libdwarf_gumsymbolutil-libdwarf.c.ll instead...") + # shutil.copyfile(ll_file, "/tmp/ll.file") + # should_copy = True + # sz = (os.path.getsize(ll_file)) /1024 + # if sz < 30: + rename_symbols_in_ll(ll_file, rename_pairs) + # else: + # print(f"[!] Skipped {ll_file} due to size: {sz}kb") + # if should_copy: + # import shutil + # shutil.copyfile(ll_file, "/tmp/ll_renamed.file") + # should_copy = True + # print(f"4. re-assembling bitcode... {ll_file}") + modified_bitcode = reassemble_bitcode(llvm_as, ll_file) + + # Re-embed the modified bitcode back into the object file + # print(f"5. re-embedding bitcode in {object_file}: {modified_bitcode}") + embed_bitcode(objcopy, object_file, modified_bitcode) + + +def rename_symbols_in_library(library_path, rename_pairs, objcopy, llvm_dis, llvm_as): + """Main function to handle symbol renaming in a bitcode-enabled static library.""" + extract_dir = extract_objects_from_archive(library_path) + output_library_path = library_path + ".out" + + # Step 1: Identify bitcode-enabled object files + bitcode_files = identify_bitcode_files(extract_dir.name) + print(f"Total files with bitcode: {len(bitcode_files)}") + from concurrent.futures import ThreadPoolExecutor, as_completed + num_threads = 1 + + # Step 2: split the workload and process the IR + with ThreadPoolExecutor(max_workers=num_threads) as executor: + # Submit each item to be processed + future_to_item = {executor.submit(process_object_file, item, rename_pairs, objcopy, llvm_dis, llvm_as): item for item in bitcode_files} + + # Collect the results as they complete + for future in as_completed(future_to_item): + item = future_to_item[future] + try: + # Retrieve the result from each future + _result = future.result() + except Exception as e: + print(f"Error processing item {item}: {e}") + + # Step 3: Rebuild the static library + print("Re-building .a archive.") + rebuild_library(extract_dir.name, output_library_path) + + # Clean up the extracted directory + extract_dir.cleanup() + + print(f"Modified library saved as {output_library_path}") + return output_library_path \ No newline at end of file diff --git a/env_android.py b/env_android.py index a1233baa..89d45887 100644 --- a/env_android.py +++ b/env_android.py @@ -64,8 +64,8 @@ def init_machine_config(machine: MachineSpec, ] c_like_flags = [ "-DANDROID", - "-ffunction-sections", - "-fdata-sections", + # "-ffunction-sections", + # "-fdata-sections", ] cxx_like_flags = [] cxx_link_flags = [ diff --git a/env_generic.py b/env_generic.py index 3ca63c0c..b83fcca3 100644 --- a/env_generic.py +++ b/env_generic.py @@ -193,8 +193,8 @@ def init_machine_config(machine: MachineSpec, c_like_flags += ARCH_C_LIKE_FLAGS_UNIX.get(machine.arch, []) c_like_flags += [ - "-ffunction-sections", - "-fdata-sections", + # "-ffunction-sections", + # "-fdata-sections", ] if linker_flavor.startswith("gnu-"): From b1c2d0a7070e946e52dbf18504800e057d8bcae2 Mon Sep 17 00:00:00 2001 From: Grant Date: Fri, 13 Sep 2024 10:54:58 +0100 Subject: [PATCH 2/7] more small changes --- devkit.py | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/devkit.py b/devkit.py index e06ee970..d2e1c1b8 100644 --- a/devkit.py +++ b/devkit.py @@ -275,9 +275,7 @@ def _do_generate_library_unix(self, library_paths): 3. Patch the IR 4. Re-assemble the .o """ - new_archive = rename_symbols_in_library(str(output_path), thirdparty_symbol_mappings, objcopy, llvm_dis, llvm_as) - if new_archive: - output_path = new_archive + # rename_symbols_in_library(str(output_path), thirdparty_symbol_mappings, objcopy, llvm_dis, llvm_as) renames = "\n".join([f"{original} {renamed}" for original, renamed in thirdparty_symbol_mappings]) + "\n" with tempfile.NamedTemporaryFile() as renames_file: @@ -285,6 +283,7 @@ def _do_generate_library_unix(self, library_paths): renames_file.flush() subprocess.run(objcopy + ["--redefine-syms=" + renames_file.name, output_path], check=True) + else: thirdparty_symbol_mappings = [] @@ -571,6 +570,7 @@ def disassemble_bitcode(llvm_dis, bitcode_file): def rename_symbols_in_ll(ll_file, rename_pairs): """Rename symbols in the LLVM IR file using sed.""" print(f"[+] Renaming symbols in {ll_file}") + for old_symbol, new_symbol in rename_pairs: with open(ll_file, 'r') as file: lines = file.readlines() @@ -582,16 +582,19 @@ def rename_symbols_in_ll(ll_file, rename_pairs): if cautious_symbol: print(f"Cautious symbol: {old_symbol}") - pattern = rf'\B\@{old_symbol}\b' - for line in lines: if cautious_symbol: if 'define ' not in line and 'declare ' not in line and 'call ' not in line: modified_lines.append(line) continue - # modified_line = line.replace(f"@{old_symbol}", f"@{new_symbol}") - modified_line = re.sub(pattern, f"@{new_symbol}", line) + modified_line = line + if f"@{old_symbol}" in modified_line: + modified_line = modified_line.replace(f"@{old_symbol}", f"@{new_symbol}") + + if f"\"{old_symbol}\"" in modified_line: + modified_line = modified_line.replace(f"\"{old_symbol}\"", f"\"{new_symbol}\"") + modified_lines.append(modified_line) # Write the modified lines back to the file @@ -651,7 +654,10 @@ def process_object_file(object_file, rename_pairs, objcopy, llvm_dis, llvm_as): # should_copy = True # sz = (os.path.getsize(ll_file)) /1024 # if sz < 30: - rename_symbols_in_ll(ll_file, rename_pairs) + # rename_symbols_in_ll(ll_file, rename_pairs) + with open("/tmp/bitcode.log", "a+") as fo: + fo.write(f"Renaming bitcode for {bitcode_path}") + # else: # print(f"[!] Skipped {ll_file} due to size: {sz}kb") # if should_copy: @@ -663,7 +669,7 @@ def process_object_file(object_file, rename_pairs, objcopy, llvm_dis, llvm_as): # Re-embed the modified bitcode back into the object file # print(f"5. re-embedding bitcode in {object_file}: {modified_bitcode}") - embed_bitcode(objcopy, object_file, modified_bitcode) + # embed_bitcode(objcopy, object_file, modified_bitcode) def rename_symbols_in_library(library_path, rename_pairs, objcopy, llvm_dis, llvm_as): @@ -672,6 +678,7 @@ def rename_symbols_in_library(library_path, rename_pairs, objcopy, llvm_dis, llv output_library_path = library_path + ".out" # Step 1: Identify bitcode-enabled object files + print(f"Tmpdir: {extract_dir.name}") bitcode_files = identify_bitcode_files(extract_dir.name) print(f"Total files with bitcode: {len(bitcode_files)}") from concurrent.futures import ThreadPoolExecutor, as_completed @@ -689,7 +696,7 @@ def rename_symbols_in_library(library_path, rename_pairs, objcopy, llvm_dis, llv # Retrieve the result from each future _result = future.result() except Exception as e: - print(f"Error processing item {item}: {e}") + print(f"Error processing item {item}: {e}") # Step 3: Rebuild the static library print("Re-building .a archive.") @@ -698,5 +705,6 @@ def rename_symbols_in_library(library_path, rename_pairs, objcopy, llvm_dis, llv # Clean up the extracted directory extract_dir.cleanup() - print(f"Modified library saved as {output_library_path}") - return output_library_path \ No newline at end of file + import shutil + shutil.move(output_library_path, library_path) + print(f"Modified library saved as {library_path}") \ No newline at end of file From d7f04d71590660ec95761e02c45b717c797b8067 Mon Sep 17 00:00:00 2001 From: Grant Date: Fri, 13 Sep 2024 13:02:40 +0100 Subject: [PATCH 3/7] small fixes --- devkit.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/devkit.py b/devkit.py index d2e1c1b8..fdea20fb 100644 --- a/devkit.py +++ b/devkit.py @@ -263,10 +263,10 @@ def _do_generate_library_unix(self, library_paths): objcopy = meson_config.get("objcopy", None) if objcopy is not None: - llvm_dis = [objcopy[0].replace("-objcopy", "-dis")] - llvm_as = [llvm_dis[0].replace("-dis", "-as")] - print("Generating symbol mappings...") - thirdparty_symbol_mappings = get_thirdparty_symbol_mappings(output_path, meson_config) + # llvm_dis = [objcopy[0].replace("-objcopy", "-dis")] + # llvm_as = [llvm_dis[0].replace("-dis", "-as")] + # print("Generating symbol mappings...") + # thirdparty_symbol_mappings = get_thirdparty_symbol_mappings(output_path, meson_config) """ If there is llvmbc then we must patch the bitcode too. @@ -277,13 +277,13 @@ def _do_generate_library_unix(self, library_paths): """ # rename_symbols_in_library(str(output_path), thirdparty_symbol_mappings, objcopy, llvm_dis, llvm_as) - renames = "\n".join([f"{original} {renamed}" for original, renamed in thirdparty_symbol_mappings]) + "\n" - with tempfile.NamedTemporaryFile() as renames_file: - renames_file.write(renames.encode("utf-8")) - renames_file.flush() - subprocess.run(objcopy + ["--redefine-syms=" + renames_file.name, output_path], - check=True) - + # renames = "\n".join([f"{original} {renamed}" for original, renamed in thirdparty_symbol_mappings]) + "\n" + # with tempfile.NamedTemporaryFile() as renames_file: + # renames_file.write(renames.encode("utf-8")) + # renames_file.flush() + # subprocess.run(objcopy + ["--redefine-syms=" + renames_file.name, output_path], + # check=True) + thirdparty_symbol_mappings = [] else: thirdparty_symbol_mappings = [] From 63c71492aef2eb26cb30521cdf5bf373defd7f9b Mon Sep 17 00:00:00 2001 From: Grant Date: Sun, 15 Sep 2024 09:06:15 +0100 Subject: [PATCH 4/7] support for optional renaming of symbols, on by default which is currently the case --- devkit.py | 199 ++++------------------------------------------------ mkdevkit.py | 9 ++- 2 files changed, 22 insertions(+), 186 deletions(-) diff --git a/devkit.py b/devkit.py index fdea20fb..6875c9bf 100644 --- a/devkit.py +++ b/devkit.py @@ -31,7 +31,8 @@ def __init__(self, kit: str, machine: MachineSpec, meson_config: Mapping[str, Union[str, Sequence[str]]], - output_dir: Path): + output_dir: Path, + prefix_syms: bool = True): self.kit = kit package, umbrella_header = DEVKITS[kit] self.package = package @@ -42,6 +43,7 @@ def __init__(self, self.compiler_argument_syntax = None self.output_dir = output_dir self.library_filename = None + self.prefix_syms = prefix_syms def run(self): output_dir = self.output_dir @@ -263,27 +265,16 @@ def _do_generate_library_unix(self, library_paths): objcopy = meson_config.get("objcopy", None) if objcopy is not None: - # llvm_dis = [objcopy[0].replace("-objcopy", "-dis")] - # llvm_as = [llvm_dis[0].replace("-dis", "-as")] - # print("Generating symbol mappings...") - # thirdparty_symbol_mappings = get_thirdparty_symbol_mappings(output_path, meson_config) - - """ - If there is llvmbc then we must patch the bitcode too. - 1. Check for the llvmbc section - 2. Extract the IR from the .o - 3. Patch the IR - 4. Re-assemble the .o - """ - # rename_symbols_in_library(str(output_path), thirdparty_symbol_mappings, objcopy, llvm_dis, llvm_as) - - # renames = "\n".join([f"{original} {renamed}" for original, renamed in thirdparty_symbol_mappings]) + "\n" - # with tempfile.NamedTemporaryFile() as renames_file: - # renames_file.write(renames.encode("utf-8")) - # renames_file.flush() - # subprocess.run(objcopy + ["--redefine-syms=" + renames_file.name, output_path], - # check=True) - thirdparty_symbol_mappings = [] + # New option to prevent renaming, to avoid erroneous bitcode symbols + if self.prefix_syms: + renames = "\n".join([f"{original} {renamed}" for original, renamed in thirdparty_symbol_mappings]) + "\n" + with tempfile.NamedTemporaryFile() as renames_file: + renames_file.write(renames.encode("utf-8")) + renames_file.flush() + subprocess.run(objcopy + ["--redefine-syms=" + renames_file.name, output_path], + check=True) + else: + thirdparty_symbol_mappings = [] else: thirdparty_symbol_mappings = [] @@ -545,166 +536,4 @@ def tweak_flags(cflags, ldflags): def deduplicate(items): - return list(OrderedDict.fromkeys(items)) - - -def identify_bitcode_files(extract_dir): - """Identify bitcode-enabled object files.""" - bitcode_files = [] - for obj_file in os.listdir(extract_dir): - obj_path = os.path.join(extract_dir, obj_file) - # Check if the file is LLVM bitcode - result = subprocess.run(['readelf', '-S', obj_path], capture_output=True, text=True) - if '.llvmbc' in result.stdout: - bitcode_files.append(obj_path) - return bitcode_files - - -def disassemble_bitcode(llvm_dis, bitcode_file): - """Disassemble a bitcode object file into LLVM IR (.ll).""" - ll_file = bitcode_file.replace('.bc', '.ll') - subprocess.run(llvm_dis + [bitcode_file, '-o', ll_file], check=True) - return ll_file - - -def rename_symbols_in_ll(ll_file, rename_pairs): - """Rename symbols in the LLVM IR file using sed.""" - print(f"[+] Renaming symbols in {ll_file}") - - for old_symbol, new_symbol in rename_pairs: - with open(ll_file, 'r') as file: - lines = file.readlines() - - # Regular expression to match function declarations, definitions, and calls for the specific symbol - # Prepare a list to hold modified lines - modified_lines = [] - cautious_symbol = "file" == old_symbol or "load" == old_symbol or "store" == old_symbol or "free" == old_symbol or "alloca" == old_symbol - if cautious_symbol: - print(f"Cautious symbol: {old_symbol}") - - for line in lines: - if cautious_symbol: - if 'define ' not in line and 'declare ' not in line and 'call ' not in line: - modified_lines.append(line) - continue - - modified_line = line - if f"@{old_symbol}" in modified_line: - modified_line = modified_line.replace(f"@{old_symbol}", f"@{new_symbol}") - - if f"\"{old_symbol}\"" in modified_line: - modified_line = modified_line.replace(f"\"{old_symbol}\"", f"\"{new_symbol}\"") - - modified_lines.append(modified_line) - - # Write the modified lines back to the file - with open(ll_file, 'w') as file: - file.writelines(modified_lines) - - -def reassemble_bitcode(llvm_as, ll_file): - """Reassemble the modified LLVM IR file back to a bitcode object file.""" - bitcode_file = ll_file.replace('.ll', '.bc') - subprocess.run(llvm_as + [ll_file, '-o', bitcode_file], check=True) - return bitcode_file - - -def rebuild_library(extract_dir, output_library_path): - """Rebuild the static library from modified object files.""" - object_files = [f for f in os.listdir(extract_dir) if f.endswith('.o')] - subprocess.run(['ar', 'rcs', output_library_path] + object_files, cwd=extract_dir, check=True) - - -def extract_bitcode(objcopy, obj_file, bitcode_path): - """Extract the .llvmbc section from an object file.""" - subprocess.run(objcopy + ['--dump-section=.llvmbc=' + bitcode_path, obj_file], check=True) - - -def embed_bitcode(objcopy, obj_file, bitcode_path): - """Re-embed the modified bitcode back into the object file.""" - subprocess.run(objcopy + ['--update-section', f'.llvmbc={bitcode_path}', obj_file], check=True) - - -def extract_objects_from_archive(archive_path): - """Extract object files from a static library.""" - extract_dir = tempfile.TemporaryDirectory() - if not os.path.exists(extract_dir.name): - os.makedirs(extract_dir.name) - if os.path.exists(extract_dir.name): - print(f"Path does exist: {extract_dir.name}") - subprocess.run(['ar', 'x', archive_path], cwd=extract_dir.name, check=True) - return extract_dir - - -def process_object_file(object_file, rename_pairs, objcopy, llvm_dis, llvm_as): - # print(f"Processing {object_file}") - bitcode_path = object_file.replace('.o', '.bc') - # print(f"1. extracting bitcode from {object_file}: {bitcode_path}") - extract_bitcode(objcopy, object_file, bitcode_path) - - # Disassemble, rename symbols, and reassemble the bitcode - # print(f"2. disassembling bitcode: {bitcode_path}") - ll_file = disassemble_bitcode(llvm_dis, bitcode_path) - # print(f"3. renaming... {ll_file}") - # should_copy = False - # if "backend-libdwarf_gumsymbolutil-libdwarf.c." in ll_file: - # import shutil - # print("Copying backend-libdwarf_gumsymbolutil-libdwarf.c.ll instead...") - # shutil.copyfile(ll_file, "/tmp/ll.file") - # should_copy = True - # sz = (os.path.getsize(ll_file)) /1024 - # if sz < 30: - # rename_symbols_in_ll(ll_file, rename_pairs) - with open("/tmp/bitcode.log", "a+") as fo: - fo.write(f"Renaming bitcode for {bitcode_path}") - - # else: - # print(f"[!] Skipped {ll_file} due to size: {sz}kb") - # if should_copy: - # import shutil - # shutil.copyfile(ll_file, "/tmp/ll_renamed.file") - # should_copy = True - # print(f"4. re-assembling bitcode... {ll_file}") - modified_bitcode = reassemble_bitcode(llvm_as, ll_file) - - # Re-embed the modified bitcode back into the object file - # print(f"5. re-embedding bitcode in {object_file}: {modified_bitcode}") - # embed_bitcode(objcopy, object_file, modified_bitcode) - - -def rename_symbols_in_library(library_path, rename_pairs, objcopy, llvm_dis, llvm_as): - """Main function to handle symbol renaming in a bitcode-enabled static library.""" - extract_dir = extract_objects_from_archive(library_path) - output_library_path = library_path + ".out" - - # Step 1: Identify bitcode-enabled object files - print(f"Tmpdir: {extract_dir.name}") - bitcode_files = identify_bitcode_files(extract_dir.name) - print(f"Total files with bitcode: {len(bitcode_files)}") - from concurrent.futures import ThreadPoolExecutor, as_completed - num_threads = 1 - - # Step 2: split the workload and process the IR - with ThreadPoolExecutor(max_workers=num_threads) as executor: - # Submit each item to be processed - future_to_item = {executor.submit(process_object_file, item, rename_pairs, objcopy, llvm_dis, llvm_as): item for item in bitcode_files} - - # Collect the results as they complete - for future in as_completed(future_to_item): - item = future_to_item[future] - try: - # Retrieve the result from each future - _result = future.result() - except Exception as e: - print(f"Error processing item {item}: {e}") - - # Step 3: Rebuild the static library - print("Re-building .a archive.") - rebuild_library(extract_dir.name, output_library_path) - - # Clean up the extracted directory - extract_dir.cleanup() - - import shutil - shutil.move(output_library_path, library_path) - print(f"Modified library saved as {library_path}") \ No newline at end of file + return list(OrderedDict.fromkeys(items)) \ No newline at end of file diff --git a/mkdevkit.py b/mkdevkit.py index 91a22959..07bcc4b8 100755 --- a/mkdevkit.py +++ b/mkdevkit.py @@ -45,6 +45,12 @@ def main(): dest="flavor", const="_thin", default="") + parser.add_argument("-p", "--prefixsyms", + help="redefine the produced symbols with the frida_ prefix", + action="store_const", + dest="prefix_syms", + const="prefix_syms", + default=True) parser.add_argument("--cc", help="C compiler to use", type=lambda v: parse_array_option_value(v, ool_optvals)) @@ -61,6 +67,7 @@ def main(): machine = options.machine outdir = options.outdir.resolve() flavor = options.flavor + prefix_syms = options.prefix_syms cc = options.cc if cc is not None: @@ -82,7 +89,7 @@ def main(): assert meson_config is not None try: - app = devkit.CompilerApplication(kit, machine, meson_config, outdir) + app = devkit.CompilerApplication(kit, machine, meson_config, outdir, prefix_syms) app.run() except subprocess.CalledProcessError as e: print(e, file=sys.stderr) From d70d5b88cf533a8944296216504c148c39692189 Mon Sep 17 00:00:00 2001 From: Grant Date: Thu, 19 Sep 2024 11:01:10 -0700 Subject: [PATCH 5/7] enable passthrough of bitcode options --- devkit.py | 1 + env.py | 7 ++++++- env_android.py | 8 +++++--- env_apple.py | 3 ++- env_generic.py | 7 ++++--- mkdevkit.py | 2 +- 6 files changed, 19 insertions(+), 9 deletions(-) diff --git a/devkit.py b/devkit.py index 6875c9bf..03fe69cd 100644 --- a/devkit.py +++ b/devkit.py @@ -267,6 +267,7 @@ def _do_generate_library_unix(self, library_paths): if objcopy is not None: # New option to prevent renaming, to avoid erroneous bitcode symbols if self.prefix_syms: + print("PREFIX IS GO!!!!") renames = "\n".join([f"{original} {renamed}" for original, renamed in thirdparty_symbol_mappings]) + "\n" with tempfile.NamedTemporaryFile() as renames_file: renames_file.write(renames.encode("utf-8")) diff --git a/env.py b/env.py index 2d850ea1..5bad87de 100644 --- a/env.py +++ b/env.py @@ -142,6 +142,10 @@ def generate_machine_config(machine: MachineSpec, else: impl = env_generic + options = {} + if 1 == 1: + options["EMBED_BITCODE"] = True + impl.init_machine_config(machine, build_machine, is_cross_build, @@ -152,7 +156,8 @@ def generate_machine_config(machine: MachineSpec, config, outpath, outenv, - outdir) + outdir, + options) if machine.toolchain_is_msvc: builtin_options["b_vscrt"] = str_to_meson(machine.config) diff --git a/env_android.py b/env_android.py index 89d45887..4cac922c 100644 --- a/env_android.py +++ b/env_android.py @@ -17,7 +17,8 @@ def init_machine_config(machine: MachineSpec, config: ConfigParser, outpath: list[str], outenv: dict[str, str], - outdir: Path): + outdir: Path, + options: dict = {}): ndk_found = False try: ndk_root = Path(environ["ANDROID_NDK_ROOT"]) @@ -64,9 +65,10 @@ def init_machine_config(machine: MachineSpec, ] c_like_flags = [ "-DANDROID", - # "-ffunction-sections", - # "-fdata-sections", ] + if "EMBED_BITCODE" not in options: + c_like_flags.append("-ffunction-sections") + c_like_flags.append("-fdata-sections") cxx_like_flags = [] cxx_link_flags = [ "-static-libstdc++", diff --git a/env_apple.py b/env_apple.py index 6a34e5d8..75588a1f 100644 --- a/env_apple.py +++ b/env_apple.py @@ -18,7 +18,8 @@ def init_machine_config(machine: MachineSpec, config: ConfigParser, outpath: list[str], outenv: dict[str, str], - outdir: Path): + outdir: Path, + options: dict = {}): xcenv = {**environ} if machine.arch == "arm64eoabi": try: diff --git a/env_generic.py b/env_generic.py index b83fcca3..abb51a34 100644 --- a/env_generic.py +++ b/env_generic.py @@ -21,7 +21,8 @@ def init_machine_config(machine: MachineSpec, config: ConfigParser, outpath: list[str], outenv: dict[str, str], - outdir: Path): + outdir: Path, + options: dict = {}): allow_undefined_symbols = machine.os == "freebsd" options = config["built-in options"] @@ -193,8 +194,8 @@ def init_machine_config(machine: MachineSpec, c_like_flags += ARCH_C_LIKE_FLAGS_UNIX.get(machine.arch, []) c_like_flags += [ - # "-ffunction-sections", - # "-fdata-sections", + "-ffunction-sections", + "-fdata-sections", ] if linker_flavor.startswith("gnu-"): diff --git a/mkdevkit.py b/mkdevkit.py index 07bcc4b8..7c7f7981 100755 --- a/mkdevkit.py +++ b/mkdevkit.py @@ -50,7 +50,7 @@ def main(): action="store_const", dest="prefix_syms", const="prefix_syms", - default=True) + default=False) parser.add_argument("--cc", help="C compiler to use", type=lambda v: parse_array_option_value(v, ool_optvals)) From 8e0cd59d1998733a44e31425c55134703bf0cda4 Mon Sep 17 00:00:00 2001 From: Grant Date: Thu, 19 Sep 2024 11:15:47 -0700 Subject: [PATCH 6/7] enable passthrough of bitcode options --- env.py | 16 ++++++++-------- meson_configure.py | 7 ++++++- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/env.py b/env.py index 5bad87de..ac656f14 100644 --- a/env.py +++ b/env.py @@ -69,7 +69,8 @@ def generate_machine_configs(build_machine: MachineSpec, host_sdk_prefix: Optional[Path], call_selected_meson: Callable, default_library: DefaultLibrary, - outdir: Path) -> tuple[MachineConfig, MachineConfig]: + outdir: Path, + options: dict = {}) -> tuple[MachineConfig, MachineConfig]: is_cross_build = host_machine != build_machine if is_cross_build: @@ -86,7 +87,8 @@ def generate_machine_configs(build_machine: MachineSpec, build_sdk_prefix, call_selected_meson, default_library, - outdir) + outdir, + options) if is_cross_build: host_config = generate_machine_config(host_machine, @@ -97,7 +99,8 @@ def generate_machine_configs(build_machine: MachineSpec, host_sdk_prefix, call_selected_meson, default_library, - outdir) + outdir, + options) else: host_config = build_config @@ -112,7 +115,8 @@ def generate_machine_config(machine: MachineSpec, sdk_prefix: Optional[Path], call_selected_meson: Callable, default_library: DefaultLibrary, - outdir: Path) -> MachineConfig: + outdir: Path, + options: dict = {}) -> MachineConfig: config = ConfigParser(dict_type=OrderedDict) config["constants"] = OrderedDict() config["binaries"] = OrderedDict() @@ -142,10 +146,6 @@ def generate_machine_config(machine: MachineSpec, else: impl = env_generic - options = {} - if 1 == 1: - options["EMBED_BITCODE"] = True - impl.init_machine_config(machine, build_machine, is_cross_build, diff --git a/meson_configure.py b/meson_configure.py index cf90ade5..7fd9fc61 100644 --- a/meson_configure.py +++ b/meson_configure.py @@ -199,6 +199,10 @@ def configure(sourcedir: Path, except deps.BundleNotFoundError as e: raise_sdk_not_found(e, "host", host_machine) + options = {} + if '-Dbitcode=true' in extra_meson_options: + options["EMBED_BITCODE"] = True + build_config, host_config = \ env.generate_machine_configs(build_machine, host_machine, @@ -208,7 +212,8 @@ def configure(sourcedir: Path, host_sdk_prefix, call_selected_meson, default_library, - builddir) + builddir, + options) meson_options += [f"--native-file={build_config.machine_file}"] if host_config is not build_config: From 85e2221ffd237ae092f347e27a1712ba2e00bdbf Mon Sep 17 00:00:00 2001 From: Grant Date: Thu, 19 Sep 2024 11:19:15 -0700 Subject: [PATCH 7/7] fix erroneous removal of symbol mapping generation --- devkit.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/devkit.py b/devkit.py index 03fe69cd..e2103163 100644 --- a/devkit.py +++ b/devkit.py @@ -267,7 +267,8 @@ def _do_generate_library_unix(self, library_paths): if objcopy is not None: # New option to prevent renaming, to avoid erroneous bitcode symbols if self.prefix_syms: - print("PREFIX IS GO!!!!") + thirdparty_symbol_mappings = get_thirdparty_symbol_mappings(output_path, meson_config) + renames = "\n".join([f"{original} {renamed}" for original, renamed in thirdparty_symbol_mappings]) + "\n" with tempfile.NamedTemporaryFile() as renames_file: renames_file.write(renames.encode("utf-8"))