Add script to generate input code snippets for llvm-exegesis

gematria/datasets/BUILD

@@ -54,6 +54,20 @@ cc_binary(
     ],
 )
 
+cc_binary(
+    name = "convert_bhive_to_llvm_exegesis_input",
+    srcs = ["convert_bhive_to_llvm_exegesis_input.cc"],
+    deps = [
+        ":bhive_importer",
+        ":find_accessed_addrs",
+        "//gematria/llvm:canonicalizer",
+        "//gematria/llvm:llvm_architecture_support",
+        "//gematria/utils:string",
+        "@com_google_absl//absl/flags:flag",
+        "@com_google_absl//absl/flags:parse",
+    ],
+)
+
 cc_library(
     name = "find_accessed_addrs",
     srcs = ["find_accessed_addrs.cc"],

gematria/datasets/convert_bhive_to_llvm_exegesis_input.cc

@@ -0,0 +1,147 @@
+// Copyright 2023 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <fstream>
+#include <iostream>
+#include <memory>
+#include <sstream>
+#include <string>
+#include <string_view>
+
+#include "absl/flags/flag.h"
+#include "absl/flags/parse.h"
+#include "gematria/datasets/bhive_importer.h"
+#include "gematria/datasets/find_accessed_addrs.h"
+#include "gematria/llvm/canonicalizer.h"
+#include "gematria/llvm/llvm_architecture_support.h"
+#include "gematria/utils/string.h"
+
+#define INITIAL_REG_VALUE 0x10000
+#define INITIAL_MEM_VALUE 2147483647
+#define LLVM_EXEGESIS_MEM_DEF_PREFIX "# LLVM-EXEGESIS-MEM-DEF "
+#define LLVM_EXEGESIS_MEM_MAP_PREFIX "# LLVM-EXEGESIS-MEM-MAP "
+#define LLVM_EXEGESIS_MEM_NAME_PREFIX "MEM"
+
+static unsigned int file_counter = 0;
+
+ABSL_FLAG(std::string, bhive_csv, "",
+          "Filename of the input file containing code hex");
+ABSL_FLAG(
+    std::string, output_dir, "",
+    "Directory containing output files that can be executed by llvm-exegesis");
+ABSL_FLAG(std::string, exegesis_template,
+          "gematria/datasets/llvm-exegesis_warpper.S",
+          "Template file for llvm-exegesis");
+
+int main(int argc, char* argv[]) {
+  absl::ParseCommandLine(argc, argv);
+
+  std::string bhive_filename = absl::GetFlag(FLAGS_bhive_csv);
+  if (bhive_filename.empty()) {
+    std::cerr << "Error: --bhive_csv is required\n";
+    return 1;
+  }
+
+  std::string output_dir = absl::GetFlag(FLAGS_output_dir);
+  if (output_dir.empty()) {
+    std::cerr << "Error: --output_dir is required\n";
+    return 1;
+  }
+
+  // open template llvm-exegesis file
+  std::string template_file_path = absl::GetFlag(FLAGS_exegesis_template);
+  std::string template_content;
+
+  // Read the content of the template file
+  {
+    std::ifstream template_file(template_file_path);
+    std::stringstream buffer;
+    buffer << template_file.rdbuf();
+    template_content = buffer.str();
+  }
+
+  const std::unique_ptr<gematria::LlvmArchitectureSupport> llvm_support =
+      gematria::LlvmArchitectureSupport::X86_64();
+  gematria::X86Canonicalizer canonicalizer(&llvm_support->target_machine());
+  gematria::BHiveImporter bhive_importer(&canonicalizer);
+
+  std::ifstream bhive_csv_file(bhive_filename);
+  for (std::string line; std::getline(bhive_csv_file, line);) {
+    auto comma_index = line.find(',');
+    if (comma_index == std::string::npos) {
+      std::cerr << "Invalid CSV file: no comma in line '" << line << "'\n";
+      return 2;
+    }
+
+    std::string_view hex = std::string_view(line).substr(0, comma_index);
+    auto bytes_or = gematria::ParseHexString(hex);
+    if (!bytes_or.has_value()) {
+      std::cerr << "could not parse: " << hex << "\n";
+      return 3;
+    }
+
+    // for each line, find the accessed addresses & disassemble instructions
+    const auto& bytes = bytes_or.value();
+
+    // this will only get the first segfault address
+    auto addrs_or = gematria::FindAccessedAddrs(bytes);
+    auto proto = bhive_importer.BasicBlockProtoFromMachineCode(bytes);
+    if (addrs_or.ok() && proto.ok()) {
+      // Create output file path
+      std::string output_file_path =
+          output_dir + "/" + std::to_string(file_counter) + ".test";
+
+      // Open output file for writing
+      std::ofstream output_file(output_file_path);
+      if (!output_file.is_open()) {
+        std::cerr << "Failed to open output file: " << output_file_path << "\n";
+        return 4;
+      }
+
+      // Write the template content into the output file
+      output_file << template_content;
+
+      // Append memory annotations
+      auto addrs = addrs_or.value();
+
+      // Multiple mappings can point to the same definition
+      if (addrs.accessed_blocks.size() > 0) {
+        output_file << LLVM_EXEGESIS_MEM_DEF_PREFIX
+                    << LLVM_EXEGESIS_MEM_NAME_PREFIX << " " << addrs.block_size
+                    << " " << INITIAL_MEM_VALUE << "\n";
+      }
+      for (const auto& addr : addrs.accessed_blocks) {
+        output_file << LLVM_EXEGESIS_MEM_MAP_PREFIX
+                    << LLVM_EXEGESIS_MEM_NAME_PREFIX << " " << std::dec << addr
+                    << "\n";
+      }
+
+      // Append disassembled instructions
+      for (const auto& instr : proto->machine_instructions()) {
+        output_file << instr.assembly() << "\n";
+      }
+
+      file_counter++;
+    } else if (proto.ok()) {
+      std::cerr << "Failed to find addresses for block '" << hex
+                << "': " << addrs_or.status() << "\n";
+      std::cerr << "Block disassembly:\n";
+      for (const auto& instr : proto->machine_instructions()) {
+        std::cerr << "\t" << instr.assembly() << "\n";
+      }
+    } else {
+      std::cerr << "Failed to disassemble block '" << hex << "\n";
+    }
+  }
+}

gematria/datasets/llvm-exegesis_warpper.S

@@ -0,0 +1,33 @@
+# LLVM-EXEGESIS-DEFREG RAX 10000
+# LLVM-EXEGESIS-DEFREG RBX 10000
+# LLVM-EXEGESIS-DEFREG RCX 10000
+# LLVM-EXEGESIS-DEFREG RDX 10000
+# LLVM-EXEGESIS-DEFREG RSI 10000
+# LLVM-EXEGESIS-DEFREG RDI 10000
+# LLVM-EXEGESIS-DEFREG RBP 10000
+# LLVM-EXEGESIS-DEFREG RSP 10000
+# LLVM-EXEGESIS-DEFREG R8 10000
+# LLVM-EXEGESIS-DEFREG R9 10000
+# LLVM-EXEGESIS-DEFREG R10 10000
+# LLVM-EXEGESIS-DEFREG R11 10000
+# LLVM-EXEGESIS-DEFREG R12 10000
+# LLVM-EXEGESIS-DEFREG R13 10000
+# LLVM-EXEGESIS-DEFREG R14 10000
+# LLVM-EXEGESIS-DEFREG R15 10000
+# LLVM-EXEGESIS-DEFREG XMM0 10000
+# LLVM-EXEGESIS-DEFREG XMM1 10000
+# LLVM-EXEGESIS-DEFREG XMM2 10000
+# LLVM-EXEGESIS-DEFREG XMM3 10000
+# LLVM-EXEGESIS-DEFREG XMM4 10000
+# LLVM-EXEGESIS-DEFREG XMM5 10000
+# LLVM-EXEGESIS-DEFREG XMM6 10000
+# LLVM-EXEGESIS-DEFREG XMM7 10000
+# LLVM-EXEGESIS-DEFREG XMM8 10000
+# LLVM-EXEGESIS-DEFREG XMM9 10000
+# LLVM-EXEGESIS-DEFREG XMM10 10000
+# LLVM-EXEGESIS-DEFREG XMM11 10000
+# LLVM-EXEGESIS-DEFREG XMM12 10000
+# LLVM-EXEGESIS-DEFREG XMM13 10000
+# LLVM-EXEGESIS-DEFREG XMM14 10000
+# LLVM-EXEGESIS-DEFREG XMM15 10000
+

gematria/testing/testdata/test.csv

@@ -0,0 +1,154 @@
+4883c2014883fa40,0.00001339
+f30f6f0ff30f6f06660f74c1660fdac1660fefc9660f74c1660fd7c04885c0,0.00001339
+baffffffff,0.00001339
+498b4508488b75a0488906488d65d84c89e85b415c415d415e415f5d,0.00001339
+480fbcd00fb604170fb6141629d0,0.00001339
+ba60d264008b0a4883c2048d81fffefefef7d121c82580808080,0.00001339
+4c8b0a4c8b5208488d7c2410488b4210488b521848897c24084c894c24104c8954241848894424204889542428,0.00001339
+4883c4285b5d415c415d,0.00002678
+41544989fc5553488b9f48040000,0.00002678
+4157415641554154554c89cd534881ecc80300004d85c989542418894c241c,0.00002678
+534883ec104885c0,0.00002678
+f3440f6f2066450f74e066410fd7d485d2,0.00002678
+31d24889ee4489e7,0.00002678
+4983bf3802000000,0.00002678
+488b78080f1f40004883c010488b104885d2,0.00002678
+0fbe170fbe06662e0f1f84000000000084d2,0.00002678
+4c89d94829c14883f90a,0.00002678
+4c89d94829c14883f90b,0.00002678
+0fb70e66890f4883c6024883c702f6c204,0.00002678
+31c90fb77a384889d0480342204885ff49898424a0020000664189bc24b0020000,0.00002678
+4883c310488b034885c0,0.00002678
+be01000000,0.00002678
+89c1c1e910a9808000000f44c1488d4a02480f44d100c04883da034881ea60d264004889d0483dff030000,0.00002678
+415741564155415455534883ec284885ff48897c2418,0.00002678
+488b02498d4f404885c0,0.00002678
+488b55c0488b45c831f644895db04c8945b84c29ca4a8d3c704801d2,0.00002678
+488d7b11ba07000000,0.00002678
+4c8b6d804d85ed,0.00002678
+488b8548ffffffc7401801000000,0.00002678
+448b9d50ffffff4585db,0.00002678
+4531e44531ed488b8578ffffff4885c0,0.00002678
+4983bc24c000000000,0.00002678
+4889e8489949f7fb4889c5,0.00002678
+498b45084889c248c1e2054885c0498d4c15f0,0.00002678
+4801f8488b3e4889384889d748037e084889780849898424c80100004889c8498b8c24980200004885c9,0.00002678
+4889e84080e50048d1f85b4889e848d1f85d415c,0.00002678
+4501c84883c7014883ff0e,0.00002678
+4885c0488945c0,0.00002678
+644c8b242510000000,0.00002678
+488b70184c8d68104885f6,0.00002678
+660f1f8400000000004883c2010fb60a84c9,0.00002678
+4154554889fd53498b1c244885db,0.00002678
+488948184885d2,0.00002678
+488b53104989542410488b532848c1ea046641899424b2020000,0.00002678
+f6451004,0.00002678
+498b8424700100004885c0,0.00002678
+4863d1ffc148c1e2044889441608488b4018,0.00002678
+4801cf4983bfe000000000,0.00002678
+c1e80583e00783f801,0.00002678
+8d470383f804,0.00002678
+48897810498b87880200004885c0,0.00002678
+4883ec384889042448894c24084889542410488974241848897c24204c894424284c894c2430488b742440488b7c2438,0.00002678
+48837d2000,0.00002678
+85c00f94c00fb6c041894728488b459041c74718010000000fb6003c3a4d8b6f20b8000000000f45458c4d85ed89458c,0.00002678
+c1e00531d04883c4085b5d,0.00002678
+488b8d40ffffff4531e448898d38ffffff,0.00002678
+488b4c24384885c9,0.00002678
+49b8fffefefefefefefe660f1f440000488b084883c0084c89c24801ca,0.00002678
+ff0b,0.00002678
+4889c5648b04251800000085c0,0.00002678
+4154ba000900004989fc55534883ec1085c9,0.00002678
+83c20139ca,0.00002678
+48838540ffffff20488b8510ffffff48398540ffffff,0.00002678
+53bb0100000085d2,0.00002678
+4c89efbbc0426100,0.00002678
+4c8bb00003000031ff31f60f1f8000000000418b0cbe85c9,0.00002678
+31c085d20f94c0,0.00002678
+4883e80884c9,0.00002678
+4183fd02,0.00002678
+488b45804885c0,0.00002678
+488d471ebb1000000031d248897db048f7f348c1e0044829c44c8d7c240f4983e7f0c745cc00000000,0.00002678
+4889febf08000000,0.00002678
+0f1f80000000008b50084889d94803084883fa26,0.00002678
+4c01d84d8b1a4c89184989d34d035a084c89580849894424684889c84889f14889fe4c89c74d89c84d8b4c24704d85c9,0.00002678
+488d51ff482353104989942438040000488b53204989942420040000488b53104989942418040000410fb694241403000083e20380fa01,0.00002678
+683a000000,0.00002678
+31d24489f1488bb3800300004889df85c00f95c2,0.00002678
+488b8d60ffffff483901,0.00002678
+0fb68d70ffffffbe0100000031c048d3e64889f248c1e2044885f6,0.00002678
+488b7c24084531e4,0.00002678
+488b334889ef,0.00002678
+0f1f8400000000004883e80884c9,0.00002678
+4c89f64939d5,0.00002678
+891348890a890a4883c4105b,0.00002678
+4983bff000000000,0.00002678
+4156415541544189fc554889f5534881ecb0000000488b3e,0.00002678
+415541544189fc554889f5534889d34883ec284885c0,0.00002678
+8d3c00d1ff83fffc,0.00002678
+8078ff2f488d50ff,0.00002678
+4c29e04c89ef488d3403,0.00002678
+b8af46610055482da84661004883f80e4889e5,0.00002678
+488d7c2418be60506100b92000000048c744241070354000c78424980000000000000030db448b2c9d700b4100bf605061004489ee,0.00002678
+6820000000,0.00002678
+4531c94189d8b9020000004489ea31ff,0.00002678
+4989d34d0359084c01d04d8b114c8958084c891049894424704889c84889f14889fe4c89c74d8b4424784d85c0,0.00002678
+4883bd50ffffff00,0.00002678
+488b124885d2,0.00002678
+440fb6d64183fa40,0.00002678
+488dba60d26400ba00040000,0.00002678
+8b13f6c202,0.00002678
+83fa016690,0.00002678
+488d04db48c1e0044801c8488b104885d2,0.00002678
+ffc9488b064c8b46084c8b4e104c8b56184889074c8947084c894f104c895718488d7620488d7f20,0.00002678
+415641554989cd41544989d45589f5530fb687140300004889fb488b4f0883c8088887140300000fb61184d2,0.00002678
+0fb74c240e8d4201888a408a64003d00400000,0.00002678
+4531c931c931d241b8000000204889f7,0.00002678
+48891f4889e848c1c011488947084c8967104c896f184c8977204c897f28488d54240848c1c21148895730488b042448c1c0114889473831c0,0.00002678
+4939c7,0.00002678
+4531ff4585c0,0.00002678
+55534889f34883ec084885c0,0.00002678
+4883c4084489e05b5d415c415d,0.00002678
+488b034885c0,0.00002678
+4883c4084889da4889ee5b5d4889c7,0.00002678
+415531ff41545564488b2c2510000000534883ec08,0.00002678
+488db808090000,0.00002678
+4c8b52104d01d14c39cd,0.00002678
+4803034883f83f4989c6488903,0.00002678
+488b52084c8b4598480313,0.00002678
+480fb11a4839c1,0.00002678
+488b83a00000004885c0,0.00002678
+41f68424d403000020,0.00002678
+4531c931ff41b8ffffffffb922000000ba030000004889ee,0.00002678
+410fb68715030000a802,0.00002678
+4889d34889c2,0.00002678
+4c8b33498d59f04c8b5d004c8b55084883c0024883ed0889f14d89d74c8b651849d3e789f94c8b6b1849d3eb4d09df4d39f7,0.00002678
+4c89f948030c304883c010488948f04883ea01,0.00002678
+4c8b80a00200000fb780b00200004885c0488945c8,0.00002678
+488b0841bb7000000041b96000000041b850000000bf40000000be300000004889d1480348084989442460b810000000b9200000004d8b5424584d85d2,0.00002678
+418947344189473049c7472000000000,0.00002678
+31f685ed41b804000000,0.00002678
+4883c308488b2b4885ed,0.00002678
+488d42014883c408,0.00002678
+64488b00488b00488b7040488b7848488d8e000100006448890a488d8f000200006448890a488b50584881c20002000064488910,0.00002678
+83e0f7888314030000488b83100100004885c0,0.00002678
+31c0488b4c2410488b7c240848890cc5c0426100,0.00002678
+498b86b803000089da488b3cd0f6871403000008,0.00002678
+83470401,0.00002678
+488b8b100400004c8b830804000048894d80,0.00002678
+488b5b1831c985c00f95c109cd4885db,0.00002678
+49c7060000000049c74708000000004885db,0.00002678
+480fbcc0ba00000000488d0407403830480f45c2,0.00002678
+0f1f400048838540ffffff20488b8510ffffff48398540ffffff,0.00002678
+80bd58ffffff00,0.00002678
+4989fe31c0c78554ffffff01000000,0.00002678
+488b83a80000004885c0,0.00002678
+488d461881e300000002899d50ffffff48898540ffffff488d45e848898510ffffff488d458048898528ffffff488d458848898500ffffff488d8578ffffff488985f8feffff488d8570ffffff488985f0feffff488bbd40ffffff498b1f488b47e8448b27488b57f84989c34c035ff04585e4,0.00002678
+418b4734418b5730,0.00002678
+4981f8ffc99a3b,0.00002678
+48399380080000,0.00002678
+4c8b62084c036008488b3a48899568ffffff4c89e6,0.00002678
+4d85e44989c5,0.00002678
+8d50014088b08064610039ca,0.00002678
+488b0e48890f4883c6084883c70881e2f0000000,0.00002678
+4883f804,0.00002678