create-ethernet-map lib instead of bin

.gitattributes

@@ -0,0 +1 @@
+device/libs/**/*.a filter=lfs diff=lfs merge=lfs -text

.github/docker_install_common.sh

@@ -7,6 +7,7 @@ apt-get update && apt-get install -y \
     cmake \
     ninja-build \
     git \
+    git-lfs \
     libhwloc-dev \
     libgtest-dev \
     libyaml-cpp-dev \

.github/workflows/build-device.yml

@@ -44,6 +44,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
+          lfs: 'true'
 
       - name: Build ${{ env.BUILD_TARGET }}
         run: |

.github/workflows/build-image.yml

@@ -28,6 +28,9 @@ jobs:
 
       - name: Checkout repository
         uses: actions/checkout@v4
+        with:
+          submodules: recursive
+          lfs: 'true'
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3

.github/workflows/build-tests.yml

@@ -33,7 +33,6 @@ env:
   LIB_OUTPUT_DIR: ./build/lib
   DEPS_OUTPUT_DIR: ./build/_deps
   TEST_OUTPUT_DIR: ./build/test
-  CREATE_MAP_BINARIES_DIR: ./device/bin/silicon
 
 jobs:
   build:
@@ -59,6 +58,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
+          lfs: 'true'
 
       - name: Build ${{ env.BUILD_TARGET }}
         run: |
@@ -74,8 +74,7 @@ jobs:
         run: |
           tar cvf artifact.tar ${{ env.TEST_OUTPUT_DIR }} \
           ${{ env.LIB_OUTPUT_DIR }} \
-          ${{ env.DEPS_OUTPUT_DIR }} \
-          ${{ env.CREATE_MAP_BINARIES_DIR }}
+          ${{ env.DEPS_OUTPUT_DIR }}
 
       - name: Upload build artifacts archive
         uses: actions/upload-artifact@v4

device/CMakeLists.txt

@@ -54,6 +54,7 @@ target_link_libraries(
         Boost::interprocess
         fmt::fmt-header-only
         yaml-cpp::yaml-cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/libs/${CMAKE_SYSTEM_PROCESSOR}/libcreate_ethernet_map.a
 )
 
 install(

device/libs/README.md

@@ -0,0 +1,9 @@
+## Building lib from source
+
+The source can be found in the repository: https://github.com/tenstorrent/luwen
+
+To build static lib, use the following commands:
+```
+cargo build -p create-ethernet-map --release --lib
+cross build --target aarch64-unknown-linux-gnu --release --lib -p create-ethernet-map
+```

device/libs/create_ethernet_map.h

@@ -0,0 +1,4 @@
+// External function to create the cluster descriptor yaml file.
+extern "C" {
+int create_ethernet_map(char *file);
+}

device/tt_cluster_descriptor.cpp

@@ -4,6 +4,7 @@
 
 
 #include "tt_cluster_descriptor.h"
+#include "libs/create_ethernet_map.h"
 
 #include <fstream>
 #include <memory>
@@ -298,6 +299,25 @@ chip_id_t tt_ClusterDescriptor::get_closest_mmio_capable_chip(const chip_id_t ch
     return closest_chip;
 }
 
+std::string tt_ClusterDescriptor::get_cluster_descriptor_file_path() {
+    static std::string yaml_path;
+    static bool is_initialized = false;
+    if (!is_initialized){
+        // Cluster descriptor path will be created in the working directory.        
+        std::filesystem::path cluster_path = std::filesystem::path("cluster_descriptor.yaml");
+        if (!std::filesystem::exists(cluster_path)){
+            auto val = system ( ("touch " + cluster_path.string()).c_str());
+            if(val != 0) throw std::runtime_error("Cluster Generation Failed!");
+        }
+
+        int val = create_ethernet_map((char*)cluster_path.string().c_str());
+        if(val != 0) throw std::runtime_error("Cluster Generation Failed!");
+        yaml_path = cluster_path.string();
+        is_initialized = true;
+    }
+    return yaml_path;
+}
+
 std::unique_ptr<tt_ClusterDescriptor> tt_ClusterDescriptor::create_from_yaml(const std::string &cluster_descriptor_file_path) {
     std::unique_ptr<tt_ClusterDescriptor> desc = std::unique_ptr<tt_ClusterDescriptor>(new tt_ClusterDescriptor());
 

device/tt_cluster_descriptor.h

@@ -83,6 +83,10 @@ class tt_ClusterDescriptor {
   bool is_chip_remote(const chip_id_t chip_id) const;
   chip_id_t get_closest_mmio_capable_chip(const chip_id_t chip);
   chip_id_t get_shelf_local_physical_chip_coords(chip_id_t virtual_coord);
+
+  // TODO: These following functions will be removed, and ClusterDescriptor will be created without any parameters.
+  // get_cluster_descriptor_file_path will create ethernet map in the background.
+  static std::string get_cluster_descriptor_file_path();
   static std::unique_ptr<tt_ClusterDescriptor> create_from_yaml(const std::string &cluster_descriptor_file_path);
   static std::unique_ptr<tt_ClusterDescriptor> create_for_grayskull_cluster(
       const std::set<chip_id_t> &logical_mmio_device_ids,

tests/api/test_chip.cpp

@@ -54,7 +54,7 @@ inline std::unique_ptr<tt_ClusterDescriptor> get_cluster_desc() {
         cluster_desc = tt_ClusterDescriptor::create_from_yaml(yaml_path);
     } else {
         // TODO: remove getting manually cluster descriptor from yaml.
-        std::string yaml_path = test_utils::GetClusterDescYAML();
+        std::string yaml_path = tt_ClusterDescriptor::get_cluster_descriptor_file_path();
         cluster_desc = tt_ClusterDescriptor::create_from_yaml(yaml_path);
     }
 
@@ -92,7 +92,7 @@ inline std::unique_ptr<Cluster> get_cluster() {
         yaml_path = test_utils::GetAbsPath("blackhole_1chip_cluster.yaml");
     } else {
         // TODO: remove getting manually cluster descriptor from yaml.
-        yaml_path = test_utils::GetClusterDescYAML();
+        yaml_path = tt_ClusterDescriptor::get_cluster_descriptor_file_path();
     }
     // TODO: Remove the need to do this, allow default constructor to construct with all chips.
     std::unique_ptr<tt_ClusterDescriptor> cluster_desc = get_cluster_desc();
@@ -123,6 +123,11 @@ inline std::unique_ptr<Cluster> get_cluster() {
 TEST(ApiChipTest, ManualTLBConfiguration) {
     std::unique_ptr<Cluster> umd_cluster = get_cluster();
 
+    if (umd_cluster == nullptr || umd_cluster->get_all_chips_in_cluster().empty()) {
+        std::cout << "No chips found. Skipping test." << std::endl;
+        return;
+    }
+
     // Expect to throw for remote chip for any worker core
     auto remote_chips = umd_cluster->get_target_remote_device_ids();
     if (!remote_chips.empty()) {
@@ -177,6 +182,12 @@ TEST(ApiChipTest, ManualTLBConfiguration) {
 // TODO: Move to test_chip
 TEST(ApiChipTest, SimpleAPIShowcase) {
     std::unique_ptr<Cluster> umd_cluster = get_cluster();
+
+    if (umd_cluster == nullptr || umd_cluster->get_all_chips_in_cluster().empty()) {
+        std::cout << "No chips found. Skipping test." << std::endl;
+        return;
+    }
+
     chip_id_t chip_id = umd_cluster->get_cluster_description()->get_chips_with_mmio().begin()->first;
 
     // TODO: In future, will be accessed through tt::umd::Chip api.

tests/api/test_cluster.cpp

@@ -66,7 +66,7 @@ inline std::unique_ptr<tt_ClusterDescriptor> get_cluster_desc() {
         cluster_desc = tt_ClusterDescriptor::create_from_yaml(yaml_path);
     } else {
         // TODO: remove getting manually cluster descriptor from yaml.
-        std::string yaml_path = test_utils::GetClusterDescYAML();
+        std::string yaml_path = tt_ClusterDescriptor::get_cluster_descriptor_file_path();
         cluster_desc = tt_ClusterDescriptor::create_from_yaml(yaml_path);
     }
 
@@ -104,7 +104,7 @@ inline std::unique_ptr<Cluster> get_cluster() {
         yaml_path = test_utils::GetAbsPath("blackhole_1chip_cluster.yaml");
     } else {
         // TODO: remove getting manually cluster descriptor from yaml.
-        yaml_path = test_utils::GetClusterDescYAML();
+        yaml_path = tt_ClusterDescriptor::get_cluster_descriptor_file_path();
     }
     // TODO: Remove the need to do this, allow default constructor to construct with all chips.
     std::unique_ptr<tt_ClusterDescriptor> cluster_desc = get_cluster_desc();

tests/api/test_cluster_descriptor.cpp

@@ -45,7 +45,7 @@ inline std::unique_ptr<tt_ClusterDescriptor> get_cluster_desc() {
         cluster_desc = tt_ClusterDescriptor::create_from_yaml(yaml_path);
     } else {
         // TODO: remove getting manually cluster descriptor from yaml.
-        std::string yaml_path = test_utils::GetClusterDescYAML();
+        std::string yaml_path = tt_ClusterDescriptor::get_cluster_descriptor_file_path();
         cluster_desc = tt_ClusterDescriptor::create_from_yaml(yaml_path);
     }
 

tests/blackhole/test_bh_common.h

@@ -61,7 +61,7 @@ class BlackholeTestFixture : public ::testing::Test {
     std::iota(devices.begin(), devices.end(), 0);
     std::set<chip_id_t> target_devices = {devices.begin(), devices.end()};
     uint32_t num_host_mem_ch_per_mmio_device = 1;
-    device = std::make_unique<tt_SiliconDevice>(test_utils::GetAbsPath(SOC_DESC_PATH), test_utils::GetClusterDescYAML(), target_devices, num_host_mem_ch_per_mmio_device, false, true, true);
+    device = std::make_unique<tt_SiliconDevice>(test_utils::GetAbsPath(SOC_DESC_PATH), tt_ClusterDescriptor::get_cluster_descriptor_file_path(), target_devices, num_host_mem_ch_per_mmio_device, false, true, true);
     assert(device != nullptr);
     assert(device->get_cluster_description()->get_number_of_chips() == get_detected_num_chips());
 

tests/blackhole/test_silicon_driver_bh.cpp

@@ -97,14 +97,14 @@ TEST(SiliconDriverBH, CreateDestroy) {
 //     std::unordered_map<chip_id_t, uint32_t> simulated_harvesting_masks = {{0, 30}, {1, 60}};
 
 //     {
-//         std::unique_ptr<tt_ClusterDescriptor> cluster_desc_uniq = tt_ClusterDescriptor::create_from_yaml(test_utils::GetClusterDescYAML());
+//         std::unique_ptr<tt_ClusterDescriptor> cluster_desc_uniq = tt_ClusterDescriptor::create_from_yaml(tt_ClusterDescriptor::get_cluster_descriptor_file_path());
 //         if (cluster_desc_uniq->get_number_of_chips() != target_devices.size()) {
 //             GTEST_SKIP() << "SiliconDriverWH.Harvesting skipped because it can only be run on a two chip nebula system";
 //         }
 //     }
 
 //     uint32_t num_host_mem_ch_per_mmio_device = 1;
-//     tt_SiliconDevice device = tt_SiliconDevice("./tests/soc_descs/wormhole_b0_8x10.yaml", test_utils::GetClusterDescYAML(), target_devices, num_host_mem_ch_per_mmio_device, false, true, true, simulated_harvesting_masks);
+//     tt_SiliconDevice device = tt_SiliconDevice("./tests/soc_descs/wormhole_b0_8x10.yaml", tt_ClusterDescriptor::get_cluster_descriptor_file_path(), target_devices, num_host_mem_ch_per_mmio_device, false, true, true, simulated_harvesting_masks);
 //     auto sdesc_per_chip = device.get_virtual_soc_descriptors();
 
 //     ASSERT_EQ(device.using_harvested_soc_descriptors(), true) << "Expected Driver to have performed harvesting";
@@ -120,15 +120,15 @@ TEST(SiliconDriverBH, CreateDestroy) {
 //     std::set<chip_id_t> target_devices = {0, 1};
 //     std::unordered_map<chip_id_t, uint32_t> simulated_harvesting_masks = {{0, 30}, {1, 60}};
 //     {
-//         std::unique_ptr<tt_ClusterDescriptor> cluster_desc_uniq = tt_ClusterDescriptor::create_from_yaml(test_utils::GetClusterDescYAML());
+//         std::unique_ptr<tt_ClusterDescriptor> cluster_desc_uniq = tt_ClusterDescriptor::create_from_yaml(tt_ClusterDescriptor::get_cluster_descriptor_file_path());
 //         if (cluster_desc_uniq->get_number_of_chips() != target_devices.size()) {
 //             GTEST_SKIP() << "SiliconDriverWH.Harvesting skipped because it can only be run on a two chip nebula system";
 //         }
 //     }
 
 //     uint32_t num_host_mem_ch_per_mmio_device = 1;
 //     // Initialize the driver with a 1x1 descriptor and explictly do not perform harvesting
-//     tt_SiliconDevice device = tt_SiliconDevice("./tests/soc_descs/wormhole_b0_1x1.yaml", test_utils::GetClusterDescYAML(), target_devices, num_host_mem_ch_per_mmio_device, false, true, false, simulated_harvesting_masks);
+//     tt_SiliconDevice device = tt_SiliconDevice("./tests/soc_descs/wormhole_b0_1x1.yaml", tt_ClusterDescriptor::get_cluster_descriptor_file_path(), target_devices, num_host_mem_ch_per_mmio_device, false, true, false, simulated_harvesting_masks);
 //     auto sdesc_per_chip = device.get_virtual_soc_descriptors();
 
 //     ASSERT_EQ(device.using_harvested_soc_descriptors(), false) << "SOC descriptors should not be modified when harvesting is disabled";
@@ -146,15 +146,15 @@ TEST(SiliconDriverBH, CreateDestroy) {
 //     std::set<chip_id_t> target_devices = {0, 1};
 //     std::unordered_map<chip_id_t, uint32_t> simulated_harvesting_masks = {{0, 30}, {1, 60}};
 //     {
-//         std::unique_ptr<tt_ClusterDescriptor> cluster_desc_uniq = tt_ClusterDescriptor::create_from_yaml(test_utils::GetClusterDescYAML());
+//         std::unique_ptr<tt_ClusterDescriptor> cluster_desc_uniq = tt_ClusterDescriptor::create_from_yaml(tt_ClusterDescriptor::get_cluster_descriptor_file_path());
 //         if (cluster_desc_uniq->get_number_of_chips() != target_devices.size()) {
 //             GTEST_SKIP() << "SiliconDriverWH.Harvesting skipped because it can only be run on a two chip nebula system";
 //         }
 //     }
 
 //     uint32_t num_host_mem_ch_per_mmio_device = 1;
 
-//     tt_SiliconDevice device = tt_SiliconDevice("./tests/soc_descs/wormhole_b0_8x10.yaml", test_utils::GetClusterDescYAML(), target_devices, num_host_mem_ch_per_mmio_device, false, true, true, simulated_harvesting_masks);
+//     tt_SiliconDevice device = tt_SiliconDevice("./tests/soc_descs/wormhole_b0_8x10.yaml", tt_ClusterDescriptor::get_cluster_descriptor_file_path(), target_devices, num_host_mem_ch_per_mmio_device, false, true, true, simulated_harvesting_masks);
 //     set_params_for_remote_txn(device);
 //     auto mmio_devices = device.get_target_mmio_device_ids();
 

tests/galaxy/test_umd_concurrent_threads.cpp

@@ -25,7 +25,7 @@ static const std::string SOC_DESC_PATH = "tests/soc_descs/wormhole_b0_8x10.yaml"
 TEST(GalaxyConcurrentThreads, WriteToAllChipsL1) {
     // Galaxy Setup
 
-    std::string cluster_desc_path = test_utils::GetClusterDescYAML();
+    std::string cluster_desc_path = tt_ClusterDescriptor::get_cluster_descriptor_file_path();
     std::shared_ptr<tt_ClusterDescriptor> cluster_desc = tt_ClusterDescriptor::create_from_yaml(cluster_desc_path);
     std::set<chip_id_t> target_devices_th1 = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
     std::set<chip_id_t> target_devices_th2 = {17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32};
@@ -113,7 +113,7 @@ TEST(GalaxyConcurrentThreads, WriteToAllChipsL1) {
 
 TEST(GalaxyConcurrentThreads, WriteToAllChipsDram) {
     // Galaxy Setup
-    std::string cluster_desc_path = test_utils::GetClusterDescYAML();
+    std::string cluster_desc_path = tt_ClusterDescriptor::get_cluster_descriptor_file_path();
     std::shared_ptr<tt_ClusterDescriptor> cluster_desc = tt_ClusterDescriptor::create_from_yaml(cluster_desc_path);
     std::set<chip_id_t> target_devices_th1 = {0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32};
     std::set<chip_id_t> target_devices_th2 = {1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31};
@@ -204,7 +204,7 @@ TEST(GalaxyConcurrentThreads, WriteToAllChipsDram) {
 
 TEST(GalaxyConcurrentThreads, PushInputsWhileSignalingCluster) {
     // Galaxy Setup
-    std::string cluster_desc_path = test_utils::GetClusterDescYAML();
+    std::string cluster_desc_path = tt_ClusterDescriptor::get_cluster_descriptor_file_path();
     std::shared_ptr<tt_ClusterDescriptor> cluster_desc = tt_ClusterDescriptor::create_from_yaml(cluster_desc_path);
     std::set<chip_id_t> target_devices = {0, 1, 2, 3, 4, 5, 6, 7, 8};
     for (const auto& chip : target_devices) {

tests/galaxy/test_umd_remote_api.cpp

@@ -22,7 +22,7 @@ static const std::string SOC_DESC_PATH = "tests/soc_descs/wormhole_b0_8x10.yaml"
 
 void run_remote_read_write_test(uint32_t vector_size, bool dram_write) {
     // Galaxy Setup
-    std::string cluster_desc_path = test_utils::GetClusterDescYAML();
+    std::string cluster_desc_path = tt_ClusterDescriptor::get_cluster_descriptor_file_path();
     std::shared_ptr<tt_ClusterDescriptor> cluster_desc = tt_ClusterDescriptor::create_from_yaml(cluster_desc_path);
     std::set<chip_id_t> target_devices = {};
     for (const auto& chip : cluster_desc->get_all_chips()) {
@@ -126,7 +126,7 @@ TEST(GalaxyBasicReadWrite, LargeRemoteDramBlockReadWrite) { run_remote_read_writ
 void run_data_mover_test(
     uint32_t vector_size, tt_multichip_core_addr sender_core, tt_multichip_core_addr receiver_core) {
     // Galaxy Setup
-    std::string cluster_desc_path = test_utils::GetClusterDescYAML();
+    std::string cluster_desc_path = tt_ClusterDescriptor::get_cluster_descriptor_file_path();
     std::shared_ptr<tt_ClusterDescriptor> cluster_desc = tt_ClusterDescriptor::create_from_yaml(cluster_desc_path);
     std::set<chip_id_t> target_devices = {};
     for (const auto& chip : cluster_desc->get_all_chips()) {
@@ -240,7 +240,7 @@ TEST(GalaxyDataMovement, TwoChipMoveData4) {
 void run_data_broadcast_test(
     uint32_t vector_size, tt_multichip_core_addr sender_core, std::vector<tt_multichip_core_addr> receiver_cores) {
     // Galaxy Setup
-    std::string cluster_desc_path = test_utils::GetClusterDescYAML();
+    std::string cluster_desc_path = tt_ClusterDescriptor::get_cluster_descriptor_file_path();
     std::shared_ptr<tt_ClusterDescriptor> cluster_desc = tt_ClusterDescriptor::create_from_yaml(cluster_desc_path);
     std::set<chip_id_t> target_devices = {};
     for (const auto& chip : cluster_desc->get_all_chips()) {

tests/galaxy/test_umd_remote_api_stability.cpp

@@ -37,7 +37,7 @@ class WormholeGalaxyStabilityTestFixture : public WormholeTestFixture {
   static uint32_t scale_number_of_tests;
 
   static void SetUpTestSuite() {
-    std::unique_ptr<tt_ClusterDescriptor> cluster_desc = tt_ClusterDescriptor::create_from_yaml(test_utils::GetClusterDescYAML());
+    std::unique_ptr<tt_ClusterDescriptor> cluster_desc = tt_ClusterDescriptor::create_from_yaml(tt_ClusterDescriptor::get_cluster_descriptor_file_path());
     detected_num_chips = cluster_desc->get_number_of_chips();
     if (detected_num_chips < EXPECTED_MIN_CHIPS) {
         skip_tests = true;

tests/test_utils/generate_cluster_desc.hpp

@@ -27,29 +27,4 @@ inline std::string GetAbsPath(std::string path_){
     std::filesystem::path abs_path = umd_root / path_;
     return abs_path.string();
 }
-
-inline std::string GetClusterDescYAML(){
-    static std::string yaml_path;
-    static bool is_initialized = false;
-    if (!is_initialized){
-        std::filesystem::path umd_path = std::filesystem::path(test_utils::GetAbsPath(""));
-        std::filesystem::path cluster_path = umd_path / ".umd";
-        std::filesystem::create_directories( cluster_path );
-
-        cluster_path /= "cluster_desc.yaml";
-        if (!std::filesystem::exists(cluster_path)){
-            auto val = system ( ("touch " + cluster_path.string()).c_str());
-            if(val != 0) throw std::runtime_error("Cluster Generation Failed!");
-        }
-        // Generates the cluster descriptor in the CWD
-
-        std::filesystem::path eth_fpath = umd_path / "device/bin/silicon/x86/create-ethernet-map";
-        std::string cmd = fmt::format("{} {}", eth_fpath.string(), cluster_path.string());
-        int val = system(cmd.c_str());
-        if(val != 0) throw std::runtime_error("Cluster Generation Failed!");
-        yaml_path = cluster_path.string();
-        is_initialized = true;
-    }
-    return yaml_path;
-}
 } // namespace test_utils