From f18a6453991c56990ec40e62159fdcda75e987f9 Mon Sep 17 00:00:00 2001 From: Julio Gonzalez Date: Mon, 4 Nov 2024 14:30:14 +0100 Subject: [PATCH] Refactor to support different target compilations. --- Cargo.lock | 3 +- builder/.cargo/config.toml | 5 - builder/Cargo.toml | 22 +- builder/build/data_pipeline.rs | 23 -- builder/build/main.rs | 278 ++----------------------- builder/build/symbolizer.rs | 23 -- builder/{build => src}/arch/apple.rs | 0 builder/{build => src}/arch/linux.rs | 7 +- builder/{build => src}/arch/mod.rs | 0 builder/{build => src}/arch/musl.rs | 0 builder/{build => src}/arch/windows.rs | 0 builder/src/bin/dummy.rs | 4 - builder/src/bin/release.rs | 126 +++++++++++ builder/{build => src}/common.rs | 22 ++ builder/{build => src}/crashtracker.rs | 32 ++- builder/src/dd_builder.rs | 172 +++++++++++++++ builder/src/lib.rs | 13 ++ builder/{build => src}/module.rs | 1 + builder/{build => src}/profiling.rs | 64 +++++- builder/{build => src}/utils.rs | 9 + ddcommon/src/rate_limiter.rs | 7 +- tools/src/bin/dedup_headers.rs | 2 +- tools/src/lib.rs | 4 +- 23 files changed, 470 insertions(+), 347 deletions(-) delete mode 100644 builder/.cargo/config.toml delete mode 100644 builder/build/data_pipeline.rs delete mode 100644 builder/build/symbolizer.rs rename builder/{build => src}/arch/apple.rs (100%) rename builder/{build => src}/arch/linux.rs (93%) rename builder/{build => src}/arch/mod.rs (100%) rename builder/{build => src}/arch/musl.rs (100%) rename builder/{build => src}/arch/windows.rs (100%) delete mode 100644 builder/src/bin/dummy.rs create mode 100644 builder/src/bin/release.rs rename builder/{build => src}/common.rs (53%) rename builder/{build => src}/crashtracker.rs (53%) create mode 100644 builder/src/dd_builder.rs create mode 100644 builder/src/lib.rs rename builder/{build => src}/module.rs (97%) rename builder/{build => src}/profiling.rs (61%) rename builder/{build => src}/utils.rs (76%) diff --git a/Cargo.lock b/Cargo.lock index 76b777bf5..073716ced 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -789,8 +789,7 @@ dependencies = [ "anyhow", "build_common", "cmake", - "datadog-profiling-ffi", - "ddcommon-ffi", + "pico-args", "tar", "tools", ] diff --git a/builder/.cargo/config.toml b/builder/.cargo/config.toml deleted file mode 100644 index 054956dfd..000000000 --- a/builder/.cargo/config.toml +++ /dev/null @@ -1,5 +0,0 @@ -[build] -jobs = 1 - -[env] -LIBDD_OUTPUT_FOLDER = { value = "../release", relative = true } diff --git a/builder/Cargo.toml b/builder/Cargo.toml index 2873a9e57..794aaad9e 100644 --- a/builder/Cargo.toml +++ b/builder/Cargo.toml @@ -8,24 +8,22 @@ license.workspace = true [features] default = [] -crashtracker = ["datadog-profiling-ffi?/crashtracker-receiver", "datadog-profiling-ffi?/crashtracker-collector", "datadog-profiling-ffi?/demangler"] -profiling = ["dep:datadog-profiling-ffi"] -telemetry = ["profiling", "datadog-profiling-ffi?/ddtelemetry-ffi"] -data-pipeline = ["telemetry", "datadog-profiling-ffi?/data-pipeline-ffi"] -symbolizer = ["profiling", "datadog-profiling-ffi?/symbolizer"] +crashtracker = [] +profiling = [] +telemetry = [] +data-pipeline = [] +symbolizer = [] -[build-dependencies] + +[dependencies] anyhow = { version = "1.0" } build_common = { path = "../build-common", features = ["cbindgen"] } cmake = "0.1.50" -tools = { path = "../tools" } -ddcommon-ffi = { path = "../ddcommon-ffi" } +pico-args = "0.5.0" tar = "0.4.41" - -[dependencies] -datadog-profiling-ffi = { path = "../profiling-ffi", optional = true, features = ["cbindgen"] } +tools = { path = "../tools" } [[bin]] -name = "dummy" +name = "release" test = false bench = false diff --git a/builder/build/data_pipeline.rs b/builder/build/data_pipeline.rs deleted file mode 100644 index 9c3e8ad7b..000000000 --- a/builder/build/data_pipeline.rs +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ -// SPDX-License-Identifier: Apache-2.0 - -use crate::module::Module; -use anyhow::Result; -use std::fs; -use std::path::PathBuf; -use std::rc::Rc; - -pub struct DataPipeline { - pub source_include: Rc, - pub target_include: Rc, -} - -impl Module for DataPipeline { - fn install(&self) -> Result<()> { - let origin_path: PathBuf = [&self.source_include, "data-pipeline.h"].iter().collect(); - let target_path: PathBuf = [&self.target_include, "data-pipeline.h"].iter().collect(); - - fs::copy(origin_path, target_path).expect("Failed to copy data pipeline header"); - Ok(()) - } -} diff --git a/builder/build/main.rs b/builder/build/main.rs index 58b15d3d9..67444d67c 100644 --- a/builder/build/main.rs +++ b/builder/build/main.rs @@ -1,266 +1,24 @@ // Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 -pub mod arch; -mod common; -#[cfg(feature = "crashtracker")] -mod crashtracker; -mod module; - -#[cfg(feature = "data-pipeline")] -mod data_pipeline; - -#[cfg(feature = "profiling")] -mod profiling; - -#[cfg(feature = "symbolizer")] -mod symbolizer; - -mod utils; - -use anyhow::Result; -use std::path::{Path, PathBuf}; -use std::rc::Rc; -use std::{env, fs}; -use utils::file_replace; - -use build_common::{determine_paths, HEADER_PATH}; -use tools::headers::dedup_headers; - -use crate::common::Common; -#[cfg(feature = "crashtracker")] -use crate::crashtracker::CrashTracker; -#[cfg(feature = "data-pipeline")] -use crate::data_pipeline::DataPipeline; -#[cfg(feature = "profiling")] -use crate::profiling::Profiling; -#[cfg(feature = "symbolizer")] -use crate::symbolizer::Symbolizer; -use module::Module; - -/// [`Builder`] is a structure that holds all the information required to assemble the final -/// workspace artifact. It will manage the different modules which will be in charge of producing -/// the different binaries and source files that will be part of the artifact. The builder will -/// provide the needed information: paths, version, etc, to the different modules so they can -/// install their sub-artifacts on the target folder. -/// The target folder is set through `LIBDD_OUTPUT_FOLDER` environment variable if it is not -/// provided the default target folder will be the builder output directory. -/// -/// # Example -/// -/// ```rust -/// use crate::core::Core; -/// -/// let mut builder = Builder::new(&path, &profile, &version); -/// let core = Box::new(Core { -/// version: builder.version.clone(), -/// }); -/// builder.add_module(core); -/// builder.build()?; -/// builder.pack()?; -/// ``` -struct Builder { - modules: Vec>, - main_header: Rc, - source_inc: Rc, - source_lib: Rc, - target_dir: Rc, - target_lib: Rc, - target_include: Rc, - target_bin: Rc, - target_pkconfig: Rc, - version: Rc, -} - -impl Builder { - /// Creates a new Builder instance - /// - /// # Aguments - /// - /// * `target_dir`: artifact folder. - /// * `profile`: Release configuration: debug or release; - /// * `version`: artifact's version. - /// - /// # Returns - /// - /// A new Builder instance. - fn new(source_dir: &str, target_dir: &str, target_arch: &str, host_arch: &str, profile: &str, version: &str) -> Self { - Builder { - modules: Vec::new(), - main_header: "common.h".into(), - source_lib: if host_arch == target_arch { - (source_dir.to_string() + "/" + profile + "/deps").into() - } else { - (source_dir.to_string() + "/" + target_arch + "/" + profile + "/deps").into() - }, - source_inc: (source_dir.to_string() + "/" + HEADER_PATH).into(), - target_dir: target_dir.into(), - target_lib: (target_dir.to_string() + "/lib").into(), - target_include: (target_dir.to_string() + "/" + HEADER_PATH).into(), - target_bin: (target_dir.to_string() + "/bin").into(), - target_pkconfig: (target_dir.to_string() + "/lib/pkgconfig").into(), - version: version.into(), - } - } - - /// Adds a boxed object which implements Module trait. - fn add_module(&mut self, module: Box) { - self.modules.push(module); - } - - fn create_dir_structure(&self) { - let target = Path::new(self.target_dir.as_ref()); - if fs::metadata(target).is_ok() { - fs::remove_dir_all(Path::new(self.target_dir.as_ref())) - .expect("Failed to clean preexisting target folder"); - } - fs::create_dir_all(Path::new(self.target_dir.as_ref())) - .expect("Failed to create target directory"); - fs::create_dir_all(Path::new(self.target_include.as_ref())) - .expect("Failed to create include directory"); - fs::create_dir_all(Path::new(self.target_lib.as_ref())) - .expect("Failed to create include directory"); - fs::create_dir_all(Path::new(self.target_bin.as_ref())) - .expect("Failed to create include directory"); - fs::create_dir_all(Path::new(self.target_pkconfig.as_ref())) - .expect("Failed to create include directory"); - } - - fn deduplicate_headers(&self) { - let datadog_inc_dir = Path::new(self.source_inc.as_ref()); - - let mut headers: Vec = Vec::new(); - let inc_files = fs::read_dir(datadog_inc_dir).unwrap(); - for file in inc_files.flatten() { - let name = file.file_name().into_string().unwrap(); - if name.ends_with(".h") && !name.eq("common.h") && !name.eq("blazesym.h") { - headers.push(file.path().to_string_lossy().to_string()); - } - } - - let base_header = self.source_inc.to_string() + "/" + self.main_header.as_ref(); - dedup_headers(&base_header, &headers); - } - - // TODO: maybe do this in module's build.rs - fn sanitize_libraries(&self) { - let datadog_lib_dir = Path::new(self.source_lib.as_ref()); - - let libs = fs::read_dir(datadog_lib_dir).unwrap(); - for lib in libs.flatten() { - let name = lib.file_name().into_string().unwrap(); - if name.ends_with(".so") { - arch::fix_rpath(lib.path().to_str().unwrap()); - } - } - } - - fn add_cmake(&self) { - let libs = arch::NATIVE_LIBS.to_owned(); - let cmake_path: PathBuf = [&self.target_dir, "DatadogConfig.cmake"].iter().collect(); - file_replace( - "../cmake/DatadogConfig.cmake.in", - cmake_path.to_str().unwrap(), - "@Datadog_LIBRARIES@", - libs.trim(), - ) - .expect("Failed to modify the cmake"); - } - - /// Builds the final artifact by going through all modules and instancing their install method. - /// - /// #Returns - /// - /// Ok(()) if success Err(_) if failure. - fn build(&self) -> Result<()> { - for module in &self.modules { - module.install()?; - } - Ok(()) - } - - /// Generate a tar file with all the intermediate artifacts generated by all the modules.k - /// - /// #Returns - /// - /// Ok(()) if success Err(_) if failure. - fn pack(&self) -> Result<()> { - let tarname = "libdatadog".to_string() + "_v" + &self.version + ".tar"; - let path: PathBuf = [self.target_dir.as_ref(), &tarname].iter().collect(); - let artifact = fs::File::create(path).expect("Failed to create tarfile"); - let mut ar = tar::Builder::new(artifact); - ar.append_dir_all("lib", self.target_lib.as_ref())?; - ar.append_dir("bin", self.target_bin.as_ref())?; - ar.append_dir_all("include/datadog", self.target_include.as_ref())?; - - ar.finish().expect("Failed to write the tarfile"); - Ok(()) - } -} - fn main() { // Rerun build script if any of the env vars change. - println!("cargo:rerun-if-env-changed=LIBDD_OUTPUT_FOLDER"); - println!("cargo:rerun-if-env-changed=PROFILE"); - - let (_, source_path) = determine_paths(); - let mut path = env::var("OUT_DIR").unwrap(); - if let Ok(libdd_path) = env::var("LIBDD_OUTPUT_FOLDER") { - path = libdd_path; - } - - let profile = env::var("PROFILE").unwrap(); - let version = env::var("CARGO_PKG_VERSION").unwrap(); - let target = env::var("TARGET").unwrap(); - let host = env::var("HOST").unwrap(); - - println!("Target: {}, Host: {}", target, host); - let mut builder = Builder::new(source_path.to_str().unwrap(), &path, &target, &host, &profile, &version); - - builder.create_dir_structure(); - builder.deduplicate_headers(); - builder.sanitize_libraries(); - builder.add_cmake(); - - // add modules based on features - builder.add_module(Box::new(Common { - source_include: builder.source_inc.clone(), - target_include: builder.target_include.clone(), - })); - - #[cfg(feature = "profiling")] - builder.add_module(Box::new(Profiling { - source_include: builder.source_inc.clone(), - source_lib: builder.source_lib.clone(), - target_include: builder.target_include.clone(), - target_lib: builder.target_lib.clone(), - target_pkconfig: builder.target_pkconfig.clone(), - version: builder.version.clone(), - })); - - #[cfg(feature = "data-pipeline")] - builder.add_module(Box::new(DataPipeline { - source_include: builder.source_inc.clone(), - target_include: builder.target_include.clone(), - })); - - #[cfg(feature = "symbolizer")] - builder.add_module(Box::new(Symbolizer { - source_include: builder.source_inc.clone(), - target_include: builder.target_include.clone(), - })); - - #[cfg(feature = "crashtracker")] - builder.add_module(Box::new(CrashTracker { - source_include: builder.source_inc.clone(), - target_include: builder.target_include.clone(), - target_dir: builder.target_dir.clone(), - })); - - // Build artifacts. - let res = builder.build(); - match res { - Ok(_) => builder.pack().unwrap(), - Err(err) => panic!("{}", format!("Building failed: {}", err)), - } + println!( + "cargo:rustc-env=TARGET={}", + std::env::var("TARGET").unwrap() + ); + println!( + "cargo:rustc-env=PROFILE={}", + std::env::var("PROFILE").unwrap() + ); + println!( + "cargo:rustc-env=OPT_LEVEL={}", + std::env::var("OPT_LEVEL").unwrap() + ); + println!("cargo:rustc-env=DEBUG={}", std::env::var("DEBUG").unwrap()); + println!( + "cargo:rustc-env=CARGO_PKG_VERSION={}", + std::env::var("CARGO_PKG_VERSION").unwrap() + ); + println!("cargo:rustc-env=HOST={}", std::env::var("HOST").unwrap()); } diff --git a/builder/build/symbolizer.rs b/builder/build/symbolizer.rs deleted file mode 100644 index e176c0ca0..000000000 --- a/builder/build/symbolizer.rs +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ -// SPDX-License-Identifier: Apache-2.0 - -use crate::module::Module; -use anyhow::Result; -use std::fs; -use std::path::PathBuf; -use std::rc::Rc; - -pub struct Symbolizer { - pub source_include: Rc, - pub target_include: Rc, -} - -impl Module for Symbolizer { - fn install(&self) -> Result<()> { - let origin_path: PathBuf = [&self.source_include, "blazesym.h"].iter().collect(); - let target_path: PathBuf = [&self.target_include, "blazesym.h"].iter().collect(); - - fs::copy(origin_path, target_path).expect("Failed to copy data pipeline header"); - Ok(()) - } -} diff --git a/builder/build/arch/apple.rs b/builder/src/arch/apple.rs similarity index 100% rename from builder/build/arch/apple.rs rename to builder/src/arch/apple.rs diff --git a/builder/build/arch/linux.rs b/builder/src/arch/linux.rs similarity index 93% rename from builder/build/arch/linux.rs rename to builder/src/arch/linux.rs index 2e1eb188e..6f3e87e31 100644 --- a/builder/build/arch/linux.rs +++ b/builder/src/arch/linux.rs @@ -13,14 +13,15 @@ pub const PROF_STATIC_LIB_FFI: &str = "libdatadog_profiling_ffi.a"; pub const REMOVE_RPATH: bool = false; pub const BUILD_CRASHTRACKER: bool = true; -#[allow(clippy::zombie_processes)] pub fn fix_rpath(lib_path: &str) { if REMOVE_RPATH { - Command::new("patchelf") + let mut patchelf = Command::new("patchelf") .arg("--remove-rpath") .arg(lib_path) .spawn() - .expect("failed to remove rpath"); + .expect("failed to spawn patchelf"); + + patchelf.wait().expect("failed to remove rpath"); } } diff --git a/builder/build/arch/mod.rs b/builder/src/arch/mod.rs similarity index 100% rename from builder/build/arch/mod.rs rename to builder/src/arch/mod.rs diff --git a/builder/build/arch/musl.rs b/builder/src/arch/musl.rs similarity index 100% rename from builder/build/arch/musl.rs rename to builder/src/arch/musl.rs diff --git a/builder/build/arch/windows.rs b/builder/src/arch/windows.rs similarity index 100% rename from builder/build/arch/windows.rs rename to builder/src/arch/windows.rs diff --git a/builder/src/bin/dummy.rs b/builder/src/bin/dummy.rs deleted file mode 100644 index 15e274398..000000000 --- a/builder/src/bin/dummy.rs +++ /dev/null @@ -1,4 +0,0 @@ -// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ -// SPDX-License-Identifier: Apache-2.0 - -pub fn main() {} diff --git a/builder/src/bin/release.rs b/builder/src/bin/release.rs new file mode 100644 index 000000000..de24527de --- /dev/null +++ b/builder/src/bin/release.rs @@ -0,0 +1,126 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use std::env; + +use build_common::determine_paths; + +use builder::common::Common; +#[cfg(feature = "crashtracker")] +use builder::crashtracker::CrashTracker; +use builder::dd_builder::Builder; +#[cfg(feature = "profiling")] +use builder::profiling::Profiling; +use builder::utils::project_root; + +#[derive(Debug)] +struct ReleaseArgs { + pub out_dir: Option, + pub target: Option, +} + +impl From for ReleaseArgs { + fn from(mut args: pico_args::Arguments) -> Self { + let release_args = ReleaseArgs { + // out_dir: args.value_from_str("--out").map_or(None, Some), + out_dir: match args.value_from_str("--out") { + Ok(v) => Some(v), + Err(_) => None + }, + target: match args.value_from_str("--target") { + Ok(v) => Some(v), + Err(_) => None + } + }; + + args.finish(); + release_args + } +} + +pub fn main() { + let args: ReleaseArgs = pico_args::Arguments::from_env().into(); + + let (_, source_path) = determine_paths(); + + let profile = env::var("PROFILE").unwrap(); + let version = env::var("CARGO_PKG_VERSION").unwrap(); + let host = env::var("HOST").unwrap(); + let out_dir = if let Some(out) = args.out_dir { + out + } else { + project_root().to_string_lossy().to_string() + "/" + "release" + }; + + let target = if let Some(target) = args.target { + target + } else { + host.clone() + }; + + #[allow(clippy::vec_init_then_push)] + let features = { + let mut f: Vec = vec![]; + #[cfg(feature = "telemetry")] + f.push("ddtelemetry-ffi".to_string()); + #[cfg(feature = "data-pipeline")] + f.push("data-pipeline-ffi".to_string()); + #[cfg(feature = "crashtracker")] + f.push("crashtracker-ffi".to_string()); + #[cfg(feature = "symbolizer")] + f.push("symbolizer".to_string()); + f + }; + + let mut builder = Builder::new( + source_path.to_str().unwrap(), + &out_dir, + &target, + &profile, + &features.join(","), + &version, + ); + + builder.create_dir_structure(); + builder.add_cmake(); + + // add modules based on features + builder.add_module(Box::new(Common { + arch: builder.arch.clone(), + source_include: builder.source_inc.clone(), + target_include: builder.target_include.clone(), + })); + + #[cfg(feature = "profiling")] + builder.add_module(Box::new(Profiling { + arch: builder.arch.clone(), + base_header: builder.main_header.clone(), + features: builder.features.clone(), + source_include: builder.source_inc.clone(), + source_lib: builder.source_lib.clone(), + target_include: builder.target_include.clone(), + target_lib: builder.target_lib.clone(), + target_pkconfig: builder.target_pkconfig.clone(), + version: builder.version.clone(), + })); + + #[cfg(feature = "crashtracker")] + builder.add_module(Box::new(CrashTracker { + arch: builder.arch.clone(), + base_header: builder.main_header.clone(), + source_include: builder.source_inc.clone(), + target_include: builder.target_include.clone(), + target_dir: builder.target_dir.clone(), + })); + + // Build artifacts. + let res = builder.build(); + match res { + Ok(_) => { + builder.sanitize_libraries(); + // builder.deduplicate_headers(); + builder.pack().unwrap() + } + Err(err) => panic!("{}", format!("Building failed: {}", err)), + } +} diff --git a/builder/build/common.rs b/builder/src/common.rs similarity index 53% rename from builder/build/common.rs rename to builder/src/common.rs index b9f8d9750..096e8f3bf 100644 --- a/builder/build/common.rs +++ b/builder/src/common.rs @@ -2,17 +2,39 @@ // SPDX-License-Identifier: Apache-2.0 use crate::module::Module; +use crate::utils::project_root; use anyhow::Result; use std::fs; use std::path::PathBuf; +use std::process::Command; use std::rc::Rc; pub struct Common { + pub arch: Rc, pub source_include: Rc, pub target_include: Rc, } impl Module for Common { + fn build(&self) -> Result<()> { + let mut cargo = Command::new("cargo") + .current_dir(project_root()) + .args([ + "build", + "-p", + "ddcommon-ffi", + "--features", + "cbindgen", + "--target", + &self.arch, + ]) + .spawn() + .expect("failed to spawn cargo"); + + cargo.wait().expect("Cargo failed"); + Ok(()) + } + fn install(&self) -> Result<()> { let target_path: PathBuf = [self.target_include.as_ref(), "common.h"].iter().collect(); diff --git a/builder/build/crashtracker.rs b/builder/src/crashtracker.rs similarity index 53% rename from builder/build/crashtracker.rs rename to builder/src/crashtracker.rs index 6de4dbfc2..5833bb929 100644 --- a/builder/build/crashtracker.rs +++ b/builder/src/crashtracker.rs @@ -3,12 +3,17 @@ use crate::arch; use crate::module::Module; +use crate::utils::project_root; use anyhow::Result; use std::fs; use std::path::PathBuf; +use std::process::Command; use std::rc::Rc; +use tools::headers::dedup_headers; pub struct CrashTracker { + pub arch: Rc, + pub base_header: Rc, pub source_include: Rc, pub target_dir: Rc, pub target_include: Rc, @@ -16,7 +21,9 @@ pub struct CrashTracker { impl CrashTracker { fn add_binaries(&self) -> Result<()> { - let _dst = cmake::Config::new("../crashtracker") + let mut crashtracker_dir = project_root(); + crashtracker_dir.push("crashtracker"); + let _dst = cmake::Config::new(crashtracker_dir.to_str().unwrap()) .define("Datadog_ROOT", self.target_dir.as_ref()) .define("CMAKE_INSTALL_PREFIX", self.target_dir.as_ref()) .build(); @@ -31,13 +38,34 @@ impl CrashTracker { let target_path: PathBuf = [self.target_include.as_ref(), "crashtracker.h"] .iter() .collect(); - fs::copy(origin_path, target_path).expect("Failed to copy crashtracker.h"); + + let headers = vec![target_path.to_str().unwrap()]; + fs::copy(origin_path, &target_path).expect("Failed to copy crashtracker.h"); + + dedup_headers(self.base_header.as_ref(), &headers); Ok(()) } } impl Module for CrashTracker { + fn build(&self) -> Result<()> { + let mut cargo = Command::new("cargo") + .current_dir(project_root()) + .args([ + "build", + "-p", + "datadog-crashtracker-ffi", + "--target", + &self.arch, + ]) + .spawn() + .expect("failed to spawn cargo"); + + cargo.wait().expect("Cargo failed"); + Ok(()) + } + fn install(&self) -> Result<()> { self.add_headers()?; if arch::BUILD_CRASHTRACKER { diff --git a/builder/src/dd_builder.rs b/builder/src/dd_builder.rs new file mode 100644 index 000000000..d05e9904d --- /dev/null +++ b/builder/src/dd_builder.rs @@ -0,0 +1,172 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use std::fs; +use std::path::{Path, PathBuf}; +use std::rc::Rc; + +use anyhow::Result; +use build_common::HEADER_PATH; +use tar; + +use crate::arch; +use crate::module::Module; +use crate::utils::{file_replace, project_root}; + +/// [`Builder`] is a structure that holds all the information required to assemble the final +/// workspace artifact. It will manage the different modules which will be in charge of producing +/// the different binaries and source files that will be part of the artifact. The builder will +/// provide the needed information: paths, version, etc, to the different modules so they can +/// install their sub-artifacts on the target folder. +/// The target folder is set through `LIBDD_OUTPUT_FOLDER` environment variable if it is not +/// provided the default target folder will be the builder output directory. +/// +/// # Example +/// +/// ```rust +/// use crate::core::Core; +/// +/// let mut builder = Builder::new(&path, &profile, &version); +/// let core = Box::new(Core { +/// version: builder.version.clone(), +/// }); +/// builder.add_module(core); +/// builder.build()?; +/// builder.pack()?; +/// ``` +pub struct Builder { + modules: Vec>, + pub arch: Rc, + pub features: Rc, + pub main_header: Rc, + pub source_inc: Rc, + pub source_lib: Rc, + pub target_dir: Rc, + pub target_lib: Rc, + pub target_include: Rc, + pub target_bin: Rc, + pub target_pkconfig: Rc, + pub version: Rc, +} + +impl Builder { + /// Creates a new Builder instance + /// + /// # Aguments + /// + /// * `target_dir`: artifact folder. + /// * `profile`: Release configuration: debug or release; + /// * `version`: artifact's version. + /// + /// # Returns + /// + /// A new Builder instance. + pub fn new( + source_dir: &str, + target_dir: &str, + target_arch: &str, + profile: &str, + features: &str, + version: &str, + ) -> Self { + Builder { + modules: Vec::new(), + arch: target_arch.into(), + features: features.into(), + main_header: (target_dir.to_string() + "/" + HEADER_PATH + "/" + "common.h").into(), + source_lib: (source_dir.to_string() + "/" + target_arch + "/" + profile + "/deps") + .into(), + source_inc: (source_dir.to_string() + "/" + HEADER_PATH).into(), + target_dir: target_dir.into(), + target_lib: (target_dir.to_string() + "/" + "/lib").into(), + target_include: (target_dir.to_string() + "/" + HEADER_PATH).into(), + target_bin: (target_dir.to_string() + "/bin").into(), + target_pkconfig: (target_dir.to_string() + "/lib/pkgconfig").into(), + version: version.into(), + } + } + + /// Adds a boxed object which implements Module trait. + pub fn add_module(&mut self, module: Box) { + self.modules.push(module); + } + + pub fn create_dir_structure(&self) { + let target = Path::new(self.target_dir.as_ref()); + if fs::metadata(target).is_ok() { + fs::remove_dir_all(Path::new(self.target_dir.as_ref())) + .expect("Failed to clean preexisting target folder"); + } + fs::create_dir_all(Path::new(self.target_dir.as_ref())) + .expect("Failed to create target directory"); + fs::create_dir_all(Path::new(self.target_include.as_ref())) + .expect("Failed to create include directory"); + fs::create_dir_all(Path::new(self.target_lib.as_ref())) + .expect("Failed to create include directory"); + fs::create_dir_all(Path::new(self.target_bin.as_ref())) + .expect("Failed to create include directory"); + fs::create_dir_all(Path::new(self.target_pkconfig.as_ref())) + .expect("Failed to create include directory"); + } + + // TODO: maybe do this in module's build.rs + pub fn sanitize_libraries(&self) { + let datadog_lib_dir = Path::new(self.source_lib.as_ref()); + + let libs = fs::read_dir(datadog_lib_dir).unwrap(); + for lib in libs.flatten() { + let name = lib.file_name().into_string().unwrap(); + if name.ends_with(".so") { + arch::fix_rpath(lib.path().to_str().unwrap()); + } + } + } + + pub fn add_cmake(&self) { + let libs = arch::NATIVE_LIBS.to_owned(); + let cmake_path: PathBuf = [&self.target_dir, "DatadogConfig.cmake"].iter().collect(); + let mut origin = project_root(); + origin.push("cmake"); + origin.push("DatadogConfig.cmake.in"); + + file_replace( + origin.to_str().unwrap(), + cmake_path.to_str().unwrap(), + "@Datadog_LIBRARIES@", + libs.trim(), + ) + .expect("Failed to modify the cmake"); + } + + /// Builds the final artifact by going through all modules and instancing their install method. + /// + /// #Returns + /// + /// Ok(()) if success Err(_) if failure. + pub fn build(&self) -> Result<()> { + for module in &self.modules { + module.build()?; + // self.deduplicate_headers(); + module.install()?; + } + Ok(()) + } + + /// Generate a tar file with all the intermediate artifacts generated by all the modules.k + /// + /// #Returns + /// + /// Ok(()) if success Err(_) if failure. + pub fn pack(&self) -> Result<()> { + let tarname = "libdatadog".to_string() + "_v" + &self.version + ".tar"; + let path: PathBuf = [self.target_dir.as_ref(), &tarname].iter().collect(); + let artifact = fs::File::create(path).expect("Failed to create tarfile"); + let mut ar = tar::Builder::new(artifact); + ar.append_dir_all("lib", self.target_lib.as_ref())?; + ar.append_dir("bin", self.target_bin.as_ref())?; + ar.append_dir_all("include/datadog", self.target_include.as_ref())?; + + ar.finish().expect("Failed to write the tarfile"); + Ok(()) + } +} diff --git a/builder/src/lib.rs b/builder/src/lib.rs new file mode 100644 index 000000000..d403f3620 --- /dev/null +++ b/builder/src/lib.rs @@ -0,0 +1,13 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +pub mod arch; +pub mod common; +#[cfg(feature = "crashtracker")] +pub mod crashtracker; +pub mod dd_builder; +mod module; +pub mod utils; + +#[cfg(feature = "profiling")] +pub mod profiling; diff --git a/builder/build/module.rs b/builder/src/module.rs similarity index 97% rename from builder/build/module.rs rename to builder/src/module.rs index c7fd5ff22..97ca65d2a 100644 --- a/builder/build/module.rs +++ b/builder/src/module.rs @@ -43,5 +43,6 @@ use anyhow::Result; /// } /// ``` pub trait Module { + fn build(&self) -> Result<()>; fn install(&self) -> Result<()>; } diff --git a/builder/build/profiling.rs b/builder/src/profiling.rs similarity index 61% rename from builder/build/profiling.rs rename to builder/src/profiling.rs index ef12d9e35..7bfd0a556 100644 --- a/builder/build/profiling.rs +++ b/builder/src/profiling.rs @@ -3,14 +3,20 @@ use crate::arch; use crate::module::Module; -use crate::utils::file_replace; +use crate::utils::{file_replace, project_root}; use anyhow::Result; use std::ffi::OsStr; use std::fs; +use std::ops::Add; use std::path::{Path, PathBuf}; +use std::process::Command; use std::rc::Rc; +use tools::headers::dedup_headers; pub struct Profiling { + pub arch: Rc, + pub base_header: Rc, + pub features: Rc, pub source_include: Rc, pub source_lib: Rc, pub target_include: Rc, @@ -26,16 +32,31 @@ impl Profiling { let mut headers = vec!["profiling.h"]; #[cfg(feature = "telemetry")] headers.push("telemetry.h"); + #[cfg(feature = "data-pipeline")] + headers.push("data-pipeline.h"); + #[cfg(feature = "symbolizer")] + headers.push("blazesym.h"); let mut origin_path: PathBuf = [&self.source_include, "dummy.h"].iter().collect(); let mut target_path: PathBuf = [&self.target_include, "dummy.h"].iter().collect(); - for header in headers { + let mut to_dedup = vec![]; + for header in &headers { origin_path.set_file_name(header); target_path.set_file_name(header); fs::copy(&origin_path, &target_path).expect("Failed to copy the header"); + + // Exclude blazesym header from deduplication + if !target_path.to_str().unwrap().contains("blazesym.h") { + to_dedup.push(target_path.clone()); + } } + dedup_headers( + self.base_header.as_ref(), + &(to_dedup.iter().map(|i| i.to_str().unwrap()).collect()), + ); + Ok(()) } @@ -82,21 +103,26 @@ impl Profiling { // Create files for file in files.iter() { - let file_in = "../profiling-ffi/".to_string() + file + ".in"; + // let file_in = "../profiling-ffi/".to_string() + file + ".in"; + let file_in = file.to_string() + ".in"; - let pc_file: PathBuf = [pc_dir.as_os_str(), OsStr::new(file)].iter().collect(); + let mut pc_origin: PathBuf = project_root(); + pc_origin.push("profiling-ffi"); + pc_origin.push(file_in); + + let pc_target: PathBuf = [pc_dir.as_os_str(), OsStr::new(file)].iter().collect(); file_replace( - &file_in, - pc_file.to_str().unwrap(), + pc_origin.to_str().unwrap(), + pc_target.to_str().unwrap(), "@Datadog_VERSION@", &self.version, )?; if *file == files[2] { file_replace( - &file_in, - pc_file.to_str().unwrap(), + pc_origin.to_str().unwrap(), + pc_target.to_str().unwrap(), "@Datadog_LIBRARIES@", arch::NATIVE_LIBS, )?; @@ -107,6 +133,28 @@ impl Profiling { } impl Module for Profiling { + fn build(&self) -> Result<()> { + //TODO: create anywhow error + let features = self.features.to_string() + "," + "cbindgen"; + #[cfg(feature = "crashtracker")] + let features = features.add(",crashtracker-collector,crashtracker-receiver,demangler"); + let mut cargo = Command::new("cargo") + .current_dir(project_root()) + .args([ + "build", + "-p", + "datadog-profiling-ffi", + "--features", + &features, + "--target", + &self.arch, + ]) + .spawn() + .expect("failed to spawn cargo"); + + cargo.wait().expect("Cargo failed"); + Ok(()) + } fn install(&self) -> Result<()> { self.add_headers()?; self.add_libs()?; diff --git a/builder/build/utils.rs b/builder/src/utils.rs similarity index 76% rename from builder/build/utils.rs rename to builder/src/utils.rs index 22825ab09..89200fb02 100644 --- a/builder/build/utils.rs +++ b/builder/src/utils.rs @@ -4,6 +4,7 @@ use anyhow::{anyhow, Result}; use std::fs::{self, OpenOptions}; use std::io::Write; +use std::path::{Path, PathBuf}; pub fn file_replace(file_in: &str, file_out: &str, target: &str, replace: &str) -> Result<()> { let content = fs::read_to_string(file_in)?; @@ -17,3 +18,11 @@ pub fn file_replace(file_in: &str, file_out: &str, target: &str, replace: &str) file.write_all(new.as_bytes()) .map_err(|err| anyhow!("failed to write file: {}", err)) } + +pub fn project_root() -> PathBuf { + Path::new(&env!("CARGO_MANIFEST_DIR")) + .ancestors() + .nth(1) + .unwrap() + .to_path_buf() +} diff --git a/ddcommon/src/rate_limiter.rs b/ddcommon/src/rate_limiter.rs index 2c983e9a5..158cf4015 100644 --- a/ddcommon/src/rate_limiter.rs +++ b/ddcommon/src/rate_limiter.rs @@ -28,6 +28,9 @@ pub struct LocalLimiter { granularity: i64, } +#[cfg(target_arch = "x86")] +const TIME_PER_SECOND: i32 = 1_000_000_000; // nanoseconds +#[cfg(target_arch = "x86_64")] const TIME_PER_SECOND: i64 = 1_000_000_000; // nanoseconds fn now() -> u64 { @@ -65,7 +68,7 @@ impl Default for LocalLimiter { hit_count: Default::default(), last_update: AtomicU64::from(now()), last_limit: Default::default(), - granularity: TIME_PER_SECOND, + granularity: TIME_PER_SECOND as i64, } } } @@ -83,7 +86,7 @@ impl LocalLimiter { self.last_update.store(now(), Ordering::Relaxed); self.hit_count.store(0, Ordering::Relaxed); self.last_limit.store(0, Ordering::Relaxed); - self.granularity = TIME_PER_SECOND * seconds as i64; + self.granularity = TIME_PER_SECOND as i64 * seconds as i64; } fn update(&self, limit: u32, inc: i64) -> i64 { diff --git a/tools/src/bin/dedup_headers.rs b/tools/src/bin/dedup_headers.rs index 27c5c9a0b..ad2f03ac1 100644 --- a/tools/src/bin/dedup_headers.rs +++ b/tools/src/bin/dedup_headers.rs @@ -12,5 +12,5 @@ fn main() { let args: Vec<_> = std::env::args_os() .flat_map(|arg| arg.into_string()) .collect(); - dedup_headers(&args[1], &args[2..]); + dedup_headers(&args[1], &(args[2..].iter().map(|i| i.as_str()).collect())); } diff --git a/tools/src/lib.rs b/tools/src/lib.rs index 480f69c4a..bc87ec589 100644 --- a/tools/src/lib.rs +++ b/tools/src/lib.rs @@ -45,9 +45,10 @@ pub mod headers { new_content_parts } - pub fn dedup_headers(base: &str, headers: &[String]) { + pub fn dedup_headers(base: &str, headers: &Vec<&str>) { let mut unique_child_defs: Vec = Vec::new(); let mut present = HashSet::new(); + for child_def in headers.iter().flat_map(|p| { let child_header = OpenOptions::new().read(true).write(true).open(p).unwrap(); @@ -69,7 +70,6 @@ pub mod headers { present.insert(child_def); } - println!("base: {:?}", base); let base_header = OpenOptions::new() .read(true) .write(true)