diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 308bc55ead7..b04ea1550ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -124,10 +124,19 @@ jobs: - name: Run tests run: | # TODO: remove --features master when it is back to the default. - ./test.sh --features master --release --clean --build-sysroot ${{ matrix.commands }} + ./y.sh test --features master --release --clean --build-sysroot ${{ matrix.commands }} duplicates: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - run: python tools/check_intrinsics_duplicates.py + + build_system: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Test build system + run: | + cd build_system + cargo test diff --git a/.github/workflows/failures.yml b/.github/workflows/failures.yml index ae8de79b773..b411b9a1784 100644 --- a/.github/workflows/failures.yml +++ b/.github/workflows/failures.yml @@ -128,5 +128,5 @@ jobs: - name: Run tests id: tests run: | - ${{ matrix.libgccjit_version.env_extra }} ./test.sh --release --clean --build-sysroot --test-failing-rustc ${{ matrix.libgccjit_version.extra }} | tee output_log + ${{ matrix.libgccjit_version.env_extra }} ./y.sh test --release --clean --build-sysroot --test-failing-rustc ${{ matrix.libgccjit_version.extra }} | tee output_log rg --text "test result" output_log >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/gcc12.yml b/.github/workflows/gcc12.yml index a0d363cf1fb..a27ef913c21 100644 --- a/.github/workflows/gcc12.yml +++ b/.github/workflows/gcc12.yml @@ -28,9 +28,6 @@ jobs: # FIXME: re-enable asm tests when GCC can emit in the right syntax. # "--asm-tests", "--test-libcore", - "--extended-rand-tests", - "--extended-regex-example-tests", - "--extended-regex-tests", "--test-successful-rustc --nb-parts 2 --current-part 0", "--test-successful-rustc --nb-parts 2 --current-part 1", ] @@ -112,4 +109,4 @@ jobs: - name: Run tests run: | - ./test.sh --release --clean --build-sysroot ${{ matrix.commands }} --no-default-features + ./y.sh test --release --clean --build-sysroot ${{ matrix.commands }} --no-default-features diff --git a/.github/workflows/m68k.yml b/.github/workflows/m68k.yml index 4d9d7e23dc2..ac141e06247 100644 --- a/.github/workflows/m68k.yml +++ b/.github/workflows/m68k.yml @@ -139,4 +139,4 @@ jobs: - name: Run tests run: | # TODO: remove --features master when it is back to the default. - ./test.sh --release --features master --clean --build-sysroot ${{ matrix.commands }} + ./y.sh test --release --features master --clean --build-sysroot ${{ matrix.commands }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 43b90fcec93..9798bc338f3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -105,4 +105,4 @@ jobs: - name: Run tests run: | # TODO: remove --features master when it is back to the default. - EMBED_LTO_BITCODE=1 ./test.sh --release --clean --release-sysroot --build-sysroot ${{ matrix.commands }} --features master + EMBED_LTO_BITCODE=1 ./y.sh test --release --clean --release-sysroot --build-sysroot ${{ matrix.commands }} --features master diff --git a/.github/workflows/stdarch.yml b/.github/workflows/stdarch.yml index 42109ba3e02..d290f1d0562 100644 --- a/.github/workflows/stdarch.yml +++ b/.github/workflows/stdarch.yml @@ -120,7 +120,7 @@ jobs: if: ${{ !matrix.cargo_runner }} run: | # TODO: remove `--features master` when it is back to the default. - ./test.sh --release --clean --release-sysroot --build-sysroot --mini-tests --std-tests --test-libcore --features master + ./y.sh test --release --clean --release-sysroot --build-sysroot --mini-tests --std-tests --test-libcore --features master - name: Run stdarch tests if: ${{ !matrix.cargo_runner }} diff --git a/.rustfmt.toml b/.rustfmt.toml index c7ad93bafe3..87f034950e3 100644 --- a/.rustfmt.toml +++ b/.rustfmt.toml @@ -1 +1 @@ -disable_all_formatting = true +ignore = ["/src", "/tests"] diff --git a/Readme.md b/Readme.md index 9db1dec1032..68effb2bf78 100644 --- a/Readme.md +++ b/Readme.md @@ -65,7 +65,7 @@ $ LIBRARY_PATH=$(cat gcc_path) LD_LIBRARY_PATH=$(cat gcc_path) ./y.sh build --re To run the tests: ```bash -$ ./test.sh --release --features master +$ ./y.sh test --release --features master ``` ## Usage @@ -82,7 +82,7 @@ export CG_GCCJIT_DIR=[the full path to rustc_codegen_gcc] $ CHANNEL="release" $CG_GCCJIT_DIR/cargo.sh run ``` -If you compiled cg_gccjit in debug mode (aka you didn't pass `--release` to `./test.sh`) you should use `CHANNEL="debug"` instead or omit `CHANNEL="release"` completely. +If you compiled cg_gccjit in debug mode (aka you didn't pass `--release` to `./y.sh test`) you should use `CHANNEL="debug"` instead or omit `CHANNEL="release"` completely. ### LTO diff --git a/build_system/src/build.rs b/build_system/src/build.rs index f1c3701a946..d264aac7eff 100644 --- a/build_system/src/build.rs +++ b/build_system/src/build.rs @@ -1,7 +1,5 @@ -use crate::config::{set_config, ConfigInfo}; -use crate::utils::{ - get_gcc_path, run_command, run_command_with_output_and_env, walk_dir, -}; +use crate::config::{Channel, ConfigInfo}; +use crate::utils::{get_gcc_path, run_command, run_command_with_output_and_env, walk_dir}; use std::collections::HashMap; use std::ffi::OsStr; use std::fs; @@ -9,11 +7,9 @@ use std::path::Path; #[derive(Default)] struct BuildArg { - codegen_release_channel: bool, - sysroot_release_channel: bool, - sysroot_panic_abort: bool, flags: Vec, gcc_path: String, + config_info: ConfigInfo, } impl BuildArg { @@ -28,14 +24,9 @@ impl BuildArg { while let Some(arg) = args.next() { match arg.as_str() { - "--release" => build_arg.codegen_release_channel = true, - "--release-sysroot" => build_arg.sysroot_release_channel = true, "--no-default-features" => { build_arg.flags.push("--no-default-features".to_string()); } - "--sysroot-panic-abort" => { - build_arg.sysroot_panic_abort = true; - }, "--features" => { if let Some(arg) = args.next() { build_arg.flags.push("--features".to_string()); @@ -50,25 +41,11 @@ impl BuildArg { Self::usage(); return Ok(None); } - "--target-triple" => { - if args.next().is_some() { - // Handled in config.rs. - } else { - return Err( - "Expected a value after `--target-triple`, found nothing".to_string() - ); - } - } - "--target" => { - if args.next().is_some() { - // Handled in config.rs. - } else { - return Err( - "Expected a value after `--target`, found nothing".to_string() - ); + arg => { + if !build_arg.config_info.parse_argument(arg, &mut args)? { + return Err(format!("Unknown argument `{}`", arg)); } } - arg => return Err(format!("Unknown argument `{}`", arg)), } } Ok(Some(build_arg)) @@ -79,29 +56,20 @@ impl BuildArg { r#" `build` command help: - --release : Build codegen in release mode - --release-sysroot : Build sysroot in release mode - --sysroot-panic-abort : Build the sysroot without unwinding support. --no-default-features : Add `--no-default-features` flag - --features [arg] : Add a new feature [arg] - --target-triple [arg] : Set the target triple to [arg] - --help : Show this help -"# - ) + --features [arg] : Add a new feature [arg]"# + ); + ConfigInfo::show_usage(); + println!(" --help : Show this help"); } } -fn build_sysroot( - env: &mut HashMap, - args: &BuildArg, - config: &ConfigInfo, -) -> Result<(), String> { - std::env::set_current_dir("build_sysroot") - .map_err(|error| format!("Failed to go to `build_sysroot` directory: {:?}", error))?; +pub fn build_sysroot(env: &HashMap, config: &ConfigInfo) -> Result<(), String> { + let start_dir = Path::new("build_sysroot"); // Cleanup for previous run // Clean target dir except for build scripts and incremental cache let _ = walk_dir( - "target", + start_dir.join("target"), |dir: &Path| { for top in &["debug", "release"] { let _ = fs::remove_dir_all(dir.join(top).join("build")); @@ -138,79 +106,93 @@ fn build_sysroot( |_| Ok(()), ); - let _ = fs::remove_file("Cargo.lock"); - let _ = fs::remove_file("test_target/Cargo.lock"); - let _ = fs::remove_dir_all("sysroot"); + let _ = fs::remove_file(start_dir.join("Cargo.lock")); + let _ = fs::remove_file(start_dir.join("test_target/Cargo.lock")); + let _ = fs::remove_dir_all(start_dir.join("sysroot")); // Builds libs - let mut rustflags = env - .get("RUSTFLAGS") - .cloned() - .unwrap_or_default(); - if args.sysroot_panic_abort { + let mut rustflags = env.get("RUSTFLAGS").cloned().unwrap_or_default(); + if config.sysroot_panic_abort { rustflags.push_str(" -Cpanic=abort -Zpanic-abort-tests"); } - env.insert( - "RUSTFLAGS".to_string(), - format!("{} -Zmir-opt-level=3", rustflags), - ); - let channel = if args.sysroot_release_channel { + rustflags.push_str(" -Z force-unstable-if-unmarked"); + let mut env = env.clone(); + let channel = if config.sysroot_release_channel { + env.insert( + "RUSTFLAGS".to_string(), + format!("{} -Zmir-opt-level=3", rustflags), + ); run_command_with_output_and_env( &[ &"cargo", &"build", + &"--release", &"--target", &config.target, - &"--release", ], - None, + Some(start_dir), Some(&env), )?; "release" } else { + env.insert("RUSTFLAGS".to_string(), rustflags); + run_command_with_output_and_env( - &[ - &"cargo", - &"build", - &"--target", - &config.target, - ], - None, - Some(env), + &[&"cargo", &"build", &"--target", &config.target], + Some(start_dir), + Some(&env), )?; "debug" }; // Copy files to sysroot - let sysroot_path = format!("sysroot/lib/rustlib/{}/lib/", config.target_triple); - fs::create_dir_all(&sysroot_path) - .map_err(|error| format!("Failed to create directory `{}`: {:?}", sysroot_path, error))?; + let sysroot_path = start_dir.join(format!("sysroot/lib/rustlib/{}/lib/", config.target_triple)); + fs::create_dir_all(&sysroot_path).map_err(|error| { + format!( + "Failed to create directory `{}`: {:?}", + sysroot_path.display(), + error + ) + })?; let copier = |dir_to_copy: &Path| { + // FIXME: should not use shell command! run_command(&[&"cp", &"-r", &dir_to_copy, &sysroot_path], None).map(|_| ()) }; walk_dir( - &format!("target/{}/{}/deps", config.target_triple, channel), + start_dir.join(&format!("target/{}/{}/deps", config.target_triple, channel)), copier, copier, )?; // Copy the source files to the sysroot (Rust for Linux needs this). let sysroot_src_path = "sysroot/lib/rustlib/src/rust"; - fs::create_dir_all(&sysroot_src_path) - .map_err(|error| format!("Failed to create directory `{}`: {:?}", sysroot_src_path, error))?; - run_command(&[&"cp", &"-r", &"sysroot_src/library/", &sysroot_src_path], None)?; + fs::create_dir_all(&sysroot_src_path).map_err(|error| { + format!( + "Failed to create directory `{}`: {:?}", + sysroot_src_path, error + ) + })?; + run_command( + &[ + &"cp", + &"-r", + &start_dir.join("sysroot_src/library/"), + &sysroot_src_path, + ], + None, + )?; Ok(()) } -fn build_codegen(args: &BuildArg) -> Result<(), String> { +fn build_codegen(args: &mut BuildArg) -> Result<(), String> { let mut env = HashMap::new(); env.insert("LD_LIBRARY_PATH".to_string(), args.gcc_path.clone()); env.insert("LIBRARY_PATH".to_string(), args.gcc_path.clone()); let mut command: Vec<&dyn AsRef> = vec![&"cargo", &"rustc"]; - if args.codegen_release_channel { + if args.config_info.channel == Channel::Release { command.push(&"--release"); env.insert("CHANNEL".to_string(), "release".to_string()); env.insert("CARGO_INCREMENTAL".to_string(), "1".to_string()); @@ -223,7 +205,7 @@ fn build_codegen(args: &BuildArg) -> Result<(), String> { } run_command_with_output_and_env(&command, None, Some(&env))?; - let config = set_config(&mut env, &[], Some(&args.gcc_path))?; + args.config_info.setup(&mut env, Some(&args.gcc_path))?; // We voluntarily ignore the error. let _ = fs::remove_dir_all("target/out"); @@ -236,19 +218,15 @@ fn build_codegen(args: &BuildArg) -> Result<(), String> { })?; println!("[BUILD] sysroot"); - build_sysroot( - &mut env, - args, - &config, - )?; + build_sysroot(&env, &args.config_info)?; Ok(()) } pub fn run() -> Result<(), String> { - let args = match BuildArg::new()? { + let mut args = match BuildArg::new()? { Some(args) => args, None => return Ok(()), }; - build_codegen(&args)?; + build_codegen(&mut args)?; Ok(()) } diff --git a/build_system/src/config.rs b/build_system/src/config.rs index 64d9bd73e01..1824bdd292f 100644 --- a/build_system/src/config.rs +++ b/build_system/src/config.rs @@ -1,149 +1,271 @@ -use crate::utils::{get_gcc_path, get_os_name, get_rustc_host_triple}; +use crate::utils::{get_gcc_path, get_os_name, rustc_version_info, split_args}; use std::collections::HashMap; use std::env as std_env; +use std::ffi::OsStr; +#[derive(Default, PartialEq, Eq, Clone, Copy, Debug)] +pub enum Channel { + #[default] + Debug, + Release, +} + +impl Channel { + pub fn as_str(self) -> &'static str { + match self { + Self::Debug => "debug", + Self::Release => "release", + } + } +} + +#[derive(Default, Debug)] pub struct ConfigInfo { pub target: String, pub target_triple: String, + pub host_triple: String, pub rustc_command: Vec, + pub run_in_vm: bool, + pub cargo_target_dir: String, + pub dylib_ext: String, + pub sysroot_release_channel: bool, + pub channel: Channel, + pub sysroot_panic_abort: bool, + pub cg_backend_path: String, + pub sysroot_path: String, } -// Returns the beginning for the command line of rustc. -pub fn set_config( - env: &mut HashMap, - test_flags: &[String], - gcc_path: Option<&str>, -) -> Result { - env.insert("CARGO_INCREMENTAL".to_string(), "0".to_string()); - - let gcc_path = match gcc_path { - Some(path) => path.to_string(), - None => get_gcc_path()?, - }; - env.insert("GCC_PATH".to_string(), gcc_path.clone()); - - let os_name = get_os_name()?; - let dylib_ext = match os_name.as_str() { - "Linux" => "so", - "Darwin" => "dylib", - os => return Err(format!("unsupported OS `{}`", os)), - }; - let host_triple = get_rustc_host_triple()?; - let mut linker = None; - let mut target_triple = host_triple.clone(); - let mut target = target_triple.clone(); - - // We skip binary name and the command. - let mut args = std::env::args().skip(2); - - let mut set_target_triple = false; - let mut set_target = false; - while let Some(arg) = args.next() { - match arg.as_str() { - "--target-triple" => { +impl ConfigInfo { + /// Returns `true` if the argument was taken into account. + pub fn parse_argument( + &mut self, + arg: &str, + args: &mut impl Iterator, + ) -> Result { + match arg { + "--target" => { if let Some(arg) = args.next() { - target_triple = arg; - set_target_triple = true; + self.target = arg; } else { + return Err("Expected a value after `--target`, found nothing".to_string()); + } + } + "--target-triple" => match args.next() { + Some(arg) if !arg.is_empty() => self.target_triple = arg.to_string(), + _ => { return Err( "Expected a value after `--target-triple`, found nothing".to_string() - ); + ) } }, - "--target" => { - if let Some(arg) = args.next() { - target = arg; - set_target = true; - } else { - return Err( - "Expected a value after `--target`, found nothing".to_string() - ); + "--out-dir" => match args.next() { + Some(arg) if !arg.is_empty() => { + self.cargo_target_dir = arg.to_string(); } + _ => return Err("Expected a value after `--out-dir`, found nothing".to_string()), }, - _ => (), + "--release-sysroot" => self.sysroot_release_channel = true, + "--release" => self.channel = Channel::Release, + "--sysroot-panic-abort" => self.sysroot_panic_abort = true, + _ => return Ok(false), } + Ok(true) } - if set_target_triple && !set_target { - target = target_triple.clone(); + pub fn rustc_command_vec(&self) -> Vec<&dyn AsRef> { + let mut command: Vec<&dyn AsRef> = Vec::with_capacity(self.rustc_command.len()); + for arg in self.rustc_command.iter() { + command.push(arg); + } + command } - if host_triple != target_triple { - linker = Some(format!("-Clinker={}-gcc", target_triple)); - } - let current_dir = - std_env::current_dir().map_err(|error| format!("`current_dir` failed: {:?}", error))?; - let channel = if let Some(channel) = env.get("CHANNEL") { - channel.as_str() - } else { - "debug" - }; - let cg_backend_path = current_dir - .join("target") - .join(channel) - .join(&format!("librustc_codegen_gcc.{}", dylib_ext)); - let sysroot_path = current_dir.join("build_sysroot/sysroot"); - let mut rustflags = Vec::new(); - if let Some(cg_rustflags) = env.get("CG_RUSTFLAGS") { - rustflags.push(cg_rustflags.clone()); - } - if let Some(linker) = linker { - rustflags.push(linker.to_string()); - } - rustflags.extend_from_slice(&[ - "-Csymbol-mangling-version=v0".to_string(), - "-Cdebuginfo=2".to_string(), - format!("-Zcodegen-backend={}", cg_backend_path.display()), - "--sysroot".to_string(), - sysroot_path.display().to_string(), - ]); - - // Since we don't support ThinLTO, disable LTO completely when not trying to do LTO. - // TODO(antoyo): remove when we can handle ThinLTO. - if !env.contains_key(&"FAT_LTO".to_string()) { - rustflags.push("-Clto=off".to_string()); - } - rustflags.extend_from_slice(test_flags); - // FIXME(antoyo): remove once the atomic shim is gone - if os_name == "Darwin" { + pub fn setup( + &mut self, + env: &mut HashMap, + gcc_path: Option<&str>, + ) -> Result<(), String> { + env.insert("CARGO_INCREMENTAL".to_string(), "0".to_string()); + + let gcc_path = match gcc_path { + Some(path) => path.to_string(), + None => get_gcc_path()?, + }; + env.insert("GCC_PATH".to_string(), gcc_path.clone()); + + if self.cargo_target_dir.is_empty() { + match env.get("CARGO_TARGET_DIR").filter(|dir| !dir.is_empty()) { + Some(cargo_target_dir) => self.cargo_target_dir = cargo_target_dir.clone(), + None => self.cargo_target_dir = "target/out".to_string(), + } + } + + let os_name = get_os_name()?; + self.dylib_ext = match os_name.as_str() { + "Linux" => "so", + "Darwin" => "dylib", + os => return Err(format!("unsupported OS `{}`", os)), + } + .to_string(); + let rustc = match env.get("RUSTC") { + Some(r) if !r.is_empty() => r.to_string(), + _ => "rustc".to_string(), + }; + self.host_triple = match rustc_version_info(Some(&rustc))?.host { + Some(host) => host, + None => return Err("no host found".to_string()), + }; + + if self.target_triple.is_empty() { + if let Some(overwrite) = env.get("OVERWRITE_TARGET_TRIPLE") { + self.target_triple = overwrite.clone(); + } + } + if self.target_triple.is_empty() { + self.target_triple = self.host_triple.clone(); + } + if self.target.is_empty() && !self.target_triple.is_empty() { + self.target = self.target_triple.clone(); + } + + let mut linker = None; + + if self.host_triple != self.target_triple { + if self.target_triple.is_empty() { + return Err("Unknown non-native platform".to_string()); + } + linker = Some(format!("-Clinker={}-gcc", self.target_triple)); + self.run_in_vm = true; + } + + let current_dir = + std_env::current_dir().map_err(|error| format!("`current_dir` failed: {:?}", error))?; + let channel = if self.channel == Channel::Release { + "release" + } else if let Some(channel) = env.get("CHANNEL") { + channel.as_str() + } else { + "debug" + }; + + let has_builtin_backend = env + .get("BUILTIN_BACKEND") + .map(|backend| !backend.is_empty()) + .unwrap_or(false); + + let mut rustflags = Vec::new(); + if has_builtin_backend { + // It means we're building inside the rustc testsuite, so some options need to be handled + // a bit differently. + self.cg_backend_path = "gcc".to_string(); + + match env.get("RUSTC_SYSROOT") { + Some(rustc_sysroot) if !rustc_sysroot.is_empty() => { + rustflags.extend_from_slice(&["--sysroot".to_string(), rustc_sysroot.clone()]); + } + _ => {} + } + // This should not be needed, but is necessary for the CI in the rust repository. + // FIXME: Remove when the rust CI switches to the master version of libgccjit. + rustflags.push("-Cpanic=abort".to_string()); + } else { + self.cg_backend_path = current_dir + .join("target") + .join(channel) + .join(&format!("librustc_codegen_gcc.{}", self.dylib_ext)) + .display() + .to_string(); + self.sysroot_path = current_dir + .join("build_sysroot/sysroot") + .display() + .to_string(); + rustflags.extend_from_slice(&["--sysroot".to_string(), self.sysroot_path.clone()]); + }; + + // This environment variable is useful in case we want to change options of rustc commands. + if let Some(cg_rustflags) = env.get("CG_RUSTFLAGS") { + rustflags.extend_from_slice(&split_args(&cg_rustflags)?); + } + if let Some(test_flags) = env.get("TEST_FLAGS") { + rustflags.extend_from_slice(&split_args(&test_flags)?); + } + + if let Some(linker) = linker { + rustflags.push(linker.to_string()); + } rustflags.extend_from_slice(&[ - "-Clink-arg=-undefined".to_string(), - "-Clink-arg=dynamic_lookup".to_string(), + "-Csymbol-mangling-version=v0".to_string(), + "-Cdebuginfo=2".to_string(), + format!("-Zcodegen-backend={}", self.cg_backend_path), + ]); + + // Since we don't support ThinLTO, disable LTO completely when not trying to do LTO. + // TODO(antoyo): remove when we can handle ThinLTO. + if !env.contains_key(&"FAT_LTO".to_string()) { + rustflags.push("-Clto=off".to_string()); + } + // FIXME(antoyo): remove once the atomic shim is gone + if os_name == "Darwin" { + rustflags.extend_from_slice(&[ + "-Clink-arg=-undefined".to_string(), + "-Clink-arg=dynamic_lookup".to_string(), + ]); + } + env.insert("RUSTFLAGS".to_string(), rustflags.join(" ")); + // display metadata load errors + env.insert("RUSTC_LOG".to_string(), "warn".to_string()); + + let sysroot = current_dir.join(&format!( + "build_sysroot/sysroot/lib/rustlib/{}/lib", + self.target_triple, + )); + let ld_library_path = format!( + "{target}:{sysroot}:{gcc_path}", + // FIXME: It's possible to pick another out directory. Would be nice to have a command + // line option to change it. + target = current_dir.join("target/out").display(), + sysroot = sysroot.display(), + ); + env.insert("LD_LIBRARY_PATH".to_string(), ld_library_path.clone()); + env.insert("DYLD_LIBRARY_PATH".to_string(), ld_library_path); + + // NOTE: To avoid the -fno-inline errors, use /opt/gcc/bin/gcc instead of cc. + // To do so, add a symlink for cc to /opt/gcc/bin/gcc in our PATH. + // Another option would be to add the following Rust flag: -Clinker=/opt/gcc/bin/gcc + let path = std::env::var("PATH").unwrap_or_default(); + env.insert( + "PATH".to_string(), + format!( + "/opt/gcc/bin:/opt/m68k-unknown-linux-gnu/bin{}{}", + if path.is_empty() { "" } else { ":" }, + path + ), + ); + + self.rustc_command = vec![rustc]; + self.rustc_command.extend_from_slice(&rustflags); + self.rustc_command.extend_from_slice(&[ + "-L".to_string(), + "crate=target/out".to_string(), + "--out-dir".to_string(), + self.cargo_target_dir.clone(), ]); + + if !env.contains_key("RUSTC_LOG") { + env.insert("RUSTC_LOG".to_string(), "warn".to_string()); + } + Ok(()) + } + + pub fn show_usage() { + println!( + "\ + --target-triple [arg] : Set the target triple to [arg] + --target [arg] : Set the target to [arg] + --out-dir : Location where the files will be generated + --release : Build in release mode + --release-sysroot : Build sysroot in release mode + --sysroot-panic-abort : Build the sysroot without unwinding support." + ); } - env.insert("RUSTFLAGS".to_string(), rustflags.join(" ")); - // display metadata load errors - env.insert("RUSTC_LOG".to_string(), "warn".to_string()); - - let sysroot = current_dir.join(&format!( - "build_sysroot/sysroot/lib/rustlib/{}/lib", - target_triple - )); - let ld_library_path = format!( - "{target}:{sysroot}:{gcc_path}", - target = current_dir.join("target/out").display(), - sysroot = sysroot.display(), - ); - env.insert("LD_LIBRARY_PATH".to_string(), ld_library_path.clone()); - env.insert("DYLD_LIBRARY_PATH".to_string(), ld_library_path); - - // NOTE: To avoid the -fno-inline errors, use /opt/gcc/bin/gcc instead of cc. - // To do so, add a symlink for cc to /opt/gcc/bin/gcc in our PATH. - // Another option would be to add the following Rust flag: -Clinker=/opt/gcc/bin/gcc - let path = std::env::var("PATH").unwrap_or_default(); - env.insert("PATH".to_string(), format!("/opt/gcc/bin:{}", path)); - - let mut rustc_command = vec!["rustc".to_string()]; - rustc_command.extend_from_slice(&rustflags); - rustc_command.extend_from_slice(&[ - "-L".to_string(), - "crate=target/out".to_string(), - "--out-dir".to_string(), - "target/out".to_string(), - ]); - Ok(ConfigInfo { - target, - target_triple, - rustc_command, - }) } diff --git a/build_system/src/main.rs b/build_system/src/main.rs index bff82b6e3e5..e0091ff6977 100644 --- a/build_system/src/main.rs +++ b/build_system/src/main.rs @@ -61,7 +61,7 @@ fn main() { Command::Build => build::run(), Command::Test => test::run(), } { - eprintln!("Command failed to run: {e:?}"); + eprintln!("Command failed to run: {e}"); process::exit(1); } } diff --git a/build_system/src/prepare.rs b/build_system/src/prepare.rs index 6c7c8586834..ce9b440be05 100644 --- a/build_system/src/prepare.rs +++ b/build_system/src/prepare.rs @@ -1,10 +1,16 @@ use crate::rustc_info::get_rustc_path; -use crate::utils::{cargo_install, git_clone, run_command, run_command_with_output, walk_dir}; +use crate::utils::{ + cargo_install, git_clone, remove_file, run_command, run_command_with_output, walk_dir, +}; use std::fs; use std::path::Path; -fn prepare_libcore(sysroot_path: &Path, libgccjit12_patches: bool, cross_compile: bool) -> Result<(), String> { +fn prepare_libcore( + sysroot_path: &Path, + libgccjit12_patches: bool, + cross_compile: bool, +) -> Result<(), String> { let rustc_path = match get_rustc_path() { Some(path) => path, None => return Err("`rustc` path not found".to_string()), @@ -88,10 +94,14 @@ fn prepare_libcore(sysroot_path: &Path, libgccjit12_patches: bool, cross_compile }, )?; if cross_compile { - walk_dir("cross_patches", |_| Ok(()), |file_path: &Path| { - patches.push(file_path.to_path_buf()); - Ok(()) - })?; + walk_dir( + "cross_patches", + |_| Ok(()), + |file_path: &Path| { + patches.push(file_path.to_path_buf()); + Ok(()) + }, + )?; } if libgccjit12_patches { walk_dir( @@ -129,8 +139,7 @@ fn build_raytracer(repo_dir: &Path) -> Result<(), String> { run_command(&[&"cargo", &"build"], Some(repo_dir))?; let mv_target = repo_dir.join("raytracer_cg_llvm"); if mv_target.is_file() { - std::fs::remove_file(&mv_target) - .map_err(|e| format!("Failed to remove file `{}`: {e:?}", mv_target.display()))?; + remove_file(&mv_target)?; } run_command( &[&"mv", &"target/debug/main", &"raytracer_cg_llvm"], diff --git a/build_system/src/test.rs b/build_system/src/test.rs index 4c8c63e59ab..f66b16c3149 100644 --- a/build_system/src/test.rs +++ b/build_system/src/test.rs @@ -1,15 +1,1133 @@ -use crate::utils::run_command_with_output; +use crate::build; +use crate::config::{Channel, ConfigInfo}; +use crate::utils::{ + get_gcc_path, get_toolchain, remove_file, run_command, run_command_with_env, + run_command_with_output_and_env, rustc_version_info, split_args, walk_dir, +}; -fn get_args<'a>(args: &mut Vec<&'a dyn AsRef>, extra_args: &'a Vec) { - for extra_arg in extra_args { - args.push(extra_arg); +use std::collections::{BTreeSet, HashMap}; +use std::ffi::OsStr; +use std::fs::{remove_dir_all, File}; +use std::io::{BufRead, BufReader}; +use std::path::{Path, PathBuf}; +use std::str::FromStr; + +type Env = HashMap; +type Runner = fn(&Env, &TestArg) -> Result<(), String>; +type Runners = HashMap<&'static str, (&'static str, Runner)>; + +fn get_runners() -> Runners { + let mut runners = HashMap::new(); + + runners.insert( + "--test-rustc", + ("Run all rustc tests", test_rustc as Runner), + ); + runners.insert( + "--test-successful-rustc", + ("Run successful rustc tests", test_successful_rustc), + ); + runners.insert( + "--test-failing-rustc", + ("Run failing rustc tests", test_failing_rustc), + ); + runners.insert("--test-libcore", ("Run libcore tests", test_libcore)); + runners.insert("--clean-ui-tests", ("Clean ui tests", clean_ui_tests)); + runners.insert("--clean", ("Empty cargo target directory", clean)); + runners.insert("--build-sysroot", ("Build sysroot", build_sysroot)); + runners.insert("--std-tests", ("Run std tests", std_tests)); + runners.insert("--asm-tests", ("Run asm tests", asm_tests)); + runners.insert( + "--extended-tests", + ("Run extended sysroot tests", extended_sysroot_tests), + ); + runners.insert( + "--extended-rand-tests", + ("Run extended rand tests", extended_rand_tests), + ); + runners.insert( + "--extended-regex-example-tests", + ( + "Run extended regex example tests", + extended_regex_example_tests, + ), + ); + runners.insert( + "--extended-regex-tests", + ("Run extended regex tests", extended_regex_tests), + ); + runners.insert("--mini-tests", ("Run mini tests", mini_tests)); + + runners +} + +fn get_number_after_arg( + args: &mut impl Iterator, + option: &str, +) -> Result { + match args.next() { + Some(nb) if !nb.is_empty() => match usize::from_str(&nb) { + Ok(nb) => Ok(nb), + Err(_) => Err(format!( + "Expected a number after `{}`, found `{}`", + option, nb + )), + }, + _ => Err(format!( + "Expected a number after `{}`, found nothing", + option + )), + } +} + +fn show_usage() { + println!( + r#" +`test` command help: + + --release : Build codegen in release mode + --sysroot-panic-abort : Build the sysroot without unwinding support. + --no-default-features : Add `--no-default-features` flag + --features [arg] : Add a new feature [arg] + --use-system-gcc : Use system installed libgccjit + --build-only : Only build rustc_codegen_gcc then exits + --use-backend : Useful only for rustc testsuite + --nb-parts : Used to split rustc_tests (for CI needs) + --current-part : Used with `--nb-parts`, allows you to specify which parts to test"# + ); + ConfigInfo::show_usage(); + for (option, (doc, _)) in get_runners() { + // FIXME: Instead of using the hard-coded `23` value, better to compute it instead. + let needed_spaces = 23_usize.saturating_sub(option.len()); + let spaces: String = std::iter::repeat(' ').take(needed_spaces).collect(); + println!(" {}{}: {}", option, spaces, doc); + } + println!(" --help : Show this help"); +} + +#[derive(Default, Debug)] +struct TestArg { + no_default_features: bool, + build_only: bool, + gcc_path: String, + runners: BTreeSet, + flags: Vec, + backend: Option, + nb_parts: Option, + current_part: Option, + sysroot_panic_abort: bool, + config_info: ConfigInfo, +} + +impl TestArg { + fn new() -> Result, String> { + let mut use_system_gcc = false; + let mut test_arg = Self::default(); + + // We skip binary name and the `test` command. + let mut args = std::env::args().skip(2); + let runners = get_runners(); + + while let Some(arg) = args.next() { + match arg.as_str() { + "--no-default-features" => { + // To prevent adding it more than once. + if !test_arg.no_default_features { + test_arg.flags.push("--no-default-features".into()); + } + test_arg.no_default_features = true; + } + "--features" => match args.next() { + Some(feature) if !feature.is_empty() => { + test_arg + .flags + .extend_from_slice(&["--features".into(), feature]); + } + _ => { + return Err("Expected an argument after `--features`, found nothing".into()) + } + }, + "--use-system-gcc" => use_system_gcc = true, + "--build-only" => test_arg.build_only = true, + "--use-backend" => match args.next() { + Some(backend) if !backend.is_empty() => test_arg.backend = Some(backend), + _ => { + return Err( + "Expected an argument after `--use-backend`, found nothing".into() + ) + } + }, + "--nb-parts" => { + test_arg.nb_parts = Some(get_number_after_arg(&mut args, "--nb-parts")?); + } + "--current-part" => { + test_arg.current_part = + Some(get_number_after_arg(&mut args, "--current-part")?); + } + "--sysroot-panic-abort" => { + test_arg.sysroot_panic_abort = true; + } + "--help" => { + show_usage(); + return Ok(None); + } + x if runners.contains_key(x) => { + test_arg.runners.insert(x.into()); + } + arg => { + if !test_arg.config_info.parse_argument(arg, &mut args)? { + return Err(format!("Unknown option {}", arg)); + } + } + } + + test_arg.gcc_path = if use_system_gcc { + println!("Using system GCC"); + "gcc".to_string() + } else { + get_gcc_path()? + }; + } + match (test_arg.current_part, test_arg.nb_parts) { + (Some(_), Some(_)) | (None, None) => {} + _ => { + return Err( + "If either `--current-part` or `--nb-parts` is specified, the other one \ + needs to be specified as well!" + .to_string(), + ); + } + } + Ok(Some(test_arg)) + } + + pub fn is_using_gcc_master_branch(&self) -> bool { + !self.no_default_features + } +} + +fn build_if_no_backend(env: &Env, args: &TestArg) -> Result<(), String> { + if args.backend.is_some() { + return Ok(()); + } + let mut command: Vec<&dyn AsRef> = vec![&"cargo", &"rustc"]; + let mut tmp_env; + let env = if args.config_info.channel == Channel::Release { + tmp_env = env.clone(); + tmp_env.insert("CARGO_INCREMENTAL".to_string(), "1".to_string()); + command.push(&"--release"); + &tmp_env + } else { + &env + }; + for flag in args.flags.iter() { + command.push(flag); + } + run_command_with_output_and_env(&command, None, Some(env)) +} + +fn clean(_env: &Env, args: &TestArg) -> Result<(), String> { + let _ = std::fs::remove_dir_all(&args.config_info.cargo_target_dir); + let path = Path::new(&args.config_info.cargo_target_dir).join("gccjit"); + std::fs::create_dir_all(&path) + .map_err(|error| format!("failed to create folder `{}`: {:?}", path.display(), error)) +} + +fn mini_tests(env: &Env, args: &TestArg) -> Result<(), String> { + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[BUILD] mini_core"); + let crate_types = if args.config_info.host_triple != args.config_info.target_triple { + "lib" + } else { + "lib,dylib" + } + .to_string(); + let mut command = args.config_info.rustc_command_vec(); + command.extend_from_slice(&[ + &"example/mini_core.rs", + &"--crate-name", + &"mini_core", + &"--crate-type", + &crate_types, + &"--target", + &args.config_info.target_triple, + ]); + run_command_with_output_and_env(&command, None, Some(&env))?; + + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[BUILD] example"); + let mut command = args.config_info.rustc_command_vec(); + command.extend_from_slice(&[ + &"example/example.rs", + &"--crate-type", + &"lib", + &"--target", + &args.config_info.target_triple, + ]); + run_command_with_output_and_env(&command, None, Some(&env))?; + + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[AOT] mini_core_hello_world"); + let mut command = args.config_info.rustc_command_vec(); + command.extend_from_slice(&[ + &"example/mini_core_hello_world.rs", + &"--crate-name", + &"mini_core_hello_world", + &"--crate-type", + &"bin", + &"-g", + &"--target", + &args.config_info.target_triple, + ]); + run_command_with_output_and_env(&command, None, Some(&env))?; + + let command: &[&dyn AsRef] = &[ + &Path::new(&args.config_info.cargo_target_dir).join("mini_core_hello_world"), + &"abc", + &"bcd", + ]; + maybe_run_command_in_vm(&command, env, args)?; + Ok(()) +} + +fn build_sysroot(env: &Env, args: &TestArg) -> Result<(), String> { + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[BUILD] sysroot"); + build::build_sysroot(env, &args.config_info)?; + Ok(()) +} + +// TODO(GuillaumeGomez): when rewriting in Rust, refactor with the code in tests/lang_tests_common.rs if possible. +fn maybe_run_command_in_vm( + command: &[&dyn AsRef], + env: &Env, + args: &TestArg, +) -> Result<(), String> { + if !args.config_info.run_in_vm { + run_command_with_output_and_env(command, None, Some(env))?; + return Ok(()); + } + let vm_parent_dir = match env.get("CG_GCC_VM_DIR") { + Some(dir) if !dir.is_empty() => PathBuf::from(dir.clone()), + _ => std::env::current_dir().unwrap(), + }; + let vm_dir = "vm"; + let exe_to_run = command.first().unwrap(); + let exe = Path::new(&exe_to_run); + let exe_filename = exe.file_name().unwrap(); + let vm_home_dir = vm_parent_dir.join(vm_dir).join("home"); + let vm_exe_path = vm_home_dir.join(exe_filename); + let inside_vm_exe_path = Path::new("/home").join(exe_filename); + + let sudo_command: &[&dyn AsRef] = &[&"sudo", &"cp", &exe, &vm_exe_path]; + run_command_with_env(sudo_command, None, Some(env))?; + + let mut vm_command: Vec<&dyn AsRef> = vec![ + &"sudo", + &"chroot", + &vm_dir, + &"qemu-m68k-static", + &inside_vm_exe_path, + ]; + vm_command.extend_from_slice(command); + run_command_with_output_and_env(&vm_command, Some(&vm_parent_dir), Some(env))?; + Ok(()) +} + +fn std_tests(env: &Env, args: &TestArg) -> Result<(), String> { + let cargo_target_dir = Path::new(&args.config_info.cargo_target_dir); + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[AOT] arbitrary_self_types_pointers_and_wrappers"); + let mut command = args.config_info.rustc_command_vec(); + command.extend_from_slice(&[ + &"example/arbitrary_self_types_pointers_and_wrappers.rs", + &"--crate-name", + &"arbitrary_self_types_pointers_and_wrappers", + &"--crate-type", + &"bin", + &"--target", + &args.config_info.target_triple, + ]); + run_command_with_env(&command, None, Some(env))?; + maybe_run_command_in_vm( + &[&cargo_target_dir.join("arbitrary_self_types_pointers_and_wrappers")], + env, + args, + )?; + + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[AOT] alloc_system"); + let mut command = args.config_info.rustc_command_vec(); + command.extend_from_slice(&[ + &"example/alloc_system.rs", + &"--crate-type", + &"lib", + &"--target", + &args.config_info.target_triple, + ]); + if args.is_using_gcc_master_branch() { + command.extend_from_slice(&[&"--cfg", &"feature=\"master\""]); + } + run_command_with_env(&command, None, Some(env))?; + + // FIXME: doesn't work on m68k. + if args.config_info.host_triple == args.config_info.target_triple { + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[AOT] alloc_example"); + let mut command = args.config_info.rustc_command_vec(); + command.extend_from_slice(&[ + &"example/alloc_example.rs", + &"--crate-type", + &"bin", + &"--target", + &args.config_info.target_triple, + ]); + run_command_with_env(&command, None, Some(env))?; + maybe_run_command_in_vm(&[&cargo_target_dir.join("alloc_example")], env, args)?; + } + + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[AOT] dst_field_align"); + // FIXME(antoyo): Re-add -Zmir-opt-level=2 once rust-lang/rust#67529 is fixed. + let mut command = args.config_info.rustc_command_vec(); + command.extend_from_slice(&[ + &"example/dst-field-align.rs", + &"--crate-name", + &"dst_field_align", + &"--crate-type", + &"bin", + &"--target", + &args.config_info.target_triple, + ]); + run_command_with_env(&command, None, Some(env))?; + maybe_run_command_in_vm(&[&cargo_target_dir.join("dst_field_align")], env, args)?; + + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[AOT] std_example"); + let mut command = args.config_info.rustc_command_vec(); + command.extend_from_slice(&[ + &"example/std_example.rs", + &"--crate-type", + &"bin", + &"--target", + &args.config_info.target_triple, + ]); + if args.is_using_gcc_master_branch() { + command.extend_from_slice(&[&"--cfg", &"feature=\"master\""]); + } + run_command_with_env(&command, None, Some(env))?; + maybe_run_command_in_vm( + &[ + &cargo_target_dir.join("std_example"), + &"--target", + &args.config_info.target_triple, + ], + env, + args, + )?; + + let test_flags = if let Some(test_flags) = env.get("TEST_FLAGS") { + split_args(test_flags)? + } else { + Vec::new() + }; + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[AOT] subslice-patterns-const-eval"); + let mut command = args.config_info.rustc_command_vec(); + command.extend_from_slice(&[ + &"example/subslice-patterns-const-eval.rs", + &"--crate-type", + &"bin", + &"--target", + &args.config_info.target_triple, + ]); + for test_flag in &test_flags { + command.push(test_flag); + } + run_command_with_env(&command, None, Some(env))?; + maybe_run_command_in_vm( + &[&cargo_target_dir.join("subslice-patterns-const-eval")], + env, + args, + )?; + + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[AOT] track-caller-attribute"); + let mut command = args.config_info.rustc_command_vec(); + command.extend_from_slice(&[ + &"example/track-caller-attribute.rs", + &"--crate-type", + &"bin", + &"--target", + &args.config_info.target_triple, + ]); + for test_flag in &test_flags { + command.push(test_flag); + } + run_command_with_env(&command, None, Some(env))?; + maybe_run_command_in_vm( + &[&cargo_target_dir.join("track-caller-attribute")], + env, + args, + )?; + + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[AOT] mod_bench"); + let mut command = args.config_info.rustc_command_vec(); + command.extend_from_slice(&[ + &"example/mod_bench.rs", + &"--crate-type", + &"bin", + &"--target", + &args.config_info.target_triple, + ]); + run_command_with_env(&command, None, Some(env))?; + // FIXME: the compiled binary is not run. + + Ok(()) +} + +fn setup_rustc(env: &mut Env, args: &TestArg) -> Result<(), String> { + let toolchain = get_toolchain()?; + + let rust_dir = Some(Path::new("rust")); + // If the repository was already cloned, command will fail, so doesn't matter. + let _ = run_command_with_output_and_env( + &[&"git", &"clone", &"https://github.com/rust-lang/rust.git"], + None, + Some(env), + ); + run_command(&[&"git", &"checkout", &"--", &"tests/"], rust_dir)?; + run_command_with_output_and_env(&[&"git", &"fetch"], rust_dir, Some(env))?; + let rustc_commit = match rustc_version_info(env.get("RUSTC").map(|s| s.as_str()))?.commit_hash { + Some(commit_hash) => commit_hash, + None => return Err("Couldn't retrieve rustc commit hash".to_string()), + }; + run_command_with_output_and_env(&[&"git", &"checkout", &rustc_commit], rust_dir, Some(env))?; + let cargo = String::from_utf8( + run_command_with_env(&[&"rustup", &"which", &"cargo"], rust_dir, Some(env))?.stdout, + ) + .map_err(|error| format!("Failed to retrieve cargo path: {:?}", error)) + .and_then(|cargo| { + let cargo = cargo.trim().to_owned(); + if cargo.is_empty() { + Err(format!("`cargo` path is empty")) + } else { + Ok(cargo) + } + })?; + let llvm_filecheck = String::from_utf8( + run_command_with_env( + &[ + &"bash", + &"-c", + &"which FileCheck-10 || \ + which FileCheck-11 || \ + which FileCheck-12 || \ + which FileCheck-13 || \ + which FileCheck-14", + ], + rust_dir, + Some(env), + )? + .stdout, + ) + .map_err(|error| format!("Failed to retrieve LLVM FileCheck: {:?}", error))?; + std::fs::write( + "rust/config.toml", + &format!( + r#"change-id = 115898 + +[rust] +codegen-backends = [] +deny-warnings = false +verbose-tests = true + +[build] +cargo = "{cargo}" +local-rebuild = true +rustc = "{home}/.rustup/toolchains/{toolchain}-{host_triple}/bin/rustc" + +[target.x86_64-unknown-linux-gnu] +llvm-filecheck = "{llvm_filecheck}" + +[llvm] +download-ci-llvm = false +"#, + cargo = cargo.trim(), + home = env.get("HOME").unwrap(), + toolchain = toolchain, + host_triple = args.config_info.host_triple, + llvm_filecheck = llvm_filecheck.trim(), + ), + ) + .map_err(|error| format!("Failed to write into `rust/config.toml`: {:?}", error))?; + Ok(()) +} + +fn asm_tests(env: &Env, args: &TestArg) -> Result<(), String> { + let mut env = env.clone(); + setup_rustc(&mut env, args)?; + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[TEST] rustc asm test suite"); + + env.insert("COMPILETEST_FORCE_STAGE0".to_string(), "1".to_string()); + + run_command_with_env( + &[ + &"./x.py", + &"test", + &"--run", + &"always", + &"--stage", + &"0", + &"tests/assembly/asm", + &"--rustc-args", + &format!( + r#"-Zpanic-abort-tests -Csymbol-mangling-version=v0 \ + -Zcodegen-backend="{pwd}/target/{channel}/librustc_codegen_gcc.{dylib_ext}" \ + --sysroot "{pwd}/build_sysroot/sysroot" -Cpanic=abort"#, + pwd = std::env::current_dir() + .map_err(|error| format!("`current_dir` failed: {:?}", error))? + .display(), + channel = args.config_info.channel.as_str(), + dylib_ext = args.config_info.dylib_ext, + ) + .as_str(), + ], + Some(Path::new("rust")), + Some(&env), + )?; + Ok(()) +} + +fn run_cargo_command( + command: &[&dyn AsRef], + cwd: Option<&Path>, + env: &Env, + args: &TestArg, +) -> Result<(), String> { + run_cargo_command_with_callback(command, cwd, env, args, |cargo_command, cwd, env| { + run_command_with_output_and_env(cargo_command, cwd, Some(env))?; + Ok(()) + }) +} + +fn run_cargo_command_with_callback( + command: &[&dyn AsRef], + cwd: Option<&Path>, + env: &Env, + args: &TestArg, + callback: F, +) -> Result<(), String> +where + F: Fn(&[&dyn AsRef], Option<&Path>, &Env) -> Result<(), String>, +{ + let toolchain = get_toolchain()?; + let toolchain_arg = format!("+{}", toolchain); + let rustc_version = String::from_utf8( + run_command_with_env(&[&args.config_info.rustc_command[0], &"-V"], cwd, Some(env))?.stdout, + ) + .map_err(|error| format!("Failed to retrieve rustc version: {:?}", error))?; + let rustc_toolchain_version = String::from_utf8( + run_command_with_env( + &[&args.config_info.rustc_command[0], &toolchain_arg, &"-V"], + cwd, + Some(env), + )? + .stdout, + ) + .map_err(|error| format!("Failed to retrieve rustc +toolchain version: {:?}", error))?; + + if rustc_version != rustc_toolchain_version { + eprintln!( + "rustc_codegen_gcc is built for `{}` but the default rustc version is `{}`.", + rustc_toolchain_version, rustc_version, + ); + eprintln!("Using `{}`.", rustc_toolchain_version); + } + let mut env = env.clone(); + let rustflags = env.get("RUSTFLAGS").cloned().unwrap_or_default(); + env.insert("RUSTDOCFLAGS".to_string(), rustflags); + let mut cargo_command: Vec<&dyn AsRef> = vec![&"cargo", &toolchain_arg]; + cargo_command.extend_from_slice(&command); + callback(&cargo_command, cwd, &env) +} + +// FIXME(antoyo): linker gives multiple definitions error on Linux +// echo "[BUILD] sysroot in release mode" +// ./build_sysroot/build_sysroot.sh --release + +fn test_libcore(env: &Env, args: &TestArg) -> Result<(), String> { + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[TEST] libcore"); + let path = Path::new("build_sysroot/sysroot_src/library/core/tests"); + let _ = remove_dir_all(path.join("target")); + run_cargo_command(&[&"test"], Some(path), env, args)?; + Ok(()) +} + +// echo "[BENCH COMPILE] mod_bench" +// +// COMPILE_MOD_BENCH_INLINE="$RUSTC example/mod_bench.rs --crate-type bin -Zmir-opt-level=3 -O --crate-name mod_bench_inline" +// COMPILE_MOD_BENCH_LLVM_0="rustc example/mod_bench.rs --crate-type bin -Copt-level=0 -o $cargo_target_dir/mod_bench_llvm_0 -Cpanic=abort" +// COMPILE_MOD_BENCH_LLVM_1="rustc example/mod_bench.rs --crate-type bin -Copt-level=1 -o $cargo_target_dir/mod_bench_llvm_1 -Cpanic=abort" +// COMPILE_MOD_BENCH_LLVM_2="rustc example/mod_bench.rs --crate-type bin -Copt-level=2 -o $cargo_target_dir/mod_bench_llvm_2 -Cpanic=abort" +// COMPILE_MOD_BENCH_LLVM_3="rustc example/mod_bench.rs --crate-type bin -Copt-level=3 -o $cargo_target_dir/mod_bench_llvm_3 -Cpanic=abort" +// +// Use 100 runs, because a single compilations doesn't take more than ~150ms, so it isn't very slow +// hyperfine --runs ${COMPILE_RUNS:-100} "$COMPILE_MOD_BENCH_INLINE" "$COMPILE_MOD_BENCH_LLVM_0" "$COMPILE_MOD_BENCH_LLVM_1" "$COMPILE_MOD_BENCH_LLVM_2" "$COMPILE_MOD_BENCH_LLVM_3" +// echo "[BENCH RUN] mod_bench" +// hyperfine --runs ${RUN_RUNS:-10} $cargo_target_dir/mod_bench{,_inline} $cargo_target_dir/mod_bench_llvm_* + +fn extended_rand_tests(env: &Env, args: &TestArg) -> Result<(), String> { + if !args.is_using_gcc_master_branch() { + println!("Not using GCC master branch. Skipping `extended_rand_tests`."); + return Ok(()); + } + let path = Path::new("rand"); + run_cargo_command(&[&"clean"], Some(path), env, args)?; + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[TEST] rust-random/rand"); + run_cargo_command(&[&"test", &"--workspace"], Some(path), env, args)?; + Ok(()) +} + +fn extended_regex_example_tests(env: &Env, args: &TestArg) -> Result<(), String> { + if !args.is_using_gcc_master_branch() { + println!("Not using GCC master branch. Skipping `extended_regex_example_tests`."); + return Ok(()); + } + let path = Path::new("regex"); + run_cargo_command(&[&"clean"], Some(path), env, args)?; + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[TEST] rust-lang/regex example shootout-regex-dna"); + let mut env = env.clone(); + // newer aho_corasick versions throw a deprecation warning + let rustflags = format!( + "{} --cap-lints warn", + env.get("RUSTFLAGS").cloned().unwrap_or_default() + ); + env.insert("RUSTFLAGS".to_string(), rustflags); + // Make sure `[codegen mono items] start` doesn't poison the diff + run_cargo_command( + &[&"build", &"--example", &"shootout-regex-dna"], + Some(path), + &env, + args, + )?; + + run_cargo_command_with_callback( + &[&"run", &"--example", &"shootout-regex-dna"], + Some(path), + &env, + args, + |cargo_command, cwd, env| { + // FIXME: rewrite this with `child.stdin.write_all()` because + // `examples/regexdna-input.txt` is very small. + let mut command: Vec<&dyn AsRef> = vec![&"bash", &"-c"]; + let cargo_args = cargo_command + .iter() + .map(|s| s.as_ref().to_str().unwrap()) + .collect::>(); + let bash_command = format!( + "cat examples/regexdna-input.txt | {} | grep -v 'Spawned thread' > res.txt", + cargo_args.join(" "), + ); + command.push(&bash_command); + run_command_with_output_and_env(&command, cwd, Some(env))?; + run_command_with_output_and_env( + &[&"diff", &"-u", &"res.txt", &"examples/regexdna-output.txt"], + cwd, + Some(env), + )?; + Ok(()) + }, + )?; + + Ok(()) +} + +fn extended_regex_tests(env: &Env, args: &TestArg) -> Result<(), String> { + if !args.is_using_gcc_master_branch() { + println!("Not using GCC master branch. Skipping `extended_regex_tests`."); + return Ok(()); + } + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[TEST] rust-lang/regex tests"); + let mut env = env.clone(); + // newer aho_corasick versions throw a deprecation warning + let rustflags = format!( + "{} --cap-lints warn", + env.get("RUSTFLAGS").cloned().unwrap_or_default() + ); + env.insert("RUSTFLAGS".to_string(), rustflags); + run_cargo_command( + &[ + &"test", + &"--tests", + &"--", + // FIXME: try removing `--exclude-should-panic` argument + &"--exclude-should-panic", + &"--test-threads", + &"1", + &"-Zunstable-options", + &"-q", + ], + Some(Path::new("regex")), + &env, + args, + )?; + Ok(()) +} + +fn extended_sysroot_tests(env: &Env, args: &TestArg) -> Result<(), String> { + // pushd simple-raytracer + // echo "[BENCH COMPILE] ebobby/simple-raytracer" + // hyperfine --runs "${RUN_RUNS:-10}" --warmup 1 --prepare "cargo clean" \ + // "RUSTC=rustc RUSTFLAGS='' cargo build" \ + // "../cargo.sh build" + + // echo "[BENCH RUN] ebobby/simple-raytracer" + // cp ./target/debug/main ./raytracer_cg_gcc + // hyperfine --runs "${RUN_RUNS:-10}" ./raytracer_cg_llvm ./raytracer_cg_gcc + // popd + extended_rand_tests(env, args)?; + extended_regex_example_tests(env, args)?; + extended_regex_tests(env, args)?; + + Ok(()) +} + +fn should_not_remove_test(file: &str) -> bool { + // contains //~ERROR, but shouldn't be removed + [ + "issues/auxiliary/issue-3136-a.rs", + "type-alias-impl-trait/auxiliary/cross_crate_ice.rs", + "type-alias-impl-trait/auxiliary/cross_crate_ice2.rs", + "macros/rfc-2011-nicer-assert-messages/auxiliary/common.rs", + "imports/ambiguous-1.rs", + "imports/ambiguous-4-extern.rs", + "entry-point/auxiliary/bad_main_functions.rs", + ] + .iter() + .any(|to_ignore| file.ends_with(to_ignore)) +} + +fn should_remove_test(file_path: &Path) -> Result { + // Tests generating errors. + let file = File::open(file_path) + .map_err(|error| format!("Failed to read `{}`: {:?}", file_path.display(), error))?; + for line in BufReader::new(file).lines().filter_map(|line| line.ok()) { + let line = line.trim(); + if line.is_empty() { + continue; + } + if [ + "// error-pattern:", + "// build-fail", + "// run-fail", + "-Cllvm-args", + "//~", + "thread", + ] + .iter() + .any(|check| line.contains(check)) + { + return Ok(true); + } + if line.contains("//[") && line.contains("]~") { + return Ok(true); + } + } + if file_path + .display() + .to_string() + .contains("ambiguous-4-extern.rs") + { + eprintln!("nothing found for {file_path:?}"); } + Ok(false) +} + +fn test_rustc_inner(env: &Env, args: &TestArg, prepare_files_callback: F) -> Result<(), String> +where + F: Fn() -> Result, +{ + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[TEST] rust-lang/rust"); + let mut env = env.clone(); + setup_rustc(&mut env, args)?; + + let rust_path = Path::new("rust"); + + walk_dir( + "rust/tests/ui", + |dir| { + let dir_name = dir.file_name().and_then(|name| name.to_str()).unwrap_or(""); + if [ + "abi", + "extern", + "unsized-locals", + "proc-macro", + "threads-sendsync", + "borrowck", + "test-attrs", + ] + .iter() + .any(|name| *name == dir_name) + { + std::fs::remove_dir_all(dir).map_err(|error| { + format!("Failed to remove folder `{}`: {:?}", dir.display(), error) + })?; + } + Ok(()) + }, + |_| Ok(()), + )?; + + // These two functions are used to remove files that are known to not be working currently + // with the GCC backend to reduce noise. + fn dir_handling(dir: &Path) -> Result<(), String> { + if dir + .file_name() + .map(|name| name == "auxiliary") + .unwrap_or(true) + { + return Ok(()); + } + walk_dir(dir, dir_handling, file_handling) + } + fn file_handling(file_path: &Path) -> Result<(), String> { + if !file_path + .extension() + .map(|extension| extension == "rs") + .unwrap_or(false) + { + return Ok(()); + } + let path_str = file_path.display().to_string().replace("\\", "/"); + if should_not_remove_test(&path_str) { + return Ok(()); + } else if should_remove_test(file_path)? { + return remove_file(&file_path); + } + Ok(()) + } + + remove_file(&rust_path.join("tests/ui/consts/const_cmp_type_id.rs"))?; + remove_file(&rust_path.join("tests/ui/consts/issue-73976-monomorphic.rs"))?; + // this test is oom-killed in the CI. + remove_file(&rust_path.join("tests/ui/consts/issue-miri-1910.rs"))?; + // Tests generating errors. + remove_file(&rust_path.join("tests/ui/consts/issue-94675.rs"))?; + remove_file(&rust_path.join("tests/ui/mir/mir_heavy_promoted.rs"))?; + + walk_dir(rust_path.join("tests/ui"), dir_handling, file_handling)?; + + if !prepare_files_callback()? { + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("Keeping all UI tests"); + } + + let nb_parts = args.nb_parts.unwrap_or(0); + if nb_parts > 0 { + let current_part = args.current_part.unwrap(); + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!( + "Splitting ui_test into {} parts (and running part {})", + nb_parts, current_part + ); + let out = String::from_utf8( + run_command( + &[ + &"find", + &"tests/ui", + &"-type", + &"f", + &"-name", + &"*.rs", + &"-not", + &"-path", + &"*/auxiliary/*", + ], + Some(rust_path), + )? + .stdout, + ) + .map_err(|error| format!("Failed to retrieve output of find command: {:?}", error))?; + let mut files = out + .split('\n') + .map(|line| line.trim()) + .filter(|line| !line.is_empty()) + .collect::>(); + // To ensure it'll be always the same sub files, we sort the content. + files.sort(); + // We increment the number of tests by one because if this is an odd number, we would skip + // one test. + let count = files.len() / nb_parts + 1; + let start = current_part * count; + let end = current_part * count + count; + // We remove the files we don't want to test. + for path in files + .iter() + .enumerate() + .filter(|(pos, _)| *pos < start || *pos >= end) + .map(|(_, path)| path) + { + remove_file(&rust_path.join(path))?; + } + } + + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[TEST] rustc test suite"); + env.insert("COMPILETEST_FORCE_STAGE0".to_string(), "1".to_string()); + let rustc_args = format!( + "{} -Csymbol-mangling-version=v0 -Zcodegen-backend={} --sysroot {}", + env.get("TEST_FLAGS").unwrap_or(&String::new()), + args.config_info.cg_backend_path, + args.config_info.sysroot_path, + ); + + env.get_mut("RUSTFLAGS").unwrap().clear(); + run_command_with_output_and_env( + &[ + &"./x.py", + &"test", + &"--run", + &"always", + &"--stage", + &"0", + &"tests/ui", + &"--rustc-args", + &rustc_args, + ], + Some(rust_path), + Some(&env), + )?; + Ok(()) +} + +fn test_rustc(env: &Env, args: &TestArg) -> Result<(), String> { + test_rustc_inner(env, args, || Ok(false)) +} + +fn test_failing_rustc(env: &Env, args: &TestArg) -> Result<(), String> { + test_rustc_inner(env, args, || { + // Removing all tests. + run_command( + &[ + &"find", + &"tests/ui", + &"-type", + &"f", + &"-name", + &"*.rs", + &"-not", + &"-path", + &"*/auxiliary/*", + &"-delete", + ], + Some(Path::new("rust")), + )?; + // Putting back only the failing ones. + let path = "failing-ui-tests.txt"; + if let Ok(files) = std::fs::read_to_string(path) { + for file in files + .split('\n') + .map(|line| line.trim()) + .filter(|line| !line.is_empty()) + { + run_command( + &[&"git", &"checkout", &"--", &file], + Some(Path::new("rust")), + )?; + } + } else { + println!( + "Failed to read `{}`, not putting back failing ui tests", + path + ); + } + Ok(true) + }) +} + +fn test_successful_rustc(env: &Env, args: &TestArg) -> Result<(), String> { + test_rustc_inner(env, args, || { + // Removing the failing tests. + let path = "failing-ui-tests.txt"; + if let Ok(files) = std::fs::read_to_string(path) { + for file in files + .split('\n') + .map(|line| line.trim()) + .filter(|line| !line.is_empty()) + { + let path = Path::new("rust").join(file); + remove_file(&path)?; + } + } else { + println!( + "Failed to read `{}`, not putting back failing ui tests", + path + ); + } + Ok(true) + }) +} + +fn clean_ui_tests(_env: &Env, _args: &TestArg) -> Result<(), String> { + run_command( + &[ + &"find", + &"rust/build/x86_64-unknown-linux-gnu/test/ui/", + &"-name", + &"stamp", + &"-delete", + ], + None, + )?; + Ok(()) +} + +fn run_all(env: &Env, args: &TestArg) -> Result<(), String> { + clean(env, args)?; + mini_tests(env, args)?; + build_sysroot(env, args)?; + std_tests(env, args)?; + // asm_tests(env, args)?; + test_libcore(env, args)?; + extended_sysroot_tests(env, args)?; + test_rustc(env, args)?; + Ok(()) } pub fn run() -> Result<(), String> { - let mut args: Vec<&dyn AsRef> = vec![&"bash", &"test.sh"]; - let extra_args = std::env::args().skip(2).collect::>(); - get_args(&mut args, &extra_args); - let current_dir = std::env::current_dir().map_err(|error| format!("`current_dir` failed: {:?}", error))?; - run_command_with_output(args.as_slice(), Some(¤t_dir)) + let mut args = match TestArg::new()? { + Some(args) => args, + None => return Ok(()), + }; + let mut env: HashMap = std::env::vars().collect(); + + env.insert("LD_LIBRARY_PATH".to_string(), args.gcc_path.clone()); + env.insert("LIBRARY_PATH".to_string(), args.gcc_path.clone()); + + build_if_no_backend(&env, &args)?; + if args.build_only { + println!("Since it's build only, exiting..."); + return Ok(()); + } + + args.config_info.setup(&mut env, Some(&args.gcc_path))?; + + if args.runners.is_empty() { + run_all(&env, &args)?; + } else { + let runners = get_runners(); + for runner in args.runners.iter() { + runners.get(runner.as_str()).unwrap().1(&env, &args)?; + } + } + + Ok(()) } diff --git a/build_system/src/utils.rs b/build_system/src/utils.rs index 536f33a8029..9c3a86ad68a 100644 --- a/build_system/src/utils.rs +++ b/build_system/src/utils.rs @@ -29,22 +29,37 @@ fn check_exit_status( input: &[&dyn AsRef], cwd: Option<&Path>, exit_status: ExitStatus, + output: Option<&Output>, ) -> Result<(), String> { if exit_status.success() { - Ok(()) - } else { - Err(format!( - "Command `{}`{} exited with status {:?}", - input - .iter() - .map(|s| s.as_ref().to_str().unwrap()) - .collect::>() - .join(" "), - cwd.map(|cwd| format!(" (running in folder `{}`)", cwd.display())) - .unwrap_or_default(), - exit_status.code(), - )) + return Ok(()); + } + let mut error = format!( + "Command `{}`{} exited with status {:?}", + input + .iter() + .map(|s| s.as_ref().to_str().unwrap()) + .collect::>() + .join(" "), + cwd.map(|cwd| format!(" (running in folder `{}`)", cwd.display())) + .unwrap_or_default(), + exit_status.code() + ); + let input = input.iter().map(|i| i.as_ref()).collect::>(); + eprintln!("Command `{:?}` failed", input); + if let Some(output) = output { + let stdout = String::from_utf8_lossy(&output.stdout); + if !stdout.is_empty() { + error.push_str("\n==== STDOUT ====\n"); + error.push_str(&*stdout); + } + let stderr = String::from_utf8_lossy(&output.stderr); + if !stderr.is_empty() { + error.push_str("\n==== STDERR ====\n"); + error.push_str(&*stderr); + } } + Err(error) } fn command_error(input: &[&dyn AsRef], cwd: &Option<&Path>, error: D) -> String { @@ -73,7 +88,7 @@ pub fn run_command_with_env( let output = get_command_inner(input, cwd, env) .output() .map_err(|e| command_error(input, &cwd, e))?; - check_exit_status(input, cwd, output.status)?; + check_exit_status(input, cwd, output.status, Some(&output))?; Ok(output) } @@ -86,7 +101,7 @@ pub fn run_command_with_output( .map_err(|e| command_error(input, &cwd, e))? .wait() .map_err(|e| command_error(input, &cwd, e))?; - check_exit_status(input, cwd, exit_status)?; + check_exit_status(input, cwd, exit_status, None)?; Ok(()) } @@ -100,7 +115,7 @@ pub fn run_command_with_output_and_env( .map_err(|e| command_error(input, &cwd, e))? .wait() .map_err(|e| command_error(input, &cwd, e))?; - check_exit_status(input, cwd, exit_status)?; + check_exit_status(input, cwd, exit_status, None)?; Ok(()) } @@ -143,17 +158,56 @@ pub fn get_os_name() -> Result { } } -pub fn get_rustc_host_triple() -> Result { - let output = run_command(&[&"rustc", &"-vV"], None)?; +#[derive(Default)] +pub struct RustcVersionInfo { + pub version: String, + pub host: Option, + pub commit_hash: Option, + pub commit_date: Option, +} + +pub fn rustc_version_info(rustc: Option<&str>) -> Result { + let output = run_command(&[&rustc.unwrap_or("rustc"), &"-vV"], None)?; let content = std::str::from_utf8(&output.stdout).unwrap_or(""); + let mut info = RustcVersionInfo::default(); + for line in content.split('\n').map(|line| line.trim()) { - if !line.starts_with("host:") { - continue; + match line.split_once(':') { + Some(("host", data)) => info.host = Some(data.trim().to_string()), + Some(("release", data)) => info.version = data.trim().to_string(), + Some(("commit-hash", data)) => info.commit_hash = Some(data.trim().to_string()), + Some(("commit-date", data)) => info.commit_date = Some(data.trim().to_string()), + _ => {} } - return Ok(line.split(':').nth(1).unwrap().trim().to_string()); } - Err("Cannot find host triple".to_string()) + if info.version.is_empty() { + Err("failed to retrieve rustc version".to_string()) + } else { + Ok(info) + } +} + +pub fn get_toolchain() -> Result { + let content = match fs::read_to_string("rust-toolchain") { + Ok(content) => content, + Err(_) => return Err("No `rust-toolchain` file found".to_string()), + }; + match content + .split('\n') + .map(|line| line.trim()) + .filter(|line| !line.is_empty()) + .filter_map(|line| { + if !line.starts_with("channel") { + return None; + } + line.split('"').skip(1).next() + }) + .next() + { + Some(toolchain) => Ok(toolchain.to_string()), + None => Err("Couldn't find `channel` in `rust-toolchain` file".to_string()), + } } pub fn get_gcc_path() -> Result { @@ -238,3 +292,89 @@ where } Ok(()) } + +pub fn split_args(args: &str) -> Result, String> { + let mut out = Vec::new(); + let mut start = 0; + let args = args.trim(); + let mut iter = args.char_indices().peekable(); + + while let Some((pos, c)) = iter.next() { + if c == ' ' { + out.push(args[start..pos].to_string()); + let mut found_start = false; + while let Some((pos, c)) = iter.peek() { + if *c != ' ' { + start = *pos; + found_start = true; + break; + } else { + iter.next(); + } + } + if !found_start { + return Ok(out); + } + } else if c == '"' || c == '\'' { + let end = c; + let mut found_end = false; + while let Some((_, c)) = iter.next() { + if c == end { + found_end = true; + break; + } else if c == '\\' { + // We skip the escaped character. + iter.next(); + } + } + if !found_end { + return Err(format!( + "Didn't find `{}` at the end of `{}`", + end, + &args[start..] + )); + } + } else if c == '\\' { + // We skip the escaped character. + iter.next(); + } + } + let s = args[start..].trim(); + if !s.is_empty() { + out.push(s.to_string()); + } + Ok(out) +} + +pub fn remove_file>(file_path: &P) -> Result<(), String> { + std::fs::remove_file(file_path).map_err(|error| { + format!( + "Failed to remove `{}`: {:?}", + file_path.as_ref().display(), + error + ) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_split_args() { + // Missing `"` at the end. + assert!(split_args("\"tada").is_err()); + // Missing `'` at the end. + assert!(split_args("\'tada").is_err()); + + assert_eq!( + split_args("a \"b\" c"), + Ok(vec!["a".to_string(), "\"b\"".to_string(), "c".to_string()]) + ); + // Trailing whitespace characters. + assert_eq!( + split_args(" a \"b\" c "), + Ok(vec!["a".to_string(), "\"b\"".to_string(), "c".to_string()]) + ); + } +} diff --git a/config.sh b/config.sh deleted file mode 100644 index 7ae2175d41d..00000000000 --- a/config.sh +++ /dev/null @@ -1,85 +0,0 @@ -set -e - -export CARGO_INCREMENTAL=0 - -if [ -f ./gcc_path ]; then - export GCC_PATH=$(cat gcc_path) -elif (( $use_system_gcc == 1 )); then - echo 'Using system GCC' -else - echo 'Please put the path to your custom build of libgccjit in the file `gcc_path`, see Readme.md for details' - exit 1 -fi - -if [[ -z "$RUSTC" ]]; then - export RUSTC="rustc" -fi - -unamestr=`uname` -if [[ "$unamestr" == 'Linux' ]]; then - dylib_ext='so' -elif [[ "$unamestr" == 'Darwin' ]]; then - dylib_ext='dylib' -else - echo "Unsupported os" - exit 1 -fi - -HOST_TRIPLE=$($RUSTC -vV | grep host | cut -d: -f2 | tr -d " ") -# TODO: remove $OVERWRITE_TARGET_TRIPLE when config.sh is removed. -TARGET_TRIPLE="${OVERWRITE_TARGET_TRIPLE:-$HOST_TRIPLE}" - -linker='' -RUN_WRAPPER='' -if [[ "$HOST_TRIPLE" != "$TARGET_TRIPLE" ]]; then - RUN_WRAPPER=run_in_vm - if [[ "$TARGET_TRIPLE" == "m68k-unknown-linux-gnu" ]]; then - linker='-Clinker=m68k-unknown-linux-gnu-gcc' - elif [[ "$TARGET_TRIPLE" == "aarch64-unknown-linux-gnu" ]]; then - # We are cross-compiling for aarch64. Use the correct linker and run tests in qemu. - linker='-Clinker=aarch64-linux-gnu-gcc' - else - echo "Unknown non-native platform" - fi -fi - -# Since we don't support ThinLTO, disable LTO completely when not trying to do LTO. -# TODO(antoyo): remove when we can handle ThinLTO. -disable_lto_flags='' -if [[ ! -v FAT_LTO ]]; then - disable_lto_flags='-Clto=off' -fi - -if [[ -z "$BUILTIN_BACKEND" ]]; then - export RUSTFLAGS="$CG_RUSTFLAGS $linker -Csymbol-mangling-version=v0 -Cdebuginfo=2 $disable_lto_flags -Zcodegen-backend=$(pwd)/target/${CHANNEL:-debug}/librustc_codegen_gcc.$dylib_ext --sysroot $(pwd)/build_sysroot/sysroot $TEST_FLAGS" -else - export RUSTFLAGS="$CG_RUSTFLAGS $linker -Csymbol-mangling-version=v0 -Cdebuginfo=2 $disable_lto_flags -Zcodegen-backend=gcc $TEST_FLAGS -Cpanic=abort" - - if [[ ! -z "$RUSTC_SYSROOT" ]]; then - export RUSTFLAGS="$RUSTFLAGS --sysroot $RUSTC_SYSROOT" - fi -fi - -# FIXME(antoyo): remove once the atomic shim is gone -if [[ unamestr == 'Darwin' ]]; then - export RUSTFLAGS="$RUSTFLAGS -Clink-arg=-undefined -Clink-arg=dynamic_lookup" -fi - -if [[ -z "$cargo_target_dir" ]]; then - RUST_CMD="$RUSTC $RUSTFLAGS -L crate=target/out --out-dir target/out" - cargo_target_dir="target/out" -else - RUST_CMD="$RUSTC $RUSTFLAGS -L crate=$cargo_target_dir --out-dir $cargo_target_dir" -fi -export RUSTC_LOG=warn # display metadata load errors - -export LD_LIBRARY_PATH="$(pwd)/target/out:$(pwd)/build_sysroot/sysroot/lib/rustlib/$TARGET_TRIPLE/lib" -if [[ ! -z "$:$GCC_PATH" ]]; then - export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$GCC_PATH" -fi - -export DYLD_LIBRARY_PATH=$LD_LIBRARY_PATH -# NOTE: To avoid the -fno-inline errors, use /opt/gcc/bin/gcc instead of cc. -# To do so, add a symlink for cc to /opt/gcc/bin/gcc in our PATH. -# Another option would be to add the following Rust flag: -Clinker=/opt/gcc/bin/gcc -export PATH="/opt/gcc/bin:/opt/m68k-unknown-linux-gnu/bin:$PATH" diff --git a/test.sh b/test.sh deleted file mode 100755 index e896237a1ea..00000000000 --- a/test.sh +++ /dev/null @@ -1,479 +0,0 @@ -#!/usr/bin/env bash - -# TODO(antoyo): rewrite to cargo-make (or just) or something like that to only rebuild the sysroot when needed? - -set -e -#set -x - -flags= -gcc_master_branch=1 -channel="debug" -funcs=() -build_only=0 -nb_parts=0 -current_part=0 -use_system_gcc=0 -use_backend=0 -cargo_target_dir="" - -export CHANNEL='debug' - -while [[ $# -gt 0 ]]; do - case $1 in - --release) - codegen_channel=release - channel="release" - export CHANNEL='release' - shift - ;; - --release-sysroot) - sysroot_channel="--release" - shift - ;; - --no-default-features) - gcc_master_branch=0 - flags="$flags --no-default-features" - shift - ;; - --features) - shift - flags="$flags --features $1" - shift - ;; - "--test-rustc") - funcs+=(test_rustc) - shift - ;; - "--test-successful-rustc") - funcs+=(test_successful_rustc) - shift - ;; - "--test-failing-rustc") - funcs+=(test_failing_rustc) - shift - ;; - - "--test-libcore") - funcs+=(test_libcore) - shift - ;; - - "--clean-ui-tests") - funcs+=(clean_ui_tests) - shift - ;; - "--clean") - funcs+=(clean) - shift - ;; - - "--std-tests") - funcs+=(std_tests) - shift - ;; - - "--asm-tests") - funcs+=(asm_tests) - shift - ;; - - "--extended-tests") - funcs+=(extended_sysroot_tests) - shift - ;; - "--extended-rand-tests") - funcs+=(extended_rand_tests) - shift - ;; - "--extended-regex-example-tests") - funcs+=(extended_regex_example_tests) - shift - ;; - "--extended-regex-tests") - funcs+=(extended_regex_tests) - shift - ;; - - "--mini-tests") - funcs+=(mini_tests) - shift - ;; - - "--build-sysroot") - funcs+=(build_sysroot) - shift - ;; - "--build") - build_only=1 - shift - ;; - "--use-system-gcc") - use_system_gcc=1 - shift - ;; - "--use-backend") - use_backend=1 - shift - export BUILTIN_BACKEND=$1 - shift - ;; - "--out-dir") - shift - export CARGO_TARGET_DIR=$1 - cargo_target_dir=$1 - shift - ;; - "--nb-parts") - shift - nb_parts=$1 - shift - ;; - "--current-part") - shift - current_part=$1 - shift - ;; - *) - echo "Unknown option $1" - exit 1 - ;; - esac -done - -if [ -f ./gcc_path ]; then - export GCC_PATH=$(cat gcc_path) -elif (( $use_system_gcc == 1 )); then - echo 'Using system GCC' -else - echo 'Please put the path to your custom build of libgccjit in the file `gcc_path`, see Readme.md for details' - exit 1 -fi - -export LD_LIBRARY_PATH="$GCC_PATH" -export LIBRARY_PATH="$GCC_PATH" - -if [[ $use_backend == 0 ]]; then - if [[ $channel == "release" ]]; then - CARGO_INCREMENTAL=1 cargo rustc --release $flags - else - echo $LD_LIBRARY_PATH - cargo rustc $flags - fi -fi - -if (( $build_only == 1 )); then - echo "Since it's 'build-only', exiting..." - exit -fi - -source config.sh - -function clean() { - rm -r $cargo_target_dir || true - mkdir -p $cargo_target_dir/gccjit -} - -function mini_tests() { - echo "[BUILD] mini_core" - crate_types="lib,dylib" - - if [[ "$HOST_TRIPLE" != "$TARGET_TRIPLE" ]]; then - crate_types="lib" - fi - - $RUST_CMD example/mini_core.rs --crate-name mini_core --crate-type $crate_types --target $TARGET_TRIPLE - - echo "[BUILD] example" - $RUST_CMD example/example.rs --crate-type lib --target $TARGET_TRIPLE - - echo "[AOT] mini_core_hello_world" - $RUST_CMD example/mini_core_hello_world.rs --crate-name mini_core_hello_world --crate-type bin -g --target $TARGET_TRIPLE - $RUN_WRAPPER $cargo_target_dir/mini_core_hello_world abc bcd -} - -function build_sysroot() { - echo "[BUILD] sysroot" - time ./build_sysroot/build_sysroot.sh $sysroot_channel -} - -# TODO(GuillaumeGomez): when rewriting in Rust, refactor with the code in tests/lang_tests_common.rs if possible. -function run_in_vm() { - vm_parent_dir=${CG_GCC_VM_DIR:-$(pwd)} - vm_dir=vm - exe=$1 - exe_filename=$(basename $exe) - vm_home_dir=$vm_parent_dir/$vm_dir/home - vm_exe_path=$vm_home_dir/$exe_filename - inside_vm_exe_path=/home/$exe_filename - sudo cp $exe $vm_exe_path - - shift - pushd $vm_parent_dir - sudo chroot $vm_dir qemu-m68k-static $inside_vm_exe_path $@ - popd -} - -function std_tests() { - echo "[AOT] arbitrary_self_types_pointers_and_wrappers" - $RUST_CMD example/arbitrary_self_types_pointers_and_wrappers.rs --crate-name arbitrary_self_types_pointers_and_wrappers --crate-type bin --target $TARGET_TRIPLE - $RUN_WRAPPER $cargo_target_dir/arbitrary_self_types_pointers_and_wrappers - - echo "[AOT] alloc_system" - $RUST_CMD example/alloc_system.rs --crate-type lib --target "$TARGET_TRIPLE" - - # FIXME: doesn't work on m68k. - if [[ "$HOST_TRIPLE" == "$TARGET_TRIPLE" ]]; then - echo "[AOT] alloc_example" - $RUST_CMD example/alloc_example.rs --crate-type bin --target $TARGET_TRIPLE - $RUN_WRAPPER $cargo_target_dir/alloc_example - fi - - echo "[AOT] dst_field_align" - # FIXME(antoyo): Re-add -Zmir-opt-level=2 once rust-lang/rust#67529 is fixed. - $RUST_CMD example/dst-field-align.rs --crate-name dst_field_align --crate-type bin --target $TARGET_TRIPLE - $RUN_WRAPPER $cargo_target_dir/dst_field_align || (echo $?; false) - - echo "[AOT] std_example" - std_flags="--cfg feature=\"master\"" - if (( $gcc_master_branch == 0 )); then - std_flags="" - fi - $RUST_CMD example/std_example.rs --crate-type bin --target $TARGET_TRIPLE $std_flags - $RUN_WRAPPER $cargo_target_dir/std_example --target $TARGET_TRIPLE - - echo "[AOT] subslice-patterns-const-eval" - $RUST_CMD example/subslice-patterns-const-eval.rs --crate-type bin $TEST_FLAGS --target $TARGET_TRIPLE - $RUN_WRAPPER $cargo_target_dir/subslice-patterns-const-eval - - echo "[AOT] track-caller-attribute" - $RUST_CMD example/track-caller-attribute.rs --crate-type bin $TEST_FLAGS --target $TARGET_TRIPLE - $RUN_WRAPPER $cargo_target_dir/track-caller-attribute - - echo "[BUILD] mod_bench" - $RUST_CMD example/mod_bench.rs --crate-type bin --target $TARGET_TRIPLE -} - -function setup_rustc() { - rust_toolchain=$(cat rust-toolchain | grep channel | sed 's/channel = "\(.*\)"/\1/') - - git clone https://github.com/rust-lang/rust.git || true - cd rust - git fetch - git checkout $($RUSTC -V | cut -d' ' -f3 | tr -d '(') - export RUSTFLAGS= - - rm config.toml || true - - cat > config.toml < res.txt - diff -u res.txt examples/regexdna-output.txt - popd -} - -function extended_regex_tests() { - if (( $gcc_master_branch == 0 )); then - return - fi - - pushd regex - echo "[TEST] rust-lang/regex tests" - export CG_RUSTFLAGS="--cap-lints warn" # newer aho_corasick versions throw a deprecation warning - ../cargo.sh test --tests -- --exclude-should-panic --test-threads 1 -Zunstable-options -q - popd -} - -function extended_sysroot_tests() { - #pushd simple-raytracer - #echo "[BENCH COMPILE] ebobby/simple-raytracer" - #hyperfine --runs "${RUN_RUNS:-10}" --warmup 1 --prepare "cargo clean" \ - #"RUSTC=rustc RUSTFLAGS='' cargo build" \ - #"../cargo.sh build" - - #echo "[BENCH RUN] ebobby/simple-raytracer" - #cp ./target/debug/main ./raytracer_cg_gcc - #hyperfine --runs "${RUN_RUNS:-10}" ./raytracer_cg_llvm ./raytracer_cg_gcc - #popd - - extended_rand_tests - extended_regex_example_tests - extended_regex_tests -} - -function test_rustc() { - echo - echo "[TEST] rust-lang/rust" - - setup_rustc - - for test in $(rg -i --files-with-matches "//(\[\w+\])?~|// error-pattern:|// build-fail|// run-fail|-Cllvm-args" tests/ui); do - rm $test - done - rm tests/ui/consts/const_cmp_type_id.rs - rm tests/ui/consts/issue-73976-monomorphic.rs - - git checkout -- tests/ui/issues/auxiliary/issue-3136-a.rs # contains //~ERROR, but shouldn't be removed - - rm -r tests/ui/{abi*,extern/,unsized-locals/,proc-macro/,threads-sendsync/,borrowck/,test*,consts/issue-miri-1910.rs} || true - rm tests/ui/mir/mir_heavy_promoted.rs # this test is oom-killed in the CI. - # Tests generating errors. - rm tests/ui/consts/issue-94675.rs - for test in $(rg --files-with-matches "thread" tests/ui); do - rm $test - done - git checkout tests/ui/type-alias-impl-trait/auxiliary/cross_crate_ice.rs - git checkout tests/ui/type-alias-impl-trait/auxiliary/cross_crate_ice2.rs - git checkout tests/ui/macros/rfc-2011-nicer-assert-messages/auxiliary/common.rs - git checkout tests/ui/imports/ambiguous-1.rs - git checkout tests/ui/imports/ambiguous-4-extern.rs - git checkout tests/ui/entry-point/auxiliary/bad_main_functions.rs - - RUSTC_ARGS="$TEST_FLAGS -Csymbol-mangling-version=v0 -Zcodegen-backend="$(pwd)"/../target/"$CHANNEL"/librustc_codegen_gcc."$dylib_ext" --sysroot "$(pwd)"/../build_sysroot/sysroot" - - if [ $# -eq 0 ]; then - # No argument supplied to the function. Doing nothing. - echo "No argument provided. Keeping all UI tests" - elif [ $1 = "0" ]; then - # Removing the failing tests. - xargs -a ../failing-ui-tests.txt -d'\n' rm - else - # Removing all tests. - find tests/ui -type f -name '*.rs' -not -path '*/auxiliary/*' -delete - # Putting back only the failing ones. - xargs -a ../failing-ui-tests.txt -d'\n' git checkout -- - fi - - if [ $nb_parts -gt 0 ]; then - echo "Splitting ui_test into $nb_parts parts (and running part $current_part)" - find tests/ui -type f -name '*.rs' -not -path "*/auxiliary/*" > ui_tests - # To ensure it'll be always the same sub files, we sort the content. - sort ui_tests -o ui_tests - count=$((`wc -l < ui_tests` / $nb_parts)) - # We increment the number of tests by one because if this is an odd number, we would skip - # one test. - count=$((count + 1)) - split -d -l $count -a 1 ui_tests ui_tests.split - # Removing all tests. - find tests/ui -type f -name '*.rs' -not -path "*/auxiliary/*" -delete - # Putting back only the ones we want to test. - xargs -a "ui_tests.split$current_part" -d'\n' git checkout -- - fi - - echo "[TEST] rustc test suite" - COMPILETEST_FORCE_STAGE0=1 ./x.py test --run always --stage 0 tests/ui/ --rustc-args "$RUSTC_ARGS" # --target $TARGET_TRIPLE -} - -function test_failing_rustc() { - test_rustc "1" -} - -function test_successful_rustc() { - test_rustc "0" -} - -function clean_ui_tests() { - find rust/build/x86_64-unknown-linux-gnu/test/ui/ -name stamp -delete -} - -function all() { - clean - mini_tests - build_sysroot - std_tests - #asm_tests - test_libcore - extended_sysroot_tests - test_rustc -} - -if [ ${#funcs[@]} -eq 0 ]; then - echo "No command passed, running '--all'..." - all -else - for t in ${funcs[@]}; do - $t - done -fi