diff --git a/secp256k1-sys/build.rs b/secp256k1-sys/build.rs
index 31e6dba7c..4a9122b88 100644
--- a/secp256k1-sys/build.rs
+++ b/secp256k1-sys/build.rs
@@ -39,22 +39,25 @@ fn main() {
                // TODO these three should be changed to use libgmp, at least until secp PR 290 is merged
                .define("USE_NUM_NONE", Some("1"))
                .define("USE_FIELD_INV_BUILTIN", Some("1"))
-               .define("USE_SCALAR_INV_BUILTIN", Some("1"))
-               .compiler("/usr/local/opt/llvm/bin/clang")
-               .flag("--sysroot=/opt/riscv/riscv32-unknown-elf") // https://github.com/riscv-collab/riscv-gnu-toolchain has been built and stored in /opt/riscv
-               .flag("--gcc-toolchain=/opt/riscv") // https://github.com/riscv-collab/riscv-gnu-toolchain has been built and stored in /opt/riscv
-               .no_default_flags(true)
-               .flag("-O3")
-               .flag("--target=riscv32-unknown-none-elf")
-               .flag("-mabi=ilp32")
-               .flag("-mcmodel=medany")
-               .flag("-Os")
-               .flag("-fdata-sections")
-               .flag("-ffunction-sections")
-               .flag("-dead_strip")
-               .flag("-flto")
-               .flag("-march=rv32im")
-               .target("riscv32-unknown-none-elf");
+               .define("USE_SCALAR_INV_BUILTIN", Some("1"));
+
+    if env::var("CARGO_CFG_TARGET_ARCH").unwrap() == "riscv32" {
+        base_config.compiler("/usr/local/opt/llvm/bin/clang")
+        .flag("--sysroot=/opt/riscv/riscv32-unknown-elf") // https://github.com/riscv-collab/riscv-gnu-toolchain has been built and stored in /opt/riscv
+        .flag("--gcc-toolchain=/opt/riscv") // https://github.com/riscv-collab/riscv-gnu-toolchain has been built and stored in /opt/riscv
+        .no_default_flags(true)
+        .flag("-O3")
+        .flag("--target=riscv32-unknown-none-elf")
+        .flag("-mabi=ilp32")
+        .flag("-mcmodel=medany")
+        .flag("-Os")
+        .flag("-fdata-sections")
+        .flag("-ffunction-sections")
+        .flag("-dead_strip")
+        .flag("-flto")
+        .flag("-march=rv32im")
+        .target("riscv32-unknown-none-elf");
+    }
 
     if cfg!(feature = "lowmemory") {
         base_config.define("ECMULT_WINDOW_SIZE", Some("4")); // A low-enough value to consume negligible memory