diff --git a/.travis.yml b/.travis.yml
index 187079602114..a2a67e66e614 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -61,14 +61,12 @@ matrix:
       if: repo =~ /^rust-lang\/rust-clippy$/ AND branch IN (auto, try)
     - env: INTEGRATION=rust-lang/cargo
       if: repo =~ /^rust-lang\/rust-clippy$/ AND branch IN (auto, try)
-    # FIXME: Output too large
-    # - env: INTEGRATION=rust-lang-nursery/chalk
-    #   if: repo =~ /^rust-lang\/rust-clippy$/ AND branch IN (auto, try)
+    - env: INTEGRATION=rust-lang/chalk
+      if: repo =~ /^rust-lang\/rust-clippy$/ AND branch IN (auto, try)
     - env: INTEGRATION=Geal/nom
       if: repo =~ /^rust-lang\/rust-clippy$/ AND branch IN (auto, try)
-    # FIXME blocked on https://github.com/rust-lang/rust-clippy/issues/4727
-    #- env: INTEGRATION=rust-lang/rustfmt
-    #  if: repo =~ /^rust-lang\/rust-clippy$/ AND branch IN (auto, try)
+    - env: INTEGRATION=rust-lang/rustfmt
+      if: repo =~ /^rust-lang\/rust-clippy$/ AND branch IN (auto, try)
     - env: INTEGRATION=hyperium/hyper
       if: repo =~ /^rust-lang\/rust-clippy$/ AND branch IN (auto, try)
     - env: INTEGRATION=bluss/rust-itertools
@@ -127,7 +125,7 @@ before_script:
 script:
   - |
     if [[ -n ${INTEGRATION} ]]; then
-      ./ci/integration-tests.sh && sleep 5
+      cargo test --test integration --features integration && sleep 5
     else
       ./ci/base-tests.sh && sleep 5
     fi
diff --git a/Cargo.toml b/Cargo.toml
index b572aebdbbe3..dabc4a62e6d6 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -38,6 +38,8 @@ clippy_lints = { version = "0.0.212", path = "clippy_lints" }
 regex = "1"
 semver = "0.9"
 rustc_tools_util = { version = "0.2.0", path = "rustc_tools_util"}
+git2 = { version = "0.10", optional = true }
+tempfile = { version = "3.1.0", optional = true }
 
 [dev-dependencies]
 cargo_metadata = "0.9.0"
@@ -58,3 +60,4 @@ rustc_tools_util = { version = "0.2.0", path = "rustc_tools_util"}
 
 [features]
 deny-warnings = []
+integration = ["git2", "tempfile"]
diff --git a/ci/integration-tests.sh b/ci/integration-tests.sh
deleted file mode 100755
index f6540769bb03..000000000000
--- a/ci/integration-tests.sh
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/usr/bin/env bash
-set -ex
-
-if [[ -z "$INTEGRATION" ]]; then
-    exit 0
-fi
-
-CARGO_TARGET_DIR=$(pwd)/target/
-export CARGO_TARGET_DIR
-
-rm ~/.cargo/bin/cargo-clippy
-cargo install --force --debug --path .
-
-echo "Running integration test for crate ${INTEGRATION}"
-
-git clone --depth=1 "https://github.com/${INTEGRATION}.git" checkout
-cd checkout
-
-# run clippy on a project, try to be verbose and trigger as many warnings
-# as possible for greater coverage
-# NOTE: we use `tee` to print any warnings and errors to stdout
-#       to avoid build timeout in Travis
-RUST_BACKTRACE=full \
-cargo clippy \
-    --all-targets \
-    --all-features \
-    -- \
-    --cap-lints warn \
-    -W clippy::pedantic \
-    -W clippy::nursery \
-    2>&1 | tee clippy_output
-
-cargo uninstall clippy
-
-if grep -q "internal compiler error\|query stack during panic\|E0463" clippy_output; then
-    exit 1
-fi
diff --git a/tests/integration.rs b/tests/integration.rs
new file mode 100644
index 000000000000..455965436d63
--- /dev/null
+++ b/tests/integration.rs
@@ -0,0 +1,75 @@
+#![cfg(feature = "integration")]
+
+use git2::Repository;
+use tempfile;
+
+use std::env;
+use std::process::Command;
+
+#[cfg_attr(feature = "integration", test)]
+fn integration_test() {
+    let repo_name = env::var("INTEGRATION").expect("`INTEGRATION` var not set");
+    let repo_url = format!("https://github.com/{}", repo_name);
+    let crate_name = repo_name
+        .split('/')
+        .nth(1)
+        .expect("repo name should have format `<org>/<name>`");
+
+    let repo_dir = tempfile::tempdir()
+        .expect("couldn't create temp dir")
+        .path()
+        .join(crate_name);
+
+    Repository::clone(&repo_url, &repo_dir).expect("clone of repo failed");
+
+    let root_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    let target_dir = std::path::Path::new(&root_dir).join("target");
+    let clippy_binary = target_dir.join(env!("PROFILE")).join("cargo-clippy");
+
+    let output = Command::new(clippy_binary)
+        .current_dir(repo_dir)
+        .env("RUST_BACKTRACE", "full")
+        .env("CARGO_TARGET_DIR", target_dir)
+        .args(&[
+            "clippy",
+            "--all-targets",
+            "--all-features",
+            "--",
+            "--cap-lints",
+            "warn",
+            "-Wclippy::pedantic",
+            "-Wclippy::nursery",
+        ])
+        .output()
+        .expect("unable to run clippy");
+
+    let stderr = String::from_utf8_lossy(&output.stderr);
+    if stderr.contains("internal compiler error") {
+        let backtrace_start = stderr
+            .find("thread 'rustc' panicked at")
+            .expect("start of backtrace not found");
+        let backtrace_end = stderr
+            .rfind("error: internal compiler error")
+            .expect("end of backtrace not found");
+
+        panic!(
+            "internal compiler error\nBacktrace:\n\n{}",
+            &stderr[backtrace_start..backtrace_end]
+        );
+    } else if stderr.contains("query stack during panic") {
+        panic!("query stack during panic in the output");
+    } else if stderr.contains("E0463") {
+        panic!("error: E0463");
+    }
+
+    match output.status.code() {
+        Some(code) => {
+            if code == 0 {
+                println!("Compilation successful");
+            } else {
+                eprintln!("Compilation failed. Exit code: {}", code);
+            }
+        },
+        None => panic!("Process terminated by signal"),
+    }
+}