diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 0f5d6b7b..add5d6bf 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -30,15 +30,14 @@ jobs:
       matrix:
         ruby_version: ["3.2"]
         os: ["ubuntu-latest","windows-latest","macos-latest"]
-    env:
-      BUNDLE_GEMFILE: rack-2.gemfile
     steps:
       - uses: actions/checkout@v3
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby_version }}
           bundler-cache: true
-      - run: "bundle exec rake"
+      - run: "bundle exec appraisal install"
+      - run: "bundle exec appraisal rack-2 rake"
   test-with-active-support:
     runs-on: ${{ matrix.os }}
     env:
@@ -57,4 +56,5 @@ jobs:
         with:
           ruby-version: ${{ matrix.ruby_version }}
           bundler-cache: true
-      - run: "bundle exec rake spec_with_active_support"
+      - run: "bundle exec appraisal install"
+      - run: "bundle exec appraisal activesupport rake spec_with_active_support"
diff --git a/Appraisals b/Appraisals
new file mode 100644
index 00000000..b62fb71d
--- /dev/null
+++ b/Appraisals
@@ -0,0 +1,11 @@
+appraise "rack-2" do
+  gem "rack", "~> 2.0"
+
+  group :test do
+    remove_gem "rackup"
+  end
+end
+
+appraise "activesupport" do
+  gem "activesupport", "~> 5.1"
+end
diff --git a/Gemfile b/Gemfile
index d5cf760f..649524a1 100644
--- a/Gemfile
+++ b/Gemfile
@@ -11,6 +11,7 @@ gemspec
 # Diff:
 
 gem "rspec-mocks", "3.10.2"
+gem "appraisal", "~> 2.5"
 
 if ENV['X_PACT_DEVELOPMENT']
   gem "pact-support", path: '../pact-support'
diff --git a/active-support.gemfile b/active-support.gemfile
deleted file mode 100644
index 32c3218d..00000000
--- a/active-support.gemfile
+++ /dev/null
@@ -1,10 +0,0 @@
-# This gemfile is used to test that the pact library works ok when active support is loaded.
-# It is in a separate file because active support will not work on ruby 2.4, and the only
-# place that should be using ruby 2.4 is the pact-ruby-standalone, which will be guaranteed not
-# to have active support on it. I think :D
-
-instance_eval File.read("Gemfile")
-
-group :test do
-  gem 'activesupport', '~> 5.1'
-end
diff --git a/rack-2.gemfile b/rack-2.gemfile
deleted file mode 100644
index 4640f348..00000000
--- a/rack-2.gemfile
+++ /dev/null
@@ -1,3 +0,0 @@
-instance_eval File.read("Gemfile")
-
-gem "rack", "~> 2.0"
diff --git a/spec/support/ssl_server.rb b/spec/support/ssl_server.rb
index b8bbed84..37f75865 100644
--- a/spec/support/ssl_server.rb
+++ b/spec/support/ssl_server.rb
@@ -32,7 +32,11 @@ def webrick_opts port
   require "webrick"
   require "webrick/https"
   require "rack"
-  require "rackup/handler/webrick"
+  begin
+    require "rackup/handler/webrick" # rack 3
+  rescue LoadError
+    require "rack/handler/webrick"  # rack 2
+  end
 
   opts = webrick_opts(4444)