diff --git a/.gitignore b/.gitignore
index bb036c18..d427c406 100644
--- a/.gitignore
+++ b/.gitignore
@@ -48,5 +48,6 @@ venv/
 doc/tags
 scripts/nvim_doc_tools
 scripts/nvim-typecheck-action
-tests/perf/
+scripts/benchmark.nvim
+perf/tmp/
 profile.json
diff --git a/Makefile b/Makefile
index bd7bf6cd..10f01d11 100644
--- a/Makefile
+++ b/Makefile
@@ -37,20 +37,19 @@ fastlint: scripts/nvim_doc_tools venv
 
 ## profile: use LuaJIT profiler to profile the plugin
 .PHONY: profile
-profile:
-	nvim --clean -u tests/perf_harness.lua -c 'lua jit_profile()'
+profile: scripts/benchmark.nvim
+	nvim --clean -u perf/bootstrap.lua -c 'lua jit_profile()'
 
 ## flame_profile: create a trace in the chrome profiler format
 .PHONY: flame_profile
-flame_profile:
-	nvim --clean -u tests/perf_harness.lua -c 'lua flame_profile()'
-	@echo "Visit https://ui.perfetto.dev/ and load the profile.json file"
+flame_profile: scripts/benchmark.nvim
+	nvim --clean -u perf/bootstrap.lua -c 'lua flame_profile()'
 
 ## benchmark: benchmark performance opening directory with many files
 .PHONY: benchmark
-benchmark:
-	nvim --clean -u tests/perf_harness.lua -c 'lua benchmark(10)'
-	@cat tests/perf/benchmark.txt
+benchmark: scripts/benchmark.nvim
+	nvim --clean -u perf/bootstrap.lua -c 'lua benchmark()'
+	@cat perf/tmp/benchmark.txt
 
 scripts/nvim_doc_tools:
 	git clone https://github.com/stevearc/nvim_doc_tools scripts/nvim_doc_tools
@@ -58,7 +57,10 @@ scripts/nvim_doc_tools:
 scripts/nvim-typecheck-action:
 	git clone https://github.com/stevearc/nvim-typecheck-action scripts/nvim-typecheck-action
 
+scripts/benchmark.nvim:
+	git clone https://github.com/stevearc/benchmark.nvim scripts/benchmark.nvim
+
 ## clean: reset the repository to a clean state
 .PHONY: clean
 clean:
-	rm -rf scripts/nvim_doc_tools scripts/nvim-typecheck-action venv .testenv tests/perf profile.json
+	rm -rf scripts/nvim_doc_tools scripts/nvim-typecheck-action venv .testenv perf/tmp profile.json
diff --git a/perf/bootstrap.lua b/perf/bootstrap.lua
new file mode 100644
index 00000000..5f10c065
--- /dev/null
+++ b/perf/bootstrap.lua
@@ -0,0 +1,63 @@
+vim.opt.runtimepath:prepend("scripts/benchmark.nvim")
+vim.opt.runtimepath:prepend(".")
+
+local bm = require("benchmark")
+bm.sandbox()
+
+---@module 'oil'
+---@type oil.SetupOpts
+local setup_opts = {
+  -- columns = { "icon", "permissions", "size", "mtime" },
+}
+
+local DIR_SIZE = tonumber(vim.env.DIR_SIZE) or 100000
+local ITERATIONS = tonumber(vim.env.ITERATIONS) or 10
+local WARM_UP = tonumber(vim.env.WARM_UP) or 1
+local OUTLIERS = tonumber(vim.env.OUTLIERS) or math.floor(ITERATIONS / 10)
+local TEST_DIR = "perf/tmp/test_" .. DIR_SIZE
+
+vim.fn.mkdir(TEST_DIR, "p")
+require("benchmark.files").create_files(TEST_DIR, "file %d.txt", DIR_SIZE)
+
+function _G.jit_profile()
+  require("oil").setup(setup_opts)
+  local finish = bm.jit_profile({ filename = TEST_DIR .. "/profile.txt" })
+  bm.wait_for_user_event("OilEnter", function()
+    finish()
+  end)
+  require("oil").open(TEST_DIR)
+end
+
+function _G.flame_profile()
+  local start, stop = bm.flame_profile({
+    pattern = "oil*",
+    filename = "profile.json",
+  })
+  require("oil").setup(setup_opts)
+  start()
+  bm.wait_for_user_event("OilEnter", function()
+    stop(function()
+      vim.cmd.qall({ mods = { silent = true } })
+    end)
+  end)
+  require("oil").open(TEST_DIR)
+end
+
+function _G.benchmark()
+  require("oil").setup(setup_opts)
+  bm.run({ title = "oil.nvim", iterations = ITERATIONS, warm_up = WARM_UP }, function(callback)
+    bm.wait_for_user_event("OilEnter", callback)
+    require("oil").open(TEST_DIR)
+  end, function(times)
+    local avg = bm.avg(times, { trim_outliers = OUTLIERS })
+    local std_dev = bm.std_dev(times, { trim_outliers = OUTLIERS })
+    local lines = {
+      table.concat(vim.tbl_map(bm.format_time, times), " "),
+      string.format("Average: %s", bm.format_time(avg)),
+      string.format("Std deviation: %s", bm.format_time(std_dev)),
+    }
+
+    vim.fn.writefile(lines, "perf/tmp/benchmark.txt")
+    vim.cmd.qall({ mods = { silent = true } })
+  end)
+end
diff --git a/tests/perf_harness.lua b/tests/perf_harness.lua
deleted file mode 100644
index 679004d2..00000000
--- a/tests/perf_harness.lua
+++ /dev/null
@@ -1,122 +0,0 @@
-vim.fn.mkdir("tests/perf/.env", "p")
-local root = vim.fn.fnamemodify("./tests/perf/.env", ":p")
-
-for _, name in ipairs({ "config", "data", "state", "runtime", "cache" }) do
-  vim.env[("XDG_%s_HOME"):format(name:upper())] = root .. name
-end
-
-vim.opt.runtimepath:prepend(vim.fn.fnamemodify(".", ":p"))
-
----@module 'oil'
----@type oil.SetupOpts
-local setup_opts = {
-  -- columns = { "icon", "permissions", "size", "mtime" },
-}
-
-local num_files = 100000
-
-if not vim.uv.fs_stat(string.format("tests/perf/file %d.txt", num_files)) then
-  vim.notify("Creating files")
-  for i = 1, num_files, 1 do
-    local filename = ("tests/perf/file %d.txt"):format(i)
-    local fd = vim.uv.fs_open(filename, "a", 420)
-    assert(fd)
-    vim.uv.fs_close(fd)
-  end
-end
-
-local function wait_for_done(callback)
-  vim.api.nvim_create_autocmd("User", {
-    pattern = "OilEnter",
-    once = true,
-    callback = callback,
-  })
-end
-
-function _G.jit_profile()
-  require("oil").setup(setup_opts)
-  local outfile = "tests/perf/profile.txt"
-  require("jit.p").start("3Fpli1s", outfile)
-  local start = vim.uv.hrtime()
-  require("oil").open("tests/perf")
-
-  wait_for_done(function()
-    local delta = vim.uv.hrtime() - start
-    require("jit.p").stop()
-    print("Elapsed:", delta / 1e6, "ms")
-    vim.cmd.edit({ args = { outfile } })
-  end)
-end
-
-function _G.benchmark(iterations)
-  require("oil").setup(setup_opts)
-  local num_outliers = math.floor(0.1 * iterations)
-  local times = {}
-
-  local run_profile
-  run_profile = function()
-    -- Clear out state
-    vim.cmd.enew()
-    for _, bufnr in ipairs(vim.api.nvim_list_bufs()) do
-      if vim.api.nvim_buf_is_valid(bufnr) and bufnr ~= vim.api.nvim_get_current_buf() then
-        vim.api.nvim_buf_delete(bufnr, { force = true })
-      end
-    end
-
-    local start = vim.uv.hrtime()
-    wait_for_done(function()
-      local delta = vim.uv.hrtime() - start
-      table.insert(times, delta / 1e6)
-      if #times < iterations then
-        vim.schedule(run_profile)
-      else
-        -- Remove the outliers
-        table.sort(times)
-        for _ = 1, num_outliers do
-          table.remove(times, 1)
-          table.remove(times)
-        end
-
-        local total = 0
-        for _, time in ipairs(times) do
-          total = total + time
-        end
-
-        local lines = {
-          table.concat(
-            vim.tbl_map(function(t)
-              return string.format("%dms", math.floor(t))
-            end, times),
-            " "
-          ),
-          string.format("Average: %dms", math.floor(total / #times)),
-        }
-        vim.fn.writefile(lines, "tests/perf/benchmark.txt")
-        vim.cmd.qall()
-      end
-    end)
-    require("oil").open("tests/perf")
-  end
-
-  run_profile()
-end
-
-function _G.flame_profile()
-  if not vim.uv.fs_stat("tests/perf/profile.nvim") then
-    vim
-      .system({ "git", "clone", "https://github.com/stevearc/profile.nvim", "tests/perf/profile.nvim" })
-      :wait()
-  end
-  vim.opt.runtimepath:prepend(vim.fn.fnamemodify("./tests/perf/profile.nvim", ":p"))
-  local profile = require("profile")
-  profile.instrument_autocmds()
-  profile.instrument("oil*")
-
-  require("oil").setup(setup_opts)
-  profile.start()
-  require("oil").open("tests/perf")
-  wait_for_done(function()
-    profile.stop("profile.json")
-    vim.cmd.qall()
-  end)
-end