diff --git a/app/controllers/errors_controller.rb b/app/controllers/errors_controller.rb
index 1628d3bc14..1f247f1235 100644
--- a/app/controllers/errors_controller.rb
+++ b/app/controllers/errors_controller.rb
@@ -7,7 +7,10 @@
 class ErrorsController < ApplicationController
   before_action :update_locale, :set_error_name
   def show
-    render :show, status: status_for(@error_name)
+    respond_to do |format|
+      format.html { render :show, status: status_for(@error_name) }
+      format.all { render plain: "Not found", status: :not_found }
+    end
   end
 
 private
diff --git a/spec/requests/errors_controller_spec.rb b/spec/requests/errors_controller_spec.rb
index 87459ca5d9..2426acb112 100644
--- a/spec/requests/errors_controller_spec.rb
+++ b/spec/requests/errors_controller_spec.rb
@@ -9,7 +9,7 @@
     let(:get_invalid_path) { get("/unknown/path") }
 
     context "with default locale" do
-      it "responds with http status" do
+      it "responds with expected http status" do
         get_invalid_path
         expect(response).to have_http_status(:not_found)
       end
@@ -29,6 +29,30 @@
     end
   end
 
+  context "when page not found due to non-html format" do
+    let(:get_invalid_path) { get("/unknown/path.xml") }
+
+    context "with default locale" do
+      it "responds with expected http status" do
+        get_invalid_path
+        expect(response).to have_http_status(:not_found)
+      end
+
+      it "renders not found plain text" do
+        get_invalid_path
+        expect(response.body).to eq("Not found")
+      end
+    end
+
+    context "with Welsh locale", :use_welsh_locale do
+      it "displays the correct content" do
+        get("/unknown/path.xml", params: { locale: :cy })
+
+        expect(response.body).to eq("Not found")
+      end
+    end
+  end
+
   context "when page not found due to object not found" do
     let(:get_invalid_id) { get feedback_path(SecureRandom.uuid) }