From 8aaec910a74a82f5682134bc80cf5e4c19d2b9b7 Mon Sep 17 00:00:00 2001
From: JackBlackLight <jd2148@columbia.edu>
Date: Wed, 11 Sep 2024 11:44:31 -0400
Subject: [PATCH] add x-robots-tag header to search endpoints (#286)

---
 app/api/v1/search.rb                  |  4 ++++
 app/controllers/catalog_controller.rb |  5 +++++
 spec/api/v1/search_spec.rb            |  5 +++++
 spec/requests/catalog_spec.rb         | 12 ++++++++++++
 4 files changed, 26 insertions(+)

diff --git a/app/api/v1/search.rb b/app/api/v1/search.rb
index aa1d32610..42afe308b 100644
--- a/app/api/v1/search.rb
+++ b/app/api/v1/search.rb
@@ -3,6 +3,10 @@ class Search < Grape::API
     content_type :json, 'application/json'
     default_format :json
 
+    before do
+      header['X-Robots-Tag'] = 'none'
+    end
+
     params do
       optional :search_type, coerce: Symbol, default: :keyword,     values: V1::Helpers::Solr::SEARCH_TYPES,
                              desc: 'type of search to be conducted, in most cases a keyword search should be sufficient'
diff --git a/app/controllers/catalog_controller.rb b/app/controllers/catalog_controller.rb
index b9b27be13..7c920eac4 100644
--- a/app/controllers/catalog_controller.rb
+++ b/app/controllers/catalog_controller.rb
@@ -11,6 +11,7 @@ class CatalogController < ApplicationController
 
   # rubocop:disable Rails/LexicallyScopedActionFilter
   before_action :record_view_stats, only: :show
+  before_action :set_xrobots_tag_to_none, only: :index
   # rubocop:enable Rails/LexicallyScopedActionFilter
 
   class LazyFeatureQueryFacet < Hash
@@ -352,6 +353,10 @@ def home
 
   private
 
+  def set_xrobots_tag_to_none
+    response.headers['X-Robots-Tag'] = 'none'
+  end
+
   def record_view_stats
     record_stats(params['id'], Statistic::VIEW)
   end
diff --git a/spec/api/v1/search_spec.rb b/spec/api/v1/search_spec.rb
index 1423580ac..e4eb9115b 100644
--- a/spec/api/v1/search_spec.rb
+++ b/spec/api/v1/search_spec.rb
@@ -83,6 +83,11 @@
         'error' => 'search_type does not have a valid value'
       )
     end
+
+    it 'returns x-robots-tag none in headers' do
+      get '/api/v1/search?search_type=title'
+      expect(response.headers['X-Robots-Tag']).to eq('none')
+    end
   end
 
   context 'applies sort order' do
diff --git a/spec/requests/catalog_spec.rb b/spec/requests/catalog_spec.rb
index 640ca4df5..efa4b5e6e 100644
--- a/spec/requests/catalog_spec.rb
+++ b/spec/requests/catalog_spec.rb
@@ -19,5 +19,17 @@
       get '/search?per_page=Bad+Value&sort=Published+Latest'
       expect(response.status).to eq(400)
     end
+
+    it 'returns x-robots-tag none in headers' do
+      get '/search?per_page=20&sort=Published+Latest'
+      expect(response.headers['X-Robots-Tag']).to eq('none')
+    end
+  end
+
+  describe '/show' do
+    it 'does not return x-robots-tag none in headers' do
+      get '/doi/10.7916/ALICE'
+      expect(response.headers['X-Robots-Tag']).to be_nil
+    end
   end
 end