[#160463344] fix repeating segment regex (#147)

app/models/searchgov_crawler.rb

@@ -77,7 +77,7 @@ def skip_extensions_regex
 
   # avoid infinite loops caused by malformed urls
   def repeating_segments_regex
-    /(\/\w+)(?:.+?\1){2,}/
+    /(\/[[:alpha:]]+)(?=\/)(.*\1(?=\/)){2,}/
   end
 
   def supported_content_type(type)

spec/models/searchgov_crawler_spec.rb

@@ -316,4 +316,17 @@
       end
     end
   end
+
+  # Testing this private method directly for test speed
+  describe '.repeating_segments_regex' do
+    subject(:regex) { crawler.send(:repeating_segments_regex) }
+
+    it { is_expected.to match 'http://www.agency.gov/foo/foo/foo/' }
+    it { is_expected.to match 'http://www.agency.gov/foo/baz/foo/biz/foo/qux/' }
+    it { is_expected.not_to match 'http://www.agency.gov/fee/fie/foe/' }
+    it { is_expected.not_to match 'http://www.agency.gov/foo/foo/' }
+    it { is_expected.not_to match 'http://www.agency.gov/f/foo/foo' }
+    it { is_expected.not_to match 'http://www.agency.gov/foofoofoo/' }
+    it { is_expected.not_to match 'http://www.agency.gov/09/09/09/' }
+  end
 end