Refactoring

Hronom · Mar 17, 2016 · b696bac · b696bac
1 parent 5a90e9c
commit b696bac
Showing 1 changed file with 6 additions and 1 deletion.
diff --git a/...n/java/com/github/hronom/scrape/dat/rooms/core/webpage/html/grabbers/HtmlUnitGrabber.java b/...n/java/com/github/hronom/scrape/dat/rooms/core/webpage/html/grabbers/HtmlUnitGrabber.java
@@ -20,12 +20,17 @@
 public class HtmlUnitGrabber implements Grabber {
     private static final Logger logger = LogManager.getLogger();
 
+    private final String userAgent =
+        "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
+
     private final ProxyConfig defaultProxyConfig = new ProxyConfig();
 
     private final WebClient webClient;
 
     public HtmlUnitGrabber() {
-        webClient = new WebClient(BrowserVersion.FIREFOX_38);
+        BrowserVersion browserVersion = BrowserVersion.FIREFOX_38;
+        //browserVersion.setUserAgent(userAgent);
+        webClient = new WebClient(browserVersion);
         webClient.getOptions().setCssEnabled(true);
         webClient.getOptions().setJavaScriptEnabled(true);
         webClient.getOptions().setPopupBlockerEnabled(false);