diff --git a/scrape-dat-rooms-core/src/main/java/com/github/hronom/scrape/dat/rooms/core/webpage/html/grabbers/HtmlUnitGrabber.java b/scrape-dat-rooms-core/src/main/java/com/github/hronom/scrape/dat/rooms/core/webpage/html/grabbers/HtmlUnitGrabber.java index e5bd9e6..fc56017 100755 --- a/scrape-dat-rooms-core/src/main/java/com/github/hronom/scrape/dat/rooms/core/webpage/html/grabbers/HtmlUnitGrabber.java +++ b/scrape-dat-rooms-core/src/main/java/com/github/hronom/scrape/dat/rooms/core/webpage/html/grabbers/HtmlUnitGrabber.java @@ -20,12 +20,17 @@ public class HtmlUnitGrabber implements Grabber { private static final Logger logger = LogManager.getLogger(); + private final String userAgent = + "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"; + private final ProxyConfig defaultProxyConfig = new ProxyConfig(); private final WebClient webClient; public HtmlUnitGrabber() { - webClient = new WebClient(BrowserVersion.FIREFOX_38); + BrowserVersion browserVersion = BrowserVersion.FIREFOX_38; + //browserVersion.setUserAgent(userAgent); + webClient = new WebClient(browserVersion); webClient.getOptions().setCssEnabled(true); webClient.getOptions().setJavaScriptEnabled(true); webClient.getOptions().setPopupBlockerEnabled(false);