Skip to content

Commit

Permalink
Add possibility to set proxy for HtmlUnit
Browse files Browse the repository at this point in the history
  • Loading branch information
Hronom committed Mar 15, 2016
1 parent e31c182 commit e0d20ec
Show file tree
Hide file tree
Showing 5 changed files with 217 additions and 55 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,9 @@
import com.github.hronom.scrape.dat.rooms.core.html.parsers.utils.NetworkUtils;
import com.github.hronom.scrape.dat.rooms.core.html.parsers.utils.PathsUtils;
import com.github.hronom.scrape.dat.rooms.core.webpage.html.grabbers.JxBrowserGrabber;
import com.github.hronom.scrape.dat.rooms.view.controllers.BrowserEngineSelectionController;
import com.github.hronom.scrape.dat.rooms.view.controllers.ScrapeButtonController;
import com.github.hronom.scrape.dat.rooms.view.controllers.WebsiteUrlTypingController;
import com.github.hronom.scrape.dat.rooms.view.views.ScrapeMainView;
import com.github.hronom.scrape.dat.rooms.view.views.ScrapeView;

Expand All @@ -35,6 +37,8 @@ public static void main(String[] args) {

ScrapeView scrapeView = new ScrapeView();
new ScrapeButtonController(scrapeView);
new WebsiteUrlTypingController(scrapeView);
new BrowserEngineSelectionController(scrapeView);
new ScrapeMainView(scrapeView);

// testMotel6();
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
package com.github.hronom.scrape.dat.rooms.view.controllers;

import com.github.hronom.scrape.dat.rooms.view.views.ScrapeView;

import java.awt.event.ItemEvent;
import java.awt.event.ItemListener;

public class BrowserEngineSelectionController {
private final ScrapeView scrapeView;

public BrowserEngineSelectionController(ScrapeView scrapeViewArg) {
scrapeView = scrapeViewArg;
if (scrapeView.getSelectedBrowserEngine().equals(ScrapeView.BrowserEngine.HtmlUnit)) {
scrapeView.setProxyHostTextFieldEnabled(true);
scrapeView.setProxyPortTextFieldEnabled(true);
} else {
scrapeView.setProxyHostTextFieldEnabled(false);
scrapeView.setProxyPortTextFieldEnabled(false);
}

scrapeView.addBrowserEngineItemListener(new ItemListener() {
@Override
public void itemStateChanged(ItemEvent e) {
if (e.getStateChange() == ItemEvent.SELECTED && e.getItem().equals(ScrapeView.BrowserEngine.HtmlUnit)) {
scrapeView.setProxyHostTextFieldEnabled(true);
scrapeView.setProxyPortTextFieldEnabled(true);
} else {
scrapeView.setProxyHostTextFieldEnabled(false);
scrapeView.setProxyPortTextFieldEnabled(false);
}
}
});
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -87,15 +87,23 @@ public void run() {
logger.info("Requesting page...");

String html = null;
String selectedBrowserEngine = scrapeView.getSelectedBrowserEngine();
ScrapeView.BrowserEngine selectedBrowserEngine = scrapeView.getSelectedBrowserEngine();
switch (selectedBrowserEngine) {
case "HtmlUnit":
case HtmlUnit:
String proxyHost = scrapeView.getProxyHost();
String proxyPort = scrapeView.getProxyPort();
if (!proxyHost.trim().isEmpty() && !proxyPort.trim().isEmpty()) {
htmlUnitGrabber
.setProxyParameters(proxyHost, Integer.valueOf(proxyPort));
} else {
htmlUnitGrabber.setProxyParameters(null, 0);
}
html = htmlUnitGrabber.grabHtml(scrapeView.getWebsiteUrl());
break;
case "Ui4j":
case Ui4j:
html = ui4jGrabber.grabHtml(scrapeView.getWebsiteUrl());
break;
case "JxBrowser":
case JxBrowser:
html = jxBrowserGrabber.grabHtml(scrapeView.getWebsiteUrl());
break;
default:
Expand All @@ -107,9 +115,9 @@ public void run() {
scrapeView.setProgressBarTaskText("parsing HTML");
logger.info("Parse HTML");
ArrayList<RoomInfo> roomInfos;
String selectedParser = scrapeView.getSelectedParser();
ScrapeView.Parser selectedParser = scrapeView.getSelectedParser();
switch (selectedParser) {
case "Motel6": {
case Motel6: {
prepareFolder(motel6ResultsDir, motel6ResultsPhotosDir);
RoomPhotoDownloader downloader = createRoomPhotoDownloader(
motel6ResultsPhotosDir);
Expand All @@ -119,7 +127,7 @@ public void run() {
}
break;
}
case "RedRoof": {
case RedRoof: {
prepareFolder(redRoofResultsDir, redRoofResultsPhotosDir);
RoomPhotoDownloader downloader = createRoomPhotoDownloader(
redRoofResultsPhotosDir);
Expand All @@ -129,7 +137,7 @@ public void run() {
}
break;
}
case "RedLion": {
case RedLion: {
prepareFolder(redLionResultsDir, redLionResultsPhotosDir);
RoomPhotoDownloader downloader = createRoomPhotoDownloader(
redLionResultsPhotosDir);
Expand All @@ -139,7 +147,7 @@ public void run() {
}
break;
}
case "ebookers": {
case ebookers: {
prepareFolder(ebookersResultsDir, ebookersResultsPhotosDir);
RoomPhotoDownloader downloader =
createRoomPhotoDownloader(ebookersResultsPhotosDir);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
package com.github.hronom.scrape.dat.rooms.view.controllers;

import com.github.hronom.scrape.dat.rooms.view.views.ScrapeView;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import javax.swing.event.DocumentEvent;
import javax.swing.event.DocumentListener;
import javax.swing.text.BadLocationException;

public class WebsiteUrlTypingController {
private static final Logger logger = LogManager.getLogger();

private final ScrapeView scrapeView;

public WebsiteUrlTypingController(ScrapeView scrapeViewArg) {
scrapeView = scrapeViewArg;
scrapeView.addWebsiteUrlDocumentListener(new DocumentListener() {
@Override
public void insertUpdate(DocumentEvent e) {
try {
String str = e.getDocument().getText(0, e.getDocument().getLength());
if (str.contains("motel6.com")) {
scrapeView.selectParser(ScrapeView.Parser.Motel6);
} else if (str.contains("redroof.com")) {
scrapeView.selectParser(ScrapeView.Parser.RedRoof);
} else if (str.contains("redlion.com")) {
scrapeView.selectParser(ScrapeView.Parser.RedLion);
} else if (str.contains("ebookers.com")) {
scrapeView.selectParser(ScrapeView.Parser.ebookers);
}
} catch (BadLocationException exception) {
logger.error(exception);
}
}

@Override
public void removeUpdate(DocumentEvent e) {
}

@Override
public void changedUpdate(DocumentEvent e) {
}
});
}
}
Loading

0 comments on commit e0d20ec

Please sign in to comment.