From c8a9af4f503f3be9613d2e0f7c88defe95fe8022 Mon Sep 17 00:00:00 2001 From: Shadyzpop Date: Wed, 1 Feb 2017 13:30:27 +0200 Subject: [PATCH] Update --- Internalx/~Other/Spider.cs | 50 +++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/Internalx/~Other/Spider.cs b/Internalx/~Other/Spider.cs index c6b8160..38051ff 100644 --- a/Internalx/~Other/Spider.cs +++ b/Internalx/~Other/Spider.cs @@ -60,25 +60,31 @@ public class Proxies private int _ptype { set; get; } #region Methods(Open) - public string[] ScrapeProxies(string[] urls) + public Task ScrapeProxies(string[] urls) { - List data = new List(); - string pattern = @"\d{1,3}(\.\d{1,3}){3}:\d{1,5}"; - - using (HttpRequest req = new HttpRequest()) + return Task.Run(() => { - req.UserAgent = HttpHelper.ChromeUserAgent(); - req.IgnoreProtocolErrors = true; - req.AllowAutoRedirect = true; - foreach (var url in urls) + List data = new List(); + string pattern = @"\d{1,3}(\.\d{1,3}){3}:\d{1,5}"; + try { - var res = req.Get(url); - MatchCollection proxies = Regex.Matches(res.ToString(), pattern); - foreach (var proxy in proxies) - data.Add(proxy.ToString()); + using (HttpRequest req = new HttpRequest()) + { + req.UserAgent = HttpHelper.ChromeUserAgent(); + req.IgnoreProtocolErrors = true; + req.AllowAutoRedirect = true; + foreach (var url in urls) + { + var res = req.Get(url); + MatchCollection proxies = Regex.Matches(res.ToString(), pattern); + foreach (var proxy in proxies) + data.Add(proxy.ToString()); + } + } } - } - return data.ToArray(); + catch { } + return data.ToArray(); + }); } public string[,] ProxyCheck(string proxy, string url, bool autoredirect, bool reconnect = false, int timeout = 0 , int ptype = 0) @@ -234,13 +240,17 @@ public class Scrape public string[] BlogSpotUrls(string url) { List uris = new List(); - using (HttpRequest req = new HttpRequest()) + try { - var res = req.Get(url); - var purl = Html.HTMLparse(res.ToString(), "href", "source", "blogger:blog:plusone", "g:plusone", 1, 1); - foreach (var a in purl) - uris.Add(a); + using (HttpRequest req = new HttpRequest()) + { + var res = req.Get(url); + var purl = Html.HTMLparse(res.ToString(), "href", "source", "blogger:blog:plusone", "g:plusone", 1, 1); + foreach (var a in purl) + uris.Add(a); + } } + catch { } return uris.ToArray(); } #endregion