From 05b33a71c1e40e5fb6a323dfecedb26da677097e Mon Sep 17 00:00:00 2001 From: Shadyzpop Date: Thu, 2 Feb 2017 16:27:13 +0200 Subject: [PATCH] V7 Update --- Internalx/~Http/HttpRequest.cs | 434 ++++++++++++++++++++++++- Internalx/~Other/Helper.cs | 56 +++- Internalx/~Other/Spider.cs | 81 ++--- Internalx/~Threading/MultiThreading.cs | 6 + Properties/AssemblyInfo.cs | 4 +- 5 files changed, 526 insertions(+), 55 deletions(-) diff --git a/Internalx/~Http/HttpRequest.cs b/Internalx/~Http/HttpRequest.cs index 92e76d7..43c42b0 100644 --- a/Internalx/~Http/HttpRequest.cs +++ b/Internalx/~Http/HttpRequest.cs @@ -11,7 +11,9 @@ using System.Security.Authentication; using System.Text; using System.Threading; +using System.Threading.Tasks; using Shadynet.Proxy; +using Shadynet.Threading; namespace Shadynet.Http { @@ -917,6 +919,7 @@ public HttpRequest(Uri baseAddress) #region Methods (open) #region Get + #region Normal /// /// It sends a GET-request to the HTTP-server. @@ -957,7 +960,46 @@ public HttpResponse Get(Uri address, RequestParams urlParams = null) #endregion + #region Async + + /// + /// It sends a GET-request to the HTTP-server asynchronously. + /// + /// Address Internet resource. + /// Parameters URL-addresses, or value . + /// The object is designed to download a response from the HTTP-server. + /// parameter equally . + /// parameter is an empty string. + /// Error when working with the HTTP-report. + public async Task GetAsync(string address, RequestParams urlParams = null) + { + return await Task.Run(() => { + return this.Get(address, urlParams); + }); + } + + /// + /// It sends a GET-request to the HTTP-server asynchronously. + /// + /// Address of the Internet resource. + /// Parameters URL-addresses, or value . + /// The object is designed to download a response from the HTTP-server. + /// parameter equally . + /// Error when working with the HTTP-report. + public async Task GetAsync(Uri address, RequestParams urlParams = null) + { + return await Task.Run(() => + { + return this.Get(address, urlParams); + }); + } + + #endregion + #endregion + + #region Post + #region Normal /// /// Sends the HTTP POST-search-server. @@ -971,7 +1013,7 @@ public HttpResponse Post(string address) { return Raw(HttpMethod.POST, address); } - + /// /// Sends the HTTP POST-search-server. /// @@ -1457,9 +1499,357 @@ public HttpResponse Post(Uri address, HttpContent content) return Raw(HttpMethod.POST, address, content); } + + #endregion + + #region Async + + /// + /// Sends the HTTP POST-search-server asynchronously. + /// + /// Address Internet resource. + /// The object is designed to download a response from the HTTP-server. + /// parameter equally . + /// parameter is an empty string. + /// Error when working with the HTTP-report. + public async Task PostAsync(string address) + { + return await Task.Run(() => + { + return this.Post(address); + }); + } + + /// + /// Sends the HTTP POST-search-server asynchronously. + /// + /// Address Internet resource. + /// The object is designed to download a response from the HTTP-server. + /// parameter equally . + /// Error when working with the HTTP-report. + public async Task PostAsync(Uri address) + { + return await Task.Run(() => + { + return this.Raw(HttpMethod.POST, address); + }); + } + + /// + /// Sends the HTTP POST-search-server asynchronously. + /// + /// Address Internet resource. + /// Request parameters sent to the HTTP-server. + /// Specifies whether to encode the request parameters need. + /// The object is designed to download a response from the HTTP-server. + /// + /// parameter equally . + /// -or- + /// parameter equally . + /// + /// parameter is an empty string. + /// Error when working with the HTTP-report. + public async Task PostAsync(string address, RequestParams reqParams, bool dontEscape = false) + { + #region Check settings + + if (reqParams == null) + { + throw new ArgumentNullException("reqParams"); + } + + #endregion + + return await Task.Run(() => + { + return this.Raw(HttpMethod.POST, address, new FormUrlEncodedContent(reqParams, dontEscape, CharacterSet)); + }); + } + + /// + /// Sends the HTTP POST-search-server asynchronously. + /// + /// Address Internet resource. + /// Request parameters sent to the HTTP-server. + /// Specifies whether to encode the request parameters need. + /// The object is designed to download a response from the HTTP-server. + /// + /// parameter equally . + /// -or- + /// parameter equally . + /// + /// Error when working with the HTTP-report. + public async Task PostAsync(Uri address, RequestParams reqParams, bool dontEscape = false) + { + #region Check settings + + if (reqParams == null) + { + throw new ArgumentNullException("reqParams"); + } + + #endregion + + return await Task.Run(() => + { + return this.Raw(HttpMethod.POST, address, new FormUrlEncodedContent(reqParams, dontEscape, CharacterSet)); + }); + } + + /// + /// Sends the HTTP POST-search-server asynchronously. + /// + /// Address Internet resource. + /// The string sent by the HTTP-server. + /// Type of data to be sent. + /// The object is designed to download a response from the HTTP-server. + /// + /// parameter equally . + /// -or- + /// parameter equally . + /// -or- + /// parameter equally . + /// + /// + /// parameter is an empty string. + /// -or- + /// parameter is an empty string. + /// -or + /// parameter is an empty string. + /// + /// Error when working with the HTTP-report. + public async Task PostAsync(string address, string str, string contentType) + { + return await Task.Run(() => + { + return this.Post(address, str, contentType); + }); + } + + /// + /// Sends the HTTP POST-search-server asynchronously. + /// + /// Address Internet resource. + /// The string sent by the HTTP-server. + /// Type of data to be sent. + /// The object is designed to download a response from the HTTP-server. + /// + /// parameter equally . + /// -or- + /// parameter equally . + /// -or- + /// parameter equally . + /// + /// + /// parameter is an empty string. + /// -or- + /// parameter is an empty string. + /// + /// Error when working with the HTTP-report. + public async Task PostAsync(Uri address, string str, string contentType) + { + return await Task.Run(() => + { + return this.Post(address, str, contentType); + }); + } + + /// + /// Sends the HTTP POST-search-server asynchronously. + /// + /// Address Internet resource. + /// An array of bytes sent to the HTTP-server. + /// Type of data to be sent. + /// The object is designed to download a response from the HTTP-server. + /// + /// parameter equally . + /// -or- + /// parameter equally . + /// -or- + /// parameter equally . + /// + /// + /// parameter is an empty string. + /// -or- + /// parameter is an empty string. + /// + /// Error when working with the HTTP-report. + public async Task PostAsync(string address, byte[] bytes, string contentType = "application/octet-stream") + { + return await Task.Run(() => + { + return this.Post(address, bytes, contentType); + }); + } + + /// + /// Sends the HTTP POST-search-server asynchronously. + /// + /// Address Internet resource. + /// An array of bytes sent to the HTTP-server. + /// Type of data to be sent. + /// The object is designed to download a response from the HTTP-server. + /// + /// parameter equally . + /// -or- + /// parameter equally . + /// -or- + /// parameter equally . + /// + /// parameter is an empty string. + /// Error when working with the HTTP-report. + public async Task PostAsync(Uri address, byte[] bytes, string contentType = "application/octet-stream") + { + return await Task.Run(() => + { + return this.Post(address, bytes, contentType); + }); + } + + /// + /// Sends the HTTP POST-search-server asynchronously. + /// + /// Address Internet resource. + /// The data stream sent HTTP-server. + /// Type of data to be sent. + /// The object is designed to download a response from the HTTP-server. + /// + /// parameter equally . + /// -or- + /// parameter equally . + /// -or- + /// parameter equally . + /// + /// + /// parameter is an empty string. + /// -or- + /// parameter is an empty string. + /// + /// Error when working with the HTTP-report. + public async Task PostAsync(string address, Stream stream, string contentType = "application/octet-stream") + { + return await Task.Run(() => + { + return this.Post(address, stream, contentType); + }); + } + + /// + /// Sends the HTTP POST-search-server asynchronously. + /// + /// Address Internet resource. + /// The data stream sent HTTP-server. + /// Type of data to be sent. + /// The object is designed to download a response from the HTTP-server. + /// + /// parameter equally . + /// -or- + /// parameter equally . + /// -or- + /// parameter equally . + /// + /// parameter is an empty string. + /// Error when working with the HTTP-report. + public async Task PostAsync(Uri address, Stream stream, string contentType = "application/octet-stream") + { + return await Task.Run(() => + { + return this.Post(address, stream, contentType); + }); + } + + /// + /// Sends the HTTP POST-search-server asynchronously. + /// + /// Address Internet resource. + /// Path to the file from which the data will be sent to the HTTP-server. + /// The object is designed to download a response from the HTTP-server. + /// + /// parameter equally . + /// -or- + /// parameter equally . + /// + /// + /// parameter is an empty string. + /// -or- + /// parameter is an empty string. + /// + /// Error when working with the HTTP-report. + public async Task PostAsync(string address, string path) + { + return await Task.Run(() => + { + return this.Post(address, path); + }); + } + + /// + /// Sends the HTTP POST-search-server asynchronously. + /// + /// Address Internet resource. + /// Path to the file from which the data will be sent to the HTTP-server. + /// The object is designed to download a response from the HTTP-server. + /// + /// parameter equally . + /// -or- + /// parameter equally . + /// + /// parameter is an empty string. + /// Error when working with the HTTP-report. + public async Task PostAsync(Uri address, string path) + { + return await Task.Run(() => + { + return this.Post(address, path); + }); + } + + /// + /// Sends the HTTP POST-search-server asynchronously. + /// + /// Address Internet resource. + /// Content that is sent to the HTTP-server. + /// The object is designed to download a response from the HTTP-server. + /// + /// parameter equally . + /// -or- + /// parameter equally . + /// + /// parameter is an empty string. + /// Error when working with the HTTP-report. + public async Task PostAsync(string address, HttpContent content) + { + return await Task.Run(() => + { + return this.Post(address, content); + }); + } + + /// + /// Sends the HTTP POST-search-server asynchronously. + /// + /// Address Internet resource. + /// Content that is sent to the HTTP-server. + /// The object is designed to download a response from the HTTP-server. + /// + /// parameter equally . + /// -or- + /// parameter equally . + /// + /// Error when working with the HTTP-report. + public async Task PostAsync(Uri address, HttpContent content) + { + return await Task.Run(() => + { + return this.Post(address, content); + }); + } + + #endregion #endregion #region Raw + #region Normal /// /// It sends a HTTP-server inquiry. @@ -1510,7 +1900,7 @@ public HttpResponse Raw(HttpMethod method, Uri address, HttpContent content = nu } #endregion - + if (!address.IsAbsoluteUri) address = GetRequestAddress(BaseAddress, address); @@ -1554,6 +1944,46 @@ public HttpResponse Raw(HttpMethod method, Uri address, HttpContent content = nu #endregion + #region Async + + /// + /// It sends a HTTP-server inquiry, Asynchronously. + /// + /// HTTP-request method. + /// Address Internet resource. + /// Content that is sent to the HTTP-server, or value . + /// The object is designed to download a response from the HTTP-server. + /// parameter equally . + /// parameter is an empty string. + /// Error when working with the HTTP-report. + public async Task RawAsync(HttpMethod method, string address, HttpContent content = null) + { + return await Task.Run(() => + { + return this.Raw(method, address, content); + }); + } + + /// + /// It sends a HTTP-server inquiry, Asynchronously. + /// + /// HTTP-request method. + /// Address Internet resource. + /// Content that is sent to the HTTP-server, or value . + /// The object is designed to download a response from the HTTP-server. + /// parameter equally . + /// Error when working with the HTTP-report. + public async Task RawAsync(HttpMethod method, Uri address, HttpContent content = null) + { + return await Task.Run(() => + { + return this.Raw(method, address, content); + }); + } + + #endregion + #endregion + #region Adding temporal data query /// diff --git a/Internalx/~Other/Helper.cs b/Internalx/~Other/Helper.cs index 2d6d6e9..14b881d 100644 --- a/Internalx/~Other/Helper.cs +++ b/Internalx/~Other/Helper.cs @@ -83,13 +83,28 @@ public static string BetweenUrl(string URL,string strStart,string strEnd) } } + /// + /// Returns a string from a url using 'Get' between two strings, starts from to from , Asynchronously. + /// + /// The Url Returning source string of the context. + /// The head start of the param + /// The tail end of the param + /// The string between and from + public static async Task BetweenUrlAsync(string URL, string strStart, string strEnd) + { + return await Task.Run(() => + { + return BetweenUrl(URL, strStart, strEnd); + }); + } + /// /// Returns raw cookie value from a url using 'Get'. /// /// The Url to be specified. /// Given cookie in the context. /// Raw cookie value from the given 'URL' specified in 'Cookie' - public static string Cookie(string URL, string Cookie) + public static string Cookie(string URL, string cookie) { try { @@ -97,7 +112,7 @@ public static string Cookie(string URL, string Cookie) { rq.UserAgent = HttpHelper.ChromeUserAgent(); string strSource = rq.Get(URL).Cookies.ToString() + ";"; - string res = Betweenstring(strSource, Cookie + "=", ";"); + string res = Betweenstring(strSource, cookie + "=", ";"); return res; } } @@ -107,22 +122,39 @@ public static string Cookie(string URL, string Cookie) } } - public static string Htmlsource(string url) + /// + /// Returns raw cookie value from a url using 'Get'. + /// + /// The Url to be specified. + /// Given cookie in the context. + /// Raw cookie value from the given 'URL' specified in 'Cookie' + public static async Task CookieAsync(string URL, string cookie) { - using(HttpRequest req = new HttpRequest(url)) + return await Task.Run(() => { - req.UserAgent = HttpHelper.ChromeUserAgent(); - req.AllowAutoRedirect = true; - req.IgnoreProtocolErrors = true; - var res = req.Get("/"); - return res.ToString(); - } + return Cookie(URL, cookie); + }); } - public static string[] tester(string source,string left,string right) + public static async Task Htmlsource(string url) { - return Html.Substrings(source, left, right); + return await Task.Run(() => + { + try + { + using (HttpRequest req = new HttpRequest(url)) + { + req.UserAgent = HttpHelper.ChromeUserAgent(); + req.AllowAutoRedirect = true; + req.IgnoreProtocolErrors = true; + var res = req.Get("/"); + return res.ToString(); + } + } + catch { return ""; } + }); } + #endregion } diff --git a/Internalx/~Other/Spider.cs b/Internalx/~Other/Spider.cs index 38051ff..75ad8ab 100644 --- a/Internalx/~Other/Spider.cs +++ b/Internalx/~Other/Spider.cs @@ -60,9 +60,9 @@ public class Proxies private int _ptype { set; get; } #region Methods(Open) - public Task ScrapeProxies(string[] urls) + public async Task ScrapeProxies(string[] urls) { - return Task.Run(() => + return await Task.Run(() => { List data = new List(); string pattern = @"\d{1,3}(\.\d{1,3}){3}:\d{1,5}"; @@ -87,49 +87,52 @@ public Task ScrapeProxies(string[] urls) }); } - public string[,] ProxyCheck(string proxy, string url, bool autoredirect, bool reconnect = false, int timeout = 0 , int ptype = 0) + public async Task ProxyCheck(string proxy, string url, bool autoredirect, bool reconnect = false, int timeout = 0 , int ptype = 0) { - string[,] data = new string[1,2]; - try + return await Task.Run(() => { - using (HttpRequest req = new HttpRequest(url)) + string[,] data = new string[1, 2]; + try { - req.UserAgent = HttpHelper.ChromeUserAgent(); - req.Cookies = new CookieCore(false); - req.AllowAutoRedirect = autoredirect; - req.Reconnect = reconnect; - req.SslCertificateValidatorCallback = HttpHelper.AcceptAllCertificationsCallback; - req.ConnectTimeout = timeout; - - #region proxyset - switch (ptype) + using (HttpRequest req = new HttpRequest(url)) { - case 0: - req.Proxy = HttpProxyClient.Parse(proxy); - break; - case 1: - req.Proxy = Socks4ProxyClient.Parse(proxy); - break; - case 2: - req.Proxy = Socks4aProxyClient.Parse(proxy); - break; - case 3: - req.Proxy = Socks5ProxyClient.Parse(proxy); - break; - default: - break; + req.UserAgent = HttpHelper.ChromeUserAgent(); + req.Cookies = new CookieCore(false); + req.AllowAutoRedirect = autoredirect; + req.Reconnect = reconnect; + req.SslCertificateValidatorCallback = HttpHelper.AcceptAllCertificationsCallback; + req.ConnectTimeout = timeout; + + #region proxyset + switch (ptype) + { + case 0: + req.Proxy = HttpProxyClient.Parse(proxy); + break; + case 1: + req.Proxy = Socks4ProxyClient.Parse(proxy); + break; + case 2: + req.Proxy = Socks4aProxyClient.Parse(proxy); + break; + case 3: + req.Proxy = Socks5ProxyClient.Parse(proxy); + break; + default: + break; + } + #endregion + var res = req.Get("/"); + data[0, 0] = proxy; + data[0, 1] = res.ConnectionTime.ToString(); + return data; } - #endregion - var res = req.Get("/"); - data[0, 0] = proxy; - data[0, 1] = res.ConnectionTime.ToString(); - return data; } - } - catch - { - return null; - } + catch + { + return null; + } + }); } #region MultiThreading diff --git a/Internalx/~Threading/MultiThreading.cs b/Internalx/~Threading/MultiThreading.cs index 02a1f5b..cdd3fe0 100644 --- a/Internalx/~Threading/MultiThreading.cs +++ b/Internalx/~Threading/MultiThreading.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.ComponentModel; using System.Threading; +using Shadynet.Http; namespace Shadynet.Threading { @@ -698,5 +699,10 @@ private void ThrowIfDisposed() throw new ObjectDisposedException("MultiThreading"); } } + + internal void Run(Func get) + { + throw new NotImplementedException(); + } } } diff --git a/Properties/AssemblyInfo.cs b/Properties/AssemblyInfo.cs index 70f6537..5b04c2a 100644 --- a/Properties/AssemblyInfo.cs +++ b/Properties/AssemblyInfo.cs @@ -33,6 +33,6 @@ // You can specify all the values ​​or accept the default number of construction and revision number, // Using the "*", as shown below: // [Assembly: AssemblyVersion ( "1.0 *.")] -[assembly: AssemblyVersion("6.5.1.0")] -[assembly: AssemblyFileVersion("6.5.1.0")] +[assembly: AssemblyVersion("7.0.0.0")] +[assembly: AssemblyFileVersion("7.0.0.0")] [assembly: NeutralResourcesLanguage("en")] \ No newline at end of file