优化蜘蛛检测

ldqk · Mar 9, 2022 · 2cee940 · 2cee940
1 parent f02e5ad
commit 2cee940
Show file tree

Hide file tree

Showing 6 changed files with 43 additions and 30 deletions.
diff --git a/src/Masuit.MyBlogs.Core/Common/CommonHelper.cs b/src/Masuit.MyBlogs.Core/Common/CommonHelper.cs
@@ -468,5 +468,10 @@ public bool Contains(string s)
         {
             return ToString().Contains(s, StringComparison.CurrentCultureIgnoreCase);
         }
+
+        public bool Contains(params string[] s)
+        {
+            return ToString().Contains(s);
+        }
     }
 }
diff --git a/src/Masuit.MyBlogs.Core/Common/HttpContextExtension.cs b/src/Masuit.MyBlogs.Core/Common/HttpContextExtension.cs
@@ -1,6 +1,7 @@
 using DnsClient;
 using Masuit.Tools;
 using Microsoft.Net.Http.Headers;
+using Polly;
 
 namespace Masuit.MyBlogs.Core.Common
 {
@@ -23,25 +24,31 @@ public static IPLocation Location(this HttpRequest request)
         /// <returns></returns>
         public static bool IsRobot(this HttpRequest req)
         {
-            var robotUA = UserAgent.Parse(req.Headers[HeaderNames.UserAgent].ToString()).IsRobot;
+            var robotUA = UserAgent.Parse(req.Headers[HeaderNames.UserAgent].ToString()).IsRobot || req.Location().Contains("Spider", "蜘蛛");
             if (robotUA)
             {
                 var nslookup = new LookupClient();
-                using var cts = new CancellationTokenSource(1000);
-                return nslookup.QueryReverseAsync(req.HttpContext.Connection.RemoteIpAddress, cts.Token).ContinueWith(t => t.IsCompletedSuccessfully && t.Result.Answers.Any(r => r.ToString().Trim('.').EndsWith(new[]
+                var fallbackPolicy = Policy<bool>.Handle<Exception>().FallbackAsync(false);
+                var retryPolicy = Policy<bool>.Handle<Exception>().RetryAsync(3);
+                return Policy.WrapAsync(fallbackPolicy, retryPolicy).ExecuteAsync(async () =>
                 {
-                    "baidu.com",
-                    "google.com",
-                    "googlebot.com",
-                    "googleusercontent.com",
-                    "bing.com",
-                    "search.msn.com",
-                    "sogou.com",
-                    "soso.com",
-                    "yandex.com",
-                    "apple.com",
-                    "sm.cn"
-                 }))).Result;
+                    using var cts = new CancellationTokenSource(1000);
+                    var query = await nslookup.QueryReverseAsync(req.HttpContext.Connection.RemoteIpAddress, cts.Token);
+                    return query.Answers.Any(r => r.ToString().Trim('.').EndsWith(new[]
+                    {
+                        "baidu.com",
+                        "google.com",
+                        "googlebot.com",
+                        "googleusercontent.com",
+                        "bing.com",
+                        "search.msn.com",
+                        "sogou.com",
+                        "soso.com",
+                        "yandex.com",
+                        "apple.com",
+                        "sm.cn"
+                    }));
+                }).Result;
             }
 
             return robotUA;

diff --git a/src/Masuit.MyBlogs.Core/Controllers/LinksController.cs b/src/Masuit.MyBlogs.Core/Controllers/LinksController.cs
@@ -16,6 +16,7 @@ namespace Masuit.MyBlogs.Core.Controllers
     public class LinksController : BaseController
     {
         public IHttpClientFactory HttpClientFactory { get; set; }
+
         private HttpClient HttpClient => HttpClientFactory.CreateClient();
 
         /// <summary>
@@ -65,6 +66,7 @@ public async Task<ActionResult> Apply(Links link, CancellationToken cancellation
             HttpClient.DefaultRequestHeaders.Add("X-Forwarded-For", "1.1.1.1");
             HttpClient.DefaultRequestHeaders.Add("X-Forwarded-Host", "1.1.1.1");
             HttpClient.DefaultRequestHeaders.Add("X-Real-IP", "1.1.1.1");
+            HttpClient.DefaultRequestVersion = new Version(2, 0);
             return await HttpClient.GetAsync(link.Url, cancellationToken).ContinueWith(t =>
             {
                 if (t.IsFaulted || t.IsCanceled)
@@ -108,14 +110,15 @@ public async Task<ActionResult> Add(Links links)
         /// <param name="link"></param>
         /// <returns></returns>
         [MyAuthorize]
-        public async Task<ActionResult> Check(string link)
+        public Task<ActionResult> Check(string link)
         {
             HttpClient.DefaultRequestHeaders.UserAgent.TryParseAdd("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36 Edg/93.0.961.47");
             HttpClient.DefaultRequestHeaders.Add("X-Forwarded-For", "1.1.1.1");
             HttpClient.DefaultRequestHeaders.Add("X-Forwarded-Host", "1.1.1.1");
             HttpClient.DefaultRequestHeaders.Add("X-Real-IP", "1.1.1.1");
+            HttpClient.DefaultRequestVersion = new Version(2, 0);
             using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10));
-            return await HttpClient.GetAsync(link, cts.Token).ContinueWith(t =>
+            return HttpClient.GetAsync(link, cts.Token).ContinueWith(t =>
             {
                 if (t.IsFaulted || t.IsCanceled)
                 {
@@ -222,4 +225,4 @@ public async Task<ActionResult> Toggle(int id)
             return ResultData(null, b, b ? "切换成功！" : "切换失败！");
         }
     }
-}
+}
diff --git a/src/Masuit.MyBlogs.Core/Infrastructure/Repository/BaseRepository.cs b/src/Masuit.MyBlogs.Core/Infrastructure/Repository/BaseRepository.cs
@@ -1049,4 +1049,4 @@ public static async Task<PagedList<TDto>> ToCachedPagedListAsync<T, TDto>(this I
             return new PagedList<TDto>(list.ToList(), page, size, totalCount);
         }
     }
-}
+}
diff --git a/src/Masuit.MyBlogs.Core/Infrastructure/Services/PostService.cs b/src/Masuit.MyBlogs.Core/Infrastructure/Services/PostService.cs
@@ -10,15 +10,13 @@
 using Masuit.MyBlogs.Core.Models.ViewModel;
 using Masuit.Tools;
 using Masuit.Tools.Html;
-using OpenXmlPowerTools;
 using PanGu;
 using PanGu.HighLight;
 using System.Linq.Expressions;
 using System.Reflection;
 using System.Text.RegularExpressions;
 using AngleSharp.Dom;
 using AngleSharp.Html.Parser;
-using DocumentFormat.OpenXml.Bibliography;
 
 namespace Masuit.MyBlogs.Core.Infrastructure.Services
 {

diff --git a/src/Masuit.MyBlogs.Core/Masuit.MyBlogs.Core.csproj b/src/Masuit.MyBlogs.Core/Masuit.MyBlogs.Core.csproj
@@ -38,35 +38,35 @@
 
     <ItemGroup>
         <PackageReference Include="Autofac.Extensions.DependencyInjection" Version="7.2.0" />
-        <PackageReference Include="AutoMapper.Extensions.ExpressionMapping" Version="5.0" />
+        <PackageReference Include="AutoMapper.Extensions.ExpressionMapping" Version="5.0.1" />
         <PackageReference Include="AutoMapper.Extensions.Microsoft.DependencyInjection" Version="11.0.0" />
         <PackageReference Include="CacheManager.Serialization.Json" Version="1.2.0" />
         <PackageReference Include="CacheManager.StackExchange.Redis" Version="1.2.0" />
         <PackageReference Include="CHTCHSConv" Version="1.0.0" />
         <PackageReference Include="CLRStats" Version="1.0.0" />
-        <PackageReference Include="CSRedisCore" Version="3.6.8" />
+        <PackageReference Include="CSRedisCore" Version="3.6.9" />
         <PackageReference Include="EFCoreSecondLevelCacheInterceptor" Version="3.3.0" />
         <PackageReference Include="Hangfire" Version="1.7.28" />
         <PackageReference Include="Hangfire.MemoryStorage" Version="1.7.0" />
         <PackageReference Include="htmldiff.net-core" Version="1.3.6" />
         <PackageReference Include="IP2Region" Version="1.2.0" />
         <PackageReference Include="Karambolo.AspNetCore.Bundling.NUglify" Version="3.5.1" />
         <PackageReference Include="MaxMind.GeoIP2" Version="5.1.0" />
-        <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="6.0.2" />
-        <PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="6.0.2" />
-        <PackageReference Include="Microsoft.EntityFrameworkCore.Proxies" Version="6.0.2" />
-        <PackageReference Include="Microsoft.Extensions.Http.Polly" Version="6.0.2" />
-        <PackageReference Include="Microsoft.Graph" Version="4.18.0" />
+        <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="6.0.3" />
+        <PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="6.0.3" />
+        <PackageReference Include="Microsoft.EntityFrameworkCore.Proxies" Version="6.0.3" />
+        <PackageReference Include="Microsoft.Extensions.Http.Polly" Version="6.0.3" />
+        <PackageReference Include="Microsoft.Graph" Version="4.19.0" />
         <PackageReference Include="Microsoft.Graph.Auth" Version="1.0.0-preview.7" />
         <PackageReference Include="MiniProfiler.AspNetCore.Mvc" Version="4.2.22" />
         <PackageReference Include="OpenXmlPowerTools-NetStandard" Version="4.4.21" />
         <PackageReference Include="MiniProfiler.EntityFrameworkCore" Version="4.2.22" />
         <PackageReference Include="PanGu.HighLight" Version="1.0.0" />
         <PackageReference Include="Pomelo.EntityFrameworkCore.MySql" Version="6.0.1" />
-        <PackageReference Include="System.Linq.Dynamic.Core" Version="1.2.17" />
+        <PackageReference Include="System.Linq.Dynamic.Core" Version="1.2.18" />
         <PackageReference Include="TimeZoneConverter" Version="5.0.0" />
         <PackageReference Include="WilderMinds.RssSyndication" Version="1.7.0" />
-        <PackageReference Include="Z.EntityFramework.Plus.EFCore" Version="6.13.10" />
+        <PackageReference Include="Z.EntityFramework.Plus.EFCore" Version="6.13.14" />
     </ItemGroup>
     <ItemGroup>
         <Content Update="appsettings.json">