diff --git a/c#/crawler/packages.lock.json b/c#/crawler/packages.lock.json index 8667fecb..a5cbfa38 100644 --- a/c#/crawler/packages.lock.json +++ b/c#/crawler/packages.lock.json @@ -4,9 +4,9 @@ "net7.0": { "AngleSharp": { "type": "Direct", - "requested": "[1.0.7, )", - "resolved": "1.0.7", - "contentHash": "jZg7lDcrXRiIC8VBluuGKOCMR9mc4CIRX5vpQJ8fcgafs6T6plOzKHhAn3lJEwXorrltd9p1WtRxxhFpRBACbg==", + "requested": "[1.1.0, )", + "resolved": "1.1.0", + "contentHash": "3baCuPkx2Wrpjr2b67wJAf9fCbYSbEVcdCh88S66HLdfHa2b1Mk1FsGpcH76uHLf3MPxopWngT1imHPEmDimAg==", "dependencies": { "System.Text.Encoding.CodePages": "7.0.0" } diff --git a/c#/crawler/src/GlobalUsings.cs b/c#/crawler/src/GlobalUsings.cs index 33287c36..4703671a 100644 --- a/c#/crawler/src/GlobalUsings.cs +++ b/c#/crawler/src/GlobalUsings.cs @@ -50,6 +50,7 @@ global using TbClient.Post.Common; global using TbClient.Wrapper; global using Thread = TbClient.Post.Thread; +global using User = tbm.Crawler.Db.User; global using Fid = System.UInt32; global using PostId = System.UInt64; diff --git a/c#/crawler/src/Tieba/Crawl/Facade/ReplyCrawlFacade.cs b/c#/crawler/src/Tieba/Crawl/Facade/ReplyCrawlFacade.cs index 4d8d1eec..e8ad34fe 100644 --- a/c#/crawler/src/Tieba/Crawl/Facade/ReplyCrawlFacade.cs +++ b/c#/crawler/src/Tieba/Crawl/Facade/ReplyCrawlFacade.cs @@ -33,7 +33,7 @@ protected override void PostCommitSaveHook( p => p.Pid, p => p.OriginalContents, stoppingToken); // fill the values for some field of reply from user list which is out of post list - private static void FillAuthorInfoBackToReply(IEnumerable users, IEnumerable parsedReplies) => + private static void FillAuthorInfoBackToReply(IEnumerable users, IEnumerable parsedReplies) => (from reply in parsedReplies join user in users on reply.AuthorUid equals user.Uid select (reply, user)) diff --git a/c#/crawler/src/Tieba/Crawl/Facade/ThreadCrawlFacade.cs b/c#/crawler/src/Tieba/Crawl/Facade/ThreadCrawlFacade.cs index e56b3cf6..b41f84a6 100644 --- a/c#/crawler/src/Tieba/Crawl/Facade/ThreadCrawlFacade.cs +++ b/c#/crawler/src/Tieba/Crawl/Facade/ThreadCrawlFacade.cs @@ -40,7 +40,7 @@ protected override void BeforeCommitSaveHook(CrawlerDbContext db) protected void ParseLatestRepliers(IEnumerable threads) => threads.Select(th => th.LastReplyer ?? null) // LastReplyer will be null when LivePostType != "" - .OfType() // filter out nulls + .OfType() // filter out nulls // some rare deleted thread but still visible in 6.0.2 response // will have a latest replier uid=0 name="" nameShow=".*" diff --git a/c#/crawler/src/Tieba/Crawl/Parser/BaseParser.cs b/c#/crawler/src/Tieba/Crawl/Parser/BaseParser.cs index 40afc0cf..0303980e 100644 --- a/c#/crawler/src/Tieba/Crawl/Parser/BaseParser.cs +++ b/c#/crawler/src/Tieba/Crawl/Parser/BaseParser.cs @@ -6,7 +6,7 @@ public abstract class BaseParser { public void ParsePosts( CrawlRequestFlag requestFlag, IList inPosts, - out Dictionary outPosts, out List outUsers) + out Dictionary outPosts, out List outUsers) { outUsers = new(30); if (ShouldSkipParse(requestFlag)) @@ -14,17 +14,17 @@ public void ParsePosts( outPosts = new(); return; } - var outNullableUsers = new List(); + var outNullableUsers = new List(); outPosts = ParsePostsInternal(inPosts, outNullableUsers).ToDictionary(PostIdSelector, post => post); if (outPosts.Values.Any(p => p.AuthorUid == 0)) throw new TiebaException(shouldRetry: true, "Value of IPost.AuthorUid is the protoBuf default value 0."); - outUsers.AddRange(outNullableUsers.OfType() + outUsers.AddRange(outNullableUsers.OfType() .Where(u => u.CalculateSize() != 0)); // remove empty users } protected abstract PostId PostIdSelector(TPost post); protected abstract TPost Convert(TPostProtoBuf inPost); - protected abstract IEnumerable ParsePostsInternal(IList inPosts, List outUsers); + protected abstract IEnumerable ParsePostsInternal(IList inPosts, List outUsers); protected virtual bool ShouldSkipParse(CrawlRequestFlag requestFlag) => false; } diff --git a/c#/crawler/src/Tieba/Crawl/Parser/ReplyParser.cs b/c#/crawler/src/Tieba/Crawl/Parser/ReplyParser.cs index d2ef3b26..3f9d90cd 100644 --- a/c#/crawler/src/Tieba/Crawl/Parser/ReplyParser.cs +++ b/c#/crawler/src/Tieba/Crawl/Parser/ReplyParser.cs @@ -10,7 +10,7 @@ public partial class ReplyParser(ILogger logger) protected override PostId PostIdSelector(ReplyPost post) => post.Pid; protected override IEnumerable ParsePostsInternal - (IList inPosts, List outUsers) => inPosts.Select(Convert); + (IList inPosts, List outUsers) => inPosts.Select(Convert); protected override ReplyPost Convert(Reply inPost) { diff --git a/c#/crawler/src/Tieba/Crawl/Parser/SubReplyParser.cs b/c#/crawler/src/Tieba/Crawl/Parser/SubReplyParser.cs index addb44b5..20d83af5 100644 --- a/c#/crawler/src/Tieba/Crawl/Parser/SubReplyParser.cs +++ b/c#/crawler/src/Tieba/Crawl/Parser/SubReplyParser.cs @@ -4,7 +4,7 @@ public class SubReplyParser : BaseParser { protected override PostId PostIdSelector(SubReplyPost post) => post.Spid; - protected override IEnumerable ParsePostsInternal(IList inPosts, List outUsers) + protected override IEnumerable ParsePostsInternal(IList inPosts, List outUsers) { outUsers.AddRange(inPosts.Select(sr => sr.Author)); return inPosts.Select(Convert); diff --git a/c#/crawler/src/Tieba/Crawl/Parser/ThreadParser.cs b/c#/crawler/src/Tieba/Crawl/Parser/ThreadParser.cs index 7cfc37ee..0c129d89 100644 --- a/c#/crawler/src/Tieba/Crawl/Parser/ThreadParser.cs +++ b/c#/crawler/src/Tieba/Crawl/Parser/ThreadParser.cs @@ -8,7 +8,7 @@ protected override bool ShouldSkipParse(CrawlRequestFlag requestFlag) => requestFlag == CrawlRequestFlag.ThreadClientVersion602; protected override IEnumerable ParsePostsInternal - (IList inPosts, List outUsers) => inPosts.Select(Convert); + (IList inPosts, List outUsers) => inPosts.Select(Convert); protected override ThreadPost Convert(Thread inPost) { diff --git a/c#/crawler/src/Tieba/Crawl/UserParserAndSaver.cs b/c#/crawler/src/Tieba/Crawl/UserParserAndSaver.cs index afbb65bd..0f909e39 100644 --- a/c#/crawler/src/Tieba/Crawl/UserParserAndSaver.cs +++ b/c#/crawler/src/Tieba/Crawl/UserParserAndSaver.cs @@ -40,7 +40,7 @@ public partial class UserParserAndSaver(ILogger logger) private readonly List _savedUsersId = new(); private readonly ConcurrentDictionary _users = new(); - public void ParseUsers(IEnumerable users) => + public void ParseUsers(IEnumerable users) => users.Select(el => { static (string Portrait, uint? UpdateTime) ExtractPortrait(string portrait) => diff --git a/c#/crawler/tbm.Crawler.csproj b/c#/crawler/tbm.Crawler.csproj index 0d11be88..57e6407b 100644 --- a/c#/crawler/tbm.Crawler.csproj +++ b/c#/crawler/tbm.Crawler.csproj @@ -7,7 +7,7 @@ - + diff --git a/c#/imagePipeline/packages.lock.json b/c#/imagePipeline/packages.lock.json index e3b1a17f..0eec0c4f 100644 --- a/c#/imagePipeline/packages.lock.json +++ b/c#/imagePipeline/packages.lock.json @@ -4,9 +4,9 @@ "net7.0": { "LanguageExt.Core": { "type": "Direct", - "requested": "[4.4.7, )", - "resolved": "4.4.7", - "contentHash": "dZVtfDw22sbiSXsMMgytlv+ggE2q3mf6fvLp4YNoksNkNLk2inXCe/TM6UcdXNj9tOhn3mUTtXealIxkG1gGzA==", + "requested": "[4.4.8, )", + "resolved": "4.4.8", + "contentHash": "zd3+mICicbTvf17OuOaruTpluiA5qEl9e28rXSokkI/rK9vdtZzpigYN6+D6Yx9jTkkVDZlvoYUg28RtQqtGiw==", "dependencies": { "Microsoft.Bcl.AsyncInterfaces": "7.0.0", "Microsoft.CSharp": "4.7.0", diff --git a/c#/imagePipeline/tbm.ImagePipeline.csproj b/c#/imagePipeline/tbm.ImagePipeline.csproj index cd8eaac0..81bbb82e 100644 --- a/c#/imagePipeline/tbm.ImagePipeline.csproj +++ b/c#/imagePipeline/tbm.ImagePipeline.csproj @@ -6,7 +6,7 @@ - +