diff --git a/c#/crawler/src/Db/CrawlerDbContext.cs b/c#/crawler/src/Db/CrawlerDbContext.cs index e8b58164..b0784bed 100644 --- a/c#/crawler/src/Db/CrawlerDbContext.cs +++ b/c#/crawler/src/Db/CrawlerDbContext.cs @@ -13,7 +13,7 @@ public CrawlerDbContext() : this(fid: 0) { } public delegate CrawlerDbContext New(Fid fid); public Fid Fid { get; } = fid; - public DbSet Users => Set(); + public DbSet Users => Set(); public DbSet AuthorExpGradeRevisions => Set(); public DbSet ForumModeratorRevisions => Set(); public DbSet Threads => Set(); @@ -65,7 +65,7 @@ protected override void OnModelCreating(ModelBuilder b) { base.OnModelCreating(b); OnModelCreatingWithFid(b, Fid); - b.Entity().ToTable("tbmc_user"); + b.Entity().ToTable("tbmc_user"); b.Entity().ToTable($"tbmc_f{Fid}_thread"); b.Entity().ToTable("tbmc_thread_missingFirstReply"); b.Entity().ToTable($"tbmc_f{Fid}_reply"); diff --git a/c#/crawler/src/Db/TiebaUser.cs b/c#/crawler/src/Db/User.cs similarity index 80% rename from c#/crawler/src/Db/TiebaUser.cs rename to c#/crawler/src/Db/User.cs index e1b4e4f6..37aa7e00 100644 --- a/c#/crawler/src/Db/TiebaUser.cs +++ b/c#/crawler/src/Db/User.cs @@ -1,6 +1,6 @@ namespace tbm.Crawler.Db; -public class TiebaUser : ITimestampingEntity +public class User : ITimestampingEntity { [Key] public long Uid { get; set; } public string? Name { get; set; } @@ -14,6 +14,6 @@ public class TiebaUser : ITimestampingEntity public uint CreatedAt { get; set; } public uint? UpdatedAt { get; set; } - public static TiebaUser CreateLatestReplier(long uid, string? name, string? displayName) => + public static User CreateLatestReplier(long uid, string? name, string? displayName) => new() {Uid = uid, Name = name, DisplayName = displayName}; } diff --git a/c#/crawler/src/Tieba/Crawl/Facade/BaseCrawlFacade.cs b/c#/crawler/src/Tieba/Crawl/Facade/BaseCrawlFacade.cs index 054aad8f..120d64af 100644 --- a/c#/crawler/src/Tieba/Crawl/Facade/BaseCrawlFacade.cs +++ b/c#/crawler/src/Tieba/Crawl/Facade/BaseCrawlFacade.cs @@ -42,7 +42,7 @@ public virtual void Dispose() using var transaction = db.Database.BeginTransaction(IsolationLevel.ReadCommitted); var saver = saverFactory(Posts); var savedPosts = Posts.IsEmpty ? null : saver.SavePosts(db); - Users.SaveUsers(db, saver.PostType, saver.TiebaUserFieldChangeIgnorance); + Users.SaveUsers(db, saver.PostType, saver.UserFieldChangeIgnorance); BeforeCommitSaveHook(db); try { diff --git a/c#/crawler/src/Tieba/Crawl/Facade/ThreadCrawlFacade.cs b/c#/crawler/src/Tieba/Crawl/Facade/ThreadCrawlFacade.cs index d7e9ba8d..e56b3cf6 100644 --- a/c#/crawler/src/Tieba/Crawl/Facade/ThreadCrawlFacade.cs +++ b/c#/crawler/src/Tieba/Crawl/Facade/ThreadCrawlFacade.cs @@ -10,7 +10,7 @@ public class ThreadCrawlFacade( : BaseCrawlFacade (crawler(forumName), parser, saver.Invoke, locks["thread"], new(fid), fid) { - private readonly Dictionary _latestRepliers = new(); + private readonly Dictionary _latestRepliers = new(); public delegate ThreadCrawlFacade New(Fid fid, string forumName); @@ -20,7 +20,7 @@ protected override void BeforeCommitSaveHook(CrawlerDbContext db) // note this will bypass user revision detection since not invoking CommonInSavers.SavePostsOrUsers() but directly DbContext.AddRange() // users has already been added into DbContext and tracking - var existingUsersId = db.ChangeTracker.Entries().Select(ee => ee.Entity.Uid); + var existingUsersId = db.ChangeTracker.Entries().Select(ee => ee.Entity.Uid); var newLatestRepliers = _latestRepliers .ExceptBy(existingUsersId, pair => pair.Key) .Select(pair => pair.Value) @@ -45,7 +45,7 @@ protected void ParseLatestRepliers(IEnumerable threads) => // some rare deleted thread but still visible in 6.0.2 response // will have a latest replier uid=0 name="" nameShow=".*" .Where(u => u.Uid != 0) - .Select(u => TiebaUser.CreateLatestReplier(u.Uid, u.Name.NullIfEmpty(), + .Select(u => User.CreateLatestReplier(u.Uid, u.Name.NullIfEmpty(), u.Name == u.NameShow ? null : u.NameShow)) .ForEach(u => _latestRepliers[u.Uid] = u); diff --git a/c#/crawler/src/Tieba/Crawl/Saver/BaseSaver.cs b/c#/crawler/src/Tieba/Crawl/Saver/BaseSaver.cs index e2d8f384..a620a505 100644 --- a/c#/crawler/src/Tieba/Crawl/Saver/BaseSaver.cs +++ b/c#/crawler/src/Tieba/Crawl/Saver/BaseSaver.cs @@ -14,7 +14,7 @@ public abstract class BaseSaver( protected delegate void PostSaveEventHandler(); protected event PostSaveEventHandler PostSaveEvent = () => { }; - public virtual FieldChangeIgnoranceDelegates TiebaUserFieldChangeIgnorance => + public virtual FieldChangeIgnoranceDelegates UserFieldChangeIgnorance => throw new NotImplementedException(); public string PostType { get; } = postType; protected ConcurrentDictionary Posts { get; } = posts; @@ -37,7 +37,7 @@ protected SaverChangeSet SavePosts( // deep copy before entities get mutated by CommonInSavers.SavePostsOrUsers() var existingBeforeMerge = existingPostsKeyById.Select(pair => (TPost)pair.Value.Clone()).ToList(); - SavePostsOrUsers(db, TiebaUserFieldChangeIgnorance, revisionFactory, + SavePostsOrUsers(db, UserFieldChangeIgnorance, revisionFactory, Posts.Values.ToLookup(p => existingPostsKeyById.ContainsKey(postIdSelector(p))), p => existingPostsKeyById[postIdSelector(p)]); return new(existingBeforeMerge, Posts.Values, postIdSelector); diff --git a/c#/crawler/src/Tieba/Crawl/Saver/CommonInSavers.cs b/c#/crawler/src/Tieba/Crawl/Saver/CommonInSavers.cs index 91f46cd5..6aaefa7e 100644 --- a/c#/crawler/src/Tieba/Crawl/Saver/CommonInSavers.cs +++ b/c#/crawler/src/Tieba/Crawl/Saver/CommonInSavers.cs @@ -42,7 +42,7 @@ bool IsTimestampingFieldName(string name) => name is nameof(IPost.LastSeenAt) var revision = default(TRevision); var revisionNullFieldsBitMask = 0; var whichPostType = typeof(TPostOrUser); - var entryIsUser = whichPostType == typeof(TiebaUser); + var entryIsUser = whichPostType == typeof(User); foreach (var p in entry.Properties) { var pName = p.Metadata.Name; @@ -118,21 +118,21 @@ private static bool IsLatestReplierUser(string pName, PropertyEntry p, EntityEnt // so we should ignore its revision update for all fields // ignore entire record is not possible via GlobalFieldChangeIgnorance.Revision() // since it can only determine one field at the time - if (pName != nameof(TiebaUser.Portrait) || p.OriginalValue is not "") return false; + if (pName != nameof(User.Portrait) || p.OriginalValue is not "") return false; // invokes OriginalValues.ToObject() to get a new instance // since postOrUserInTracking is reference to the changed one - var user = (TiebaUser)entry.OriginalValues.ToObject(); + var user = (User)entry.OriginalValues.ToObject(); // create another user instance with only fields of latest replier filled - var latestReplier = TiebaUser.CreateLatestReplier(user.Uid, user.Name, user.DisplayName); + var latestReplier = User.CreateLatestReplier(user.Uid, user.Name, user.DisplayName); // if they are same by fields values, the original one is a latest replier // that previously generated by ParseLatestRepliers() return IsSameUser(user, latestReplier); } - private static bool IsSameUser(TiebaUser a, TiebaUser b) => + private static bool IsSameUser(User a, User b) => (a.Uid, a.Name, a.DisplayName, a.Portrait, a.PortraitUpdatedAt, a.Gender, a.FansNickname, a.IpGeolocation) == (b.Uid, b.Name, b.DisplayName, b.Portrait, b.PortraitUpdatedAt, b.Gender, b.FansNickname, b.IpGeolocation) && (a.Icon == b.Icon || (a.Icon != null && b.Icon != null && a.Icon.SequenceEqual(b.Icon))); diff --git a/c#/crawler/src/Tieba/Crawl/Saver/ReplySaver.cs b/c#/crawler/src/Tieba/Crawl/Saver/ReplySaver.cs index 300efb95..2a02e5f6 100644 --- a/c#/crawler/src/Tieba/Crawl/Saver/ReplySaver.cs +++ b/c#/crawler/src/Tieba/Crawl/Saver/ReplySaver.cs @@ -12,15 +12,15 @@ public partial class ReplySaver( { public delegate ReplySaver New(ConcurrentDictionary posts); - public override FieldChangeIgnoranceDelegates TiebaUserFieldChangeIgnorance { get; } = new( + public override FieldChangeIgnoranceDelegates UserFieldChangeIgnorance { get; } = new( Update: (_, propName, oldValue, newValue) => propName switch { // FansNickname in reply response will always be null - nameof(TiebaUser.FansNickname) when oldValue is not null && newValue is null => true, + nameof(User.FansNickname) when oldValue is not null && newValue is null => true, _ => false }, Revision: (_, propName, oldValue, newValue) => propName switch { // user icon will be null after UserParserAndSaver.ResetUsersIcon() get invoked - nameof(TiebaUser.Icon) when oldValue is null && newValue is not null => true, + nameof(User.Icon) when oldValue is null && newValue is not null => true, _ => false }); diff --git a/c#/crawler/src/Tieba/Crawl/Saver/StaticCommonInSavers.cs b/c#/crawler/src/Tieba/Crawl/Saver/StaticCommonInSavers.cs index 1bbd5b3a..b36d9e63 100644 --- a/c#/crawler/src/Tieba/Crawl/Saver/StaticCommonInSavers.cs +++ b/c#/crawler/src/Tieba/Crawl/Saver/StaticCommonInSavers.cs @@ -12,11 +12,11 @@ public delegate bool FieldChangeIgnoranceDelegate( protected static FieldChangeIgnoranceDelegates GlobalFieldChangeIgnorance { get; } = new( Update: (whichPostType, propName, oldValue, newValue) => { - if (whichPostType == typeof(TiebaUser)) + if (whichPostType == typeof(User)) { switch (propName) { // possible randomly respond with null - case nameof(TiebaUser.IpGeolocation) when newValue is null: + case nameof(User.IpGeolocation) when newValue is null: // possible clock drift across multiple response from tieba api // they should sync their servers with NTP /* following sql can track these drift @@ -29,7 +29,7 @@ FROM tbmcr_user WHERE portraitUpdatedAt IS NOT NULL WHERE portraitUpdatedAtDiff > -100 AND portraitUpdatedAtDiff < 100 GROUP BY portraitUpdatedAtDiff ORDER BY portraitUpdatedAtDiff; */ - case nameof(TiebaUser.PortraitUpdatedAt) + case nameof(User.PortraitUpdatedAt) when Math.Abs((newValue as int? ?? 0) - (oldValue as int? ?? 0)) <= 10: return true; } @@ -74,8 +74,8 @@ when newValue is "" }, Revision: (whichPostType, propName, oldValue, _) => { // ignore revision that figures update existing old users that don't have ip geolocation - if (whichPostType == typeof(TiebaUser) - && propName == nameof(TiebaUser.IpGeolocation) && oldValue is null) return true; + if (whichPostType == typeof(User) + && propName == nameof(User.IpGeolocation) && oldValue is null) return true; if (whichPostType == typeof(ThreadPost)) { switch (propName) diff --git a/c#/crawler/src/Tieba/Crawl/Saver/SubReplySaver.cs b/c#/crawler/src/Tieba/Crawl/Saver/SubReplySaver.cs index d7e7dceb..47f63ddf 100644 --- a/c#/crawler/src/Tieba/Crawl/Saver/SubReplySaver.cs +++ b/c#/crawler/src/Tieba/Crawl/Saver/SubReplySaver.cs @@ -11,17 +11,17 @@ public class SubReplySaver( { public delegate SubReplySaver New(ConcurrentDictionary posts); - public override FieldChangeIgnoranceDelegates TiebaUserFieldChangeIgnorance { get; } = new( + public override FieldChangeIgnoranceDelegates UserFieldChangeIgnorance { get; } = new( Update: (_, propName, oldValue, newValue) => propName switch { // always ignore updates on iconinfo due to some rare user will show some extra icons // compare to reply response in the response of sub reply - nameof(TiebaUser.Icon) => true, + nameof(User.Icon) => true, // FansNickname in sub reply response will always be null - nameof(TiebaUser.FansNickname) when oldValue is not null && newValue is null => true, + nameof(User.FansNickname) when oldValue is not null && newValue is null => true, // DisplayName in users embedded in sub replies from response will be the legacy nick name - nameof(TiebaUser.DisplayName) => true, + nameof(User.DisplayName) => true, _ => false }, (_, _, _, _) => false); diff --git a/c#/crawler/src/Tieba/Crawl/Saver/ThreadSaver.cs b/c#/crawler/src/Tieba/Crawl/Saver/ThreadSaver.cs index b9c2dc13..37fcc896 100644 --- a/c#/crawler/src/Tieba/Crawl/Saver/ThreadSaver.cs +++ b/c#/crawler/src/Tieba/Crawl/Saver/ThreadSaver.cs @@ -11,13 +11,13 @@ public class ThreadSaver( { public delegate ThreadSaver New(ConcurrentDictionary posts); - public override FieldChangeIgnoranceDelegates TiebaUserFieldChangeIgnorance { get; } = new( + public override FieldChangeIgnoranceDelegates UserFieldChangeIgnorance { get; } = new( Update: (_, propName, _, _) => propName switch { // Icon.SpriteInfo will be an empty array and the icon url is a smaller one // so we should mark it as null temporarily // note this will cause we can't record when did a user update its iconinfo to null // since these null values have been ignored in reply and sub reply saver - nameof(TiebaUser.Icon) => true, + nameof(User.Icon) => true, _ => false }, (_, _, _, _) => false); diff --git a/c#/crawler/src/Tieba/Crawl/UserParserAndSaver.cs b/c#/crawler/src/Tieba/Crawl/UserParserAndSaver.cs index afc9eae5..afbb65bd 100644 --- a/c#/crawler/src/Tieba/Crawl/UserParserAndSaver.cs +++ b/c#/crawler/src/Tieba/Crawl/UserParserAndSaver.cs @@ -27,9 +27,9 @@ protected override Dictionary [SuppressMessage("StyleCop.CSharp.SpacingRules", "SA1025:Code should not contain multiple whitespace in a row")] protected override ushort GetRevisionNullFieldBitMask(string fieldName) => fieldName switch { - nameof(TiebaUser.Name) => 1, - nameof(TiebaUser.Gender) => 1 << 3, - nameof(TiebaUser.Icon) => 1 << 5, + nameof(User.Name) => 1, + nameof(User.Gender) => 1 << 3, + nameof(User.Icon) => 1 << 5, _ => 0 }; } @@ -38,7 +38,7 @@ public partial class UserParserAndSaver(ILogger logger) { private static readonly HashSet UserIdLocks = new(); private readonly List _savedUsersId = new(); - private readonly ConcurrentDictionary _users = new(); + private readonly ConcurrentDictionary _users = new(); public void ParseUsers(IEnumerable users) => users.Select(el => @@ -65,7 +65,7 @@ public void ParseUsers(IEnumerable users) => // will be an empty string when the user hasn't set a username for their baidu account yet var name = el.Name.NullIfEmpty(); var nameShow = el.NameShow.NullIfEmpty(); - var u = new TiebaUser(); + var u = new User(); try { u.Uid = uid; @@ -85,17 +85,17 @@ public void ParseUsers(IEnumerable users) => e.Data["raw"] = Helper.UnescapedJsonSerialize(el); throw new InvalidDataException("User parse error.", e); } - }).OfType().ForEach(u => _users[u.Uid] = u); + }).OfType().ForEach(u => _users[u.Uid] = u); public void ResetUsersIcon() => _users.Values.ForEach(u => u.Icon = null); public void SaveUsers - (CrawlerDbContext db, string postType, FieldChangeIgnoranceDelegates tiebaUserFieldChangeIgnorance) + (CrawlerDbContext db, string postType, FieldChangeIgnoranceDelegates userFieldChangeIgnorance) { if (_users.IsEmpty) return; lock (UserIdLocks) { - var usersExceptLocked = new Dictionary(_users.ExceptBy(UserIdLocks, pair => pair.Key)); + var usersExceptLocked = new Dictionary(_users.ExceptBy(UserIdLocks, pair => pair.Key)); if (!usersExceptLocked.Any()) return; _savedUsersId.AddRange(usersExceptLocked.Keys); UserIdLocks.UnionWith(_savedUsersId); @@ -103,7 +103,7 @@ public void SaveUsers var existingUsersKeyByUid = (from user in db.Users.AsTracking().ForUpdate() where usersExceptLocked.Keys.Contains(user.Uid) select user).ToDictionary(u => u.Uid); - SavePostsOrUsers(db, tiebaUserFieldChangeIgnorance, + SavePostsOrUsers(db, userFieldChangeIgnorance, u => new UserRevision { TakenAt = u.UpdatedAt ?? u.CreatedAt, diff --git a/fe/src/api/index.d.ts b/fe/src/api/index.d.ts index 4119ab8d..180e49e3 100644 --- a/fe/src/api/index.d.ts +++ b/fe/src/api/index.d.ts @@ -1,6 +1,6 @@ import type { Reply, SubReply, Thread } from './post'; -import type { TiebaUser, TiebaUserGenderQueryParam } from './user'; -import type { SelectTiebaUserParams } from '@/components/widgets/selectTiebaUser'; +import type { User, UserGenderQueryParam } from './user'; +import type { SelectUserParams } from '@/components/widgets/selectUser'; import type { BoolInt, Fid, Float, PostType, UInt, UnixTimestamp } from '@/shared'; import type { Mix } from '@/shared/groupBytimeGranularityUtcPlus8'; @@ -54,8 +54,8 @@ interface CursorPagination { } export type ApiUsers = Api< - CursorPagination & { users: TiebaUser[] }, - CursorPaginationQueryParam & SelectTiebaUserParams & { gender?: TiebaUserGenderQueryParam } + CursorPagination & { users: User[] }, + CursorPaginationQueryParam & SelectUserParams & { gender?: UserGenderQueryParam } >; export type JsonString = string; @@ -70,5 +70,5 @@ export type ApiPosts = Api }>, - users: TiebaUser[] + users: User[] }, CursorPaginationQueryParam & { query: JsonString }>; diff --git a/fe/src/api/user.ts b/fe/src/api/user.ts index 9194484d..9512cf9c 100644 --- a/fe/src/api/user.ts +++ b/fe/src/api/user.ts @@ -12,16 +12,16 @@ export type ForumModeratorType = 'assist' | 'videoadmin' | 'voiceadmin'; export type AuthorExpGrade = 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18; -export type TiebaUserGender = 0 | 1 | 2 | null; -export type TiebaUserGenderQueryParam = '0' | '1' | '2' | 'NULL'; +export type UserGender = 0 | 1 | 2 | null; +export type UserGenderQueryParam = '0' | '1' | '2' | 'NULL'; -export interface TiebaUser extends TimestampFields { +export interface User extends TimestampFields { uid: BaiduUserID, name: string | null, displayName: string | null, portrait: string, portraitUpdatedAt: UInt | null, - gender: TiebaUserGender, + gender: UserGender, fansNickname: string | null, icon: ObjUnknown[] | null, ipGeolocation: string | null, diff --git a/fe/src/components/Post/badges/BadgeUser.vue b/fe/src/components/Post/badges/BadgeUser.vue index 2db7157d..dd1f4bba 100644 --- a/fe/src/components/Post/badges/BadgeUser.vue +++ b/fe/src/components/Post/badges/BadgeUser.vue @@ -18,13 +18,13 @@