From e6366cd294b6dd598e11a6babb90a8cb0e4b9541 Mon Sep 17 00:00:00 2001 From: Hourunze1997 <5427081+maomao19970922@user.noreply.gitee.com> Date: Thu, 17 Oct 2024 19:26:06 +0800 Subject: [PATCH 01/11] note related user bug fix --- models/dto/raw.go | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/models/dto/raw.go b/models/dto/raw.go index 9ab6221..1187e2a 100644 --- a/models/dto/raw.go +++ b/models/dto/raw.go @@ -7,7 +7,9 @@ package dto import ( "bytes" "encoding/json" + "log" "reflect" + "regexp" "strings" "text/template" "time" @@ -127,6 +129,32 @@ func (raw *Raw) GetRelateUsers(event *CloudEvents) { } } } +func extractMentions(note string) []string { + re := regexp.MustCompile(`@([a-zA-Z0-9_]+)`) + matches := re.FindAllStringSubmatch(note, -1) + + var mentions []string + for _, match := range matches { + if len(match) > 1 { + mentions = append(mentions, match[1]) // match[1] 是捕获组 + } + } + return mentions +} + +func getNoteAboutUsers(jsonData []byte) ([]string, error) { + var data map[string]map[string]string + err := json.Unmarshal(jsonData, &data) + if err != nil { + log.Fatalf("Error parsing JSON: %v", err) + } + // 提取 Note 字段 + note := data["NoteEvent"]["Note"] + + // 提取 @xxx 格式的人名 + mentions := extractMentions(note) + return mentions, nil +} func (raw *Raw) getGiteeRelatedUsers(event *CloudEvents, sourceGroup string) []string { lSourceGroup := strings.Split(sourceGroup, "/") @@ -139,9 +167,16 @@ func (raw *Raw) getGiteeRelatedUsers(event *CloudEvents, sourceGroup string) []s return []string{} } + noteAbout, err := getNoteAboutUsers(event.Data()) + if err != nil { + logrus.Errorf("get note about users failed, err:%v", err) + return []string{} + } switch giteeType { case "pr", "push", "issue": return allAdmins + case "note": + return noteAbout default: return []string{} } From 3570411e4a58d872527fccabd26db5e27bc8e473 Mon Sep 17 00:00:00 2001 From: Hourunze1997 <5427081+maomao19970922@user.noreply.gitee.com> Date: Thu, 17 Oct 2024 19:38:49 +0800 Subject: [PATCH 02/11] debug --- models/dto/raw.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/models/dto/raw.go b/models/dto/raw.go index 1187e2a..29720d4 100644 --- a/models/dto/raw.go +++ b/models/dto/raw.go @@ -166,7 +166,10 @@ func (raw *Raw) getGiteeRelatedUsers(event *CloudEvents, sourceGroup string) []s logrus.Errorf("get admins failed, err:%v", err) return []string{} } - + logrus.SetFormatter(&logrus.JSONFormatter{ + PrettyPrint: true, // 启用美化输出 + }) + logrus.Info(event.Data()) noteAbout, err := getNoteAboutUsers(event.Data()) if err != nil { logrus.Errorf("get note about users failed, err:%v", err) From eb5d3762664e612fa186be23e302991aed3ffee0 Mon Sep 17 00:00:00 2001 From: Hourunze1997 <5427081+maomao19970922@user.noreply.gitee.com> Date: Thu, 17 Oct 2024 19:47:20 +0800 Subject: [PATCH 03/11] debug --- models/dto/raw.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/models/dto/raw.go b/models/dto/raw.go index 29720d4..fcec41d 100644 --- a/models/dto/raw.go +++ b/models/dto/raw.go @@ -143,6 +143,9 @@ func extractMentions(note string) []string { } func getNoteAboutUsers(jsonData []byte) ([]string, error) { + if len(jsonData) == 0 { + return nil, nil + } var data map[string]map[string]string err := json.Unmarshal(jsonData, &data) if err != nil { @@ -166,10 +169,6 @@ func (raw *Raw) getGiteeRelatedUsers(event *CloudEvents, sourceGroup string) []s logrus.Errorf("get admins failed, err:%v", err) return []string{} } - logrus.SetFormatter(&logrus.JSONFormatter{ - PrettyPrint: true, // 启用美化输出 - }) - logrus.Info(event.Data()) noteAbout, err := getNoteAboutUsers(event.Data()) if err != nil { logrus.Errorf("get note about users failed, err:%v", err) From d0931944ae766cf56cffb4c3888a8f034e0196a5 Mon Sep 17 00:00:00 2001 From: Hourunze1997 <5427081+maomao19970922@user.noreply.gitee.com> Date: Thu, 17 Oct 2024 19:53:12 +0800 Subject: [PATCH 04/11] debug --- models/dto/raw.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/models/dto/raw.go b/models/dto/raw.go index fcec41d..6fd5105 100644 --- a/models/dto/raw.go +++ b/models/dto/raw.go @@ -169,6 +169,10 @@ func (raw *Raw) getGiteeRelatedUsers(event *CloudEvents, sourceGroup string) []s logrus.Errorf("get admins failed, err:%v", err) return []string{} } + logrus.SetFormatter(&logrus.JSONFormatter{ + PrettyPrint: true, // 启用美化输出 + }) + logrus.WithFields(logrus.Fields{"raw": raw}).Info("raw data") noteAbout, err := getNoteAboutUsers(event.Data()) if err != nil { logrus.Errorf("get note about users failed, err:%v", err) From 198d56c74ad7ec27414399a1c7d28e239653e79b Mon Sep 17 00:00:00 2001 From: Hourunze1997 <5427081+maomao19970922@user.noreply.gitee.com> Date: Thu, 17 Oct 2024 20:22:10 +0800 Subject: [PATCH 05/11] debug --- models/dto/raw.go | 60 +++++++++++++++++++++++----------------------- service/handler.go | 20 +++++++++++++++- 2 files changed, 49 insertions(+), 31 deletions(-) diff --git a/models/dto/raw.go b/models/dto/raw.go index 6fd5105..1983bf9 100644 --- a/models/dto/raw.go +++ b/models/dto/raw.go @@ -7,7 +7,6 @@ package dto import ( "bytes" "encoding/json" - "log" "reflect" "regexp" "strings" @@ -111,6 +110,20 @@ func (raw *Raw) ToCloudEventByConfig(sourceTopic string) CloudEvents { return newEvent } +func (raw *Raw) NoteToCloudEventByConfig(sourceTopic string, noteRaw GiteeNoteRaw) CloudEvents { + newEvent := NewCloudEvents() + configs := bo.GetTransferConfigFromDb(sourceTopic) + if configs != nil { + for _, config := range configs { + raw.transferField(&newEvent, config) + } + raw.GetRelateUsers(&newEvent) + raw.GetNoteRelatedUsers(&newEvent, noteRaw) + newEvent.SetData(cloudevents.ApplicationJSON, raw) + } + return newEvent +} + // GetRelateUsers get relate users. func (raw *Raw) GetRelateUsers(event *CloudEvents) { source := event.Source() @@ -129,8 +142,22 @@ func (raw *Raw) GetRelateUsers(event *CloudEvents) { } } } + +// GetNoteRelatedUsers get relate users. +func (raw *Raw) GetNoteRelatedUsers(event *CloudEvents, noteRaw GiteeNoteRaw) { + if _, ok := event.Extensions()["sourcegroup"].(string); ok { + if result, ok := event.Extensions()["relatedusers"].(string); ok { + lResult := strings.Split(result, ",") + noteUsers := extractMentions(*noteRaw.Note) + logrus.Infof("about user is %v", noteUsers) + lResult = append(lResult, noteUsers...) + event.SetExtension("relatedusers", strings.Join(escapeCommas(raw.distinct(lResult)), ",")) + } + } +} + func extractMentions(note string) []string { - re := regexp.MustCompile(`@([a-zA-Z0-9_]+)`) + re := regexp.MustCompile(`@([a-zA-Z0-9_-]+)`) matches := re.FindAllStringSubmatch(note, -1) var mentions []string @@ -142,23 +169,6 @@ func extractMentions(note string) []string { return mentions } -func getNoteAboutUsers(jsonData []byte) ([]string, error) { - if len(jsonData) == 0 { - return nil, nil - } - var data map[string]map[string]string - err := json.Unmarshal(jsonData, &data) - if err != nil { - log.Fatalf("Error parsing JSON: %v", err) - } - // 提取 Note 字段 - note := data["NoteEvent"]["Note"] - - // 提取 @xxx 格式的人名 - mentions := extractMentions(note) - return mentions, nil -} - func (raw *Raw) getGiteeRelatedUsers(event *CloudEvents, sourceGroup string) []string { lSourceGroup := strings.Split(sourceGroup, "/") owner, repo := lSourceGroup[0], lSourceGroup[1] @@ -169,20 +179,10 @@ func (raw *Raw) getGiteeRelatedUsers(event *CloudEvents, sourceGroup string) []s logrus.Errorf("get admins failed, err:%v", err) return []string{} } - logrus.SetFormatter(&logrus.JSONFormatter{ - PrettyPrint: true, // 启用美化输出 - }) - logrus.WithFields(logrus.Fields{"raw": raw}).Info("raw data") - noteAbout, err := getNoteAboutUsers(event.Data()) - if err != nil { - logrus.Errorf("get note about users failed, err:%v", err) - return []string{} - } + switch giteeType { case "pr", "push", "issue": return allAdmins - case "note": - return noteAbout default: return []string{} } diff --git a/service/handler.go b/service/handler.go index 52abf40..6617692 100644 --- a/service/handler.go +++ b/service/handler.go @@ -37,6 +37,24 @@ func handle(raw dto.Raw, cfg kafka.ConsumeConfig) error { return nil } +func handleNote(raw dto.Raw, giteeNote dto.GiteeNoteRaw, cfg kafka.ConsumeConfig) error { + time.Sleep(utils.GetConsumeSleepTime()) + event := raw.NoteToCloudEventByConfig(cfg.Topic, giteeNote) + if event.ID() == "" { + return nil + } + err := event.SaveDb() + if err != nil { + logrus.Errorf("saveDb failed, err:%v", err) + return err + } + kafkaSendErr := kafka.SendMsg(cfg.Publish, &event) + if kafkaSendErr != nil { + return kafkaSendErr + } + return nil +} + // CVEHandle handle cve issue raw. func CVEHandle(payload []byte, _ map[string]string) error { var raw dto.CVEIssueRaw @@ -175,7 +193,7 @@ func GiteeNoteHandle(payload []byte, _ map[string]string) error { raw.SigMaintainers = sigMaintainers raw.RepoAdmins = repoAdmins rawMap := dto.StructToMap(raw) - return handle(rawMap, config.GiteeConfigInstance.Note) + return handleNote(rawMap, raw, config.GiteeConfigInstance.Note) } // EurBuildHandle handle eur build raw. From 5b593c732e25dbe02982d5d117ec809796845fd9 Mon Sep 17 00:00:00 2001 From: Hourunze1997 <5427081+maomao19970922@user.noreply.gitee.com> Date: Fri, 18 Oct 2024 09:16:19 +0800 Subject: [PATCH 06/11] debug --- models/dto/raw.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/models/dto/raw.go b/models/dto/raw.go index 1983bf9..3b4eb63 100644 --- a/models/dto/raw.go +++ b/models/dto/raw.go @@ -183,6 +183,13 @@ func (raw *Raw) getGiteeRelatedUsers(event *CloudEvents, sourceGroup string) []s switch giteeType { case "pr", "push", "issue": return allAdmins + case "note": + logrus.SetFormatter(&logrus.JSONFormatter{ + PrettyPrint: true, // 启用美化输出 + }) + a := StructToMap((*raw)["NoteEvent"])["Note"] + logrus.Info(a) + return []string{} default: return []string{} } From d8e9a8cf5f69cfda9993b4e452bd84b1dc43a017 Mon Sep 17 00:00:00 2001 From: Hourunze1997 <5427081+maomao19970922@user.noreply.gitee.com> Date: Fri, 18 Oct 2024 09:30:11 +0800 Subject: [PATCH 07/11] debug --- models/dto/raw.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/dto/raw.go b/models/dto/raw.go index 3b4eb63..f8eeb67 100644 --- a/models/dto/raw.go +++ b/models/dto/raw.go @@ -187,8 +187,8 @@ func (raw *Raw) getGiteeRelatedUsers(event *CloudEvents, sourceGroup string) []s logrus.SetFormatter(&logrus.JSONFormatter{ PrettyPrint: true, // 启用美化输出 }) - a := StructToMap((*raw)["NoteEvent"])["Note"] - logrus.Info(a) + a := (*raw)["NoteEvent"] + logrus.Infof("the data is %v type is %v", a, reflect.TypeOf(a)) return []string{} default: return []string{} From 134e05c2c0486d0cc4864e8d6bb5097278f33fec Mon Sep 17 00:00:00 2001 From: Hourunze1997 <5427081+maomao19970922@user.noreply.gitee.com> Date: Fri, 18 Oct 2024 09:34:33 +0800 Subject: [PATCH 08/11] debug --- models/dto/raw.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/models/dto/raw.go b/models/dto/raw.go index f8eeb67..e15ac0e 100644 --- a/models/dto/raw.go +++ b/models/dto/raw.go @@ -149,7 +149,6 @@ func (raw *Raw) GetNoteRelatedUsers(event *CloudEvents, noteRaw GiteeNoteRaw) { if result, ok := event.Extensions()["relatedusers"].(string); ok { lResult := strings.Split(result, ",") noteUsers := extractMentions(*noteRaw.Note) - logrus.Infof("about user is %v", noteUsers) lResult = append(lResult, noteUsers...) event.SetExtension("relatedusers", strings.Join(escapeCommas(raw.distinct(lResult)), ",")) } @@ -187,8 +186,8 @@ func (raw *Raw) getGiteeRelatedUsers(event *CloudEvents, sourceGroup string) []s logrus.SetFormatter(&logrus.JSONFormatter{ PrettyPrint: true, // 启用美化输出 }) - a := (*raw)["NoteEvent"] - logrus.Infof("the data is %v type is %v", a, reflect.TypeOf(a)) + a := (*raw)["NoteEvent"].(map[string]interface{}) + logrus.Infof("the data is %v type is %v", a["Note"], reflect.TypeOf(a)) return []string{} default: return []string{} From 1997d46c83663d3efa48b2e5405d0e4ee983863c Mon Sep 17 00:00:00 2001 From: Hourunze1997 <5427081+maomao19970922@user.noreply.gitee.com> Date: Fri, 18 Oct 2024 09:39:47 +0800 Subject: [PATCH 09/11] debug --- models/dto/raw.go | 34 ++-------------------------------- service/handler.go | 6 +++--- 2 files changed, 5 insertions(+), 35 deletions(-) diff --git a/models/dto/raw.go b/models/dto/raw.go index e15ac0e..69fb0d4 100644 --- a/models/dto/raw.go +++ b/models/dto/raw.go @@ -110,20 +110,6 @@ func (raw *Raw) ToCloudEventByConfig(sourceTopic string) CloudEvents { return newEvent } -func (raw *Raw) NoteToCloudEventByConfig(sourceTopic string, noteRaw GiteeNoteRaw) CloudEvents { - newEvent := NewCloudEvents() - configs := bo.GetTransferConfigFromDb(sourceTopic) - if configs != nil { - for _, config := range configs { - raw.transferField(&newEvent, config) - } - raw.GetRelateUsers(&newEvent) - raw.GetNoteRelatedUsers(&newEvent, noteRaw) - newEvent.SetData(cloudevents.ApplicationJSON, raw) - } - return newEvent -} - // GetRelateUsers get relate users. func (raw *Raw) GetRelateUsers(event *CloudEvents) { source := event.Source() @@ -143,18 +129,6 @@ func (raw *Raw) GetRelateUsers(event *CloudEvents) { } } -// GetNoteRelatedUsers get relate users. -func (raw *Raw) GetNoteRelatedUsers(event *CloudEvents, noteRaw GiteeNoteRaw) { - if _, ok := event.Extensions()["sourcegroup"].(string); ok { - if result, ok := event.Extensions()["relatedusers"].(string); ok { - lResult := strings.Split(result, ",") - noteUsers := extractMentions(*noteRaw.Note) - lResult = append(lResult, noteUsers...) - event.SetExtension("relatedusers", strings.Join(escapeCommas(raw.distinct(lResult)), ",")) - } - } -} - func extractMentions(note string) []string { re := regexp.MustCompile(`@([a-zA-Z0-9_-]+)`) matches := re.FindAllStringSubmatch(note, -1) @@ -183,12 +157,8 @@ func (raw *Raw) getGiteeRelatedUsers(event *CloudEvents, sourceGroup string) []s case "pr", "push", "issue": return allAdmins case "note": - logrus.SetFormatter(&logrus.JSONFormatter{ - PrettyPrint: true, // 启用美化输出 - }) - a := (*raw)["NoteEvent"].(map[string]interface{}) - logrus.Infof("the data is %v type is %v", a["Note"], reflect.TypeOf(a)) - return []string{} + note := (*raw)["NoteEvent"].(map[string]interface{})["Note"].(string) + return extractMentions(note) default: return []string{} } diff --git a/service/handler.go b/service/handler.go index 6617692..09bbc03 100644 --- a/service/handler.go +++ b/service/handler.go @@ -37,9 +37,9 @@ func handle(raw dto.Raw, cfg kafka.ConsumeConfig) error { return nil } -func handleNote(raw dto.Raw, giteeNote dto.GiteeNoteRaw, cfg kafka.ConsumeConfig) error { +func handleNote(raw dto.Raw, cfg kafka.ConsumeConfig) error { time.Sleep(utils.GetConsumeSleepTime()) - event := raw.NoteToCloudEventByConfig(cfg.Topic, giteeNote) + event := raw.ToCloudEventByConfig(cfg.Topic) if event.ID() == "" { return nil } @@ -193,7 +193,7 @@ func GiteeNoteHandle(payload []byte, _ map[string]string) error { raw.SigMaintainers = sigMaintainers raw.RepoAdmins = repoAdmins rawMap := dto.StructToMap(raw) - return handleNote(rawMap, raw, config.GiteeConfigInstance.Note) + return handleNote(rawMap, config.GiteeConfigInstance.Note) } // EurBuildHandle handle eur build raw. From 8083af80255fc495d1db1b58c0494c1927b8604a Mon Sep 17 00:00:00 2001 From: Hourunze1997 <5427081+maomao19970922@user.noreply.gitee.com> Date: Fri, 18 Oct 2024 09:44:18 +0800 Subject: [PATCH 10/11] note about bugfix --- service/handler.go | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/service/handler.go b/service/handler.go index 09bbc03..52abf40 100644 --- a/service/handler.go +++ b/service/handler.go @@ -37,24 +37,6 @@ func handle(raw dto.Raw, cfg kafka.ConsumeConfig) error { return nil } -func handleNote(raw dto.Raw, cfg kafka.ConsumeConfig) error { - time.Sleep(utils.GetConsumeSleepTime()) - event := raw.ToCloudEventByConfig(cfg.Topic) - if event.ID() == "" { - return nil - } - err := event.SaveDb() - if err != nil { - logrus.Errorf("saveDb failed, err:%v", err) - return err - } - kafkaSendErr := kafka.SendMsg(cfg.Publish, &event) - if kafkaSendErr != nil { - return kafkaSendErr - } - return nil -} - // CVEHandle handle cve issue raw. func CVEHandle(payload []byte, _ map[string]string) error { var raw dto.CVEIssueRaw @@ -193,7 +175,7 @@ func GiteeNoteHandle(payload []byte, _ map[string]string) error { raw.SigMaintainers = sigMaintainers raw.RepoAdmins = repoAdmins rawMap := dto.StructToMap(raw) - return handleNote(rawMap, config.GiteeConfigInstance.Note) + return handle(rawMap, config.GiteeConfigInstance.Note) } // EurBuildHandle handle eur build raw. From d3312d794e713a752122f3715c2a9af49cd72ab9 Mon Sep 17 00:00:00 2001 From: Hourunze1997 <5427081+maomao19970922@user.noreply.gitee.com> Date: Fri, 18 Oct 2024 14:28:59 +0800 Subject: [PATCH 11/11] handle error --- models/dto/raw.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/models/dto/raw.go b/models/dto/raw.go index 69fb0d4..27f45e5 100644 --- a/models/dto/raw.go +++ b/models/dto/raw.go @@ -157,8 +157,17 @@ func (raw *Raw) getGiteeRelatedUsers(event *CloudEvents, sourceGroup string) []s case "pr", "push", "issue": return allAdmins case "note": - note := (*raw)["NoteEvent"].(map[string]interface{})["Note"].(string) - return extractMentions(note) + if noteEvent, ok := (*raw)["NoteEvent"].(map[string]interface{}); ok { + if note, ok := noteEvent["Note"].(string); ok { + return extractMentions(note) + } else { + logrus.Error("Note does not exist or is not a string") + return []string{} + } + } else { + logrus.Error("NoteEvent does not exist or is not a map") + return []string{} + } default: return []string{} }