From c04895166b6b87ff05355c15f14232db8d886232 Mon Sep 17 00:00:00 2001 From: Lynwee <1507509064@qq.com> Date: Thu, 12 Dec 2024 10:48:36 +0800 Subject: [PATCH 1/4] fix(framework): fix errors when updating `is_failed` in PG (#8238) --- backend/core/runner/run_task.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/core/runner/run_task.go b/backend/core/runner/run_task.go index 0a4bbcd8ca2..d488b3b721f 100644 --- a/backend/core/runner/run_task.go +++ b/backend/core/runner/run_task.go @@ -335,7 +335,7 @@ func RunPluginSubTasks( logger.Error(err, "") where := dal.Where("task_id = ? and name = ?", task.ID, subtaskCtx.GetName()) if err := basicRes.GetDal().UpdateColumns(subtask, []dal.DalSet{ - {ColumnName: "is_failed", Value: 1}, + {ColumnName: "is_failed", Value: true}, {ColumnName: "message", Value: err.Error()}, }, where); err != nil { basicRes.GetLogger().Error(err, "error writing subtask %v status to DB", subtaskCtx.GetName()) From 47b40143cfa536f4650eb5968a065577d76ec411 Mon Sep 17 00:00:00 2001 From: Claudio Mascaro Date: Thu, 12 Dec 2024 03:21:48 -0300 Subject: [PATCH 2/4] fix: add pagination to job_collector task (#8233) --- .../github_graphql/tasks/job_collector.go | 73 +++++++++++++------ 1 file changed, 50 insertions(+), 23 deletions(-) diff --git a/backend/plugins/github_graphql/tasks/job_collector.go b/backend/plugins/github_graphql/tasks/job_collector.go index 73c914a8b74..af25d589790 100644 --- a/backend/plugins/github_graphql/tasks/job_collector.go +++ b/backend/plugins/github_graphql/tasks/job_collector.go @@ -51,7 +51,11 @@ type GraphqlQueryCheckSuite struct { // equal to Job in rest CheckRuns struct { TotalCount int - Nodes []struct { + PageInfo struct { + EndCursor string `graphql:"endCursor"` + HasNextPage bool `graphql:"hasNextPage"` + } + Nodes []struct { Id string Name string DetailsUrl string @@ -79,7 +83,7 @@ type GraphqlQueryCheckSuite struct { } } `graphql:"steps(first: 50)"` } - } `graphql:"checkRuns(first: 50)"` + } `graphql:"checkRuns(first: $pageSize, after: $skipCursor)"` } `graphql:"... on CheckSuite"` } @@ -95,7 +99,45 @@ var CollectJobsMeta = plugin.SubTaskMeta{ DomainTypes: []string{plugin.DOMAIN_TYPE_CICD}, } -var _ plugin.SubTaskEntryPoint = CollectAccount +var _ plugin.SubTaskEntryPoint = CollectJobs + +func getPageInfo(query interface{}, args *helper.GraphqlCollectorArgs) (*helper.GraphqlQueryPageInfo, error) { + queryWrapper := query.(*GraphqlQueryCheckRunWrapper) + hasNextPage := false + endCursor := "" + for _, node := range queryWrapper.Node { + if node.CheckSuite.CheckRuns.PageInfo.HasNextPage { + hasNextPage = true + endCursor = node.CheckSuite.CheckRuns.PageInfo.EndCursor + break + } + } + return &helper.GraphqlQueryPageInfo{ + EndCursor: endCursor, + HasNextPage: hasNextPage, + }, nil +} + +func buildQuery(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) { + query := &GraphqlQueryCheckRunWrapper{} + if reqData == nil { + return query, map[string]interface{}{}, nil + } + workflowRuns := reqData.Input.([]interface{}) + checkSuiteIds := []map[string]interface{}{} + for _, iWorkflowRuns := range workflowRuns { + workflowRun := iWorkflowRuns.(*SimpleWorkflowRun) + checkSuiteIds = append(checkSuiteIds, map[string]interface{}{ + `id`: graphql.ID(workflowRun.CheckSuiteNodeID), + }) + } + variables := map[string]interface{}{ + "node": checkSuiteIds, + "pageSize": graphql.Int(reqData.Pager.Size), + "skipCursor": (*graphql.String)(reqData.Pager.SkipCursor), + } + return query, variables, nil +} func CollectJobs(taskCtx plugin.SubTaskContext) errors.Error { db := taskCtx.GetDal() @@ -137,26 +179,10 @@ func CollectJobs(taskCtx plugin.SubTaskContext) errors.Error { err = apiCollector.InitGraphQLCollector(helper.GraphqlCollectorArgs{ Input: iterator, - InputStep: 20, + InputStep: 10, GraphqlClient: data.GraphqlClient, - BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) { - query := &GraphqlQueryCheckRunWrapper{} - if reqData == nil { - return query, map[string]interface{}{}, nil - } - workflowRuns := reqData.Input.([]interface{}) - checkSuiteIds := []map[string]interface{}{} - for _, iWorkflowRuns := range workflowRuns { - workflowRun := iWorkflowRuns.(*SimpleWorkflowRun) - checkSuiteIds = append(checkSuiteIds, map[string]interface{}{ - `id`: graphql.ID(workflowRun.CheckSuiteNodeID), - }) - } - variables := map[string]interface{}{ - "node": checkSuiteIds, - } - return query, variables, nil - }, + BuildQuery: buildQuery, + GetPageInfo: getPageInfo, ResponseParser: func(queryWrapper any) (messages []json.RawMessage, err errors.Error) { query := queryWrapper.(*GraphqlQueryCheckRunWrapper) for _, node := range query.Node { @@ -168,12 +194,13 @@ func CollectJobs(taskCtx plugin.SubTaskContext) errors.Error { if apiCollector.GetSince() != nil && !apiCollector.GetSince().Before(*updatedAt) { return messages, helper.ErrFinishCollect } - messages = append(messages, errors.Must1(json.Marshal(node))) + messages = append(messages, errors.Must1(json.Marshal(checkRun))) } } return }, IgnoreQueryErrors: true, + PageSize: 20, }) if err != nil { return err From 3de98407374f8c30a64af9a76899b2c7e9012686 Mon Sep 17 00:00:00 2001 From: Lynwee <1507509064@qq.com> Date: Wed, 18 Dec 2024 15:07:46 +0800 Subject: [PATCH 3/4] fix(tapd): fix overflow when converting lead time minutes (#8244) --- backend/plugins/tapd/tasks/bug_converter.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/backend/plugins/tapd/tasks/bug_converter.go b/backend/plugins/tapd/tasks/bug_converter.go index 635b153dcac..1ae6eaefeab 100644 --- a/backend/plugins/tapd/tasks/bug_converter.go +++ b/backend/plugins/tapd/tasks/bug_converter.go @@ -18,6 +18,7 @@ limitations under the License. package tasks import ( + "math" "reflect" "strconv" "time" @@ -88,8 +89,12 @@ func ConvertBug(taskCtx plugin.SubTaskContext) errors.Error { results = append(results, issueAssignee) } if domainL.ResolutionDate != nil && domainL.CreatedDate != nil { - temp := uint(domainL.ResolutionDate.Sub(*domainL.CreatedDate).Minutes()) - domainL.LeadTimeMinutes = &temp + durationInMinutes := domainL.ResolutionDate.Sub(*domainL.CreatedDate).Minutes() + // we have found some issues' ResolutionDate is earlier than CreatedDate in tapd. + if durationInMinutes > 0 && durationInMinutes < math.MaxUint { + temp := uint(durationInMinutes) + domainL.LeadTimeMinutes = &temp + } } boardIssue := &ticket.BoardIssue{ BoardId: getWorkspaceIdGen().Generate(toolL.ConnectionId, toolL.WorkspaceId), From bebd0541647b8aca71d976e6bd91cc11224052dd Mon Sep 17 00:00:00 2001 From: Lynwee <1507509064@qq.com> Date: Wed, 18 Dec 2024 15:51:09 +0800 Subject: [PATCH 4/4] fix(zentao): make sure connection uncacheable (#8245) * fix(zentao): make sure connection uncacheable * fix(zentao): fix lint errors * Revert "fix(zentao): fix lint errors" This reverts commit e216a23ac1fdb0cb2297fc1bfb8e88abb0da9a90. * fix(zentao): fix lint errors * fix(zentao): remove UnCacheableConnetion interface, try to fix it with another way --- .../helpers/pluginhelper/api/ds_remote_api_proxy_api.go | 6 +----- backend/plugins/zentao/models/connection.go | 9 +++++++++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/backend/helpers/pluginhelper/api/ds_remote_api_proxy_api.go b/backend/helpers/pluginhelper/api/ds_remote_api_proxy_api.go index 9ea8b5d2a43..b0c4b0439c2 100644 --- a/backend/helpers/pluginhelper/api/ds_remote_api_proxy_api.go +++ b/backend/helpers/pluginhelper/api/ds_remote_api_proxy_api.go @@ -39,11 +39,7 @@ type DsRemoteApiProxyHelper[C plugin.ToolLayerApiConnection] struct { } // NewDsRemoteApiProxyHelper creates a new DsRemoteApiProxyHelper -func NewDsRemoteApiProxyHelper[ - C plugin.ToolLayerApiConnection, -]( - modelApiHelper *ModelApiHelper[C], -) *DsRemoteApiProxyHelper[C] { +func NewDsRemoteApiProxyHelper[C plugin.ToolLayerApiConnection](modelApiHelper *ModelApiHelper[C]) *DsRemoteApiProxyHelper[C] { return &DsRemoteApiProxyHelper[C]{ ModelApiHelper: modelApiHelper, logger: modelApiHelper.basicRes.GetLogger().Nested("remote_api_helper"), diff --git a/backend/plugins/zentao/models/connection.go b/backend/plugins/zentao/models/connection.go index 0554450e7ae..949dd36d2b4 100644 --- a/backend/plugins/zentao/models/connection.go +++ b/backend/plugins/zentao/models/connection.go @@ -73,6 +73,11 @@ type ZentaoConn struct { DbMaxConns int `json:"dbMaxConns" mapstructure:"dbMaxConns"` } +func (connection ZentaoConn) GetHash() string { + // zentao's token will expire after about 24min, so api client cannot be cached. + return "" +} + func (connection ZentaoConn) Sanitize() ZentaoConn { connection.Password = "" if connection.DbUrl != "" { @@ -106,6 +111,10 @@ func (connection ZentaoConn) SanitizeDbUrl() string { return dbUrl } +func (connection ZentaoConnection) GetHash() string { + return connection.ZentaoConn.GetHash() +} + func (connection ZentaoConnection) Sanitize() ZentaoConnection { connection.ZentaoConn = connection.ZentaoConn.Sanitize() return connection