Refs 4739: metric check for failing snap tasks on RH repos

pkg/dao/interfaces.go

@@ -134,6 +134,7 @@ type MetricsDao interface {
 	PendingTasksAverageLatency(ctx context.Context) float64
 	PendingTasksCount(ctx context.Context) int64
 	PendingTasksOldestTask(ctx context.Context) float64
+	RHReposNoSuccessfulSnapshotTaskIn36Hours(ctx context.Context) int64
 }
 
 //go:generate $GO_OUTPUT/mockery --name TaskInfoDao --filename task_info_mock.go --inpackage

pkg/dao/metrics.go

@@ -142,3 +142,24 @@ func (d metricsDaoImpl) PendingTasksOldestTask(ctx context.Context) float64 {
 	}
 	return time.Since(*task.Queued).Seconds()
 }
+
+func (d metricsDaoImpl) RHReposNoSuccessfulSnapshotTaskIn36Hours(ctx context.Context) int64 {
+	var output int64 = -1
+	date := time.Now().Add(-36 * time.Hour).Format(time.RFC3339)
+
+	subQuery := d.db.WithContext(ctx).
+		Model(&models.RepositoryConfiguration{}).
+		Select("repository_configurations.uuid, bool_or(tasks.status ILIKE ? AND tasks.finished_at > ?) AS has_successful_tasks", fmt.Sprintf("%%%s%%", config.TaskStatusCompleted), date).
+		Joins("LEFT OUTER JOIN tasks ON repository_configurations.repository_uuid = tasks.object_uuid").
+		Where("repository_configurations.org_id = ?", config.RedHatOrg).
+		Where("repository_configurations.snapshot IS TRUE").
+		Where("tasks.type = ?", config.RepositorySnapshotTask).
+		Group("repository_configurations.uuid")
+
+	d.db.WithContext(ctx).
+		Table("(?) AS sq", subQuery).
+		Where("sq.has_successful_tasks IS FALSE").
+		Count(&output)
+
+	return output
+}

pkg/dao/metrics_test.go

@@ -320,3 +320,87 @@ func (s *MetricsSuite) TestPendingTasksOldestTask() {
 	oldestQeuedAt := s.dao.PendingTasksOldestTask(context.Background())
 	assert.True(t, oldestQeuedAt > 1)
 }
+
+func (s *MetricsSuite) TestRHReposNoSuccessfulSnapshotTaskIn36Hours() {
+	t := s.T()
+
+	initialCount := s.dao.RHReposNoSuccessfulSnapshotTaskIn36Hours(context.Background())
+	assert.NotEqual(t, -1, initialCount)
+
+	rcs, err := seeds.SeedRepositoryConfigurations(s.tx, 4, seeds.SeedOptions{OrgID: config.RedHatOrg})
+	assert.NoError(t, err)
+	assert.Equal(t, 4, len(rcs))
+
+	r1, r2, r3, r4 := rcs[0], rcs[1], rcs[2], rcs[3]
+
+	_, err = seeds.SeedTasks(s.tx, 1, seeds.TaskSeedOptions{
+		RepoConfigUUID: r1.UUID,
+		RepoUUID:       r1.RepositoryUUID,
+		Typename:       config.RepositorySnapshotTask,
+		QueuedAt:       utils.Ptr(time.Now().Add(-10 * time.Hour)),
+		FinishedAt:     utils.Ptr(time.Now().Add(-9 * time.Hour)),
+		Status:         config.TaskStatusCompleted,
+	})
+	assert.NoError(t, err)
+	_, err = seeds.SeedTasks(s.tx, 1, seeds.TaskSeedOptions{
+		RepoConfigUUID: r1.UUID,
+		RepoUUID:       r1.RepositoryUUID,
+		Typename:       config.IntrospectTask,
+		QueuedAt:       utils.Ptr(time.Now().Add(-5 * time.Hour)),
+		FinishedAt:     utils.Ptr(time.Now().Add(-4 * time.Hour)),
+		Status:         config.TaskStatusFailed,
+	})
+	assert.NoError(t, err)
+
+	_, err = seeds.SeedTasks(s.tx, 1, seeds.TaskSeedOptions{
+		RepoConfigUUID: r2.UUID,
+		RepoUUID:       r2.RepositoryUUID,
+		Typename:       config.RepositorySnapshotTask,
+		QueuedAt:       utils.Ptr(time.Now().Add(-40 * time.Hour)),
+		FinishedAt:     utils.Ptr(time.Now().Add(-39 * time.Hour)),
+		Status:         config.TaskStatusCompleted,
+	})
+	assert.NoError(t, err)
+	_, err = seeds.SeedTasks(s.tx, 1, seeds.TaskSeedOptions{
+		RepoConfigUUID: r2.UUID,
+		RepoUUID:       r2.RepositoryUUID,
+		Typename:       config.RepositorySnapshotTask,
+		QueuedAt:       utils.Ptr(time.Now().Add(-30 * time.Hour)),
+		FinishedAt:     utils.Ptr(time.Now().Add(-29 * time.Hour)),
+		Status:         config.TaskStatusFailed,
+	})
+	assert.NoError(t, err)
+
+	_, err = seeds.SeedTasks(s.tx, 1, seeds.TaskSeedOptions{
+		RepoConfigUUID: r3.UUID,
+		RepoUUID:       r3.RepositoryUUID,
+		Typename:       config.RepositorySnapshotTask,
+		QueuedAt:       utils.Ptr(time.Now().Add(-30 * time.Hour)),
+		FinishedAt:     utils.Ptr(time.Now().Add(-29 * time.Hour)),
+		Status:         config.TaskStatusCompleted,
+	})
+	assert.NoError(t, err)
+	_, err = seeds.SeedTasks(s.tx, 1, seeds.TaskSeedOptions{
+		RepoConfigUUID: r3.UUID,
+		RepoUUID:       r3.RepositoryUUID,
+		Typename:       config.RepositorySnapshotTask,
+		QueuedAt:       utils.Ptr(time.Now().Add(-20 * time.Hour)),
+		FinishedAt:     utils.Ptr(time.Now().Add(-19 * time.Hour)),
+		Status:         config.TaskStatusFailed,
+	})
+	assert.NoError(t, err)
+
+	_, err = seeds.SeedTasks(s.tx, 1, seeds.TaskSeedOptions{
+		RepoConfigUUID: r4.UUID,
+		RepoUUID:       r4.RepositoryUUID,
+		Typename:       config.RepositorySnapshotTask,
+		QueuedAt:       utils.Ptr(time.Now().Add(-20 * time.Hour)),
+		FinishedAt:     utils.Ptr(time.Now().Add(-19 * time.Hour)),
+		Status:         config.TaskStatusFailed,
+	})
+	assert.NoError(t, err)
+
+	// expecting r2, r4 to be additionally counted in this metric
+	count := s.dao.RHReposNoSuccessfulSnapshotTaskIn36Hours(context.Background())
+	assert.Equal(t, 2+initialCount, count)
+}

pkg/instrumentation/custom/collector.go

@@ -14,7 +14,8 @@ import (
 	"gorm.io/gorm"
 )
 
-const tickerDelay = 30 // in seconds // could be good to match this with the scrapper frequency
+const tickerDelay = 30                     // in seconds // could be good to match this with the scrapper frequency
+const snapshottingFailCheckDelay = 60 * 60 // in seconds
 
 type Collector struct {
 	context    context.Context
@@ -88,13 +89,21 @@ func (c *Collector) iterate() {
 	}
 }
 
+func (c *Collector) snapshottingFailCheckIterate() {
+	ctx := c.context
+	c.metrics.RHRepositories36HourWithoutSuccessfulSnapTask.Set(float64(c.dao.RHReposNoSuccessfulSnapshotTaskIn36Hours(ctx)))
+}
+
 func (c *Collector) Run() {
 	log.Info().Msg("Starting metrics collector go routine")
 	ticker := time.NewTicker(tickerDelay * time.Second)
+	snapshottingFailCheckTicker := time.NewTicker(snapshottingFailCheckDelay * time.Second)
 	for {
 		select {
 		case <-ticker.C:
 			c.iterate()
+		case <-snapshottingFailCheckTicker.C:
+			c.snapshottingFailCheckIterate()
 		case <-c.context.Done():
 			log.Info().Msgf("Stopping metrics collector go routine")
 			ticker.Stop()

pkg/instrumentation/metrics.go

@@ -26,6 +26,7 @@ const (
 	TaskStatsLabelPendingCount                     = "task_stats_pending_count"
 	TaskStatsLabelOldestWait                       = "task_stats_oldest_wait"
 	TaskStatsLabelAverageWait                      = "task_stats_average_wait"
+	RHRepositories36HourWithoutSuccessfulSnapTask  = "rh_repositories_36_hour_without_successful_snap_task"
 )
 
 type Metrics struct {
@@ -43,6 +44,7 @@ type Metrics struct {
 	TaskStats                                      prometheus.GaugeVec
 	OrgTotal                                       prometheus.Gauge
 	RHCertExpiryDays                               prometheus.Gauge
+	RHRepositories36HourWithoutSuccessfulSnapTask  prometheus.Gauge
 	reg                                            *prometheus.Registry
 }
 
@@ -119,6 +121,11 @@ func NewMetrics(reg *prometheus.Registry) *Metrics {
 			Name:      RHCertExpiryDays,
 			Help:      "Number of days until the Red Hat client certificate expires",
 		}),
+		RHRepositories36HourWithoutSuccessfulSnapTask: promauto.With(reg).NewGauge(prometheus.GaugeOpts{
+			Namespace: NameSpace,
+			Name:      RHRepositories36HourWithoutSuccessfulSnapTask,
+			Help:      "Number of Red Hat repositories that haven't had successful snapshot task in the last 36 hours.",
+		}),
 	}
 
 	reg.MustRegister(collectors.NewBuildInfoCollector())

pkg/seeds/seeds.go

@@ -99,6 +99,7 @@ func SeedRepositoryConfigurations(db *gorm.DB, size int, options SeedOptions) ([
 			OrgID:                createOrgId(options.OrgID),
 			RepositoryUUID:       repos[i].UUID,
 			LastSnapshotTaskUUID: options.TaskID,
+			Snapshot:             true,
 		}
 		repoConfigurations = append(repoConfigurations, repoConfig)
 	}
@@ -404,6 +405,7 @@ type TaskSeedOptions struct {
 	RepoConfigUUID string
 	RepoUUID       string
 	QueuedAt       *time.Time
+	FinishedAt     *time.Time
 }
 
 func SeedTasks(db *gorm.DB, size int, options TaskSeedOptions) ([]models.TaskInfo, error) {
@@ -469,6 +471,10 @@ func SeedTasks(db *gorm.DB, size int, options TaskSeedOptions) ([]models.TaskInf
 		}
 		started := time.Now().Add(time.Minute * time.Duration(i+5))
 		finished := time.Now().Add(time.Minute * time.Duration(i+10))
+		if options.FinishedAt != nil {
+			started = (*options.FinishedAt).Add(-5 * time.Minute)
+			finished = *options.FinishedAt
+		}
 		tasks[i] = models.TaskInfo{
 			Id:           uuid.New(),
 			Typename:     typename,