From eecffd28b28c1e9c758868ffcadf8f157139e362 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 8 Mar 2024 14:13:20 +0530 Subject: [PATCH 1/3] feat: create a throttled logger and use that for exeeced memory warnings Signed-off-by: Manan Gupta --- go/vt/vtgate/executor.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/go/vt/vtgate/executor.go b/go/vt/vtgate/executor.go index 9f14888c846..885bb18f811 100644 --- a/go/vt/vtgate/executor.go +++ b/go/vt/vtgate/executor.go @@ -40,6 +40,7 @@ import ( "vitess.io/vitess/go/vt/callerid" "vitess.io/vitess/go/vt/key" "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/logutil" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" querypb "vitess.io/vitess/go/vt/proto/query" topodatapb "vitess.io/vitess/go/vt/proto/topodata" @@ -73,6 +74,8 @@ var ( queriesProcessedByTable = stats.NewCountersWithMultiLabels("QueriesProcessedByTable", "Queries processed at vtgate by plan type, keyspace and table", []string{"Plan", "Keyspace", "Table"}) queriesRoutedByTable = stats.NewCountersWithMultiLabels("QueriesRoutedByTable", "Queries routed from vtgate to vttablet by plan type, keyspace and table", []string{"Plan", "Keyspace", "Table"}) + + exceedMemoryRowsLogger = logutil.NewThrottledLogger("ExceedMemoryRows", 1*time.Minute) ) const ( @@ -231,7 +234,7 @@ func (e *Executor) Execute(ctx context.Context, mysqlCtx vtgateservice.MySQLConn if err != nil { piiSafeSQL = logStats.StmtType } - log.Warningf("%q exceeds warning threshold of max memory rows: %v. Actual memory rows: %v", piiSafeSQL, warnMemoryRows, len(result.Rows)) + exceedMemoryRowsLogger.Warningf("%q exceeds warning threshold of max memory rows: %v. Actual memory rows: %v", piiSafeSQL, warnMemoryRows, len(result.Rows)) } logStats.SaveEndTime() @@ -365,7 +368,7 @@ func (e *Executor) StreamExecute( if err != nil { piiSafeSQL = logStats.StmtType } - log.Warningf("%q exceeds warning threshold of max memory rows: %v. Actual memory rows: %v", piiSafeSQL, warnMemoryRows, srr.rowsReturned) + exceedMemoryRowsLogger.Warningf("%q exceeds warning threshold of max memory rows: %v. Actual memory rows: %v", piiSafeSQL, warnMemoryRows, srr.rowsReturned) } logStats.SaveEndTime() From 1b6c1391b8a74e4f59bb6899fd8ca004e775e17f Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Tue, 12 Mar 2024 17:31:54 -0600 Subject: [PATCH 2/3] return a mysql warning Signed-off-by: Florent Poinsard --- go/vt/vtgate/executor.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/go/vt/vtgate/executor.go b/go/vt/vtgate/executor.go index 885bb18f811..f83b20b2d70 100644 --- a/go/vt/vtgate/executor.go +++ b/go/vt/vtgate/executor.go @@ -33,6 +33,7 @@ import ( "vitess.io/vitess/go/acl" "vitess.io/vitess/go/cache/theine" "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/mysql/sqlerror" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/stats" "vitess.io/vitess/go/streamlog" @@ -234,7 +235,12 @@ func (e *Executor) Execute(ctx context.Context, mysqlCtx vtgateservice.MySQLConn if err != nil { piiSafeSQL = logStats.StmtType } - exceedMemoryRowsLogger.Warningf("%q exceeds warning threshold of max memory rows: %v. Actual memory rows: %v", piiSafeSQL, warnMemoryRows, len(result.Rows)) + warningMsg := fmt.Sprintf("%q exceeds warning threshold of max memory rows: %v. Actual memory rows: %v", piiSafeSQL, warnMemoryRows, len(result.Rows)) + exceedMemoryRowsLogger.Warningf(sql, warningMsg) + safeSession.RecordWarning(&querypb.QueryWarning{ + Code: uint32(sqlerror.EROutOfMemory), + Message: warningMsg, + }) } logStats.SaveEndTime() @@ -368,7 +374,12 @@ func (e *Executor) StreamExecute( if err != nil { piiSafeSQL = logStats.StmtType } - exceedMemoryRowsLogger.Warningf("%q exceeds warning threshold of max memory rows: %v. Actual memory rows: %v", piiSafeSQL, warnMemoryRows, srr.rowsReturned) + warningMsg := fmt.Sprintf("%q exceeds warning threshold of max memory rows: %v. Actual memory rows: %v", piiSafeSQL, warnMemoryRows, srr.rowsReturned) + exceedMemoryRowsLogger.Warningf(sql, warningMsg) + safeSession.RecordWarning(&querypb.QueryWarning{ + Code: uint32(sqlerror.EROutOfMemory), + Message: warningMsg, + }) } logStats.SaveEndTime() From d27dec50a6dd5b393b792fd285ae1d6b51056fb9 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Tue, 12 Mar 2024 18:20:26 -0600 Subject: [PATCH 3/3] fix typo Signed-off-by: Florent Poinsard --- go/vt/vtgate/executor.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/vt/vtgate/executor.go b/go/vt/vtgate/executor.go index f83b20b2d70..70f53bcd769 100644 --- a/go/vt/vtgate/executor.go +++ b/go/vt/vtgate/executor.go @@ -236,7 +236,7 @@ func (e *Executor) Execute(ctx context.Context, mysqlCtx vtgateservice.MySQLConn piiSafeSQL = logStats.StmtType } warningMsg := fmt.Sprintf("%q exceeds warning threshold of max memory rows: %v. Actual memory rows: %v", piiSafeSQL, warnMemoryRows, len(result.Rows)) - exceedMemoryRowsLogger.Warningf(sql, warningMsg) + exceedMemoryRowsLogger.Warningf(warningMsg) safeSession.RecordWarning(&querypb.QueryWarning{ Code: uint32(sqlerror.EROutOfMemory), Message: warningMsg, @@ -375,7 +375,7 @@ func (e *Executor) StreamExecute( piiSafeSQL = logStats.StmtType } warningMsg := fmt.Sprintf("%q exceeds warning threshold of max memory rows: %v. Actual memory rows: %v", piiSafeSQL, warnMemoryRows, srr.rowsReturned) - exceedMemoryRowsLogger.Warningf(sql, warningMsg) + exceedMemoryRowsLogger.Warningf(warningMsg) safeSession.RecordWarning(&querypb.QueryWarning{ Code: uint32(sqlerror.EROutOfMemory), Message: warningMsg,