-
Notifications
You must be signed in to change notification settings - Fork 2.1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
pass tmClient and tabletInfo to self metrics
Signed-off-by: Shlomi Noach <[email protected]>
- Loading branch information
1 parent
adfcf6d
commit d8b6ec2
Showing
5 changed files
with
308 additions
and
33 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
156 changes: 156 additions & 0 deletions
156
go/vt/vttablet/tabletserver/throttle/base/self_metric_mysqld.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,156 @@ | ||
| /* | ||
| Copyright 2024 The Vitess Authors. | ||
| Licensed under the Apache License, Version 2.0 (the "License"); | ||
| you may not use this file except in compliance with the License. | ||
| You may obtain a copy of the License at | ||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under the License is distributed on an "AS IS" BASIS, | ||
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| See the License for the specific language governing permissions and | ||
| limitations under the License. | ||
| */ | ||
|
|
||
| package base | ||
|
|
||
| import ( | ||
| "context" | ||
| "errors" | ||
| "fmt" | ||
| "sync/atomic" | ||
| "time" | ||
|
|
||
| "vitess.io/vitess/go/timer" | ||
|
|
||
| tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" | ||
| ) | ||
|
|
||
| var ( | ||
| mysqlHostMetricsRpcTimeout = 5 * time.Second | ||
| mysqlHostMetricsRateLimit = 10 * time.Second | ||
| mysqlHostMetricsRateLimiter atomic.Pointer[timer.RateLimiter] | ||
| lastMySQLHostMetricsResponse atomic.Pointer[tabletmanagerdatapb.MysqlHostMetricsResponse] | ||
| ) | ||
|
|
||
| // getMysqlMetricsRateLimiter returns a rate limiter that is active until the given context is cancelled. | ||
| // This function will be called sequentially, but nonetheless it offers _some_ concurrent safety. Namely, | ||
| // that a created rate limiter is guaranteed to be cleaned up | ||
| func getMysqlMetricsRateLimiter(ctx context.Context, rateLimit time.Duration) *timer.RateLimiter { | ||
| rateLimiter := mysqlHostMetricsRateLimiter.Load() | ||
| if rateLimiter == nil { | ||
| rateLimiter = timer.NewRateLimiter(rateLimit) | ||
| go func() { | ||
| defer mysqlHostMetricsRateLimiter.Store(nil) | ||
| defer rateLimiter.Stop() | ||
| <-ctx.Done() | ||
| }() | ||
| mysqlHostMetricsRateLimiter.Store(rateLimiter) | ||
| } | ||
| return rateLimiter | ||
| } | ||
|
|
||
| // readMysqlHostMetrics reads MySQL host metrics sporadically from the tablet manager (which in turn reads | ||
| // them from mysql deamon). The metrics are then cached, whether successful or not. | ||
| // This idea is that is is very wasteful to read these metrics for every single query. E.g. right now the throttler | ||
| // can issue 4 reads per second, which is wasteful to go through two RPCs to get the disk space usage for example. Even the load | ||
| // average on the MySQL server is not that susceptible to change. | ||
| func readMysqlHostMetrics(ctx context.Context, params *SelfMetricReadParams) error { | ||
| if params.TmClient == nil { | ||
| return fmt.Errorf("tmClient is nil") | ||
| } | ||
| if params.TabletInfo == nil { | ||
| return fmt.Errorf("tabletInfo is nil") | ||
| } | ||
| rateLimiter := getMysqlMetricsRateLimiter(ctx, mysqlHostMetricsRateLimit) | ||
| err := rateLimiter.Do(func() error { | ||
| ctx, cancel := context.WithTimeout(ctx, mysqlHostMetricsRpcTimeout) | ||
| defer cancel() | ||
|
|
||
| resp, err := params.TmClient.MysqlHostMetrics(ctx, params.TabletInfo.Tablet, &tabletmanagerdatapb.MysqlHostMetricsRequest{}) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| lastMySQLHostMetricsResponse.Store(resp) | ||
| return nil | ||
| }) | ||
| return err | ||
| } | ||
|
|
||
| // getMysqlHostMetric gets a metric from the last read MySQL host metrics. The metric will either be directly read from | ||
| // tablet manager (which then reads it from the mysql deamon), or from the cache. | ||
| func getMysqlHostMetric(ctx context.Context, params *SelfMetricReadParams, mysqlHostMetricName string) *ThrottleMetric { | ||
| metric := &ThrottleMetric{ | ||
| Scope: SelfScope, | ||
| } | ||
| if err := readMysqlHostMetrics(ctx, params); err != nil { | ||
| return metric.WithError(err) | ||
| } | ||
| resp := lastMySQLHostMetricsResponse.Load() | ||
| if resp == nil { | ||
| return metric.WithError(ErrNoResultYet) | ||
| } | ||
| mysqlMetric := resp.HostMetrics.Metrics[mysqlHostMetricName] | ||
| if mysqlMetric == nil { | ||
| return metric.WithError(ErrNoSuchMetric) | ||
| } | ||
| metric.Value = mysqlMetric.Value | ||
| if mysqlMetric.Error != nil { | ||
| metric.Err = errors.New(mysqlMetric.Error.Message) | ||
| } | ||
| return metric | ||
| } | ||
|
|
||
| var _ SelfMetric = registerSelfMetric(&MysqldLoadAvgSelfMetric{}) | ||
| var _ SelfMetric = registerSelfMetric(&MysqldDatadirUsedRatioSelfMetric{}) | ||
|
|
||
| // MysqldLoadAvgSelfMetric stands for the load average per cpu, on the MySQL host. | ||
| type MysqldLoadAvgSelfMetric struct { | ||
| } | ||
|
|
||
| func (m *MysqldLoadAvgSelfMetric) Name() MetricName { | ||
| return MysqldLoadAvgMetricName | ||
| } | ||
|
|
||
| func (m *MysqldLoadAvgSelfMetric) DefaultScope() Scope { | ||
| return SelfScope | ||
| } | ||
|
|
||
| func (m *MysqldLoadAvgSelfMetric) DefaultThreshold() float64 { | ||
| return 1.0 | ||
| } | ||
|
|
||
| func (m *MysqldLoadAvgSelfMetric) RequiresConn() bool { | ||
| return false | ||
| } | ||
|
|
||
| func (m *MysqldLoadAvgSelfMetric) Read(ctx context.Context, params *SelfMetricReadParams) *ThrottleMetric { | ||
| return getMysqlHostMetric(ctx, params, "loadavg") | ||
| } | ||
|
|
||
| // MysqldDatadirUsedRatioSelfMetric stands for the disk space usage of the mount where MySQL's datadir is located. | ||
| // Range: 0.0 (empty) - 1.0 (full) | ||
| type MysqldDatadirUsedRatioSelfMetric struct { | ||
| } | ||
|
|
||
| func (m *MysqldDatadirUsedRatioSelfMetric) Name() MetricName { | ||
| return MysqldDatadirUsedRatioMetricName | ||
| } | ||
|
|
||
| func (m *MysqldDatadirUsedRatioSelfMetric) DefaultScope() Scope { | ||
| return SelfScope | ||
| } | ||
|
|
||
| func (m *MysqldDatadirUsedRatioSelfMetric) DefaultThreshold() float64 { | ||
| return 0.98 | ||
| } | ||
|
|
||
| func (m *MysqldDatadirUsedRatioSelfMetric) RequiresConn() bool { | ||
| return false | ||
| } | ||
|
|
||
| func (m *MysqldDatadirUsedRatioSelfMetric) Read(ctx context.Context, params *SelfMetricReadParams) *ThrottleMetric { | ||
| return getMysqlHostMetric(ctx, params, "datadir-used-ratio") | ||
| } |
69 changes: 69 additions & 0 deletions
69
go/vt/vttablet/tabletserver/throttle/base/self_metric_mysqld_test.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,69 @@ | ||
| /* | ||
| Copyright 2024 The Vitess Authors. | ||
| Licensed under the Apache License, Version 2.0 (the "License"); | ||
| you may not use this file except in compliance with the License. | ||
| You may obtain a copy of the License at | ||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under the License is distributed on an "AS IS" BASIS, | ||
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| See the License for the specific language governing permissions and | ||
| limitations under the License. | ||
| */ | ||
|
|
||
| package base | ||
|
|
||
| import ( | ||
| "context" | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/stretchr/testify/assert" | ||
| ) | ||
|
|
||
| func TestGetMysqlMetricsRateLimiter(t *testing.T) { | ||
| rateLimit := 10 * time.Millisecond | ||
| for range 3 { | ||
| t.Run("iteration", func(t *testing.T) { | ||
| ctx, cancel := context.WithCancel(context.Background()) | ||
| defer cancel() | ||
| { | ||
| rateLimiter := mysqlHostMetricsRateLimiter.Load() | ||
| assert.Nil(t, rateLimiter) | ||
| } | ||
| rateLimiter := getMysqlMetricsRateLimiter(ctx, rateLimit) | ||
| assert.NotNil(t, rateLimiter) | ||
| for range 5 { | ||
| r := getMysqlMetricsRateLimiter(ctx, rateLimit) | ||
| // Returning the same rate limiter | ||
| assert.Equal(t, rateLimiter, r) | ||
| } | ||
| val := 0 | ||
| incr := func() error { | ||
| val++ | ||
| return nil | ||
| } | ||
| for range 10 { | ||
| rateLimiter.Do(incr) | ||
| time.Sleep(2 * rateLimit) | ||
| } | ||
| assert.EqualValues(t, 10, val) | ||
| cancel() | ||
| // There can be a race condition where the rate limiter still emits one final tick after the context is cancelled. | ||
| // So we wait enough time to ensure that tick is "wasted". | ||
| time.Sleep(2 * rateLimit) | ||
| for range 10 { | ||
| rateLimiter.Do(incr) | ||
| time.Sleep(time.Millisecond) | ||
| } | ||
| assert.EqualValues(t, 10, val) | ||
| { | ||
| rateLimiter := mysqlHostMetricsRateLimiter.Load() | ||
| assert.Nil(t, rateLimiter) | ||
| } | ||
| }) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.