diff --git a/contrib/resourcetuner/cgroups.go b/contrib/resourcetuner/cgroups.go new file mode 100644 index 000000000..3dc9ff58a --- /dev/null +++ b/contrib/resourcetuner/cgroups.go @@ -0,0 +1,161 @@ +// The MIT License +// +// Copyright (c) 2020 Temporal Technologies Inc. All rights reserved. +// +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +//go:build linux + +package resourcetuner + +import ( + "errors" + "fmt" + "io/fs" + "os" + "strconv" + "strings" + "time" + + "github.com/containerd/cgroups/v3/cgroup2" + "github.com/containerd/cgroups/v3/cgroup2/stats" +) + +func newCGroupInfo() cGroupInfo { + return &cGroupInfoImpl{} +} + +type cGroupInfoImpl struct { + lastCGroupMemStat *stats.MemoryStat + cgroupCpuCalc cgroupCpuCalc +} + +func (p *cGroupInfoImpl) Update() (bool, error) { + err := p.updateCGroupStats() + // Stop updates if not in a container. No need to return the error and log it. + if !errors.Is(err, fs.ErrNotExist) { + return false, nil + } else if err != nil { + return true, err + } + return true, nil +} + +func (p *cGroupInfoImpl) GetLastMemUsage() float64 { + if p.lastCGroupMemStat != nil { + return float64(p.lastCGroupMemStat.Usage) / float64(p.lastCGroupMemStat.UsageLimit) + } + return 0 +} + +func (p *cGroupInfoImpl) GetLastCPUUsage() float64 { + return p.cgroupCpuCalc.lastCalculatedPercent +} + +func (p *cGroupInfoImpl) updateCGroupStats() error { + control, err := cgroup2.Load("/") + if err != nil { + return fmt.Errorf("failed to get cgroup mem stats %v", err) + } + metrics, err := control.Stat() + if err != nil { + return fmt.Errorf("failed to get cgroup mem stats %v", err) + } + // Only update if a limit has been set + if metrics.Memory.UsageLimit != 0 { + p.lastCGroupMemStat = metrics.Memory + } + + err = p.cgroupCpuCalc.updateCpuUsage(metrics) + if err != nil { + return fmt.Errorf("failed to get cgroup cpu usage %v", err) + } + return nil +} + +type cgroupCpuCalc struct { + lastRefresh time.Time + lastCpuUsage uint64 + lastCalculatedPercent float64 +} + +func (p *cgroupCpuCalc) updateCpuUsage(metrics *stats.Metrics) error { + // Read CPU quota and period from cpu.max + cpuQuota, cpuPeriod, err := readCpuMax("/sys/fs/cgroup/cpu.max") + // We might simply be in a container with an unset cpu.max in which case we don't want to error + if err == nil { + // CPU usage calculation based on delta + currentCpuUsage := metrics.CPU.UsageUsec + now := time.Now() + + if p.lastCpuUsage == 0 || p.lastRefresh.IsZero() { + p.lastCpuUsage = currentCpuUsage + p.lastRefresh = now + return nil + } + + // Time passed between this and last check + timeDelta := now.Sub(p.lastRefresh).Microseconds() // Convert to microseconds + + // Calculate CPU usage percentage based on the delta + cpuUsageDelta := float64(currentCpuUsage - p.lastCpuUsage) + + if cpuQuota > 0 { + p.lastCalculatedPercent = cpuUsageDelta * float64(cpuPeriod) / float64(cpuQuota*timeDelta) + } + + // Update for next call + p.lastCpuUsage = currentCpuUsage + p.lastRefresh = now + } + + return nil +} + +// readCpuMax reads the cpu.max file to get the CPU quota and period +func readCpuMax(path string) (quota int64, period int64, err error) { + data, err := os.ReadFile(path) + if err != nil { + return 0, 0, err + } + parts := strings.Fields(string(data)) + if len(parts) != 2 { + return 0, 0, errors.New("invalid format in cpu.max") + } + + // Parse the quota (first value) + if parts[0] == "max" { + quota = 0 // Unlimited quota + } else { + quota, err = strconv.ParseInt(parts[0], 10, 64) + if err != nil { + return 0, 0, err + } + } + + // Parse the period (second value) + period, err = strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return 0, 0, err + } + + return quota, period, nil +} diff --git a/contrib/resourcetuner/cgroups_notlinux.go b/contrib/resourcetuner/cgroups_notlinux.go new file mode 100644 index 000000000..9b00efb15 --- /dev/null +++ b/contrib/resourcetuner/cgroups_notlinux.go @@ -0,0 +1,48 @@ +// The MIT License +// +// Copyright (c) 2020 Temporal Technologies Inc. All rights reserved. +// +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +//go:build !linux + +package resourcetuner + +import "errors" + +func newCGroupInfo() cGroupInfo { + return &cGroupInfoImpl{} +} + +type cGroupInfoImpl struct { +} + +func (p *cGroupInfoImpl) Update() (bool, error) { + return false, errors.New("cgroup is not supported on this platform") +} + +func (p *cGroupInfoImpl) GetLastMemUsage() float64 { + return 0 +} + +func (p *cGroupInfoImpl) GetLastCPUUsage() float64 { + return 0 +} diff --git a/contrib/resourcetuner/go.mod b/contrib/resourcetuner/go.mod index fcd41a5f9..5169c4d36 100644 --- a/contrib/resourcetuner/go.mod +++ b/contrib/resourcetuner/go.mod @@ -5,16 +5,20 @@ go 1.21 toolchain go1.22.5 require ( - github.com/shirou/gopsutil/v4 v4.24.6 + github.com/containerd/cgroups/v3 v3.0.3 + github.com/shirou/gopsutil/v4 v4.24.8 github.com/stretchr/testify v1.9.0 go.einride.tech/pid v0.1.3 - go.temporal.io/sdk v1.28.1 + go.temporal.io/sdk v1.29.1 ) require ( + github.com/cilium/ebpf v0.11.0 // indirect + github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect github.com/go-ole/go-ole v1.2.6 // indirect + github.com/godbus/dbus/v5 v5.0.4 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/mock v1.6.0 // indirect github.com/google/uuid v1.6.0 // indirect @@ -22,11 +26,13 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/nexus-rpc/sdk-go v0.0.10 // indirect + github.com/opencontainers/runtime-spec v1.0.2 // indirect github.com/pborman/uuid v1.2.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/robfig/cron v1.2.0 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/sirupsen/logrus v1.9.0 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect diff --git a/contrib/resourcetuner/go.sum b/contrib/resourcetuner/go.sum index 73306d9cd..047ee4a6f 100644 --- a/contrib/resourcetuner/go.sum +++ b/contrib/resourcetuner/go.sum @@ -2,8 +2,14 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cilium/ebpf v0.11.0 h1:V8gS/bTCCjX9uUnkUFUpPsksM8n1lXBAvHcpiFk1X2Y= +github.com/cilium/ebpf v0.11.0/go.mod h1:WE7CZAnqOL2RouJ4f1uyNhqr2P4CCvXFIqdRDUgWsVs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/containerd/cgroups/v3 v3.0.3 h1:S5ByHZ/h9PMe5IOQoN7E+nMc2UcLEM/V48DGDJ9kip0= +github.com/containerd/cgroups/v3 v3.0.3/go.mod h1:8HBe7V3aWGLFPd/k03swSIsGjZhHI2WzJmticMgVuz0= +github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -13,11 +19,15 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw= github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA= +github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= +github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -54,6 +64,8 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/nexus-rpc/sdk-go v0.0.10 h1:7jEPUlsghxoD4OJ2H8YbFJ1t4wbxsUef7yZgBfyY3uA= github.com/nexus-rpc/sdk-go v0.0.10/go.mod h1:TpfkM2Cw0Rlk9drGkoiSMpFqflKTiQLWUNyKJjF8mKQ= +github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0= +github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= @@ -67,13 +79,15 @@ github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= -github.com/shirou/gopsutil/v4 v4.24.6 h1:9qqCSYF2pgOU+t+NgJtp7Co5+5mHF/HyKBUckySQL64= -github.com/shirou/gopsutil/v4 v4.24.6/go.mod h1:aoebb2vxetJ/yIDZISmduFvVNPHqXQ9SEJwRXxkf0RA= +github.com/shirou/gopsutil/v4 v4.24.8 h1:pVQjIenQkIhqO81mwTaXjTzOMT7d3TZkf43PlVFHENI= +github.com/shirou/gopsutil/v4 v4.24.8/go.mod h1:wE0OrJtj4dG+hYkxqDH3QiBICdKSf04/npcvLLc/oRg= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= @@ -99,6 +113,8 @@ go.temporal.io/api v1.39.0 h1:pbhcfvNDB7mllb8lIBqPcg+m6LMG/IhTpdiFxe+0mYk= go.temporal.io/api v1.39.0/go.mod h1:1WwYUMo6lao8yl0371xWUm13paHExN5ATYT/B7QtFis= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -145,6 +161,7 @@ golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= diff --git a/contrib/resourcetuner/resourcetuner.go b/contrib/resourcetuner/resourcetuner.go index 69b2bc4f5..a52a53e28 100644 --- a/contrib/resourcetuner/resourcetuner.go +++ b/contrib/resourcetuner/resourcetuner.go @@ -25,18 +25,30 @@ package resourcetuner import ( "context" "errors" + "runtime" "sync" "time" "github.com/shirou/gopsutil/v4/cpu" "github.com/shirou/gopsutil/v4/mem" "go.einride.tech/pid" + "go.temporal.io/sdk/log" "go.temporal.io/sdk/worker" ) type ResourceBasedTunerOptions struct { + // TargetMem is the target overall system memory usage as value 0 and 1 that the controller will + // attempt to maintain. Must be set nonzero. TargetMem float64 + // TargetCpu is the target overall system CPU usage as value 0 and 1 that the controller will + // attempt to maintain. Must be set nonzero. TargetCpu float64 + // Passed to ResourceBasedSlotSupplierOptions.RampThrottle for activities. + // If not set, the default value is 50ms. + ActivityRampThrottle time.Duration + // Passed to ResourceBasedSlotSupplierOptions.RampThrottle for workflows. + // If not set, the default value is 0ms. + WorkflowRampThrottle time.Duration } // NewResourceBasedTuner creates a WorkerTuner that dynamically adjusts the number of slots based @@ -50,10 +62,19 @@ func NewResourceBasedTuner(opts ResourceBasedTunerOptions) (worker.WorkerTuner, controller := NewResourceController(options) wfSS := &ResourceBasedSlotSupplier{controller: controller, options: defaultWorkflowResourceBasedSlotSupplierOptions()} + if opts.WorkflowRampThrottle != 0 { + wfSS.options.RampThrottle = opts.WorkflowRampThrottle + } actSS := &ResourceBasedSlotSupplier{controller: controller, options: defaultActivityResourceBasedSlotSupplierOptions()} + if opts.ActivityRampThrottle != 0 { + actSS.options.RampThrottle = opts.ActivityRampThrottle + } laSS := &ResourceBasedSlotSupplier{controller: controller, options: defaultActivityResourceBasedSlotSupplierOptions()} + if opts.ActivityRampThrottle != 0 { + laSS.options.RampThrottle = opts.ActivityRampThrottle + } nexusSS := &ResourceBasedSlotSupplier{controller: controller, options: defaultWorkflowResourceBasedSlotSupplierOptions()} compositeTuner, err := worker.NewCompositeTuner(worker.CompositeTunerOptions{ @@ -163,7 +184,7 @@ func (r *ResourceBasedSlotSupplier) TryReserveSlot(info worker.SlotReservationIn numIssued := info.NumIssuedSlots() if numIssued < r.options.MinSlots || (numIssued < r.options.MaxSlots && time.Since(r.lastSlotIssuedAt) > r.options.RampThrottle) { - decision, err := r.controller.pidDecision() + decision, err := r.controller.pidDecision(info.Logger()) if err != nil { info.Logger().Error("Error calculating resource usage", "error", err) return nil @@ -188,10 +209,14 @@ func (r *ResourceBasedSlotSupplier) MaxSlots() int { type SystemInfoSupplier interface { // GetMemoryUsage returns the current system memory usage as a fraction of total memory between // 0 and 1. - GetMemoryUsage() (float64, error) + GetMemoryUsage(infoContext *SystemInfoContext) (float64, error) // GetCpuUsage returns the current system CPU usage as a fraction of total CPU usage between 0 // and 1. - GetCpuUsage() (float64, error) + GetCpuUsage(infoContext *SystemInfoContext) (float64, error) +} + +type SystemInfoContext struct { + Logger log.Logger } // ResourceControllerOptions contains configurable parameters for a ResourceController. @@ -262,7 +287,9 @@ type ResourceController struct { func NewResourceController(options ResourceControllerOptions) *ResourceController { var infoSupplier SystemInfoSupplier if options.InfoSupplier == nil { - infoSupplier = &psUtilSystemInfoSupplier{} + infoSupplier = &psUtilSystemInfoSupplier{ + cGroupInfo: newCGroupInfo(), + } } else { infoSupplier = options.InfoSupplier } @@ -286,15 +313,15 @@ func NewResourceController(options ResourceControllerOptions) *ResourceControlle } } -func (rc *ResourceController) pidDecision() (bool, error) { +func (rc *ResourceController) pidDecision(logger log.Logger) (bool, error) { rc.mu.Lock() defer rc.mu.Unlock() - memUsage, err := rc.infoSupplier.GetMemoryUsage() + memUsage, err := rc.infoSupplier.GetMemoryUsage(&SystemInfoContext{Logger: logger}) if err != nil { return false, err } - cpuUsage, err := rc.infoSupplier.GetCpuUsage() + cpuUsage, err := rc.infoSupplier.GetCpuUsage(&SystemInfoContext{Logger: logger}) if err != nil { return false, err } @@ -302,7 +329,6 @@ func (rc *ResourceController) pidDecision() (bool, error) { // Never allow going over the memory target return false, nil } - //fmt.Printf("mem: %f, cpu: %f\n", memUsage, cpuUsage) elapsedTime := time.Since(rc.lastRefresh) // This shouldn't be possible with real implementations, but if the elapsed time is 0 the // PID controller can produce NaNs. @@ -326,27 +352,54 @@ func (rc *ResourceController) pidDecision() (bool, error) { } type psUtilSystemInfoSupplier struct { - mu sync.Mutex + logger log.Logger + mu sync.Mutex + lastRefresh time.Time + lastMemStat *mem.VirtualMemoryStat lastCpuUsage float64 - lastRefresh time.Time + + stopTryingToGetCGroupInfo bool + cGroupInfo cGroupInfo } -func (p *psUtilSystemInfoSupplier) GetMemoryUsage() (float64, error) { - if err := p.maybeRefresh(); err != nil { +type cGroupInfo interface { + // Update requests an update of the cgroup stats. This is a no-op if not in a cgroup. Returns + // true if cgroup stats should continue to be updated, false if not in a cgroup or the returned + // error is considered unrecoverable. + Update() (bool, error) + // GetLastMemUsage returns last known memory usage as a fraction of the cgroup limit. 0 if not + // in a cgroup or limit is not set. + GetLastMemUsage() float64 + // GetLastCPUUsage returns last known CPU usage as a fraction of the cgroup limit. 0 if not in a + // cgroup or limit is not set. + GetLastCPUUsage() float64 +} + +func (p *psUtilSystemInfoSupplier) GetMemoryUsage(infoContext *SystemInfoContext) (float64, error) { + if err := p.maybeRefresh(infoContext); err != nil { return 0, err } + lastCGroupMem := p.cGroupInfo.GetLastMemUsage() + if lastCGroupMem != 0 { + return lastCGroupMem, nil + } return p.lastMemStat.UsedPercent / 100, nil } -func (p *psUtilSystemInfoSupplier) GetCpuUsage() (float64, error) { - if err := p.maybeRefresh(); err != nil { +func (p *psUtilSystemInfoSupplier) GetCpuUsage(infoContext *SystemInfoContext) (float64, error) { + if err := p.maybeRefresh(infoContext); err != nil { return 0, err } + + lastCGroupCPU := p.cGroupInfo.GetLastCPUUsage() + if lastCGroupCPU != 0 { + return lastCGroupCPU, nil + } return p.lastCpuUsage / 100, nil } -func (p *psUtilSystemInfoSupplier) maybeRefresh() error { +func (p *psUtilSystemInfoSupplier) maybeRefresh(infoContext *SystemInfoContext) error { if time.Since(p.lastRefresh) < 100*time.Millisecond { return nil } @@ -360,16 +413,24 @@ func (p *psUtilSystemInfoSupplier) maybeRefresh() error { defer cancelFn() memStat, err := mem.VirtualMemoryWithContext(ctx) if err != nil { - println("Refresh error: ", err) return err } cpuUsage, err := cpu.PercentWithContext(ctx, 0, false) if err != nil { - println("Refresh error: ", err) return err } + p.lastMemStat = memStat p.lastCpuUsage = cpuUsage[0] + + if runtime.GOOS == "linux" && !p.stopTryingToGetCGroupInfo { + continueUpdates, err := p.cGroupInfo.Update() + if err != nil { + infoContext.Logger.Warn("Failed to get cgroup stats", "error", err) + } + p.stopTryingToGetCGroupInfo = !continueUpdates + } + p.lastRefresh = time.Now() return nil } diff --git a/contrib/resourcetuner/resourcetuner_test.go b/contrib/resourcetuner/resourcetuner_test.go index 88840624b..f05a19f61 100644 --- a/contrib/resourcetuner/resourcetuner_test.go +++ b/contrib/resourcetuner/resourcetuner_test.go @@ -26,6 +26,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "go.temporal.io/sdk/internal/log" ) type FakeSystemInfoSupplier struct { @@ -33,15 +34,16 @@ type FakeSystemInfoSupplier struct { cpuUse float64 } -func (f FakeSystemInfoSupplier) GetMemoryUsage() (float64, error) { +func (f FakeSystemInfoSupplier) GetMemoryUsage(_ *SystemInfoContext) (float64, error) { return f.memUse, nil } -func (f FakeSystemInfoSupplier) GetCpuUsage() (float64, error) { +func (f FakeSystemInfoSupplier) GetCpuUsage(_ *SystemInfoContext) (float64, error) { return f.cpuUse, nil } func TestPidDecisions(t *testing.T) { + logger := &log.NoopLogger{} fakeSupplier := &FakeSystemInfoSupplier{memUse: 0.5, cpuUse: 0.5} rcOpts := DefaultResourceControllerOptions() rcOpts.MemTargetPercent = 0.8 @@ -50,7 +52,7 @@ func TestPidDecisions(t *testing.T) { rc := NewResourceController(rcOpts) for i := 0; i < 10; i++ { - decision, err := rc.pidDecision() + decision, err := rc.pidDecision(logger) assert.NoError(t, err) assert.True(t, decision) @@ -61,7 +63,7 @@ func TestPidDecisions(t *testing.T) { fakeSupplier.memUse = 0.8 fakeSupplier.cpuUse = 0.9 for i := 0; i < 10; i++ { - decision, err := rc.pidDecision() + decision, err := rc.pidDecision(logger) assert.NoError(t, err) assert.False(t, decision) } @@ -69,7 +71,7 @@ func TestPidDecisions(t *testing.T) { fakeSupplier.memUse = 0.7 fakeSupplier.cpuUse = 0.9 for i := 0; i < 10; i++ { - decision, err := rc.pidDecision() + decision, err := rc.pidDecision(logger) assert.NoError(t, err) assert.False(t, decision) } @@ -77,7 +79,7 @@ func TestPidDecisions(t *testing.T) { fakeSupplier.memUse = 0.7 fakeSupplier.cpuUse = 0.7 for i := 0; i < 10; i++ { - decision, err := rc.pidDecision() + decision, err := rc.pidDecision(logger) assert.NoError(t, err) assert.True(t, decision) } diff --git a/test/go.mod b/test/go.mod index a030c9cfd..3d3fe794d 100644 --- a/test/go.mod +++ b/test/go.mod @@ -16,31 +16,37 @@ require ( go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/trace v1.28.0 go.temporal.io/api v1.39.0 - go.temporal.io/sdk v1.28.1 + go.temporal.io/sdk v1.29.1 go.temporal.io/sdk/contrib/opentelemetry v0.1.0 go.temporal.io/sdk/contrib/opentracing v0.0.0-00010101000000-000000000000 go.temporal.io/sdk/contrib/resourcetuner v0.1.0 go.temporal.io/sdk/contrib/tally v0.1.0 - go.uber.org/goleak v1.1.11 + go.uber.org/goleak v1.1.12 google.golang.org/grpc v1.66.0 google.golang.org/protobuf v1.34.2 ) require ( + github.com/cilium/ebpf v0.11.0 // indirect + github.com/containerd/cgroups/v3 v3.0.3 // indirect + github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect + github.com/godbus/dbus/v5 v5.0.4 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/opencontainers/runtime-spec v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/robfig/cron v1.2.0 // indirect - github.com/shirou/gopsutil/v4 v4.24.6 // indirect + github.com/shirou/gopsutil/v4 v4.24.8 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/sirupsen/logrus v1.9.0 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect diff --git a/test/go.sum b/test/go.sum index 47a657f60..8d4a26a35 100644 --- a/test/go.sum +++ b/test/go.sum @@ -13,8 +13,14 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/cactus/go-statsd-client/statsd v0.0.0-20200423205355-cb0885a1018c/go.mod h1:l/bIBLeOl9eX+wxJAzxS4TveKRtAqlyDpHjhkfO0MEI= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cilium/ebpf v0.11.0 h1:V8gS/bTCCjX9uUnkUFUpPsksM8n1lXBAvHcpiFk1X2Y= +github.com/cilium/ebpf v0.11.0/go.mod h1:WE7CZAnqOL2RouJ4f1uyNhqr2P4CCvXFIqdRDUgWsVs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/containerd/cgroups/v3 v3.0.3 h1:S5ByHZ/h9PMe5IOQoN7E+nMc2UcLEM/V48DGDJ9kip0= +github.com/containerd/cgroups/v3 v3.0.3/go.mod h1:8HBe7V3aWGLFPd/k03swSIsGjZhHI2WzJmticMgVuz0= +github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -24,6 +30,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw= github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA= +github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= +github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -38,6 +46,8 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -104,6 +114,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nexus-rpc/sdk-go v0.0.10 h1:7jEPUlsghxoD4OJ2H8YbFJ1t4wbxsUef7yZgBfyY3uA= github.com/nexus-rpc/sdk-go v0.0.10/go.mod h1:TpfkM2Cw0Rlk9drGkoiSMpFqflKTiQLWUNyKJjF8mKQ= +github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0= +github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= @@ -135,8 +147,8 @@ github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= -github.com/shirou/gopsutil/v4 v4.24.6 h1:9qqCSYF2pgOU+t+NgJtp7Co5+5mHF/HyKBUckySQL64= -github.com/shirou/gopsutil/v4 v4.24.6/go.mod h1:aoebb2vxetJ/yIDZISmduFvVNPHqXQ9SEJwRXxkf0RA= +github.com/shirou/gopsutil/v4 v4.24.8 h1:pVQjIenQkIhqO81mwTaXjTzOMT7d3TZkf43PlVFHENI= +github.com/shirou/gopsutil/v4 v4.24.8/go.mod h1:wE0OrJtj4dG+hYkxqDH3QiBICdKSf04/npcvLLc/oRg= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= @@ -144,6 +156,8 @@ github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnj github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= @@ -185,8 +199,8 @@ go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -250,6 +264,7 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=