Skip to content

Commit

Permalink
Merge pull request #68 from MikeSpreitzer/add-observability
Browse files Browse the repository at this point in the history
✨ Offer /metrics and /debug/pprof from controller and agent
  • Loading branch information
MikeSpreitzer authored May 29, 2024
2 parents 9107a25 + 6922e8a commit 52ef690
Show file tree
Hide file tree
Showing 4 changed files with 123 additions and 15 deletions.
15 changes: 13 additions & 2 deletions cmd/ocm-status-addon/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (

"github.com/kubestellar/ocm-status-addon/pkg/agent"
"github.com/kubestellar/ocm-status-addon/pkg/controller"
"github.com/kubestellar/ocm-status-addon/pkg/observability"
)

func main() {
Expand Down Expand Up @@ -82,22 +83,31 @@ func newCommand(logConfig *logs.LoggingConfiguration, features featuregate.Featu
}

type agentController struct {
NameToWrapped map[string]*pflag.Flag
ObservabilityOptions observability.ObservabilityOptions[*pflag.FlagSet]
NameToWrapped map[string]*pflag.Flag
}

func newControllerCommand() *cobra.Command {
ac := agentController{NameToWrapped: make(map[string]*pflag.Flag)}
agentObservability := agent.NewObservabilityOptions()
ac := agentController{
ObservabilityOptions: observability.ObservabilityOptions[*pflag.FlagSet]{
MetricsBindAddr: ":9280",
PprofBindAddr: ":9282",
},
NameToWrapped: make(map[string]*pflag.Flag)}
agentLogConfig := logs.NewLoggingConfiguration()
agentUserOptions := agent.NewAgentUserOptions()
flagsOnAgent := pflag.NewFlagSet("on-agent", pflag.ContinueOnError)
flagsFromAgent := pflag.NewFlagSet("from-agent", pflag.ContinueOnError)
agentObservability.AddToFlagSet(flagsOnAgent)
logs.AddFlags(agentLogConfig, flagsOnAgent)
agentUserOptions.AddToFlagSet(flagsFromAgent)
cmd := cmdfactory.
NewControllerCommandConfig("status-addon-controller", version.Get(), ac.runController).
NewCommand()
cmd.Use = "controller"
cmd.Short = "Start the addon controller"
ac.ObservabilityOptions.AddToFlagSet(cmd.PersistentFlags())
for connector, flagSet := range map[string]*pflag.FlagSet{"on": flagsOnAgent, "from": flagsFromAgent} {
flagSet.VisitAll(func(flag *pflag.Flag) {
wrapped := *flag
Expand All @@ -123,6 +133,7 @@ func (ac *agentController) getPropagatedSettings(*clusterv1.ManagedCluster, *add
}

func (ac *agentController) runController(ctx context.Context, kubeConfig *rest.Config) error {
ac.ObservabilityOptions.StartServing(ctx)
addonClient, err := addonv1alpha1client.NewForConfig(kubeConfig)
if err != nil {
return err
Expand Down
29 changes: 19 additions & 10 deletions pkg/agent/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ package agent

import (
"context"
"flag"
"os"

// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
Expand All @@ -40,6 +39,7 @@ import (

v1alpha1 "github.com/kubestellar/ocm-status-addon/api/v1alpha1"
clientopts "github.com/kubestellar/ocm-status-addon/pkg/client-options"
"github.com/kubestellar/ocm-status-addon/pkg/observability"
)

var (
Expand Down Expand Up @@ -73,7 +73,7 @@ func NewAgentCommand(addonName string) *cobra.Command {

// AgentOptions defines the flags for workload agent
type AgentOptions struct {
MetricsAddr string
ObservabilityOptions observability.ObservabilityOptions[*pflag.FlagSet]
EnableLeaderElection bool
ProbeAddr string
HubKubeconfigFile string
Expand All @@ -91,8 +91,16 @@ type AgentUserOptions struct {
// NewAgentOptions returns the flags with default value set
func NewAgentOptions(addonName string) *AgentOptions {
return &AgentOptions{
AddonName: addonName,
AgentUserOptions: NewAgentUserOptions()}
ObservabilityOptions: NewObservabilityOptions(),
AddonName: addonName,
AgentUserOptions: NewAgentUserOptions()}
}

func NewObservabilityOptions() observability.ObservabilityOptions[*pflag.FlagSet] {
return observability.ObservabilityOptions[*pflag.FlagSet]{
MetricsBindAddr: ":8080",
PprofBindAddr: ":8082",
}
}

func NewAgentUserOptions() AgentUserOptions {
Expand All @@ -104,23 +112,23 @@ func NewAgentUserOptions() AgentUserOptions {

func (o *AgentOptions) AddFlags(cmd *cobra.Command) {
flags := cmd.PersistentFlags()
o.ObservabilityOptions.AddToFlagSet(flags)
// This command only supports reading from config
flags.StringVar(&o.HubKubeconfigFile, "hub-kubeconfig", o.HubKubeconfigFile,
"Location of kubeconfig file to connect to hub cluster.")
flags.StringVar(&o.SpokeClusterName, "cluster-name", o.SpokeClusterName, "Name of spoke cluster.")
flags.StringVar(&o.AddonNamespace, "addon-namespace", o.AddonNamespace, "Installation namespace of addon.")
flags.StringVar(&o.AddonName, "addon-name", o.AddonName, "name of the addon.")
flag.StringVar(&o.MetricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
flag.StringVar(&o.ProbeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
flag.BoolVar(&o.EnableLeaderElection, "leader-elect", false,
flags.StringVar(&o.ProbeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
flags.BoolVar(&o.EnableLeaderElection, "leader-elect", false,
"Enable leader election for controller manager. "+
"Enabling this will ensure there is only one active controller manager.")
o.AgentUserOptions.AddToFlagSet(flags)
}

func (o *AgentUserOptions) AddToFlagSet(flags *pflag.FlagSet) {
o.LocalLimits.AddFlags(flags)
o.HubLimits.AddFlags(flags)
o.LocalLimits.AddToFlagSet(flags)
o.HubLimits.AddToFlagSet(flags)
}

func (o *AgentOptions) RunAgent(ctx context.Context, kubeconfig *rest.Config) error {
Expand All @@ -132,7 +140,8 @@ func (o *AgentOptions) RunAgent(ctx context.Context, kubeconfig *rest.Config) er
managedConfig = o.LocalLimits.LimitConfig(managedConfig)
mgr, err := ctrl.NewManager(managedConfig, ctrl.Options{
Scheme: scheme,
MetricsBindAddress: o.MetricsAddr,
MetricsBindAddress: o.ObservabilityOptions.MetricsBindAddr,
PprofBindAddress: o.ObservabilityOptions.PprofBindAddr,
Port: 9443,
HealthProbeBindAddress: o.ProbeAddr,
LeaderElection: o.EnableLeaderElection,
Expand Down
6 changes: 3 additions & 3 deletions pkg/client-options/client-options.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,13 +58,13 @@ func NewClientOptions[FS FlagSet](name string, description string) *ClientOption
}
}

func (opts *ClientLimits[FS]) AddFlags(flags FS) {
func (opts *ClientLimits[FS]) AddToFlagSet(flags FS) {
flags.Float64Var(&opts.QPS, opts.name+"-qps", opts.QPS, "Max average requests/sec for "+opts.description)
flags.IntVar(&opts.Burst, opts.name+"-burst", opts.Burst, "Allowed burst in requests/sec for "+opts.description)
}

func (opts *ClientOptions[FS]) AddFlags(flags FS) {
opts.ClientLimits.AddFlags(flags)
func (opts *ClientOptions[FS]) AddToFlagSet(flags FS) {
opts.ClientLimits.AddToFlagSet(flags)
flags.StringVar(&opts.loadingRules.ExplicitPath, opts.name+"-kubeconfig", opts.loadingRules.ExplicitPath, "Path to the kubeconfig file to use for "+opts.description)
flags.StringVar(&opts.overrides.CurrentContext, opts.name+"-context", opts.overrides.CurrentContext, "The name of the kubeconfig context to use for "+opts.description)
flags.StringVar(&opts.overrides.Context.AuthInfo, opts.name+"-user", opts.overrides.Context.AuthInfo, "The name of the kubeconfig user to use for "+opts.description)
Expand Down
88 changes: 88 additions & 0 deletions pkg/observability/observability.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
/*
Copyright 2024 The KubeStellar Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package observability

import (
"context"
"net"
"net/http"

"k8s.io/apiserver/pkg/server/mux"
"k8s.io/apiserver/pkg/server/routes"
"k8s.io/component-base/metrics/legacyregistry"
_ "k8s.io/component-base/metrics/prometheus/clientgo"
_ "k8s.io/component-base/metrics/prometheus/version"
"k8s.io/klog/v2"
)

type FlagSet interface {
Float64Var(p *float64, name string, value float64, usage string)
IntVar(p *int, name string, value int, usage string)
StringVar(p *string, name string, value string, usage string)
}

// ObservabilityOptions covers offering Prometheus metrics and /debug/pprof .
type ObservabilityOptions[FS FlagSet] struct {

// MetricsBindAddr is the local `:$port` or `$host:$port`
// that the listening socket gets bound to.
// More specifically, this is the sort of string that can
// be used as the `Addr` in a `net/http.Server`.
MetricsBindAddr string

// PprofBindAddr is the local `:$port` or `$host:$port`
// that the listening socket gets bound to.
// More specifically, this is the sort of string that can
// be used as the `Addr` in a `net/http.Server`.
PprofBindAddr string
}

func (opts *ObservabilityOptions[FS]) AddToFlagSet(flags FS) {
flags.StringVar(&opts.MetricsBindAddr, "metrics-bind-addr", opts.MetricsBindAddr, "[host]:port at which to listen for HTTP requests for Prometheus /metrics requests")
flags.StringVar(&opts.PprofBindAddr, "pprof-bind-addr", opts.PprofBindAddr, "[host]:port at which to listen for HTTP requests for go /debug/pprof requests")
}

func (opts *ObservabilityOptions[FS]) StartServing(ctx context.Context) {
logger := klog.FromContext(ctx)
go func() {
metricsServer := http.Server{
Addr: opts.MetricsBindAddr,
Handler: legacyregistry.Handler(),
BaseContext: func(net.Listener) context.Context { return ctx },
}
err := metricsServer.ListenAndServe()
if err != nil {
logger.Error(err, "Failed to serve Prometheus metrics", "bindAddress", opts.MetricsBindAddr)
panic(err)
}
}()

go func() {
mymux := mux.NewPathRecorderMux("transport-controller")
pprofServer := http.Server{
Addr: opts.PprofBindAddr,
Handler: mymux,
BaseContext: func(net.Listener) context.Context { return ctx },
}
routes.Profiling{}.Install(mymux)
err := pprofServer.ListenAndServe()
if err != nil {
logger.Error(err, "Failure in serving /debug/pprof", "bindAddress", opts.PprofBindAddr)
panic(err)
}
}()
}

0 comments on commit 52ef690

Please sign in to comment.