Skip to content

Commit

Permalink
refactoring SPKS and adding base metrics
Browse files Browse the repository at this point in the history
  • Loading branch information
[email protected] committed Jul 5, 2024
1 parent 6a9bb3e commit 083bda9
Show file tree
Hide file tree
Showing 9 changed files with 251 additions and 157 deletions.
6 changes: 3 additions & 3 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ require (
github.com/exoscale/egoscale v0.90.1
github.com/go-logr/logr v1.3.0
github.com/go-logr/zapr v1.3.0
github.com/prometheus/client_golang v1.19.0
github.com/prometheus/client_golang v1.19.1
github.com/stretchr/testify v1.8.4
github.com/urfave/cli/v2 v2.24.4
github.com/vshn/provider-cloudscale v0.5.0
Expand Down Expand Up @@ -79,13 +79,13 @@ require (
golang.org/x/crypto v0.18.0 // indirect
golang.org/x/exp v0.0.0-20230307190834-24139beb5833 // indirect
golang.org/x/net v0.20.0 // indirect
golang.org/x/sys v0.16.0 // indirect
golang.org/x/sys v0.17.0 // indirect
golang.org/x/term v0.16.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/time v0.5.0 // indirect
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.32.0 // indirect
google.golang.org/protobuf v1.33.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
Expand Down
6 changes: 6 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -358,6 +358,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU=
github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k=
github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE=
github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
Expand Down Expand Up @@ -626,6 +628,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
Expand Down Expand Up @@ -814,6 +818,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
Expand Down
59 changes: 55 additions & 4 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,16 @@ package main
import (
"context"
"fmt"
"net/http"
"os"
"os/signal"
"runtime"
"syscall"
"time"

"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/urfave/cli/v2"
"github.com/vshn/billing-collector-cloudservices/pkg/cmd"
"github.com/vshn/billing-collector-cloudservices/pkg/log"
Expand All @@ -22,6 +26,40 @@ var (

appName = "billing-collector-cloudservices"
appLongName = "Metrics collector which gathers metrics information for cloud services"

odooFailed = promauto.NewCounter(prometheus.CounterOpts{
Name: "billing_cloud_collector_http_requests_odoo_failed_total",
Help: "Total number of failed HTTP requests to Odoo",
})
odooSucceeded = promauto.NewCounter(prometheus.CounterOpts{
Name: "billing_cloud_collector_http_requests_odoo_succeeded_total",
Help: "Total number of successful HTTP requests to Odoo",
})

providerFailed = promauto.NewCounter(prometheus.CounterOpts{
Name: "billing_cloud_collector_http_requests_provider_failed_total",
Help: "Total number of failed HTTP requests to the cloud provider",
})

providerSucceeded = promauto.NewCounter(prometheus.CounterOpts{
Name: "billing_cloud_collector_http_requests_provider_succeeded_total",
Help: "Total number of successful HTTP requests to the cloud provider",
})

providerMetrics = map[string]prometheus.Counter{
"providerFailed": providerFailed,
"providerSucceeded": providerSucceeded,
}

odooMetrics = map[string]prometheus.Counter{
"odooFailed": odooFailed,
"odooSucceeded": odooSucceeded,
}

allMetrics = map[string]map[string]prometheus.Counter{
"odooMetrics": odooMetrics,
"providerMetrics": providerMetrics,
}
)

func init() {
Expand All @@ -32,19 +70,33 @@ func init() {
func main() {
ctx, stop, app := newApp()
defer stop()

go func() {
http.Handle("/metrics", promhttp.Handler())
err := http.ListenAndServe(":2112", nil)
if err != nil {
fmt.Println("Error starting prometheus server: ", err.Error())
}
os.Exit(1)
}()

err := app.RunContext(ctx, os.Args)
// If required flags aren't set, it will return with error before we could set up logging
if err != nil {
_, _ = fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}

}

func newApp() (context.Context, context.CancelFunc, *cli.App) {
var (
logLevel int
logFormat string
)

ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)

app := &cli.App{
Name: appName,
Usage: appLongName,
Expand Down Expand Up @@ -120,9 +172,9 @@ func newApp() (context.Context, context.CancelFunc, *cli.App) {
return nil
},
Commands: []*cli.Command{
cmd.ExoscaleCmds(),
cmd.CloudscaleCmds(),
cmd.SpksCMD(),
cmd.ExoscaleCmds(allMetrics),
cmd.CloudscaleCmds(allMetrics),
cmd.SpksCMD(allMetrics, ctx),
},
ExitErrHandler: func(c *cli.Context, err error) {
if err != nil {
Expand All @@ -132,6 +184,5 @@ func newApp() (context.Context, context.CancelFunc, *cli.App) {
},
}

ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
return ctx, stop, app
}
8 changes: 7 additions & 1 deletion pkg/cloudscale/objectstorage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"fmt"
"time"

"github.com/prometheus/client_golang/prometheus"
"github.com/vshn/billing-collector-cloudservices/pkg/controlAPI"
"github.com/vshn/billing-collector-cloudservices/pkg/kubernetes"
"github.com/vshn/billing-collector-cloudservices/pkg/log"
Expand All @@ -30,13 +31,14 @@ type ObjectStorage struct {
clusterId string
cloudZone string
uomMapping map[string]string
providerMetrics map[string]prometheus.Counter
}

const (
namespaceLabel = "crossplane.io/claim-namespace"
)

func NewObjectStorage(client *cloudscale.Client, k8sClient k8s.Client, controlApiClient k8s.Client, salesOrder, clusterId string, cloudZone string, uomMapping map[string]string) (*ObjectStorage, error) {
func NewObjectStorage(client *cloudscale.Client, k8sClient k8s.Client, controlApiClient k8s.Client, salesOrder, clusterId string, cloudZone string, uomMapping map[string]string, providerMetrics map[string]prometheus.Counter) (*ObjectStorage, error) {
return &ObjectStorage{
client: client,
k8sClient: k8sClient,
Expand All @@ -45,6 +47,7 @@ func NewObjectStorage(client *cloudscale.Client, k8sClient k8s.Client, controlAp
clusterId: clusterId,
cloudZone: cloudZone,
uomMapping: uomMapping,
providerMetrics: providerMetrics,
}, nil
}

Expand All @@ -56,7 +59,10 @@ func (o *ObjectStorage) GetMetrics(ctx context.Context, billingDate time.Time) (
bucketMetricsRequest := cloudscale.BucketMetricsRequest{Start: billingDate, End: billingDate}
bucketMetrics, err := o.client.Metrics.GetBucketMetrics(ctx, &bucketMetricsRequest)
if err != nil {
o.providerMetrics["providerFailed"].Inc()
return nil, err
} else {
o.providerMetrics["providerSucceeded"].Inc()
}

// Fetch organisations in case salesOrder is missing
Expand Down
7 changes: 4 additions & 3 deletions pkg/cmd/cloudscale.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"sync"
"time"

"github.com/prometheus/client_golang/prometheus"
"github.com/vshn/billing-collector-cloudservices/pkg/odoo"

"github.com/cloudscale-ch/cloudscale-go-sdk/v2"
Expand All @@ -19,7 +20,7 @@ import (
const defaultTextForRequiredFlags = "<required>"
const defaultTextForOptionalFlags = "<optional>"

func CloudscaleCmds() *cli.Command {
func CloudscaleCmds(allMetrics map[string]map[string]prometheus.Counter) *cli.Command {
var (
apiToken string
kubeconfig string
Expand Down Expand Up @@ -102,14 +103,14 @@ func CloudscaleCmds() *cli.Command {
return fmt.Errorf("k8s control client: %w", err)
}

odooClient := odoo.NewOdooAPIClient(c.Context, odooURL, odooOauthTokenURL, odooClientId, odooClientSecret, logger)
odooClient := odoo.NewOdooAPIClient(c.Context, odooURL, odooOauthTokenURL, odooClientId, odooClientSecret, logger, allMetrics["odooMetrics"])

location, err := time.LoadLocation("Europe/Zurich")
if err != nil {
return fmt.Errorf("load loaction: %w", err)
}

o, err := cs.NewObjectStorage(cloudscaleClient, k8sClient, k8sControlClient, salesOrder, clusterId, cloudZone, mapping)
o, err := cs.NewObjectStorage(cloudscaleClient, k8sClient, k8sControlClient, salesOrder, clusterId, cloudZone, mapping, allMetrics["providerMetrics"])
if err != nil {
return fmt.Errorf("object storage: %w", err)
}
Expand Down
9 changes: 5 additions & 4 deletions pkg/cmd/exoscale.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"sync"
"time"

"github.com/prometheus/client_golang/prometheus"
"github.com/vshn/billing-collector-cloudservices/pkg/odoo"

"github.com/urfave/cli/v2"
Expand All @@ -19,7 +20,7 @@ func addCommandName(c *cli.Context) error {
return nil
}

func ExoscaleCmds() *cli.Command {
func ExoscaleCmds(allMetrics map[string]map[string]prometheus.Counter) *cli.Command {
var (
secret string
accessKey string
Expand Down Expand Up @@ -112,14 +113,14 @@ func ExoscaleCmds() *cli.Command {
return fmt.Errorf("k8s control client: %w", err)
}

odooClient := odoo.NewOdooAPIClient(c.Context, odooURL, odooOauthTokenURL, odooClientId, odooClientSecret, logger)
odooClient := odoo.NewOdooAPIClient(c.Context, odooURL, odooOauthTokenURL, odooClientId, odooClientSecret, logger, allMetrics["odooMetrics"])

if collectInterval < 1 || collectInterval > 23 {
// Set to run once a day after billingHour in case the collectInterval is out of boundaries
collectInterval = 23
}

o, err := exoscale.NewObjectStorage(exoscaleClient, k8sClient, k8sControlClient, salesOrder, clusterId, cloudZone, mapping)
o, err := exoscale.NewObjectStorage(exoscaleClient, k8sClient, k8sControlClient, salesOrder, clusterId, cloudZone, mapping, allMetrics["providerMetrics"])
if err != nil {
return fmt.Errorf("objectbucket service: %w", err)
}
Expand Down Expand Up @@ -192,7 +193,7 @@ func ExoscaleCmds() *cli.Command {
return fmt.Errorf("k8s control client: %w", err)
}

odooClient := odoo.NewOdooAPIClient(c.Context, odooURL, odooOauthTokenURL, odooClientId, odooClientSecret, logger)
odooClient := odoo.NewOdooAPIClient(c.Context, odooURL, odooOauthTokenURL, odooClientId, odooClientSecret, logger, allMetrics["odooMetrics"])

if collectInterval < 1 || collectInterval > 24 {
// Set to run once a day after billingHour in case the collectInterval is out of boundaries
Expand Down
Loading

0 comments on commit 083bda9

Please sign in to comment.