diff --git a/go.mod b/go.mod index a2df1a7..bc8ffd3 100644 --- a/go.mod +++ b/go.mod @@ -9,8 +9,6 @@ require ( github.com/exoscale/egoscale v0.90.1 github.com/go-logr/logr v1.3.0 github.com/go-logr/zapr v1.3.0 - github.com/prometheus/client_golang v1.17.0 - github.com/prometheus/common v0.45.0 github.com/stretchr/testify v1.8.4 github.com/urfave/cli/v2 v2.24.4 github.com/vshn/provider-cloudscale v0.5.0 @@ -69,7 +67,9 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_golang v1.17.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spf13/afero v1.9.3 // indirect diff --git a/go.sum b/go.sum index 467697f..85900f6 100644 --- a/go.sum +++ b/go.sum @@ -291,8 +291,6 @@ github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -346,8 +344,6 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg= diff --git a/pkg/cloudscale/metrics_test.go b/pkg/cloudscale/metrics_test.go deleted file mode 100644 index ddb55ca..0000000 --- a/pkg/cloudscale/metrics_test.go +++ /dev/null @@ -1,72 +0,0 @@ -//go:build integration - -package cloudscale - -import ( - "fmt" - "testing" - "time" - - "github.com/cloudscale-ch/cloudscale-go-sdk/v2" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// assertEqualfUint64 implements the functionality of assert.Equalf for uint64, because assert.Equalf cannot print uint64 correctly. -// See https://github.com/stretchr/testify/issues/400 -func assertEqualfUint64(t *testing.T, expected uint64, actual uint64, msg string, args ...interface{}) bool { - if expected != actual { - return assert.Fail(t, fmt.Sprintf("Not equal: \n"+ - "expected: %d\n"+ - "actual : %d", expected, actual)) - } - return true -} - -func TestAccumulateBucketMetricsForObjectsUser(t *testing.T) { - zone := "cloudscale" - organization := "inity" - namespace := "testnamespace" - - location, err := time.LoadLocation("Europe/Zurich") - assert.NoError(t, err) - - now := time.Now().In(location) - date := time.Date(now.Year(), now.Month(), now.Day()-1, 0, 0, 0, 0, now.Location()) - - // build input data structure - bucketMetricsInterval := []cloudscale.BucketMetricsInterval{ - { - Start: date, - End: date, - Usage: cloudscale.BucketMetricsIntervalUsage{ - Requests: 5, - StorageBytes: 1000000, - SentBytes: 2000000, - }, - }, - } - bucketMetricsData := cloudscale.BucketMetricsData{ - TimeSeries: bucketMetricsInterval, - } - - accumulated := make(map[AccumulateKey]uint64) - assert.NoError(t, accumulateBucketMetricsForObjectsUser(accumulated, bucketMetricsData, namespace)) - - require.Len(t, accumulated, 3, "incorrect amount of values 'accumulated'") - - key := AccumulateKey{ - Zone: zone, - Namespace: namespace, - Start: date, - } - - key.ProductId = "appcat_object-storage-requests" - assertEqualfUint64(t, uint64(5), accumulated[key], "incorrect value in %s", key) - - key.ProductId = "appcat_object-storage-storage" - assertEqualfUint64(t, uint64(1000000), accumulated[key], "incorrect value in %s", key) - - key.ProductId = "appcat_object-storage-traffic-out" - assertEqualfUint64(t, uint64(2000000), accumulated[key], "incorrect value in %s", key) -} diff --git a/pkg/cloudscale/objecstorage_integration_test.go b/pkg/cloudscale/objecstorage_integration_test.go deleted file mode 100644 index 0ebf09c..0000000 --- a/pkg/cloudscale/objecstorage_integration_test.go +++ /dev/null @@ -1,158 +0,0 @@ -//go:build integration - -package cloudscale - -import ( - "os" - "testing" - "time" - - "github.com/cloudscale-ch/cloudscale-go-sdk/v2" - "github.com/stretchr/testify/suite" - "github.com/vshn/billing-collector-cloudservices/pkg/prom" - "github.com/vshn/billing-collector-cloudservices/pkg/test" - cloudscalev1 "github.com/vshn/provider-cloudscale/apis/cloudscale/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -const objectStorageBind = ":9123" - -type ObjectStorageTestSuite struct { - test.Suite - billingDate time.Time -} - -func (ts *ObjectStorageTestSuite) SetupSuite() { - cloudscaleCRDsPath := os.Getenv("CLOUDSCALE_CRDS_PATH") - ts.Require().NotZero(cloudscaleCRDsPath, "missing env variable CLOUDSCALE_CRDS_PATH") - - ts.SetupEnv([]string{cloudscaleCRDsPath}, objectStorageBind) - - ts.RegisterScheme(cloudscalev1.SchemeBuilder.AddToScheme) -} - -// TestMetrics sets up a couple of buckets and associated namespaces with organizations set. -// The cloudscale client is set up with an HTTP replay recorder (go-vcr) which looks into testdata/ for recorded -// HTTP responses. -// For simplicity reasons, the recorder only uses URL and method for matching recorded responses. The upside -// of this is it doesn't matter when we execute the tests since the date used to fetch metrics doesn't matter for matching. -// Downside of course is it doesn't do any validation related to the date matching but that's not the main thing to test here. -func (ts *ObjectStorageTestSuite) TestMetrics() { - assert := ts.Assert() - ctx := ts.Context - - o, cancel := ts.setupObjectStorage() - defer cancel() - - assertMetrics := []test.PromMetric{ - { - Category: "cloudscale:example-project", - Product: "appcat_object-storage-requests:cloudscale:example-company:example-project", - Value: 100.001, - }, - { - Category: "cloudscale:example-project", - Product: "appcat_object-storage-storage:cloudscale:example-company:example-project", - Value: 1000.000004096, - }, - { - Category: "cloudscale:example-project", - Product: "appcat_object-storage-traffic-out:cloudscale:example-company:example-project", - Value: 50, - }, - { - Category: "cloudscale:next-big-thing", - Product: "appcat_object-storage-requests:cloudscale:big-corporation:next-big-thing", - Value: 0.001, - }, - { - Category: "cloudscale:next-big-thing", - Product: "appcat_object-storage-storage:cloudscale:big-corporation:next-big-thing", - Value: 0, - }, - { - Category: "cloudscale:next-big-thing", - Product: "appcat_object-storage-traffic-out:cloudscale:big-corporation:next-big-thing", - Value: 0, - }, - } - nameNsMap := map[string]string{ - "example-project-a": "example-project", - "example-project-b": "example-project", - "next-big-thing-a": "next-big-thing", - } - nsTenantMap := map[string]string{ - "example-project": "example-company", - "next-big-thing": "big-corporation", - } - ts.ensureBuckets(nameNsMap) - - createdNs := make(map[string]bool) - for _, ns := range nameNsMap { - if _, ok := createdNs[ns]; !ok { - ts.EnsureNS(ns, map[string]string{organizationLabel: nsTenantMap[ns]}) - createdNs[ns] = true - } - } - - testDate := time.Date(2023, 1, 11, 0, 0, 0, 0, time.Local) - metrics, err := o.Accumulate(ctx, testDate) - assert.NoError(err) - - // This test doesn't divide the values, it tests with 1 hour remaining for the day - assert.NoError(Export(metrics, 23)) - assert.NoError(test.AssertPromMetrics(assert, assertMetrics, objectStorageBind), "cannot assert prom metrics") - prom.ResetAppCatMetric() - - // This test simulates exporting the metrics from 06:00 for the rest of the day - // For that we'll have to divide the values by 18 to match up. - for i := range assertMetrics { - assertMetrics[i].Value = assertMetrics[i].Value / float64(18) - } - assert.NoError(Export(metrics, 6)) - assert.NoError(test.AssertPromMetrics(assert, assertMetrics, objectStorageBind), "cannot assert prom metrics") - prom.ResetAppCatMetric() -} - -func (ts *ObjectStorageTestSuite) ensureBuckets(nameNsMap map[string]string) { - resources := make([]client.Object, 0) - for name, ns := range nameNsMap { - resources = append(resources, &cloudscalev1.Bucket{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Labels: map[string]string{namespaceLabel: ns}, - }, - Spec: cloudscalev1.BucketSpec{ - ForProvider: cloudscalev1.BucketParameters{BucketName: name}, - }, - }) - } - ts.EnsureResources(resources...) -} - -func (ts *ObjectStorageTestSuite) setupObjectStorage() (*ObjectStorage, func()) { - assert := ts.Assert() - httpClient, cancel, err := test.RequestRecorder(ts.T(), "testdata/cloudscale/"+ts.T().Name()) - assert.NoError(err) - - c := cloudscale.NewClient(httpClient) - // required to be set when recording new response. - if apiToken := os.Getenv("CLOUDSCALE_API_TOKEN"); apiToken != "" { - c.AuthToken = apiToken - ts.T().Log("API token set") - } else { - ts.T().Log("no API token provided") - } - - o, err := NewObjectStorage(c, ts.Client, "") - assert.NoError(err) - - return o, cancel -} - -// In order for 'go test' to run this suite, we need to create -// a normal test function and pass our suite to suite.Run -func TestObjectStorageTestSuite(t *testing.T) { - suite.Run(t, new(ObjectStorageTestSuite)) -} diff --git a/pkg/exoscale/dbaas_integration_test.go b/pkg/exoscale/dbaas_integration_test.go deleted file mode 100644 index 2a30dcd..0000000 --- a/pkg/exoscale/dbaas_integration_test.go +++ /dev/null @@ -1,212 +0,0 @@ -//go:build integration - -package exoscale - -import ( - "os" - "strings" - "testing" - "time" - - "github.com/exoscale/egoscale/v2/oapi" - "github.com/stretchr/testify/suite" - "github.com/vshn/billing-collector-cloudservices/pkg/kubernetes" - "github.com/vshn/billing-collector-cloudservices/pkg/prom" - "github.com/vshn/billing-collector-cloudservices/pkg/test" - exoscalev1 "github.com/vshn/provider-exoscale/apis/exoscale/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime/schema" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -const dbaasBind = ":9124" - -type DBaaSTestSuite struct { - test.Suite - billingDate time.Time -} - -func (ts *DBaaSTestSuite) SetupSuite() { - exoscaleCRDPaths := os.Getenv("EXOSCALE_CRDS_PATH") - ts.Require().NotZero(exoscaleCRDPaths, "missing env variable EXOSCALE_CRDS_PATH") - - ts.SetupEnv([]string{exoscaleCRDPaths}, dbaasBind) - - ts.RegisterScheme(exoscalev1.SchemeBuilder.AddToScheme) -} - -func (ts *DBaaSTestSuite) TestMetrics() { - assert := ts.Assert() - ctx := ts.Context - - ds, cancel := ts.setupDBaaS() - defer cancel() - - type testcase struct { - gvk schema.GroupVersionKind - ns string - plan string - dbType oapi.DbaasServiceTypeName - } - - nsTenantMap := map[string]string{ - "example-project": "example-company", - "next-big-thing": "big-corporation", - } - for ns, tenant := range nsTenantMap { - ts.EnsureNS(ns, map[string]string{kubernetes.OrganizationLabel: tenant}) - } - - tests := make(map[string]testcase) - for key, gvk := range groupVersionKinds { - plan := "hobbyist-2" - // kafka has no hobbyist plan - if key == "kafka" { - plan = "startup-2" - } - tests[key+"-example-project"] = testcase{ - gvk: gvk, - ns: "example-project", - plan: plan, - dbType: oapi.DbaasServiceTypeName(key), - } - } - - tests["pg-expensive-example-project"] = testcase{ - gvk: groupVersionKinds["pg"], - ns: "example-project", - plan: "premium-225", - dbType: "pg", - } - - tests["pg-next-big-thing"] = testcase{ - gvk: groupVersionKinds["pg"], - ns: "next-big-thing", - plan: "business-225", - dbType: "pg", - } - - type expectation struct { - value float64 - tc testcase - } - expectedQuantities := make(map[Key]expectation, 0) - - for name, tc := range tests { - key := NewKey(tc.ns, tc.plan, string(tc.dbType)) - if _, ok := expectedQuantities[key]; !ok { - expectedQuantities[key] = expectation{ - value: 0, - tc: tc, - } - } - expectedQuantities[key] = expectation{ - value: expectedQuantities[key].value + 1, - tc: tc, - } - - obj := &unstructured.Unstructured{} - obj.SetUnstructuredContent(map[string]interface{}{ - "apiVersion": tc.gvk.GroupVersion().String(), - // a bit ugly, but I wanted to avoid adding more than necessary code - "kind": strings.Replace(tc.gvk.Kind, "List", "", 1), - "metadata": map[string]interface{}{ - "name": name, - "labels": map[string]string{ - namespaceLabel: tc.ns, - }, - }, - "spec": map[string]interface{}{ - "forProvider": map[string]interface{}{ - "zone": "ch-gva-2", - }, - }, - }) - ts.EnsureResources(obj) - } - - assertMetrics := []test.PromMetric{ - { - Product: "appcat_opensearch:exoscale:example-company:example-project:hobbyist-2", - Category: "exoscale:example-project", - Value: 1, - }, - { - Product: "appcat_postgres:exoscale:example-company:example-project:premium-225", - Category: "exoscale:example-project", - Value: 1, - }, - { - Product: "appcat_postgres:exoscale:big-corporation:next-big-thing:business-225", - Category: "exoscale:next-big-thing", - Value: 1, - }, - { - Product: "appcat_mysql:exoscale:example-company:example-project:hobbyist-2", - Category: "exoscale:example-project", - Value: 1, - }, - { - Product: "appcat_opensearch:exoscale:example-company:example-project:hobbyist-2", - Category: "exoscale:example-project", - Value: 1, - }, - { - Product: "appcat_redis:exoscale:example-company:example-project:hobbyist-2", - Category: "exoscale:example-project", - Value: 1, - }, - { - Product: "appcat_kafka:exoscale:example-company:example-project:startup-2", - Category: "exoscale:example-project", - Value: 1, - }, - } - - metrics, err := ds.Accumulate(ctx) - assert.NoError(err, "cannot accumulate dbaas") - - assert.NoError(Export(metrics)) - assert.NoError(test.AssertPromMetrics(assert, assertMetrics, dbaasBind), "cannot assert prom metrics") - prom.ResetAppCatMetric() - -} - -type dbaasSource struct { - dbType string - tenant string - namespace string - plan string -} - -func (ts *DBaaSTestSuite) ensureBuckets(nameNsMap map[string]string) { - resources := make([]client.Object, 0) - for name, ns := range nameNsMap { - resources = append(resources, &exoscalev1.Bucket{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Labels: map[string]string{namespaceLabel: ns}, - }, - Spec: exoscalev1.BucketSpec{ - ForProvider: exoscalev1.BucketParameters{BucketName: name}, - }, - }) - } - ts.EnsureResources(resources...) -} - -func (ts *DBaaSTestSuite) setupDBaaS() (*DBaaS, func()) { - exoClient, cancel, err := newEgoscaleClient(ts.T()) - ts.Assert().NoError(err) - - ds, err := NewDBaaS(exoClient, ts.Client, "") - ts.Assert().NoError(err) - return ds, cancel -} - -// In order for 'go test' to run this suite, we need to create -// a normal test function and pass our suite to suite.Run -func TestDBaaSTestSuite(t *testing.T) { - suite.Run(t, new(DBaaSTestSuite)) -} diff --git a/pkg/exoscale/objectstorage_integration_test.go b/pkg/exoscale/objectstorage_integration_test.go deleted file mode 100644 index 2a104b6..0000000 --- a/pkg/exoscale/objectstorage_integration_test.go +++ /dev/null @@ -1,152 +0,0 @@ -//go:build integration - -package exoscale - -import ( - "fmt" - "os" - "testing" - "time" - - egoscale "github.com/exoscale/egoscale/v2" - "github.com/stretchr/testify/suite" - "github.com/vshn/billing-collector-cloudservices/pkg/exofixtures" - "github.com/vshn/billing-collector-cloudservices/pkg/kubernetes" - "github.com/vshn/billing-collector-cloudservices/pkg/prom" - "github.com/vshn/billing-collector-cloudservices/pkg/test" - exoscalev1 "github.com/vshn/provider-exoscale/apis/exoscale/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -const objectStorageBind = ":9125" - -type ObjectStorageTestSuite struct { - test.Suite - billingDate time.Time -} - -func (ts *ObjectStorageTestSuite) SetupSuite() { - exoscaleCRDPaths := os.Getenv("EXOSCALE_CRDS_PATH") - ts.Require().NotZero(exoscaleCRDPaths, "missing env variable EXOSCALE_CRDS_PATH") - - ts.SetupEnv([]string{exoscaleCRDPaths}, objectStorageBind) - - ts.RegisterScheme(exoscalev1.SchemeBuilder.AddToScheme) -} - -type objectStorageSource struct { - namespace string - tenant string - objectType exofixtures.ObjectType - billingDate time.Time -} - -func (ts *ObjectStorageTestSuite) TestMetrics() { - assert := ts.Assert() - ctx := ts.Context - - o, cancel := ts.setupObjectStorage() - defer cancel() - - nameNsMap := map[string]string{ - "example-project-a": "example-project", - "example-project-b": "example-project", - "next-big-thing-a": "next-big-thing", - } - nsTenantMap := map[string]string{ - "example-project": "example-company", - "next-big-thing": "big-corporation", - } - ts.ensureBuckets(nameNsMap) - - for ns, tenant := range nsTenantMap { - ts.EnsureNS(ns, map[string]string{kubernetes.OrganizationLabel: tenant}) - } - - assertMetrics := []test.PromMetric{ - { - Product: "appcat_object-storage-storage:exoscale:example-company:example-project", - Category: "exoscale:example-project", - Value: 932.253897190094, - }, - { - Product: "appcat_object-storage-storage:exoscale:big-corporation:next-big-thing", - Category: "exoscale:next-big-thing", - Value: 0, - }, - } - - // This test doesn't divide the values, it tests with 1 hour remaining for the day - metrics, err := o.Accumulate(ctx, 23) - assert.NoError(err, "cannot accumulate exoscale object storage") - assert.NoError(Export(metrics)) - assert.NoError(test.AssertPromMetrics(assert, assertMetrics, objectStorageBind), "cannot assert prom metrics") - prom.ResetAppCatMetric() - - // This test simulates exporting the metrics from 06:00 for the rest of the day - // For that we'll have to divide the values by 18 to match up. - for i := range assertMetrics { - assertMetrics[i].Value = assertMetrics[i].Value / float64(18) - } - metrics, err = o.Accumulate(ctx, 6) - assert.NoError(err, "cannot accumulate exoscale object storage") - assert.NoError(Export(metrics)) - assert.NoError(test.AssertPromMetrics(assert, assertMetrics, objectStorageBind), "cannot assert prom metrics") - prom.ResetAppCatMetric() -} - -func (ts *ObjectStorageTestSuite) ensureBuckets(nameNsMap map[string]string) { - resources := make([]client.Object, 0) - for name, ns := range nameNsMap { - resources = append(resources, &exoscalev1.Bucket{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Labels: map[string]string{namespaceLabel: ns}, - }, - Spec: exoscalev1.BucketSpec{ - ForProvider: exoscalev1.BucketParameters{BucketName: name}, - }, - }) - } - ts.EnsureResources(resources...) -} - -func (ts *ObjectStorageTestSuite) setupObjectStorage() (*ObjectStorage, func()) { - exoClient, cancel, err := newEgoscaleClient(ts.T()) - ts.Assert().NoError(err) - - o, err := NewObjectStorage(exoClient, ts.Client, "") - ts.Assert().NoError(err) - return o, cancel -} - -func newEgoscaleClient(t *testing.T) (*egoscale.Client, func(), error) { - httpClient, cancel, err := test.RequestRecorder(t, "testdata/exoscale/"+t.Name()) - if err != nil { - return nil, nil, fmt.Errorf("request recorder: %w", err) - } - - apiKey := os.Getenv("EXOSCALE_API_KEY") - secret := os.Getenv("EXOSCALE_API_SECRET") - if apiKey != "" && secret != "" { - t.Log("api key & secret set") - } else { - // override empty values since otherwise egoscale complains - apiKey = "NOTVALID" - secret = "NOTVALIDSECRET" - t.Log("api key or secret not set") - } - - exoClient, err := NewClientWithOptions(apiKey, secret, egoscale.ClientOptWithHTTPClient(httpClient)) - if err != nil { - return nil, nil, fmt.Errorf("new client: %w", err) - } - return exoClient, cancel, nil -} - -// In order for 'go test' to run this suite, we need to create -// a normal test function and pass our suite to suite.Run -func TestObjectStorageTestSuite(t *testing.T) { - suite.Run(t, new(ObjectStorageTestSuite)) -} diff --git a/pkg/exoscale/objectstorage_test.go b/pkg/exoscale/objectstorage_test.go deleted file mode 100644 index a1ea313..0000000 --- a/pkg/exoscale/objectstorage_test.go +++ /dev/null @@ -1,198 +0,0 @@ -//go:build integration - -package exoscale - -import ( - "testing" - "time" - - "github.com/exoscale/egoscale/v2/oapi" - "github.com/stretchr/testify/assert" - "github.com/vshn/billing-collector-cloudservices/pkg/exofixtures" - "github.com/vshn/billing-collector-cloudservices/pkg/kubernetes" - exoscalev1 "github.com/vshn/provider-exoscale/apis/exoscale/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func TestObjectStorage_GetAggregated(t *testing.T) { - defaultKey := NewKey("default") - alphaKey := NewKey("alpha") - omegaKey := NewKey("omega") - - tests := map[string]struct { - givenSosBucketsUsage []oapi.SosBucketUsage - givenBucketDetails []BucketDetail - expectedAggregated map[Key]Aggregated - }{ - "GivenSosBucketsUsageAndBuckets_WhenMatch_ThenExpectAggregatedObjects": { - givenSosBucketsUsage: []oapi.SosBucketUsage{ - createSosBucketUsage("bucket-test-1", 1), - createSosBucketUsage("bucket-test-2", 4), - createSosBucketUsage("bucket-test-3", 9), - createSosBucketUsage("bucket-test-4", 0), - createSosBucketUsage("bucket-test-5", 5), - }, - givenBucketDetails: []BucketDetail{ - createBucketDetail("bucket-test-1", "default", "orgA"), - createBucketDetail("bucket-test-2", "alpha", "orgB"), - createBucketDetail("bucket-test-3", "alpha", "orgB"), - createBucketDetail("bucket-test-4", "omega", "orgC"), - createBucketDetail("no-metrics-bucket", "beta", "orgD"), - }, - expectedAggregated: map[Key]Aggregated{ - defaultKey: createAggregated(defaultKey, "orgA", "default", 1), - alphaKey: createAggregated(alphaKey, "orgB", "alpha", 13), - omegaKey: createAggregated(omegaKey, "orgC", "omega", 0), - }, - }, - "GivenSosBucketsUsageAndBuckets_WhenMatch_ThenExpectNoAggregatedObjects": { - givenSosBucketsUsage: []oapi.SosBucketUsage{ - createSosBucketUsage("bucket-test-1", 1), - createSosBucketUsage("bucket-test-2", 4), - }, - givenBucketDetails: []BucketDetail{ - createBucketDetail("bucket-test-3", "default", "orgA"), - createBucketDetail("bucket-test-4", "alpha", "orgB"), - createBucketDetail("bucket-test-5", "alpha", "orgB"), - }, - expectedAggregated: map[Key]Aggregated{}, - }, - "GivenSosBucketsUsageAndBuckets_WhenSosBucketsUsageEmpty_ThenExpectNoAggregatedObjects": { - givenSosBucketsUsage: []oapi.SosBucketUsage{ - createSosBucketUsage("bucket-test-1", 1), - createSosBucketUsage("bucket-test-2", 4), - }, - givenBucketDetails: []BucketDetail{}, - expectedAggregated: map[Key]Aggregated{}, - }, - "GivenSosBucketsUsageAndBuckets_WhenNoBuckets_ThenExpectNoAggregatedObjects": { - givenSosBucketsUsage: []oapi.SosBucketUsage{}, - givenBucketDetails: []BucketDetail{ - createBucketDetail("bucket-test-3", "default", "orgA"), - createBucketDetail("bucket-test-4", "alpha", "orgB"), - }, - expectedAggregated: map[Key]Aggregated{}, - }, - } - for name, tc := range tests { - t.Run(name, func(t *testing.T) { - // Given - ctx := getTestContext(t) - - // When - aggregated := getOdooMeteredBillingRecords(ctx, tc.givenSosBucketsUsage, tc.givenBucketDetails, 1) - - // Then - assert.Equal(t, tc.expectedAggregated, aggregated) - }) - } -} - -func TestObjectStorage_addOrgAndNamespaceToBucket(t *testing.T) { - tests := map[string]struct { - givenBucketList exoscalev1.BucketList - givenNamespaces map[string]string - expectedBucketDetails []BucketDetail - }{ - "GivenBucketListFromExoscale_WhenOrgAndNamespaces_ThenExpectBucketDetailObjects": { - givenBucketList: exoscalev1.BucketList{ - Items: []exoscalev1.Bucket{ - createBucket("bucket-1", "alpha", "orgA"), - createBucket("bucket-2", "beta", "orgB"), - createBucket("bucket-3", "alpha", "orgA"), - createBucket("bucket-4", "omega", "orgB"), - createBucket("bucket-5", "theta", "orgC"), - }, - }, - givenNamespaces: map[string]string{ - "alpha": "orgA", - "beta": "orgB", - "omega": "orgB", - "theta": "orgC", - }, - expectedBucketDetails: []BucketDetail{ - createBucketDetail("bucket-1", "alpha", "orgA"), - createBucketDetail("bucket-2", "beta", "orgB"), - createBucketDetail("bucket-3", "alpha", "orgA"), - createBucketDetail("bucket-4", "omega", "orgB"), - createBucketDetail("bucket-5", "theta", "orgC"), - }, - }, - "GivenBucketListFromExoscale_WhenNoOrgOrNamespaces_ThenExpectNoBucketDetailObjects": { - givenBucketList: exoscalev1.BucketList{ - Items: []exoscalev1.Bucket{ - createBucket("bucket-1", "", "orgA"), - createBucket("bucket-2", "beta", ""), - createBucket("bucket-3", "", ""), - }, - }, - givenNamespaces: map[string]string{}, - expectedBucketDetails: []BucketDetail{}, - }, - } - for name, tc := range tests { - t.Run(name, func(t *testing.T) { - // Given - ctx := getTestContext(t) - - // When - bucketDetails := addOrgAndNamespaceToBucket(ctx, tc.givenBucketList, tc.givenNamespaces) - - // Then - assert.ElementsMatch(t, tc.expectedBucketDetails, bucketDetails) - }) - } -} - -func createBucket(name, namespace, organization string) exoscalev1.Bucket { - labels := make(map[string]string) - if namespace != "" { - labels[namespaceLabel] = namespace - } - if organization != "" { - labels[kubernetes.OrganizationLabel] = organization - } - return exoscalev1.Bucket{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - Labels: labels, - }, - Spec: exoscalev1.BucketSpec{ - ForProvider: exoscalev1.BucketParameters{ - BucketName: name, - }, - }, - } -} - -func createAggregated(key Key, organization, namespace string, size float64) Aggregated { - return Aggregated{ - Key: key, - Value: size / 1024 / 1024 / 1024, - Source: exofixtures.SOSSourceString{ - Namespace: namespace, - Organization: organization, - }, - } -} - -func createBucketDetail(bucketName, namespace, organization string) BucketDetail { - return BucketDetail{ - Organization: organization, - BucketName: bucketName, - Namespace: namespace, - } -} - -func createSosBucketUsage(bucketName string, size int) oapi.SosBucketUsage { - date := time.Now() - actualSize := int64(size) - zone := oapi.ZoneName("ch-gva-2") - return oapi.SosBucketUsage{ - CreatedAt: &date, - Name: &bucketName, - Size: &actualSize, - ZoneName: &zone, - } -} diff --git a/pkg/prom/prom.go b/pkg/prom/prom.go deleted file mode 100644 index b9f6613..0000000 --- a/pkg/prom/prom.go +++ /dev/null @@ -1,54 +0,0 @@ -package prom - -import ( - "context" - "fmt" - "github.com/appuio/appuio-cloud-reporting/pkg/thanos" - "github.com/prometheus/client_golang/api" - apiv1 "github.com/prometheus/client_golang/api/prometheus/v1" - "github.com/prometheus/common/model" - "time" -) - -const ( - salesOrderPromMetrics = "control_api_organization_info" - salesOrderIDLabel = "sales_order_id" -) - -func NewPrometheusAPIClient(promURL string) (apiv1.API, error) { - rt := api.DefaultRoundTripper - rt = &thanos.PartialResponseRoundTripper{ - RoundTripper: rt, - } - - client, err := api.NewClient(api.Config{ - Address: promURL, - RoundTripper: rt, - }) - - return apiv1.NewAPI(client), err -} - -// GetSalesOrderId retrieves from prometheus the sales order id associated to orgId -func GetSalesOrderId(ctx context.Context, client apiv1.API, orgId string) (string, error) { - query := salesOrderPromMetrics + fmt.Sprintf("{name=\"%s\"}", orgId) - - res, _, err := client.Query(ctx, query, time.Now()) - if err != nil { - return "", fmt.Errorf("cannot query '%s' for organisation %s, err: %v", salesOrderPromMetrics, orgId, err) - } - samples := res.(model.Vector) - if samples.Len() > 1 { - return "", fmt.Errorf("prometheus metric '%s' has multiple results for organisation %s ", salesOrderPromMetrics, orgId) - } - - return getMetricLabel(samples[0].Metric) -} - -func getMetricLabel(m model.Metric) (string, error) { - value, ok := m[model.LabelName(salesOrderIDLabel)] - if !ok { - return "", fmt.Errorf("no '%s' label in metrics '%s'", salesOrderIDLabel, salesOrderPromMetrics) - } - return string(value), nil -} diff --git a/pkg/test/suite.go b/pkg/test/suite.go index e791e6c..7fae2fc 100644 --- a/pkg/test/suite.go +++ b/pkg/test/suite.go @@ -13,7 +13,6 @@ import ( "github.com/go-logr/logr" "github.com/stretchr/testify/suite" "github.com/vshn/billing-collector-cloudservices/pkg/log" - "github.com/vshn/billing-collector-cloudservices/pkg/prom" "gopkg.in/dnaeon/go-vcr.v3/cassette" "gopkg.in/dnaeon/go-vcr.v3/recorder" corev1 "k8s.io/api/core/v1" @@ -82,14 +81,6 @@ func (ts *Suite) SetupEnv(crdPaths []string, bindString string) { ts.Env = testEnv ts.Config = config ts.Client = k8sClient - - go func() { - assert.NoError(ts.exportMetrics(bindString), "error exportig the metrics") - }() -} - -func (ts *Suite) exportMetrics(bindString string) error { - return prom.ServeMetrics(ts.Context, bindString) } // RegisterScheme passes the current scheme to the given SchemeBuilder func.