diff --git a/CHANGELOG.md b/CHANGELOG.md index 010cfa5..87d6eef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## v1.6 + +- Save two months of cvs files instead of only current month. +- When uploading files for previous month, exporter validates that the number of files to upload match with the number of days of the previous month. + ## v1.5 - Added costAdjustments for cpuCost, gpuCost, ramCost, pvCost, networkCost and loadBalancerCost diff --git a/README.md b/README.md index e721588..a1b79f5 100644 --- a/README.md +++ b/README.md @@ -31,8 +31,8 @@ The app is configured using environment variables defined in a .env file. The fo - `IDLE_BY_NODE` - Idle allocations are created on a per node basis. - `FILE_ROTATION` - whether to delete files generated during the previous month (or the month before the previous month if INCLUDE_PREVIOUS_MONTH is set to true). Valid values are true or false. - `FILE_PATH` - the path where the CSV files are stored -- `INCLUDE_PREVIOUS_MONTH` - whether to include data from previous month to export process. Valid values are true or false. - +- `INCLUDE_PREVIOUS_MONTH` - whether to include data from previous month to export process, only if we have files from every day of the previous month.. Valid values are true or false. + To use this app, run: ```bash @@ -131,9 +131,9 @@ You should see 200/201s in the logs, which indicates that the exporter is workin | flexera.shard | string | `"NAM"` | Shard ("NAM", "EU", "AU") | | image.pullPolicy | string | `"Always"` | | | image.repository | string | `"public.ecr.aws/flexera/cbi-oi-kubecost-exporter"` | | -| image.tag | string | `"1.5"` | | +| image.tag | string | `"1.6"` | | | imagePullSecrets | list | `[]` | | -| includePreviousMonth | bool | `false` | Include data from previous month to export process | +| includePreviousMonth | bool | `false` | Include data from previous month to the export process, only if we have files from every day of the previous month. | | kubecost.aggregation | string | `"pod"` | Aggregation Level ("namespace", "controller", "pod") | | kubecost.apiPath | string | `"/model/"` | Base path for the Kubecost API endpoints | | kubecost.host | string | `"kubecost-cost-analyzer.kubecost.svc.cluster.local:9090"` | Default kubecost-cost-analyzer service host on the current cluster. For current cluster is serviceName.namespaceName.svc.cluster.local | diff --git a/helm-chart/Chart.yaml b/helm-chart/Chart.yaml index e265521..7d8694e 100644 --- a/helm-chart/Chart.yaml +++ b/helm-chart/Chart.yaml @@ -6,10 +6,10 @@ description: Kubecost exporter helm chart for Kubernetes # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.5.0 +version: 1.6.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "1.5" +appVersion: "1.6" diff --git a/helm-chart/README.md b/helm-chart/README.md index fbe3996..5cf9b3f 100644 --- a/helm-chart/README.md +++ b/helm-chart/README.md @@ -1,6 +1,6 @@ # cbi-oi-kubecost-exporter -![Version: 1.5.0](https://img.shields.io/badge/Version-1.5.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.5](https://img.shields.io/badge/AppVersion-1.5-informational?style=flat-square) +![Version: 1.6.0](https://img.shields.io/badge/Version-1.6.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.6](https://img.shields.io/badge/AppVersion-1.6-informational?style=flat-square) ### Kubecost exporter helm chart for Kubernetes @@ -94,9 +94,9 @@ You should see 200/201s in the logs, which indicates that the exporter is workin | flexera.shard | string | `"NAM"` | Shard ("NAM", "EU", "AU") | | image.pullPolicy | string | `"Always"` | | | image.repository | string | `"public.ecr.aws/flexera/cbi-oi-kubecost-exporter"` | | -| image.tag | string | `"1.5"` | | +| image.tag | string | `"1.6"` | | | imagePullSecrets | list | `[]` | | -| includePreviousMonth | bool | `false` | Include data from previous month to export process | +| includePreviousMonth | bool | `false` | Include data from previous month to the export process, only if we have files from every day of the previous month. | | kubecost.aggregation | string | `"pod"` | Aggregation Level ("namespace", "controller", "pod") | | kubecost.apiPath | string | `"/model/"` | Base path for the Kubecost API endpoints | | kubecost.host | string | `"kubecost-cost-analyzer.kubecost.svc.cluster.local:9090"` | Default kubecost-cost-analyzer service host on the current cluster. For current cluster is serviceName.namespaceName.svc.cluster.local | diff --git a/helm-chart/cbi-oi-kubecost-exporter-1.6.0.tgz b/helm-chart/cbi-oi-kubecost-exporter-1.6.0.tgz new file mode 100644 index 0000000..9695f36 Binary files /dev/null and b/helm-chart/cbi-oi-kubecost-exporter-1.6.0.tgz differ diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml index a084147..4b89388 100644 --- a/helm-chart/values.yaml +++ b/helm-chart/values.yaml @@ -5,7 +5,7 @@ image: repository: public.ecr.aws/flexera/cbi-oi-kubecost-exporter pullPolicy: Always - tag: "1.5" + tag: "1.6" imagePullSecrets: [] @@ -63,5 +63,6 @@ fileRotation: true # -- Filepath to mount persistent volume filePath: "/var/kubecost" -# -- Include data from previous month to export process +# -- Include data from previous month to the export process, only if we have files from every day of the previous month. includePreviousMonth: false + diff --git a/index.yaml b/index.yaml index 9e378ca..a6be1af 100644 --- a/index.yaml +++ b/index.yaml @@ -1,9 +1,19 @@ apiVersion: v1 entries: cbi-oi-kubecost-exporter: + - apiVersion: v2 + appVersion: "1.6" + created: "2023-11-10T11:47:21.844983-06:00" + description: Kubecost exporter helm chart for Kubernetes + digest: f998790d9229fce36398ef487edf6685a9a29e359dd2c78191c4d16344d47acd + name: cbi-oi-kubecost-exporter + type: application + urls: + - helm-chart/cbi-oi-kubecost-exporter-1.6.0.tgz + version: 1.6.0 - apiVersion: v2 appVersion: "1.5" - created: "2023-10-27T17:36:14.535347-06:00" + created: "2023-11-10T11:47:21.844695-06:00" description: Kubecost exporter helm chart for Kubernetes digest: a0503a1af7b86a8f0769009de5a71a3613e8503a5f59cf86874d483f0f5f2abb name: cbi-oi-kubecost-exporter @@ -13,7 +23,7 @@ entries: version: 1.5.0 - apiVersion: v2 appVersion: "1.4" - created: "2023-10-27T17:36:14.535157-06:00" + created: "2023-11-10T11:47:21.844329-06:00" description: Kubecost exporter helm chart for Kubernetes digest: 5b296e921178d84717345be19b92a4eea0c9ca1c65410128dec596692220c8e2 name: cbi-oi-kubecost-exporter @@ -23,7 +33,7 @@ entries: version: 1.4.2 - apiVersion: v2 appVersion: "1.4" - created: "2023-10-27T17:36:14.534766-06:00" + created: "2023-11-10T11:47:21.844085-06:00" description: Kubecost exporter helm chart for Kubernetes digest: ab6992a0af9f24d5c090db71679b7f274f3576b1cfbe91325e0d6e9af317657e name: cbi-oi-kubecost-exporter @@ -33,7 +43,7 @@ entries: version: 1.4.1 - apiVersion: v2 appVersion: "1.4" - created: "2023-10-27T17:36:14.534469-06:00" + created: "2023-11-10T11:47:21.843733-06:00" description: Kubecost exporter helm chart for Kubernetes digest: 870030e989ba274f8b0f9d122a4ada24a4d29ef2c071408bf46d86e18942effd name: cbi-oi-kubecost-exporter @@ -43,7 +53,7 @@ entries: version: 1.4.0 - apiVersion: v2 appVersion: "1.3" - created: "2023-10-27T17:36:14.534135-06:00" + created: "2023-11-10T11:47:21.843328-06:00" description: Kubecost exporter helm chart for Kubernetes digest: 0d967927bfab5f4bb6d40ce062710679e98b145c4b3f38709c0bf5d216ad8606 name: cbi-oi-kubecost-exporter @@ -53,7 +63,7 @@ entries: version: 1.3.0 - apiVersion: v2 appVersion: "1.2" - created: "2023-10-27T17:36:14.533242-06:00" + created: "2023-11-10T11:47:21.84241-06:00" description: Kubecost exporter helm chart for Kubernetes digest: d0c724c7bb085a1801e27f60ff748bc69aba14425cfd27fd132f99966692ed80 name: cbi-oi-kubecost-exporter @@ -63,7 +73,7 @@ entries: version: 1.2.0 - apiVersion: v2 appVersion: "1.1" - created: "2023-10-27T17:36:14.53289-06:00" + created: "2023-11-10T11:47:21.842028-06:00" description: Kubecost exporter helm chart for Kubernetes digest: c6f2681575b704b5934efea2357921896d55c4ad5e09692f31ab92b9e614cfea name: cbi-oi-kubecost-exporter @@ -71,4 +81,4 @@ entries: urls: - helm-chart/cbi-oi-kubecost-exporter-1.1.0.tgz version: 1.1.0 -generated: "2023-10-27T17:36:14.53225-06:00" +generated: "2023-11-10T11:47:21.841228-06:00" diff --git a/main.go b/main.go index 017e104..559d840 100644 --- a/main.go +++ b/main.go @@ -124,11 +124,12 @@ type ( App struct { Config - aggregation string - filesToUpload map[string]map[string]struct{} - client *http.Client - lastInvoiceDate time.Time - invoiceMonths []string + aggregation string + filesToUpload map[string]map[string]struct{} + client *http.Client + lastInvoiceDate time.Time + invoiceMonths []string + mandatoryFileSavingPeriodStartDate time.Time } ) @@ -198,6 +199,11 @@ func (a *App) updateFromKubecost() { monthOfData := d.Format("2006-01") var csvFile = fmt.Sprintf(path.Join(a.FilePath, "kubecost-%v.csv"), d.Format("2006-01-02")) + if j.Code != http.StatusOK { + log.Println("Kubecost API response code different than 200, skipping") + continue + } + // If the data obtained is empty, skip the iteration, because it might overwrite a previously obtained file for the same range time _, previousFileCreated := a.filesToUpload[monthOfData][csvFile] if len(data) == 0 && previousFileCreated { @@ -243,6 +249,18 @@ func (a *App) uploadToFlexera() { billUploadURL := fmt.Sprintf("https://%s/optima/orgs/%s/billUploads", shardDict[a.Shard], a.OrgID) for month, files := range a.filesToUpload { + + if len(files) == 0 { + log.Println("No files to upload for month", month) + continue + } + + // if we try to upload files for previous month, we need to check if we have files for all days in the month + if !a.isCurrentMonth(month) && a.DaysInMonth(month) != len(files) { + log.Println("Skipping month", month, "because not all days have a file to upload") + continue + } + authHeaders := map[string]string{"Authorization": "Bearer " + accessToken} billUpload := map[string]string{"billConnectId": a.BillConnectID, "billingPeriod": month} @@ -374,7 +392,7 @@ func (a *App) updateFileList() { if t, err := time.Parse("kubecost-2006-01-02.csv", file.Name()); err == nil { if a.dateInInvoiceRange(t) { a.filesToUpload[t.Format("2006-01")][path.Join(a.FilePath, file.Name())] = struct{}{} - } else if a.FileRotation { + } else if a.FileRotation && !a.dateInMandatoryFileSavingPeriod(t) { if err = os.Remove(path.Join(a.FilePath, file.Name())); err != nil { log.Printf("error removing file %s: %v", file.Name(), err) } @@ -420,6 +438,23 @@ func (a *App) dateInInvoiceRange(date time.Time) bool { return false } +func (a *App) dateInMandatoryFileSavingPeriod(date time.Time) bool { + return !date.Before(a.mandatoryFileSavingPeriodStartDate) +} + +func (a *App) isCurrentMonth(month string) bool { + return time.Now().Local().Format("2006-01") == month +} + +func (a *App) DaysInMonth(month string) int { + date, err := time.Parse("2006-01", month) + if err != nil { + return 0 + } + numDays := date.AddDate(0, 1, 0).Sub(date).Hours() / 24 + return int(numDays) +} + func newApp() *App { lastInvoiceDate := time.Now().Local().AddDate(0, 0, -1) a := App{ @@ -435,6 +470,9 @@ func newApp() *App { if a.IncludePreviousMonth { a.invoiceMonths = append(a.invoiceMonths, a.lastInvoiceDate.AddDate(0, -1, 0).Format("2006-01")) } + // The mandatory file saving period is the period since the first day of the previous month of last invoice date + previousMonthOfLastInvoiceDate := lastInvoiceDate.AddDate(0, -1, 0) + a.mandatoryFileSavingPeriodStartDate = time.Date(previousMonthOfLastInvoiceDate.Year(), previousMonthOfLastInvoiceDate.Month(), 1, 0, 0, 0, 0, previousMonthOfLastInvoiceDate.Location()) for _, month := range a.invoiceMonths { a.filesToUpload[month] = make(map[string]struct{}) diff --git a/main_test.go b/main_test.go index d50fbb8..02adc94 100644 --- a/main_test.go +++ b/main_test.go @@ -122,21 +122,22 @@ func Test_newApp(t *testing.T) { } expectedConfig := Config{ - RefreshToken: "test_refresh_token", - OrgID: "test_org_id", - BillConnectID: "test_bill_connect_id", - Shard: "NAM", - KubecostHost: "test_kubecost_host", - Aggregation: "controller", - ShareNamespaces: "test_namespace1,test_namespace2", - Idle: true, - IdleByNode: false, - ShareIdle: false, - ShareTenancyCosts: true, - Multiplier: 1.0, - FileRotation: true, - FilePath: "/var/kubecost", - KubecostAPIPath: "/model/", + RefreshToken: "test_refresh_token", + OrgID: "test_org_id", + BillConnectID: "test_bill_connect_id", + Shard: "NAM", + KubecostHost: "test_kubecost_host", + Aggregation: "controller", + ShareNamespaces: "test_namespace1,test_namespace2", + Idle: true, + IdleByNode: false, + ShareIdle: false, + ShareTenancyCosts: true, + Multiplier: 1.0, + FileRotation: true, + FilePath: "/var/kubecost", + KubecostAPIPath: "/model/", + IncludePreviousMonth: false, } if !reflect.DeepEqual(a.Config, expectedConfig) { t.Errorf("Config is %+v, expected %+v", a.Config, expectedConfig) @@ -361,3 +362,129 @@ func TestApp_getCSVRows(t *testing.T) { }) } } + +func TestApp_dateInMandatoryFileSavingPeriod(t *testing.T) { + type args struct { + customStartDateOfMandatoryPeriod *time.Time + date time.Time + } + + firstDayOfMonth, _ := time.Parse("2006-01-02", "2023-10-01") + tests := []struct { + name string + args args + want bool + }{ + { + name: "success: date in mandatory period", + args: args{ + customStartDateOfMandatoryPeriod: nil, + date: time.Now().Local().AddDate(0, 0, -1), + }, + want: true, + }, + { + name: "success: one month before current date", + args: args{ + customStartDateOfMandatoryPeriod: nil, + date: time.Now().Local().AddDate(0, -1, 0), + }, + want: true, + }, + { + name: "fail: one day before the mandatory period", + args: args{ + customStartDateOfMandatoryPeriod: &firstDayOfMonth, + date: firstDayOfMonth.AddDate(0, 0, -1), + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + a := newApp() + if tt.args.customStartDateOfMandatoryPeriod != nil { + a.mandatoryFileSavingPeriodStartDate = *tt.args.customStartDateOfMandatoryPeriod + } + if got := a.dateInMandatoryFileSavingPeriod(tt.args.date); got != tt.want { + t.Errorf("dateInMandatoryFileSavingPeriod() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestApp_isCurrentMonth(t *testing.T) { + type args struct { + month string + } + now := time.Now().Local() + tests := []struct { + name string + args args + want bool + }{ + { + name: "success: current month", + args: args{month: now.Format("2006-01")}, + want: true, + }, + { + name: "fail: previous month", + args: args{month: now.AddDate(0, -1, 0).Format("2006-01")}, + want: false, + }, + { + name: "fail: next month", + args: args{month: now.AddDate(0, 1, 0).Format("2006-01")}, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + a := newApp() + if got := a.isCurrentMonth(tt.args.month); got != tt.want { + t.Errorf("isCurrentMonth() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestApp_DaysInMonth(t *testing.T) { + type args struct { + month string + } + tests := []struct { + name string + args args + want int + }{ + { + name: "success: October 2023", + args: args{month: "2023-10"}, + want: 31, + }, + { + name: "success: February 2023", + args: args{month: "2023-02"}, + want: 28, + }, + { + name: "success: February 2024", + args: args{month: "2024-02"}, + want: 29, + }, + { + name: "success: November 2023", + args: args{month: "2023-11"}, + want: 30, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + a := newApp() + if got := a.DaysInMonth(tt.args.month); got != tt.want { + t.Errorf("DaysInMonth() = %v, want %v", got, tt.want) + } + }) + } +}