FOPTS-2404 Save 2 months of csv files. Validate number of files to up…

…load for previous month (#27)

* FOPTS-2404 Save 2 months of csv files. Validate number of files to upload for previous month

* FOPTS-2404 Removed new env var SEND_ONLY_FULL_PREVIOUS_MONTH to don't overcomplicate configuration

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## v1.6
+
+- Save two months of cvs files instead of only current month.
+- When uploading files for previous month, exporter validates that the number of files to upload match with the number of days of the previous month.
+
 ## v1.5
 
 - Added costAdjustments for cpuCost, gpuCost, ramCost, pvCost, networkCost and loadBalancerCost

README.md

@@ -31,8 +31,8 @@ The app is configured using environment variables defined in a .env file. The fo
 -   `IDLE_BY_NODE` - Idle allocations are created on a per node basis.
 -   `FILE_ROTATION` - whether to delete files generated during the previous month (or the month before the previous month if INCLUDE_PREVIOUS_MONTH is set to true). Valid values are true or false.
 -   `FILE_PATH` - the path where the CSV files are stored
--   `INCLUDE_PREVIOUS_MONTH` - whether to include data from previous month to export process. Valid values are true or false.
-
+-   `INCLUDE_PREVIOUS_MONTH` - whether to include data from previous month to export process, only if we have files from every day of the previous month.. Valid values are true or false.
+ 
 To use this app, run:
 
 ```bash
@@ -131,9 +131,9 @@ You should see 200/201s in the logs, which indicates that the exporter is workin
 | flexera.shard | string | `"NAM"` | Shard ("NAM", "EU", "AU") |
 | image.pullPolicy | string | `"Always"` |  |
 | image.repository | string | `"public.ecr.aws/flexera/cbi-oi-kubecost-exporter"` |  |
-| image.tag | string | `"1.5"` |  |
+| image.tag | string | `"1.6"` |  |
 | imagePullSecrets | list | `[]` |  |
-| includePreviousMonth | bool | `false` | Include data from previous month to export process |
+| includePreviousMonth | bool | `false` | Include data from previous month to the export process, only if we have files from every day of the previous month. |
 | kubecost.aggregation | string | `"pod"` | Aggregation Level ("namespace", "controller", "pod") |
 | kubecost.apiPath | string | `"/model/"` | Base path for the Kubecost API endpoints |
 | kubecost.host | string | `"kubecost-cost-analyzer.kubecost.svc.cluster.local:9090"` | Default kubecost-cost-analyzer service host on the current cluster. For current cluster is serviceName.namespaceName.svc.cluster.local |

helm-chart/Chart.yaml

@@ -6,10 +6,10 @@ description: Kubecost exporter helm chart for Kubernetes
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.5.0
+version: 1.6.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "1.5"
+appVersion: "1.6"

helm-chart/README.md

@@ -1,6 +1,6 @@
 # cbi-oi-kubecost-exporter
 
-![Version: 1.5.0](https://img.shields.io/badge/Version-1.5.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.5](https://img.shields.io/badge/AppVersion-1.5-informational?style=flat-square)
+![Version: 1.6.0](https://img.shields.io/badge/Version-1.6.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.6](https://img.shields.io/badge/AppVersion-1.6-informational?style=flat-square)
 
 ### Kubecost exporter helm chart for Kubernetes
 
@@ -94,9 +94,9 @@ You should see 200/201s in the logs, which indicates that the exporter is workin
 | flexera.shard | string | `"NAM"` | Shard ("NAM", "EU", "AU") |
 | image.pullPolicy | string | `"Always"` |  |
 | image.repository | string | `"public.ecr.aws/flexera/cbi-oi-kubecost-exporter"` |  |
-| image.tag | string | `"1.5"` |  |
+| image.tag | string | `"1.6"` |  |
 | imagePullSecrets | list | `[]` |  |
-| includePreviousMonth | bool | `false` | Include data from previous month to export process |
+| includePreviousMonth | bool | `false` | Include data from previous month to the export process, only if we have files from every day of the previous month. |
 | kubecost.aggregation | string | `"pod"` | Aggregation Level ("namespace", "controller", "pod") |
 | kubecost.apiPath | string | `"/model/"` | Base path for the Kubecost API endpoints |
 | kubecost.host | string | `"kubecost-cost-analyzer.kubecost.svc.cluster.local:9090"` | Default kubecost-cost-analyzer service host on the current cluster. For current cluster is serviceName.namespaceName.svc.cluster.local |

helm-chart/values.yaml

@@ -5,7 +5,7 @@
 image:
   repository: public.ecr.aws/flexera/cbi-oi-kubecost-exporter
   pullPolicy: Always
-  tag: "1.5"
+  tag: "1.6"
 
 imagePullSecrets: []
 
@@ -63,5 +63,6 @@ fileRotation: true
 # -- Filepath to mount persistent volume
 filePath: "/var/kubecost"
 
-# -- Include data from previous month to export process
+# -- Include data from previous month to the export process, only if we have files from every day of the previous month.
 includePreviousMonth: false
+

index.yaml

@@ -1,9 +1,19 @@
 apiVersion: v1
 entries:
   cbi-oi-kubecost-exporter:
+  - apiVersion: v2
+    appVersion: "1.6"
+    created: "2023-11-10T11:47:21.844983-06:00"
+    description: Kubecost exporter helm chart for Kubernetes
+    digest: f998790d9229fce36398ef487edf6685a9a29e359dd2c78191c4d16344d47acd
+    name: cbi-oi-kubecost-exporter
+    type: application
+    urls:
+    - helm-chart/cbi-oi-kubecost-exporter-1.6.0.tgz
+    version: 1.6.0
   - apiVersion: v2
     appVersion: "1.5"
-    created: "2023-10-27T17:36:14.535347-06:00"
+    created: "2023-11-10T11:47:21.844695-06:00"
     description: Kubecost exporter helm chart for Kubernetes
     digest: a0503a1af7b86a8f0769009de5a71a3613e8503a5f59cf86874d483f0f5f2abb
     name: cbi-oi-kubecost-exporter
@@ -13,7 +23,7 @@ entries:
     version: 1.5.0
   - apiVersion: v2
     appVersion: "1.4"
-    created: "2023-10-27T17:36:14.535157-06:00"
+    created: "2023-11-10T11:47:21.844329-06:00"
     description: Kubecost exporter helm chart for Kubernetes
     digest: 5b296e921178d84717345be19b92a4eea0c9ca1c65410128dec596692220c8e2
     name: cbi-oi-kubecost-exporter
@@ -23,7 +33,7 @@ entries:
     version: 1.4.2
   - apiVersion: v2
     appVersion: "1.4"
-    created: "2023-10-27T17:36:14.534766-06:00"
+    created: "2023-11-10T11:47:21.844085-06:00"
     description: Kubecost exporter helm chart for Kubernetes
     digest: ab6992a0af9f24d5c090db71679b7f274f3576b1cfbe91325e0d6e9af317657e
     name: cbi-oi-kubecost-exporter
@@ -33,7 +43,7 @@ entries:
     version: 1.4.1
   - apiVersion: v2
     appVersion: "1.4"
-    created: "2023-10-27T17:36:14.534469-06:00"
+    created: "2023-11-10T11:47:21.843733-06:00"
     description: Kubecost exporter helm chart for Kubernetes
     digest: 870030e989ba274f8b0f9d122a4ada24a4d29ef2c071408bf46d86e18942effd
     name: cbi-oi-kubecost-exporter
@@ -43,7 +53,7 @@ entries:
     version: 1.4.0
   - apiVersion: v2
     appVersion: "1.3"
-    created: "2023-10-27T17:36:14.534135-06:00"
+    created: "2023-11-10T11:47:21.843328-06:00"
     description: Kubecost exporter helm chart for Kubernetes
     digest: 0d967927bfab5f4bb6d40ce062710679e98b145c4b3f38709c0bf5d216ad8606
     name: cbi-oi-kubecost-exporter
@@ -53,7 +63,7 @@ entries:
     version: 1.3.0
   - apiVersion: v2
     appVersion: "1.2"
-    created: "2023-10-27T17:36:14.533242-06:00"
+    created: "2023-11-10T11:47:21.84241-06:00"
     description: Kubecost exporter helm chart for Kubernetes
     digest: d0c724c7bb085a1801e27f60ff748bc69aba14425cfd27fd132f99966692ed80
     name: cbi-oi-kubecost-exporter
@@ -63,12 +73,12 @@ entries:
     version: 1.2.0
   - apiVersion: v2
     appVersion: "1.1"
-    created: "2023-10-27T17:36:14.53289-06:00"
+    created: "2023-11-10T11:47:21.842028-06:00"
     description: Kubecost exporter helm chart for Kubernetes
     digest: c6f2681575b704b5934efea2357921896d55c4ad5e09692f31ab92b9e614cfea
     name: cbi-oi-kubecost-exporter
     type: application
     urls:
     - helm-chart/cbi-oi-kubecost-exporter-1.1.0.tgz
     version: 1.1.0
-generated: "2023-10-27T17:36:14.53225-06:00"
+generated: "2023-11-10T11:47:21.841228-06:00"

main.go

@@ -124,11 +124,12 @@ type (
 
 	App struct {
 		Config
-		aggregation     string
-		filesToUpload   map[string]map[string]struct{}
-		client          *http.Client
-		lastInvoiceDate time.Time
-		invoiceMonths   []string
+		aggregation                        string
+		filesToUpload                      map[string]map[string]struct{}
+		client                             *http.Client
+		lastInvoiceDate                    time.Time
+		invoiceMonths                      []string
+		mandatoryFileSavingPeriodStartDate time.Time
 	}
 )
 
@@ -198,6 +199,11 @@ func (a *App) updateFromKubecost() {
 		monthOfData := d.Format("2006-01")
 		var csvFile = fmt.Sprintf(path.Join(a.FilePath, "kubecost-%v.csv"), d.Format("2006-01-02"))
 
+		if j.Code != http.StatusOK {
+			log.Println("Kubecost API response code different than 200, skipping")
+			continue
+		}
+
 		// If the data obtained is empty, skip the iteration, because it might overwrite a previously obtained file for the same range time
 		_, previousFileCreated := a.filesToUpload[monthOfData][csvFile]
 		if len(data) == 0 && previousFileCreated {
@@ -243,6 +249,18 @@ func (a *App) uploadToFlexera() {
 	billUploadURL := fmt.Sprintf("https://%s/optima/orgs/%s/billUploads", shardDict[a.Shard], a.OrgID)
 
 	for month, files := range a.filesToUpload {
+
+		if len(files) == 0 {
+			log.Println("No files to upload for month", month)
+			continue
+		}
+
+		// if we try to upload files for previous month, we need to check if we have files for all days in the month
+		if !a.isCurrentMonth(month) && a.DaysInMonth(month) != len(files) {
+			log.Println("Skipping month", month, "because not all days have a file to upload")
+			continue
+		}
+
 		authHeaders := map[string]string{"Authorization": "Bearer " + accessToken}
 		billUpload := map[string]string{"billConnectId": a.BillConnectID, "billingPeriod": month}
 
@@ -374,7 +392,7 @@ func (a *App) updateFileList() {
 			if t, err := time.Parse("kubecost-2006-01-02.csv", file.Name()); err == nil {
 				if a.dateInInvoiceRange(t) {
 					a.filesToUpload[t.Format("2006-01")][path.Join(a.FilePath, file.Name())] = struct{}{}
-				} else if a.FileRotation {
+				} else if a.FileRotation && !a.dateInMandatoryFileSavingPeriod(t) {
 					if err = os.Remove(path.Join(a.FilePath, file.Name())); err != nil {
 						log.Printf("error removing file %s: %v", file.Name(), err)
 					}
@@ -420,6 +438,23 @@ func (a *App) dateInInvoiceRange(date time.Time) bool {
 	return false
 }
 
+func (a *App) dateInMandatoryFileSavingPeriod(date time.Time) bool {
+	return !date.Before(a.mandatoryFileSavingPeriodStartDate)
+}
+
+func (a *App) isCurrentMonth(month string) bool {
+	return time.Now().Local().Format("2006-01") == month
+}
+
+func (a *App) DaysInMonth(month string) int {
+	date, err := time.Parse("2006-01", month)
+	if err != nil {
+		return 0
+	}
+	numDays := date.AddDate(0, 1, 0).Sub(date).Hours() / 24
+	return int(numDays)
+}
+
 func newApp() *App {
 	lastInvoiceDate := time.Now().Local().AddDate(0, 0, -1)
 	a := App{
@@ -435,6 +470,9 @@ func newApp() *App {
 	if a.IncludePreviousMonth {
 		a.invoiceMonths = append(a.invoiceMonths, a.lastInvoiceDate.AddDate(0, -1, 0).Format("2006-01"))
 	}
+	// The mandatory file saving period is the period since the first day of the previous month of last invoice date
+	previousMonthOfLastInvoiceDate := lastInvoiceDate.AddDate(0, -1, 0)
+	a.mandatoryFileSavingPeriodStartDate = time.Date(previousMonthOfLastInvoiceDate.Year(), previousMonthOfLastInvoiceDate.Month(), 1, 0, 0, 0, 0, previousMonthOfLastInvoiceDate.Location())
 
 	for _, month := range a.invoiceMonths {
 		a.filesToUpload[month] = make(map[string]struct{})