From 2d301e04aa350f4735b9869b46a23ea6d46dec6b Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Wed, 20 Nov 2024 10:52:06 +0000 Subject: [PATCH] PFS-195 Fix cypress tests, generate CSV the slow way --- db/queries/aged_debt.sql | 6 +- finance-admin-api/api/download.go | 141 +++-------------- finance-admin-api/api/request_report.go | 154 +++++++++++++++++++ finance-admin-api/api/server.go | 6 +- finance-admin-api/api/server_test.go | 4 +- finance-admin-api/api/upload.go | 2 +- finance-admin-api/filestorage/client.go | 6 +- finance-admin/internal/api/download.go | 11 +- finance-admin/internal/api/request_report.go | 2 +- 9 files changed, 190 insertions(+), 142 deletions(-) create mode 100644 finance-admin-api/api/request_report.go diff --git a/db/queries/aged_debt.sql b/db/queries/aged_debt.sql index 53b019e..ecac008 100644 --- a/db/queries/aged_debt.sql +++ b/db/queries/aged_debt.sql @@ -1,5 +1,3 @@ SELECT - 'test' AS "Customer Name", - 'test' AS "Customer Number" - -FROM supervision_finance.finance_client \ No newline at end of file + 'Joseph Smith' AS "Customer Name", + '12345678' AS "Customer Number" \ No newline at end of file diff --git a/finance-admin-api/api/download.go b/finance-admin-api/api/download.go index 5afe7bf..21aad72 100644 --- a/finance-admin-api/api/download.go +++ b/finance-admin-api/api/download.go @@ -1,142 +1,43 @@ package api import ( - "bytes" - "context" - "encoding/csv" - "encoding/json" + "errors" "fmt" + "github.com/aws/smithy-go" + "github.com/opg-sirius-finance-admin/apierror" "github.com/opg-sirius-finance-admin/shared" + "io" "net/http" "os" ) func (s *Server) download(w http.ResponseWriter, r *http.Request) error { - ctx := context.Background() + ctx := r.Context() + uid := r.URL.Query().Get("uid") - var download shared.Download - defer r.Body.Close() - - if err := json.NewDecoder(r.Body).Decode(&download); err != nil { - return err - } - - c, err := os.ReadFile("/app/db/queries/aged_debt.sql") - if err != nil { - return err - } - sql := string(c) - - rows, err := s.conn.Query(ctx, sql) - - if err != nil { - return err - } - - defer rows.Close() - var items [][]string - for rows.Next() { - var i []string - var stringValue string - - values, err := rows.Values() - if err != nil { - return err - } - for _, value := range values { - stringValue, _ = value.(string) - i = append(i, stringValue) - } - items = append(items, i) - } - if err := rows.Err(); err != nil { - return err - } - - f, err := os.Create("test.csv") + var downloadRequest shared.DownloadRequest + err := downloadRequest.Decode(uid) if err != nil { return err } - writer := csv.NewWriter(f) - agedDebtHeaders := []string{ - "Customer Name", - "Customer number", - "SOP number", - "Deputy type", - "Active case?", - "Entity", - "Receivable cost centre", - "Receivable cost centre description", - "Receivable account code", - "Revenue cost centre", - "Revenue cost centre description", - "Revenue account code", - "Revenue account code description", - "Invoice type", - "Trx number", - "Transaction Description", - "Invoice date", - "Due date", - "Financial year", - "Payment terms", - "Original amount", - "Outstanding amount", - "Current", - "0-1 years", - "1-2 years", - "2-3 years", - "3-5 years", - "5+ years", - "Debt impairment years", - } - - err = writer.Write(agedDebtHeaders) + result, err := s.filestorage.GetFile(ctx, os.Getenv("REPORTS_S3_BUCKET"), downloadRequest.Key, downloadRequest.VersionId) if err != nil { - return err - } - - for _, item := range items { - err = writer.Write(item) - if err != nil { - return err + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + if apiErr.ErrorCode() == "NoSuchKey" { + return apierror.NotFoundError(err) + } } + return fmt.Errorf("failed to get object from S3: %w", err) } + defer result.Body.Close() - writer.Flush() - if writer.Error() != nil { - return writer.Error() - } - - c, err = os.ReadFile("test.csv") - if err != nil { - return err - } - - err = s.filestorage.PutFile( - ctx, - os.Getenv("ASYNC_S3_BUCKET"), - fmt.Sprintf("%s/%s", s3Directory, "test.csv"), - bytes.NewReader(c)) - - if err != nil { - return err - } - - payload := NotifyPayload{ - EmailAddress: "test@email.com", - TemplateId: "8c85cf6c-695f-493a-a25f-77b4fb5f6a8e", - Personalisation: struct { - Filename string `json:"upload_type"` - }{ - "test", - }, - } + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%s", downloadRequest.Key)) + w.Header().Set("Content-Type", *result.ContentType) - err = s.SendEmailToNotify(ctx, payload) - if err != nil { - return err - } + // Stream the S3 object to the response writer using io.Copy + _, err = io.Copy(w, result.Body) - return nil + return err } diff --git a/finance-admin-api/api/request_report.go b/finance-admin-api/api/request_report.go new file mode 100644 index 0000000..53c6b1b --- /dev/null +++ b/finance-admin-api/api/request_report.go @@ -0,0 +1,154 @@ +package api + +import ( + "context" + "encoding/csv" + "encoding/json" + "fmt" + "github.com/opg-sirius-finance-admin/shared" + "net/http" + "os" +) + +func (s *Server) requestReport(w http.ResponseWriter, r *http.Request) error { + ctx := context.Background() + + var download shared.Download + defer r.Body.Close() + + if err := json.NewDecoder(r.Body).Decode(&download); err != nil { + return err + } + + c, err := os.ReadFile("/app/db/queries/aged_debt.sql") + if err != nil { + return err + } + + agedDebtHeaders := []string{ + "Customer Name", + "Customer number", + "SOP number", + "Deputy type", + "Active case?", + "Entity", + "Receivable cost centre", + "Receivable cost centre description", + "Receivable account code", + "Revenue cost centre", + "Revenue cost centre description", + "Revenue account code", + "Revenue account code description", + "Invoice type", + "Trx number", + "Transaction Description", + "Invoice date", + "Due date", + "Financial year", + "Payment terms", + "Original amount", + "Outstanding amount", + "Current", + "0-1 years", + "1-2 years", + "2-3 years", + "3-5 years", + "5+ years", + "Debt impairment years", + } + + ef, err := os.Create("test.csv") + if err != nil { + return err + } + + ef.Close() + + query := string(c) + + rows, err := s.conn.Query(ctx, query) + if err != nil { + return err + } + + defer rows.Close() + var items [][]string + for rows.Next() { + var i []string + var stringValue string + + values, err := rows.Values() + if err != nil { + return err + } + for _, value := range values { + stringValue, _ = value.(string) + i = append(i, stringValue) + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return err + } + + writer := csv.NewWriter(ef) + + err = writer.Write(agedDebtHeaders) + if err != nil { + return err + } + + for _, item := range items { + err = writer.Write(item) + if err != nil { + return err + } + } + + writer.Flush() + if writer.Error() != nil { + return writer.Error() + } + + versionId, err := s.filestorage.PutFile( + ctx, + os.Getenv("ASYNC_S3_BUCKET"), + fmt.Sprintf("%s/%s", s3Directory, "test.csv"), + ef, + ) + + if err != nil { + return err + } + + if versionId == nil { + return fmt.Errorf("S3 version ID not found") + } + + downloadRequest := shared.DownloadRequest{ + Key: fmt.Sprintf("%s/%s", s3Directory, "test.csv"), + VersionId: *versionId, + } + + uid, err := downloadRequest.Encode() + if err != nil { + return err + } + + payload := NotifyPayload{ + EmailAddress: "test@email.com", + TemplateId: "8c85cf6c-695f-493a-a25f-77b4fb5f6a8e", + Personalisation: struct { + Uid string `json:"upload_type"` + }{ + uid, + }, + } + + err = s.SendEmailToNotify(ctx, payload) + if err != nil { + return err + } + + return nil +} diff --git a/finance-admin-api/api/server.go b/finance-admin-api/api/server.go index a0b4a5a..e63c238 100644 --- a/finance-admin-api/api/server.go +++ b/finance-admin-api/api/server.go @@ -24,7 +24,7 @@ type Dispatch interface { type FileStorage interface { GetFile(ctx context.Context, bucketName string, filename string, versionID string) (*s3.GetObjectOutput, error) - PutFile(ctx context.Context, bucketName string, fileName string, file io.Reader) error + PutFile(ctx context.Context, bucketName string, fileName string, file io.Reader) (*string, error) FileExists(ctx context.Context, bucketName string, filename string, versionID string) bool } @@ -52,8 +52,10 @@ func (s *Server) SetupRoutes(logger *slog.Logger) http.Handler { mux.Handle(pattern, handler) } - handleFunc("GET /downloads", s.download) + handleFunc("GET /download", s.download) handleFunc("HEAD /download", s.checkDownload) + + handleFunc("POST /downloads", s.requestReport) handleFunc("POST /uploads", s.upload) handleFunc("POST /events", s.handleEvents) diff --git a/finance-admin-api/api/server_test.go b/finance-admin-api/api/server_test.go index 0520d8a..620fd9e 100644 --- a/finance-admin-api/api/server_test.go +++ b/finance-admin-api/api/server_test.go @@ -37,12 +37,12 @@ func (m *MockFileStorage) GetFile(ctx context.Context, bucketName string, fileNa return m.outgoingObject, m.err } -func (m *MockFileStorage) PutFile(ctx context.Context, bucketName string, fileName string, file io.Reader) error { +func (m *MockFileStorage) PutFile(ctx context.Context, bucketName string, fileName string, file io.Reader) (*string, error) { m.bucketname = bucketName m.filename = fileName m.file = file - return nil + return nil, nil } // add a FileExists method to the MockFileStorage struct diff --git a/finance-admin-api/api/upload.go b/finance-admin-api/api/upload.go index 2a7fe02..2b3aa34 100644 --- a/finance-admin-api/api/upload.go +++ b/finance-admin-api/api/upload.go @@ -94,7 +94,7 @@ func (s *Server) upload(w http.ResponseWriter, r *http.Request) error { return err } - err = s.filestorage.PutFile( + _, err = s.filestorage.PutFile( ctx, os.Getenv("ASYNC_S3_BUCKET"), fmt.Sprintf("%s/%s", s3Directory, upload.Filename), diff --git a/finance-admin-api/filestorage/client.go b/finance-admin-api/filestorage/client.go index 781b1fe..26baa62 100644 --- a/finance-admin-api/filestorage/client.go +++ b/finance-admin-api/filestorage/client.go @@ -59,8 +59,8 @@ func (c *Client) GetFile(ctx context.Context, bucketName string, filename string }) } -func (c *Client) PutFile(ctx context.Context, bucketName string, fileName string, file io.Reader) error { - _, err := c.s3.PutObject(ctx, &s3.PutObjectInput{ +func (c *Client) PutFile(ctx context.Context, bucketName string, fileName string, file io.Reader) (*string, error) { + output, err := c.s3.PutObject(ctx, &s3.PutObjectInput{ Bucket: &bucketName, Key: &fileName, Body: file, @@ -68,7 +68,7 @@ func (c *Client) PutFile(ctx context.Context, bucketName string, fileName string SSEKMSKeyId: aws.String(os.Getenv("S3_ENCRYPTION_KEY")), }) - return err + return output.VersionId, err } func (c *Client) FileExists(ctx context.Context, bucketName string, filename string, versionID string) bool { diff --git a/finance-admin/internal/api/download.go b/finance-admin/internal/api/download.go index 08d4c67..7440b24 100644 --- a/finance-admin/internal/api/download.go +++ b/finance-admin/internal/api/download.go @@ -1,19 +1,12 @@ package api import ( - "bytes" - "encoding/json" + "fmt" "net/http" ) func (c *Client) Download(ctx Context, uid string) (*http.Response, error) { - var body bytes.Buffer - err := json.NewEncoder(&body).Encode(uid) - if err != nil { - return nil, err - } - - req, err := c.newBackendRequest(ctx, http.MethodPost, "/downloads", &body) + req, err := c.newBackendRequest(ctx, http.MethodGet, fmt.Sprintf("/download?uid=%s", uid), nil) if err != nil { return nil, err diff --git a/finance-admin/internal/api/request_report.go b/finance-admin/internal/api/request_report.go index 9e954d2..4d35588 100644 --- a/finance-admin/internal/api/request_report.go +++ b/finance-admin/internal/api/request_report.go @@ -15,7 +15,7 @@ func (c *Client) RequestReport(ctx Context, data model.ReportRequest) error { return err } - req, err := c.newBackendRequest(ctx, http.MethodGet, "/downloads", &body) + req, err := c.newBackendRequest(ctx, http.MethodPost, "/downloads", &body) if err != nil { return err