Merge pull request #141 from dotkom/feat/use-s3-for-file-transfer

Implement s3 integration

next-env.d.ts

@@ -1,3 +1,15 @@
 /// <reference types="next" />
 /// <reference types="next/types/global" />
 /// <reference types="next-images" />
+
+declare global {
+ namespace NodeJS {
+ interface ProcessEnv {
+ AWS_REGION?: string;
+ AWS_ACCESS_KEY_ID?: string;
+ AWS_SECRET_ACCESS_KEY?: string;
+ AWS_S3_BUCKET_NAME?: string;
+ LAMBDA_PRESIGN_UPLOAD_ENDPOINT?: string;
+ }
+ }
+}

package.json

@@ -3,10 +3,11 @@
  "version": "2.2.1",
  "private": true,
  "dependencies": {
- "@dotkomonline/design-system": "^0.22.0",
+ "@dotkomonline/design-system": "^0.22.2",
  "@reduxjs/toolkit": "^1.4.0",
  "@sentry/browser": "^5.0.3",
  "@sentry/node": "^5.4.3",
+ "@types/aws-sdk": "^2.7.0",
  "@types/file-saver": "^2.0.0",
  "@types/get-stream": "^3.0.2",
  "@types/jsdom": "^16.2.4",
@@ -19,6 +20,7 @@
  "@types/react-dom": "16.9.8",
  "@types/react-redux": "^7.1.0",
  "@types/styled-components": "^5.1.4",
+ "aws-sdk": "^2.1560.0",
  "core-js": "^3.0.1",
  "file-saver": "^2.0.1",
  "get-stream": "^6.0.1",

src/constants/backend.ts

@@ -1,3 +1,4 @@
 export const LAMBDA_ENDPOINT = '/api/generate-receipt';
+export const LAMBDA_PRESIGN_UPLOAD_ENDPOINT = '/api/presign-upload-url';
 
 export const OW4_ADDRESS = process.env.NEXT_PUBLIC_OW4_ADDRESS || 'https://online.ntnu.no';

src/form/state.ts

@@ -2,6 +2,8 @@ import { ApiBodyError } from './../lambda/errors';
 import { Group } from 'models/groups';
 import { readDataUrlAsFile2 } from 'utils/readDataUrlAsFile';
 import { readFileAsDataUrl } from 'utils/readFileAsDataUrl';
+import { uploadFile } from "../utils/uploadFile";
+import { downloadFileFromS3Bucket } from "../utils/downloadFileFromS3Bucket";
 
 export type ReceiptType = 'card' | 'deposit';
 export type SendMode = 'download' | 'email' | 'teapot';
@@ -56,7 +58,9 @@ export interface IDeserializedState {
 }
 
 export const deserializeReceipt = async (state: IState): Promise<IDeserializedState> => {
- const attachments = await Promise.all(state.attachments.map(async (file) => readFileAsDataUrl(file)));
+ const attachments = await Promise.all(
+ state.attachments.map(async (file) => uploadFile(file))
+ );
  const signature = await readFileAsDataUrl(state.signature || new File([], 'newfile'));
  return {
  ...state,
@@ -67,9 +71,11 @@ export const deserializeReceipt = async (state: IState): Promise<IDeserializedSt
 
 export const serializeReceipt = async (deserializedState: IDeserializedState): Promise<IState> => {
  try {
- const attachments = await Promise.all(
- deserializedState.attachments.map(async (dataUrl) => readDataUrlAsFile2(dataUrl))
- );
+ const illegalAttachment = deserializedState.attachments.find((attachment) => !attachment.startsWith("uploads/"));
+ if (illegalAttachment) {
+ throw new TypeError('Illegal attachment');
+ }
+ const attachments = await Promise.all(deserializedState.attachments.map(downloadFileFromS3Bucket));
  const signature = await readDataUrlAsFile2(deserializedState.signature);
  return {
  ...deserializedState,

src/form/validation.ts

@@ -29,8 +29,8 @@ export const ACCOUNT_NUMBER_REGEX = new RegExp(/^\d{4} \d{2} \d{5}$/);
 export const COMMITTEE_EMAIL_REGEX = new RegExp(/^.{2,50}@online\.ntnu\.no$/);
 export const EMAIL_REGEX = new RegExp(/^[a-zA-Z0-9.!#$%&’*+/=?^_{|}~-]+@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*$/);
 export const CARD_DETAIL_REGEX = new RegExp(/^.{5,30}$/);
-export const FILE_SIZE_WARN = 7 * 1024 * 1024; // 7 MB
-export const FILE_SIZE_MAX = 9 * 1024 * 1024; // 9 MB
+export const FILE_SIZE_WARN = 20 * 1024 * 1024; // 20 MB
+export const FILE_SIZE_MAX = 25 * 1024 * 1024; // 25 MB
 
 export const STATE_VALIDATION: StateValidators = {
  fullname: [

src/lambda/handler.ts

@@ -10,6 +10,7 @@ import { readFileAsDataUrl } from 'utils/readFileAsDataUrl';
 import { sendEmail } from './sendEmail';
 import { ApiBodyError, ApiValidationError } from './errors';
 import { pdfGenerator } from './browserlessGenerator';
+import { uploadFileToS3Bucket } from "../utils/uploadFileToS3Bucket";
 
 export interface SuccessBody {
  message: string;
@@ -48,12 +49,16 @@ export const generateReceipt = async (data: IDeserializedState | null): Promise<
  }
  const validState = state as NonNullableState;
  const pdf = await pdfGenerator(validState);
- const pdfFile = new File([pdf], 'receipt.pdf', { type: 'application/pdf' });
- const pdfString = await readFileAsDataUrl(pdfFile);
 
  if (state.mode === 'download') {
- return DOWNLOAD_SUCCESS_MESSAGE(pdfString);
+ const dato = `${new Date().toISOString().split('T')[0]}`;
+ const filename = `kvittering-${dato}-${+Date.now()}.pdf`;
+ const downloadUrl = await uploadFileToS3Bucket(pdf, `receipts/${filename}`);
+ return DOWNLOAD_SUCCESS_MESSAGE(downloadUrl);
  } else if (state.mode === 'email') {
+ const pdfFile = new File([pdf], 'receipt.pdf', { type: 'application/pdf' });
+ const pdfString = await readFileAsDataUrl(pdfFile);
+
  await sendEmail(pdfString, state);
  return EMAIL_SUCCESS_MESSAGE;
  } else if (state.mode === 'teapot') {

src/pages/api/presign-upload-url.ts

@@ -0,0 +1,27 @@
+import { NextApiRequest, NextApiResponse, PageConfig } from 'next';
+
+import { SuccessBody } from 'lambda/handler';
+import { sentryMiddleware } from 'lambda/sentry';
+import { ErrorData } from 'lambda/errors';
+import { getPresignedS3URL } from "../../utils/getPresignedS3URL";
+
+const handler = async (req: NextApiRequest, res: NextApiResponse<SuccessBody | ErrorData>) => {
+ const { filename, contentType } = req.body;
+
+ try {
+ const data = await getPresignedS3URL(filename, contentType);
+ res.status(200).json({ message: "Presigned URL", data: JSON.stringify(data) });
+ } catch (error) {
+ res.status(500).json({ message: "Failed to get presigned URL", data: error });
+ }
+};
+
+export const config: PageConfig = {
+ api: {
+ bodyParser: {
+ sizeLimit: '25mb',
+ },
+ },
+};
+
+export default sentryMiddleware(handler);

src/redux/actions/authActions.ts

@@ -68,7 +68,6 @@ export const loginAction = createAsyncThunk('user/login', async (_, { dispatch,
  const user: User | null = await getManager().getUser();
  if (user) {
  const newForm = await processUser(user, form);
- console.log({ newForm });
  updateForm(dispatch, newForm);
  } else {
  logInRedirect(form);
@@ -96,6 +95,7 @@ export const catchCallbackAction = createAsyncThunk('user/catchCallback', async
  window.location.hash = '';
  } catch (err) {
  /** Do nothing if no user data is present */
+ window.location.hash = '';
  return;
  }
 });

src/redux/actions/submitActions.ts

@@ -3,9 +3,7 @@ import { createAsyncThunk } from '@reduxjs/toolkit';
 
 import { NonNullableState } from 'lambda/generatePDF';
 import { getFileName } from 'lambda/tools/format';
-import { downloadFile } from 'utils/download';
 import { postReceipt } from 'utils/postReceipt';
-import { readDataUrlAsFile } from 'utils/readDataUrlAsFile';
 import { downloadFinished, downloadStarted, loadingDone, setResponse } from 'redux/reducers/statusReducer';
 import { State } from 'redux/store';
 import { SuccessBody } from 'lambda/handler';
@@ -14,10 +12,13 @@ const handleDownload = async (response: SuccessBody, state: NonNullableState) =>
  if (response.data) {
  /** Use the same filename that would be generated when sending a mail */
  const fileName = getFileName(state);
- const pdfFile = await readDataUrlAsFile(response.data, fileName);
- if (pdfFile) {
- downloadFile(pdfFile);
- }
+ // response.data is a URL to the file
+
+ const a = document.createElement('a');
+ document.body.appendChild(a);
+ a.href = response.data;
+ a.download = fileName;
+ a.click();
  }
 };
 

src/utils/downloadFileFromS3Bucket.ts

@@ -0,0 +1,32 @@
+import AWS from "aws-sdk";
+
+const credentials = (process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY) ?
+ {
+ accessKeyId: process.env.AWS_ACCESS_KEY_ID,
+ secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+ } :
+ undefined;
+
+AWS.config.update({
+ region: process.env.AWS_REGION,
+ credentials,
+})
+
+export async function downloadFileFromS3Bucket(key: string): Promise<File> {
+ if (!process.env.AWS_S3_BUCKET_NAME) {
+ throw new Error('AWS_S3_BUCKET_NAME is not set');
+ }
+
+ const s3 = new AWS.S3({
+ apiVersion: '2006-03-01',
+ });
+
+ const params = {
+ Bucket: process.env.AWS_S3_BUCKET_NAME,
+ Key: key,
+ };
+
+ const data = await s3.getObject(params).promise();
+
+ return new File([data.Body as Blob], key, { type: data.ContentType });
+}

src/utils/getPresignedS3URL.ts

@@ -0,0 +1,41 @@
+import AWS from "aws-sdk";
+
+const credentials = (process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY) ?
+ {
+ accessKeyId: process.env.AWS_ACCESS_KEY_ID,
+ secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+ } :
+ undefined;
+
+AWS.config.update({
+ region: process.env.AWS_REGION,
+ credentials,
+})
+
+const s3 = new AWS.S3({
+ apiVersion: '2006-03-01',
+});
+
+const createPresignedPost = (params: AWS.S3.PresignedPost.Params): Promise<AWS.S3.PresignedPost> => new Promise((resolve, reject) => {
+ s3.createPresignedPost(params, function (err, data) {
+ if (err) {
+ reject(err);
+ } else {
+ resolve(data)
+ }
+ });
+});
+
+export const getPresignedS3URL = async (name: string, contentType: string): Promise<{ url: string, fields: {[key: string]: string}}> => {
+ return await createPresignedPost({
+ Bucket: "receipt-form",
+ Fields: {
+ key: `uploads/${+new Date()}-${name}`,
+ "Content-Type": contentType,
+ },
+ Conditions: [
+ ["content-length-range", 0, 1024 * 1024 * 10],
+ ],
+ Expires: 60,
+ })
+}

src/utils/uploadFile.ts

@@ -0,0 +1,42 @@
+import { LAMBDA_PRESIGN_UPLOAD_ENDPOINT } from "constants/backend";
+
+export const uploadFile = async (file: File): Promise<string> => {
+ if (!LAMBDA_PRESIGN_UPLOAD_ENDPOINT) {
+ throw new Error('LAMBDA_PRESIGN_UPLOAD_ENDPOINT is not set');
+ }
+
+ // Request to get the presigned POST data
+ const response = await fetch(LAMBDA_PRESIGN_UPLOAD_ENDPOINT, {
+ method: 'POST',
+ headers: {
+ Accept: 'application/json',
+ 'Content-Type': 'application/json',
+ },
+ body: JSON.stringify({ filename: file.name, contentType: file.type }),
+ });
+
+ const body = await response.json();
+ if (!response.ok) {
+ console.error(await response.text());
+ throw new Error('Failed to get presigned post data');
+ }
+
+ const { url, fields }: {url: string, fields: {[key: string]: string}} = JSON.parse(body.data);
+
+ const formData = new FormData();
+ for (const key in fields) {
+ formData.append(key, fields[key]);
+ }
+ formData.append('file', file);
+
+ const uploadResponse = await fetch(url, {
+ method: 'POST',
+ body: formData,
+ });
+
+ if (!uploadResponse.ok) {
+ throw new Error('Failed to upload file');
+ }
+
+ return fields.key
+};

src/utils/uploadFileToS3Bucket.ts

@@ -0,0 +1,35 @@
+import AWS from "aws-sdk";
+
+const credentials = (process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY) ?
+ {
+ accessKeyId: process.env.AWS_ACCESS_KEY_ID,
+ secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+ } :
+ undefined;
+
+AWS.config.update({
+ region: process.env.AWS_REGION,
+ credentials,
+})
+
+// upload file to S3 bucket and make it publicly downloadable
+export async function uploadFileToS3Bucket(file: Uint8Array, key: string): Promise<string> {
+ if (!process.env.AWS_S3_BUCKET_NAME) {
+ throw new Error('AWS_S3_BUCKET_NAME is not set');
+ }
+ const s3 = new AWS.S3({
+ apiVersion: '2006-03-01',
+ params: { Bucket: process.env.AWS_S3_BUCKET_NAME },
+ });
+
+ const params = {
+ Bucket: process.env.AWS_S3_BUCKET_NAME,
+ Key: key,
+ Body: file,
+ ACL: 'public-read',
+ };
+
+ const result = await s3.upload(params).promise();
+
+ return result.Location;
+}