Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rs hotfix/fix corrupt image uploads #7804

Merged
merged 18 commits into from
Jul 29, 2024
Merged
Show file tree
Hide file tree
Changes from 17 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 26 additions & 3 deletions .eslintrc.yml
Original file line number Diff line number Diff line change
@@ -1,11 +1,34 @@
env:
browser: true
es2021: true
node: true
extends:
- airbnb-base
- plugin:prettier/recommended
overrides: []
parser: '@typescript-eslint/parser'
parserOptions:
ecmaVersion: latest
rules: { "prettier/prettier": "error" }
plugins: ["prettier"]
sourceType: module
plugins:
- '@typescript-eslint'
- prettier
rules:
prettier/prettier: error
settings:
import/resolver:
node:
extensions: ['.js', '.jsx', '.ts', '.tsx']
overrides:
- files: ['*.js', '*.jsx']
rules:
prettier/prettier: error
- files: ['*.ts', '*.tsx']
extends:
- plugin:@typescript-eslint/recommended
rules:
prettier/prettier: error
no-console: off # This disables the console statement warning for TypeScript files
consistent-return: off # This disables the consistent return warning for TypeScript files
'@typescript-eslint/no-explicit-any': off # This disables the explicit any warning for TypeScript files
'no-await-in-loop': off # This disables the await in loop warning for TypeScript files
'no-restricted-syntax': off # This disables the restricted syntax warning for TypeScript files
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -25,3 +25,4 @@ content/images/_inbox/Refine_Listen.png
content/images/_inbox/Triangulate.png
static/img/proxy/fedramp-logo-stamp.jpeg
.hugo_build.lock
.vscode/launch.json
11 changes: 11 additions & 0 deletions .vscode/settings.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{
"eslint.validate": [
"javascript",
"javascriptreact",
"typescript",
"typescriptreact"
],
"editor.codeActionsOnSave": {
"source.fixAll.eslint": "always"
}
}
57 changes: 33 additions & 24 deletions config/gulp/file-prep.js
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,11 @@ const allExtensions = [...imageExtensions, ...fileExtensions];
// removes . separator to read jpg,png,jpeg...
const extensionsString = allExtensions
.map((extension) => extension.replace(".", ""))
.join(",");
.join(",");

const imageRegex = /(png|jpg|jpeg)/;
const fileRegex = /(doc|docx|pdf|ppt|pptx|pptm|xls|xlsx)/;


/**
* Converts JPG images to PNG format
* @param {string} imagePath - path of the image file
Expand All @@ -36,9 +35,7 @@ async function convertJpgToPng(imagePath) {
console.log(`Converting image ${imagePath} to PNG`);
const outputPath = imagePath.replace(/\.jpe?g$/i, ".png");

await sharp(imagePath)
.toFormat("png")
.toFile(outputPath);
await sharp(imagePath).toFormat("png").toFile(outputPath);

// Check if the original JPG file exists before unlinking
if (fs.existsSync(imagePath)) {
Expand All @@ -48,8 +45,6 @@ async function convertJpgToPng(imagePath) {
return path.basename(outputPath);
}



/**
* Object containing working folder paths used for lifecycle steps of uploading
* to-process contains the normalized filename, static files are upload to s3 from here
Expand All @@ -67,7 +62,6 @@ const filePaths = {
},
};


/**
* Creates directories for each step of the file uploading process
* These directories are removed when a file has been uploaded
Expand All @@ -80,7 +74,9 @@ function fileTidy(done) {

fs.readdir(paths.uploads, async (err, files) => {
if (err) {
console.error(`Failed to read directory ${paths.uploads}: ${err.message}`);
console.error(
`Failed to read directory ${paths.uploads}: ${err.message}`
);
done(err);
return;
}
Expand All @@ -93,29 +89,44 @@ function fileTidy(done) {
// create working directories if they do not exist
createDir(paths[filetype].toProcess, 3);
if (filetype === "image") createDir(paths[filetype].processed, 3);
// copies uploaded file to /to-process with new normalized name
// copies uploaded file to /to-process with new normalized name
// convert jpg to png
if (filetype === "image" && fileExt === ".jpg" || fileExt === ".jpeg") {
let convertedPng = await convertJpgToPng(path.join(paths.uploads, file)).catch
((err) => {
console.error(`Error converting image ${file} to PNG: ${err.message}`);
if (
(filetype === "image" && fileExt === ".jpg") ||
fileExt === ".jpeg"
) {
let convertedPng = await convertJpgToPng(
path.join(paths.uploads, file)
).catch((err) => {
console.error(
`Error converting image ${file} to PNG: ${err.message}`
);
return;
});
// ensure the folder for the process image exists.
// ensure the folder for the process image exists.
if (convertedPng != file) {
file = convertedPng;
}

}
}
newfileName = cleanFileName(file);
const newFilePath = path.join(dirToProcess, newfileName);

// Rename and move the file to the new path
try {
console.log(`Moving file from ${path.join(paths.uploads, file)} to ${newFilePath}`);
fs.renameSync(path.join(paths.uploads, file), newFilePath);
console.log(
`Moving file from ${path.join(
paths.uploads,
file
)} to ${newFilePath}`
);
fs.renameSync(path.join(paths.uploads, file), newFilePath);
} catch (renameError) {
console.error(`Error moving file from ${path.join(paths.uploads, file)} to ${newFilePath}: ${renameError.message}`);
console.error(
`Error moving file from ${path.join(
paths.uploads,
file
)} to ${newFilePath}: ${renameError.message}`
);
continue;
}
}
Expand All @@ -124,8 +135,6 @@ function fileTidy(done) {
});
}



/**
* creates the originals and to-process directories for both files and images
* ./content/uploads/_working-images/to-process";
Expand Down Expand Up @@ -178,7 +187,7 @@ function fileType(extension) {
* @returns filename in string format
*/
function cleanFileName(origfilename) {
return origfilename
return origfilename
.toLowerCase()
.replace(/[ &$_#!?.]/g, "-")
.replace(/-+/g, "-") // multiple dashes to a single dash
Expand All @@ -188,7 +197,7 @@ function cleanFileName(origfilename) {
.replace(/^\d{2,4}-*x-*\d{2,4}-*/g, "") // strip leading dimensions
.replace(/-\./g, ".") // remove leading dashes
.replace(/^-/g, "") // removes dashes from start of filename
.toLowerCase();
.toLowerCase();
}

/**
Expand Down
20 changes: 10 additions & 10 deletions config/gulp/file-process.js
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ async function processImageVariants(image) {
async function processImages() {
return new Promise((resolve, reject) => {
fs.readdir(`${processImagesDirectory}`, async (err, images) => {
if (images === undefined) {
if (images === undefined) {
resolve();
return;
}
Expand Down Expand Up @@ -155,15 +155,15 @@ async function processImages() {
* Removes the /to-process temporary working folder after variants are created
*/
function removeProcessedImage() {
return new Promise((resolve, reject) => {
const imageDir = "content/uploads/_working-images/processed";
if (fs.existsSync(imageDir) && fs.readdirSync(imageDir).length > 0) {
return del([
"content/uploads/_working-images/to-process",
]).then(() => resolve()).catch((err) => reject(err));
} else {
resolve();
return new Promise((resolve, reject) => {
const imageDir = "content/uploads/_working-images/processed";

if (fs.existsSync(imageDir) && fs.readdirSync(imageDir).length > 0) {
return del([
"content/uploads/_working-images/to-process",
]).then(() => resolve()).catch((err) => reject(err));
} else {
resolve();
}
});
}
Expand Down
152 changes: 0 additions & 152 deletions config/gulp/file-upload.js

This file was deleted.

Loading