From d9072d1a914f470182a27edfd317780b6ade7407 Mon Sep 17 00:00:00 2001 From: Oliver Gut Date: Thu, 2 May 2024 11:21:19 +0200 Subject: [PATCH 1/3] Remove obsolete Docker-Compose version tag --- docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7fdfe051..6457a239 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: "3.4" - services: stac-browser: image: ghcr.io/geowerkstatt/stac-browser:latest From 93cfe101055b3b32e9341ba54e8786ca50c62ffa Mon Sep 17 00:00:00 2001 From: Oliver Gut Date: Tue, 30 Apr 2024 11:25:23 +0200 Subject: [PATCH 2/3] Sanitize user input A malicious user may be able to forge new log entries or may access unexpected resources. --- .../Controllers/MandateController.cs | 14 +++--- .../Controllers/ValidationController.cs | 17 ++++--- .../FileAccess/PhysicalFileProvider.cs | 2 +- src/Geopilot.Api/StringExtensions.cs | 30 +++++++++++++ tests/.editorconfig | 1 + .../Controllers/MandateControllerTest.cs | 6 +-- tests/Geopilot.Api.Test/StringExtensions.cs | 44 +++++++++++++++++++ 7 files changed, 94 insertions(+), 20 deletions(-) create mode 100644 src/Geopilot.Api/StringExtensions.cs create mode 100644 tests/Geopilot.Api.Test/StringExtensions.cs diff --git a/src/Geopilot.Api/Controllers/MandateController.cs b/src/Geopilot.Api/Controllers/MandateController.cs index 64b5d45b..5816f323 100644 --- a/src/Geopilot.Api/Controllers/MandateController.cs +++ b/src/Geopilot.Api/Controllers/MandateController.cs @@ -42,9 +42,9 @@ public MandateController(ILogger logger, Context context, IVa [SwaggerResponse(StatusCodes.Status200OK, "Returns list of mandates associated to the current user matching the optional filter criteria.", typeof(IEnumerable), new[] { "application/json" })] public async Task Get( [FromQuery, SwaggerParameter("Filter mandates matching validation job file extension.")] - string jobId = "") + Guid jobId = default) { - logger.LogInformation("Getting mandates for job with id <{JobId}>.", jobId.ReplaceLineEndings(string.Empty)); + logger.LogInformation("Getting mandates for job with id <{JobId}>.", jobId); var user = await context.GetUserByPrincipalAsync(User); if (user == null) @@ -53,16 +53,16 @@ public async Task Get( var mandates = context.DeliveryMandates .Where(m => m.Organisations.SelectMany(o => o.Users).Any(u => u.Id == user.Id)); - if (Guid.TryParse(jobId, out var guid)) + if (jobId != default) { - var job = validationService.GetJob(guid); + var job = validationService.GetJob(jobId); if (job is null) { - logger.LogTrace("Validation job with id <{JobId}> was not found.", guid.ToString()); + logger.LogTrace("Validation job with id <{JobId}> was not found.", jobId); return Ok(Array.Empty()); } - logger.LogTrace("Filtering mandates for job with id <{JobId}>", guid.ToString()); + logger.LogTrace("Filtering mandates for job with id <{JobId}>", jobId); var extension = Path.GetExtension(job.OriginalFileName); mandates = mandates .Where(m => m.FileTypes.Contains(".*") || m.FileTypes.Contains(extension)); @@ -70,7 +70,7 @@ public async Task Get( var result = await mandates.ToListAsync(); - logger.LogInformation("Getting mandates with for job with id <{JobId}> resulted in <{MatchingMandatesCount}> matching mandates.", guid.ToString(), result.Count); + logger.LogInformation("Getting mandates with for job with id <{JobId}> resulted in <{MatchingMandatesCount}> matching mandates.", jobId, result.Count); return Ok(mandates); } } diff --git a/src/Geopilot.Api/Controllers/ValidationController.cs b/src/Geopilot.Api/Controllers/ValidationController.cs index 298312bd..c87d1429 100644 --- a/src/Geopilot.Api/Controllers/ValidationController.cs +++ b/src/Geopilot.Api/Controllers/ValidationController.cs @@ -8,6 +8,7 @@ using Swashbuckle.AspNetCore.Annotations; using System.Diagnostics.CodeAnalysis; using System.Globalization; +using System.Web; namespace Geopilot.Api.Controllers; @@ -166,9 +167,7 @@ public IActionResult GetStatus(Guid jobId) [SwaggerResponse(StatusCodes.Status404NotFound, "The job or log file cannot be found.", typeof(ProblemDetails), new[] { "application/json" })] public IActionResult Download(Guid jobId, string file) { - var sanitizedFilename = Path.GetFileName(file.Trim().ReplaceLineEndings(string.Empty)); - - logger.LogInformation("Download file <{File}> for job <{JobId}> requested.", sanitizedFilename, jobId.ToString()); + logger.LogInformation("Download file <{File}> for job <{JobId}> requested.", HttpUtility.HtmlEncode(file), jobId); fileProvider.Initialize(jobId); var validationJob = validationService.GetJob(jobId); @@ -178,15 +177,15 @@ public IActionResult Download(Guid jobId, string file) return Problem($"No job information available for job id <{jobId}>", statusCode: StatusCodes.Status404NotFound); } - if (!fileProvider.Exists(sanitizedFilename)) + if (!fileProvider.Exists(file)) { - logger.LogTrace("No log file <{File}> found for job id <{JobId}>", sanitizedFilename, jobId); - return Problem($"No log file <{sanitizedFilename}> found for job id <{jobId}>", statusCode: StatusCodes.Status404NotFound); + logger.LogTrace("No log file <{File}> found for job id <{JobId}>", HttpUtility.HtmlEncode(file), jobId); + return Problem($"No log file <{file}> found for job id <{jobId}>", statusCode: StatusCodes.Status404NotFound); } - var logFile = fileProvider.Open(sanitizedFilename); - var contentType = contentTypeProvider.GetContentTypeAsString(sanitizedFilename); - var logFileName = Path.GetFileNameWithoutExtension(validationJob.OriginalFileName) + "_log" + Path.GetExtension(sanitizedFilename); + var logFile = fileProvider.Open(file); + var contentType = contentTypeProvider.GetContentTypeAsString(file); + var logFileName = Path.GetFileNameWithoutExtension(validationJob.OriginalFileName) + "_log" + Path.GetExtension(file); return File(logFile, contentType, logFileName); } } diff --git a/src/Geopilot.Api/FileAccess/PhysicalFileProvider.cs b/src/Geopilot.Api/FileAccess/PhysicalFileProvider.cs index 4a8cde20..6b0f782f 100644 --- a/src/Geopilot.Api/FileAccess/PhysicalFileProvider.cs +++ b/src/Geopilot.Api/FileAccess/PhysicalFileProvider.cs @@ -44,7 +44,7 @@ public Stream Open(string file) /// public bool Exists(string file) { - return File.Exists(Path.Combine(HomeDirectory.FullName, file)); + return File.Exists(Path.Combine(HomeDirectory.FullName, file.SanitizeFileName())); } /// diff --git a/src/Geopilot.Api/StringExtensions.cs b/src/Geopilot.Api/StringExtensions.cs new file mode 100644 index 00000000..974e9ace --- /dev/null +++ b/src/Geopilot.Api/StringExtensions.cs @@ -0,0 +1,30 @@ +namespace Geopilot.Api; + +/// +/// GeoPilot API string extensions. +/// +public static class StringExtensions +{ + /// + /// Sanitizes a file name by removing invalid characters. + /// + /// The file name to sanitize. + /// The sanitized file name. + /// If is null, + /// empty or white space." + public static string SanitizeFileName(this string fileName) + { + if (string.IsNullOrWhiteSpace(fileName)) throw new ArgumentNullException(nameof(fileName)); + + // Get invalid characters for file names and add some platform-specific ones. + var invalidFileNameChars = Path.GetInvalidFileNameChars() + .Concat(new[] { '?', '$', '*', '|', '<', '>', '"', ':', '\\' }).ToArray(); + + return Path.GetFileName(new string(fileName + .Trim() + .ReplaceLineEndings(string.Empty) + .Replace("..", string.Empty) + .Replace("./", string.Empty) + .Where(x => !invalidFileNameChars.Contains(x)).ToArray())); + } +} diff --git a/tests/.editorconfig b/tests/.editorconfig index 28b5b425..33372788 100644 --- a/tests/.editorconfig +++ b/tests/.editorconfig @@ -16,4 +16,5 @@ [*.cs] dotnet_diagnostic.CA1001.severity = none dotnet_diagnostic.SA0001.severity = none +dotnet_diagnostic.CS8604.severity = none dotnet_diagnostic.CS8618.severity = none diff --git a/tests/Geopilot.Api.Test/Controllers/MandateControllerTest.cs b/tests/Geopilot.Api.Test/Controllers/MandateControllerTest.cs index 5e21bf10..657f3d5c 100644 --- a/tests/Geopilot.Api.Test/Controllers/MandateControllerTest.cs +++ b/tests/Geopilot.Api.Test/Controllers/MandateControllerTest.cs @@ -70,7 +70,7 @@ public async Task GetWithJobIdIncludesMatchingMandates() .Setup(m => m.GetJob(jobId)) .Returns(new ValidationJob(jobId, "Original.xtf", "tmp.xtf")); - var result = (await mandateController.Get(jobId.ToString())) as OkObjectResult; + var result = (await mandateController.Get(jobId)) as OkObjectResult; var mandates = (result?.Value as IEnumerable)?.ToList(); Assert.IsNotNull(mandates); @@ -88,7 +88,7 @@ public async Task GetWithJobIdExcludesNonMatchinMandates() .Setup(m => m.GetJob(jobId)) .Returns(new ValidationJob(jobId, "Original.csv", "tmp.csv")); - var result = (await mandateController.Get(jobId: jobId.ToString())) as OkObjectResult; + var result = (await mandateController.Get(jobId)) as OkObjectResult; var mandates = (result?.Value as IEnumerable)?.ToList(); Assert.IsNotNull(mandates); @@ -106,7 +106,7 @@ public async Task GetWithInvalidJobIdReturnsEmptyArray() .Setup(m => m.GetJob(jobId)) .Returns(() => null); - var result = (await mandateController.Get(jobId: jobId.ToString())) as OkObjectResult; + var result = (await mandateController.Get(jobId)) as OkObjectResult; var mandates = (result?.Value as IEnumerable)?.ToList(); Assert.IsNotNull(mandates); diff --git a/tests/Geopilot.Api.Test/StringExtensions.cs b/tests/Geopilot.Api.Test/StringExtensions.cs new file mode 100644 index 00000000..0a94b820 --- /dev/null +++ b/tests/Geopilot.Api.Test/StringExtensions.cs @@ -0,0 +1,44 @@ +namespace Geopilot.Api.Controllers; + +[TestClass] +public class StringExtensions +{ + [TestMethod] + [DataRow("SQUIRRELGENESIS", "SQUIRRELGENESIS")] + [DataRow("JUNIORARK.xyz", "JUNIORARK.xyz")] + [DataRow("PEEVEDBEAM-ANT.MESS.abc", "PEEVEDBEAM-ANT.MESS.abc")] + [DataRow("WEIRD WATER.example", "WEIRD WATER.example")] + [DataRow("AUTOFIRE123.doc", "AUTOFIRE123.doc")] + [DataRow("SUNNY(1).doc", "SUNNY(1).doc")] + [DataRow("ODD_MONKEY.doc", "ODD_MONKEY.doc")] + [DataRow("SILLY,MONKEY.docx", "SILLY,MONKEY.docx")] + [DataRow("CamelCase.bat", "CamelCase.bat")] + [DataRow("SLICKER-CHIPMUNK.bat", "SLICKER-CHIPMUNK.bat")] + public void SanitizeFileNameForValidFileNames(string expected, string fileName) + => Assert.AreEqual(expected, fileName.SanitizeFileName()); + + [TestMethod] + [DataRow("CHIPMUNKWALK", " CHIPMUNKWALK ")] + [DataRow("SLEEPYBOUNCE", "SLEEPYBOUNCE\n")] + [DataRow("PLOWARK", "PLOWARK\r")] + [DataRow("JUNIORGLEE", "JUNIORGLEE\t")] + [DataRow("SILLYWATER", "SILLYWATER\r\n")] + [DataRow("LATENTROUTE34", "LATENTROUTE?34")] + [DataRow("TRAWLSOUFFLE", "/TRAWLSOUFFLE*")] + [DataRow("VIOLENTIRON", "> Assert.AreEqual(expected, fileName.SanitizeFileName()); + + [TestMethod] + public void SanitizeFileNameForInvalid() + { + Assert.ThrowsException(() => string.Empty.SanitizeFileName()); + Assert.ThrowsException(() => " ".SanitizeFileName()); + Assert.ThrowsException(() => (null as string).SanitizeFileName()); + } +} From 7a2bd3132d195e94ba373d729c12fe44a9fa1d38 Mon Sep 17 00:00:00 2001 From: Oliver Gut Date: Mon, 6 May 2024 09:30:25 +0200 Subject: [PATCH 3/3] Automatically update GitHub release notes when releasing a GitHub pre-release. --- .editorconfig | 3 +++ .gitattributes | 3 ++- .github/workflows/release.yml | 11 ++++++++++- CHANGELOG.md | 10 +++++++--- get-changelog.sh | 23 +++++++++++++++++++++++ 5 files changed, 45 insertions(+), 5 deletions(-) create mode 100755 get-changelog.sh diff --git a/.editorconfig b/.editorconfig index 4f8f79ce..e9711746 100644 --- a/.editorconfig +++ b/.editorconfig @@ -22,6 +22,9 @@ insert_final_newline = true [*.md] trim_trailing_whitespace = false +[CHANGELOG.md] +end_of_line = lf + [*.{cs,sql,config,csx,fsx}] ; https://github.com/editorconfig/editorconfig/issues/297 charset = utf-8-bom diff --git a/.gitattributes b/.gitattributes index 79531b9c..1b5239d4 100644 --- a/.gitattributes +++ b/.gitattributes @@ -75,4 +75,5 @@ *.svg text eol=lf # *nix shell scripts always use LF (see .editorconfig) -*.sh eol=lf +*.sh text eol=lf +CHANGELOG.md text eol=lf diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a6c9c339..553299d3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -64,7 +64,7 @@ jobs: patch-changelog: runs-on: ubuntu-latest - name: Patch CHANGELOG.md + name: Patch CHANGELOG.md and update GitHub release notes steps: - name: Checkout repository @@ -74,6 +74,15 @@ jobs: run: | echo GIT_BRANCH_NAME=mark-version-${TAG_NAME#v}-as-released >> $GITHUB_ENV echo GIT_COMMIT_MESSAGE=Mark version ${TAG_NAME#v} as released >> $GITHUB_ENV + echo RELEASE_ID=$(gh api -H "Accept: application/vnd.github+json" /repos/${GITHUB_REPOSITORY}/releases/tags/${TAG_NAME} | jq '.id') >> $GITHUB_ENV + + - name: Get changelog for this specific release and update release notes + run: | + gh api \ + --method PATCH \ + --header "Accept: application/vnd.github+json" \ + /repos/${GITHUB_REPOSITORY}/releases/${RELEASE_ID} \ + -f body="$(./get-changelog.sh)" - name: Checkout new branch and patch changelog run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 050a1f06..1d830bcc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,16 +2,20 @@ ## [Unreleased] +### Added + +- When releasing a GitHub pre-release, the release notes are automatically updated with the corresponding entries from the `CHANGELOG.md` file. + ## v1.0.87 - 2024-04-26 ### Added -- Add licensing information to the about page +- Add licensing information to the about page. ### Changed -- Sort delivery mandates alphabetically +- Sort delivery mandates alphabetically. ### Fixed -- Spatial extent in STAC browser +- Spatial extent in STAC browser. diff --git a/get-changelog.sh b/get-changelog.sh new file mode 100755 index 00000000..315596c7 --- /dev/null +++ b/get-changelog.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# This script gets all changelog entries from CHANGELOG.md since last release. + +set -e + +tempDir="$(mktemp -d)" +tempFile=$tempDir/gh_release_notes.md + +# Get changelog entries since last release +cat CHANGELOG.md | \ + grep -Pazo '(?s)(?<=\#{2} \[Unreleased\]\n{2}).*?(?=\n\#{2} v|$)' \ + > $tempFile + +# Improve readability and add some icons +sed -i -E 's/(###) (Added)/\1 🚀 \2/' $tempFile +sed -i -E 's/(###) (Changed)/\1 🔨 \2/' $tempFile +sed -i -E 's/(###) (Fixed)/\1 🐛 \2/' $tempFile +sed -i 's/\x0//g' $tempFile + +cat $tempFile + +# Cleanup temporary files +trap 'rm -rf -- "$tempDir"' EXIT