From a21b6e1e44feb687e5451f62052390f592952ab8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Luthi?= Date: Sat, 17 Feb 2024 14:13:16 +0100 Subject: [PATCH 1/3] feat: Add URL-based resource mapping container builder API (#1118) Co-authored-by: Andre Hofmeister <9199345+HofmeisterAn@users.noreply.github.com> --- .../Builders/ContainerBuilder`3.cs | 18 ++++++ .../Builders/IContainerBuilder`2.cs | 33 +++++++++- .../Volumes/UriResourceMapping.cs | 60 +++++++++++++++++++ .../TarOutputMemoryStreamTest.cs | 54 ++++++++++++++--- 4 files changed, 153 insertions(+), 12 deletions(-) create mode 100644 src/Testcontainers/Configurations/Volumes/UriResourceMapping.cs diff --git a/src/Testcontainers/Builders/ContainerBuilder`3.cs b/src/Testcontainers/Builders/ContainerBuilder`3.cs index b2288d44c..9fed8de2d 100644 --- a/src/Testcontainers/Builders/ContainerBuilder`3.cs +++ b/src/Testcontainers/Builders/ContainerBuilder`3.cs @@ -197,6 +197,11 @@ public TBuilderEntity WithResourceMapping(byte[] resourceContent, string filePat /// public TBuilderEntity WithResourceMapping(string source, string target, UnixFileModes fileMode = Unix.FileMode644) { + if (Uri.IsWellFormedUriString(source, UriKind.Absolute) && Uri.TryCreate(source, UriKind.Absolute, out var uri) && new[] { Uri.UriSchemeHttp, Uri.UriSchemeHttps, Uri.UriSchemeFile }.Contains(uri.Scheme)) + { + return WithResourceMapping(uri, target, fileMode); + } + var fileAttributes = File.GetAttributes(source); if ((fileAttributes & FileAttributes.Directory) == FileAttributes.Directory) @@ -234,6 +239,19 @@ public TBuilderEntity WithResourceMapping(FileInfo source, FileInfo target, Unix } } + /// + public TBuilderEntity WithResourceMapping(Uri source, string target, UnixFileModes fileMode = Unix.FileMode644) + { + if (source.IsFile) + { + return WithResourceMapping(new FileResourceMapping(source.AbsolutePath, target, fileMode)); + } + else + { + return WithResourceMapping(new UriResourceMapping(source, target, fileMode)); + } + } + /// public TBuilderEntity WithMount(IMount mount) { diff --git a/src/Testcontainers/Builders/IContainerBuilder`2.cs b/src/Testcontainers/Builders/IContainerBuilder`2.cs index 6516c3c22..6fade9c82 100644 --- a/src/Testcontainers/Builders/IContainerBuilder`2.cs +++ b/src/Testcontainers/Builders/IContainerBuilder`2.cs @@ -219,10 +219,18 @@ public interface IContainerBuilder : I TBuilderEntity WithResourceMapping(byte[] resourceContent, string filePath, UnixFileModes fileMode = Unix.FileMode644); /// - /// Copies a test host directory or file to the container before it starts. + /// Copies the contents of a URL, a test host directory or file to the container before it starts. /// - /// The source directory or file to be copied. - /// The target directory path to copy the files to. + /// + /// If the source corresponds to a file or the Uri scheme corresponds to a file, + /// the content is copied to the target directory path. If the Uri scheme + /// corresponds to HTTP or HTTPS, the content is copied to the target file path. + /// + /// If you prefer to copy a file to a specific target file path instead of a + /// directory, use: . + /// + /// The source URL, directory or file to be copied. + /// The target directory or file path to copy the file to. /// The POSIX file mode permission. /// A configured instance of . [PublicAPI] @@ -258,6 +266,25 @@ public interface IContainerBuilder : I [PublicAPI] TBuilderEntity WithResourceMapping(FileInfo source, FileInfo target, UnixFileModes fileMode = Unix.FileMode644); + /// + /// Copies a file from a URL to the container before it starts. + /// + /// + /// If the Uri scheme corresponds to a file, the content is copied to the target + /// directory path. If the Uri scheme corresponds to HTTP or HTTPS, the content is + /// copied to the target file path. + /// + /// The Uri scheme must be either http, https or file. + /// + /// If you prefer to copy a file to a specific target file path instead of a + /// directory, use: . + /// + /// The source URL of the file to be copied. + /// The target directory or file path to copy the file to. + /// The POSIX file mode permission. + /// A configured instance of . + TBuilderEntity WithResourceMapping(Uri source, string target, UnixFileModes fileMode = Unix.FileMode644); + /// /// Assigns the mount configuration to manage data in the container. /// diff --git a/src/Testcontainers/Configurations/Volumes/UriResourceMapping.cs b/src/Testcontainers/Configurations/Volumes/UriResourceMapping.cs new file mode 100644 index 000000000..132a56ed3 --- /dev/null +++ b/src/Testcontainers/Configurations/Volumes/UriResourceMapping.cs @@ -0,0 +1,60 @@ +namespace DotNet.Testcontainers.Configurations +{ + using System; + using System.Net.Http; + using System.Threading; + using System.Threading.Tasks; + + /// + internal sealed class UriResourceMapping : IResourceMapping + { + private readonly Uri _uri; + + /// + /// Initializes a new instance of the class. + /// + /// The URL of the file to download. + /// The absolute path of the file to map in the container. + /// The POSIX file mode permission. + public UriResourceMapping(Uri uri, string containerPath, UnixFileModes fileMode) + { + _uri = uri; + Type = MountType.Bind; + Source = uri.AbsoluteUri; + Target = containerPath; + FileMode = fileMode; + AccessMode = AccessMode.ReadOnly; + } + + /// + public MountType Type { get; } + + /// + public AccessMode AccessMode { get; } + + /// + public string Source { get; } + + /// + public string Target { get; } + + /// + public UnixFileModes FileMode { get; } + + /// + public Task CreateAsync(CancellationToken ct = default) => Task.CompletedTask; + + /// + public Task DeleteAsync(CancellationToken ct = default) => Task.CompletedTask; + + /// + public async Task GetAllBytesAsync(CancellationToken ct = default) + { + using (var httpClient = new HttpClient()) + { + return await httpClient.GetByteArrayAsync(_uri) + .ConfigureAwait(false); + } + } + } +} diff --git a/tests/Testcontainers.Platform.Linux.Tests/TarOutputMemoryStreamTest.cs b/tests/Testcontainers.Platform.Linux.Tests/TarOutputMemoryStreamTest.cs index d1c986644..98ae58dd5 100644 --- a/tests/Testcontainers.Platform.Linux.Tests/TarOutputMemoryStreamTest.cs +++ b/tests/Testcontainers.Platform.Linux.Tests/TarOutputMemoryStreamTest.cs @@ -33,8 +33,18 @@ public void TestFileExistsInTarFile() } [UsedImplicitly] - public sealed class FromResourceMapping : TarOutputMemoryStreamTest, IResourceMapping, IAsyncLifetime, IDisposable + public sealed class FromResourceMapping : TarOutputMemoryStreamTest, IResourceMapping, IClassFixture, IAsyncLifetime, IDisposable { + private readonly string _testHttpUri; + + private readonly string _testFileUri; + + public FromResourceMapping(FromResourceMapping.HttpFixture httpFixture) + { + _testHttpUri = httpFixture.BaseAddress; + _testFileUri = new Uri(_testFile.FullName).ToString(); + } + public MountType Type => MountType.Bind; @@ -86,24 +96,23 @@ public async Task TestFileExistsInContainer() { // Given var targetFilePath1 = string.Join("/", string.Empty, "tmp", Guid.NewGuid(), _testFile.Name); - var targetFilePath2 = string.Join("/", string.Empty, "tmp", Guid.NewGuid(), _testFile.Name); - + var targetFilePath3 = string.Join("/", string.Empty, "tmp", Guid.NewGuid(), _testFile.Name); var targetDirectoryPath1 = string.Join("/", string.Empty, "tmp", Guid.NewGuid()); - var targetDirectoryPath2 = string.Join("/", string.Empty, "tmp", Guid.NewGuid()); - var targetDirectoryPath3 = string.Join("/", string.Empty, "tmp", Guid.NewGuid()); - var targetDirectoryPath4 = string.Join("/", string.Empty, "tmp", Guid.NewGuid()); + var targetDirectoryPath5 = string.Join("/", string.Empty, "tmp", Guid.NewGuid()); var targetFilePaths = new List(); targetFilePaths.Add(targetFilePath1); targetFilePaths.Add(targetFilePath2); + targetFilePaths.Add(targetFilePath3); targetFilePaths.Add(string.Join("/", targetDirectoryPath1, _testFile.Name)); targetFilePaths.Add(string.Join("/", targetDirectoryPath2, _testFile.Name)); targetFilePaths.Add(string.Join("/", targetDirectoryPath3, _testFile.Name)); targetFilePaths.Add(string.Join("/", targetDirectoryPath4, _testFile.Name)); + targetFilePaths.Add(string.Join("/", targetDirectoryPath5, _testFile.Name)); await using var container = new ContainerBuilder() .WithImage(CommonImages.Alpine) @@ -111,6 +120,8 @@ public async Task TestFileExistsInContainer() .WithResourceMapping(_testFile, new FileInfo(targetFilePath1)) .WithResourceMapping(_testFile.FullName, targetDirectoryPath1) .WithResourceMapping(_testFile.Directory.FullName, targetDirectoryPath2) + .WithResourceMapping(_testHttpUri, targetFilePath2) + .WithResourceMapping(_testFileUri, targetDirectoryPath3) .Build(); // When @@ -120,13 +131,13 @@ public async Task TestFileExistsInContainer() await container.StartAsync() .ConfigureAwait(true); - await container.CopyAsync(fileContent, targetFilePath2) + await container.CopyAsync(fileContent, targetFilePath3) .ConfigureAwait(true); - await container.CopyAsync(_testFile.FullName, targetDirectoryPath3) + await container.CopyAsync(_testFile.FullName, targetDirectoryPath4) .ConfigureAwait(true); - await container.CopyAsync(_testFile.Directory.FullName, targetDirectoryPath4) + await container.CopyAsync(_testFile.Directory.FullName, targetDirectoryPath5) .ConfigureAwait(true); // Then @@ -135,6 +146,31 @@ await container.CopyAsync(_testFile.Directory.FullName, targetDirectoryPath4) Assert.All(execResults, result => Assert.Equal(0, result.ExitCode)); } + + public sealed class HttpFixture : IAsyncLifetime + { + private const ushort HttpPort = 80; + + private readonly IContainer _container = new ContainerBuilder() + .WithImage(CommonImages.Alpine) + .WithEntrypoint("/bin/sh", "-c") + .WithCommand($"while true; do echo \"HTTP/1.1 200 OK\r\n\" | nc -l -p {HttpPort}; done") + .WithPortBinding(HttpPort, true) + .Build(); + + public string BaseAddress + => new UriBuilder(Uri.UriSchemeHttp, _container.Hostname, _container.GetMappedPublicPort(HttpPort)).ToString(); + + public Task InitializeAsync() + { + return _container.StartAsync(); + } + + public Task DisposeAsync() + { + return _container.DisposeAsync().AsTask(); + } + } } [UsedImplicitly] From 2766fec36518f9016e073f61099cc2d7c6ebd049 Mon Sep 17 00:00:00 2001 From: Andre Hofmeister <9199345+HofmeisterAn@users.noreply.github.com> Date: Mon, 19 Feb 2024 19:00:04 +0100 Subject: [PATCH 2/3] fix: Prevent .dockerignore entry from matching everything if it ends with globstart (#1122) --- src/Testcontainers/Images/IgnoreFile.cs | 19 +++++++++++-------- .../Fixtures/Images/IgnoreFileFixture.cs | 14 +++++++++++++- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/Testcontainers/Images/IgnoreFile.cs b/src/Testcontainers/Images/IgnoreFile.cs index d5d3ec336..ae135a6be 100644 --- a/src/Testcontainers/Images/IgnoreFile.cs +++ b/src/Testcontainers/Images/IgnoreFile.cs @@ -73,14 +73,17 @@ public IgnoreFile(IEnumerable patterns, ILogger logger) // Prepare exact and partial patterns. .Aggregate(new List>(), (lines, line) => { - var key = line.Key; - var value = line.Value; - - lines.AddRange(key - .Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries) - .Skip(1) - .Prepend(key) - .Select(ignorePattern => new KeyValuePair(ignorePattern, value))); + const string globstar = "**/"; + + if (line.Key.Contains(globstar)) + { + lines.Add(line); + lines.Add(new KeyValuePair(line.Key.Replace(globstar, string.Empty), line.Value)); + } + else + { + lines.Add(line); + } return lines; }) diff --git a/tests/Testcontainers.Tests/Fixtures/Images/IgnoreFileFixture.cs b/tests/Testcontainers.Tests/Fixtures/Images/IgnoreFileFixture.cs index fdb1a3870..48ca59d4b 100644 --- a/tests/Testcontainers.Tests/Fixtures/Images/IgnoreFileFixture.cs +++ b/tests/Testcontainers.Tests/Fixtures/Images/IgnoreFileFixture.cs @@ -13,7 +13,7 @@ public IgnoreFileFixture() var ignoreAllFilesAndDirectories = new IgnoreFile(new[] { "*", "!README*.md" }, logger); var ignoreNonRecursiveFiles = new IgnoreFile(new[] { "*/temp*" }, logger); var ignoreNonRecursiveNestedFiles = new IgnoreFile(new[] { "*/*/temp*" }, logger); - var ignoreRecursiveFiles = new IgnoreFile(new[] { "**/*.txt" }, logger); + var ignoreRecursiveFiles = new IgnoreFile(new[] { "**/*.txt", "**/.idea", "**/.vs", "**/.git", "!**/.gitignore", "!.git/HEAD", "!.git/refs/heads/**", "src/**/lipsum.config" }, logger); var ignoreSingleCharacterFiles = new IgnoreFile(new[] { "temp?" }, logger); var ignoreExceptionFiles = new IgnoreFile(new[] { "*.md", "!README*.md", "README-secret.md" }, logger); Add(ignoreFilesAndDirectories, "bin/Debug", false); @@ -35,6 +35,18 @@ public IgnoreFileFixture() Add(ignoreRecursiveFiles, "lorem/lipsum.txt", false); Add(ignoreRecursiveFiles, "lorem/lipsum/lipsum.config", true); Add(ignoreRecursiveFiles, "lorem/lipsum.config", true); + Add(ignoreRecursiveFiles, "src/.idea/../v17/../lipsum.log", false); + Add(ignoreRecursiveFiles, "src/.vs/../v17/../lipsum.log", false); + Add(ignoreRecursiveFiles, ".git/logs", false); + Add(ignoreRecursiveFiles, "src/lorem/lipsum/lipsum.config", false); + Add(ignoreRecursiveFiles, "src/lorem/lipsum.config", false); + Add(ignoreRecursiveFiles, "src/lipsum.config", false); + Add(ignoreRecursiveFiles, ".gitignore", true); + Add(ignoreRecursiveFiles, ".git/HEAD", true); + Add(ignoreRecursiveFiles, ".git/refs/heads/main", true); + Add(ignoreRecursiveFiles, ".git/refs/heads/bugfix/gh-1119", true); + Add(ignoreRecursiveFiles, "src/lorem/temp", true); + Add(ignoreRecursiveFiles, "lipsum.config", true); Add(ignoreSingleCharacterFiles, "temp", false); Add(ignoreSingleCharacterFiles, "temp1", false); Add(ignoreSingleCharacterFiles, "temp12", true); From 8aee376aa98018d07d270264920161113dfd8490 Mon Sep 17 00:00:00 2001 From: Andre Hofmeister <9199345+HofmeisterAn@users.noreply.github.com> Date: Wed, 21 Feb 2024 20:17:12 +0100 Subject: [PATCH 3/3] feat: Add remote container registry identity token support (#1124) --- Directory.Packages.props | 8 +++---- examples/Flyway/Directory.Packages.props | 4 ++-- .../WeatherForecast/Directory.Packages.props | 4 ++-- global.json | 2 +- .../PostgreSqlBuilder.cs | 21 ++++++++++--------- src/Testcontainers/Builders/Base64Provider.cs | 11 ++++++++++ ...ockerRegistryAuthenticationProviderTest.cs | 17 +++++++++------ 7 files changed, 42 insertions(+), 25 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 3a9e4dbd2..6eeb39533 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -4,7 +4,7 @@ true - + @@ -15,8 +15,8 @@ - - + + @@ -59,4 +59,4 @@ - \ No newline at end of file + diff --git a/examples/Flyway/Directory.Packages.props b/examples/Flyway/Directory.Packages.props index 6fee13d9c..8549bb998 100644 --- a/examples/Flyway/Directory.Packages.props +++ b/examples/Flyway/Directory.Packages.props @@ -7,8 +7,8 @@ - - + + diff --git a/examples/WeatherForecast/Directory.Packages.props b/examples/WeatherForecast/Directory.Packages.props index edbae9614..a281383a5 100644 --- a/examples/WeatherForecast/Directory.Packages.props +++ b/examples/WeatherForecast/Directory.Packages.props @@ -11,8 +11,8 @@ - - + + diff --git a/global.json b/global.json index b75deba61..d031a7632 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "sdk": { - "version": "8.0.100", + "version": "8.0.200", "rollForward": "latestPatch" } } diff --git a/src/Testcontainers.PostgreSql/PostgreSqlBuilder.cs b/src/Testcontainers.PostgreSql/PostgreSqlBuilder.cs index 0cbb00e9c..c7a0334ac 100644 --- a/src/Testcontainers.PostgreSql/PostgreSqlBuilder.cs +++ b/src/Testcontainers.PostgreSql/PostgreSqlBuilder.cs @@ -126,7 +126,7 @@ protected override PostgreSqlBuilder Merge(PostgreSqlConfiguration oldValue, Pos /// private sealed class WaitUntil : IWaitUntil { - private readonly string[] _command; + private readonly IList _command; /// /// Initializes a new instance of the class. @@ -134,18 +134,19 @@ private sealed class WaitUntil : IWaitUntil /// The container configuration. public WaitUntil(PostgreSqlConfiguration configuration) { - _command = new[] { - "pg_isready", - "--host", "localhost", // Explicitly specify localhost in order to be ready only after the initdb scripts have run and the server is listening over TCP/IP - "--dbname", configuration.Database, - "--username", configuration.Username, - }; + // Explicitly specify the host to ensure readiness only after the initdb scripts have executed, and the server is listening on TCP/IP. + _command = new List { "pg_isready", "--host", "localhost", "--dbname", configuration.Database, "--username", configuration.Username }; } /// - /// Test whether the database is ready to accept connections or not with the pg_isready command. + /// Checks whether the database is ready and accepts connections or not. /// - /// if the database is ready to accept connections; if the database is not yet ready. + /// + /// The wait strategy uses pg_isready to check the connection status of PostgreSql. + /// + /// The starting container instance. + /// Task that completes and returns true when the database is ready and accepts connections, otherwise false. + /// Thrown when the PostgreSql image does not contain pg_isready. public async Task UntilAsync(IContainer container) { var execResult = await container.ExecAsync(_command) @@ -153,7 +154,7 @@ public async Task UntilAsync(IContainer container) if (execResult.Stderr.Contains("pg_isready was not found")) { - throw new NotSupportedException($"The {container.Image.FullName} image is not supported. Please use postgres:9.3 onwards."); + throw new NotSupportedException($"The '{container.Image.FullName}' image does not contain: pg_isready. Please use 'postgres:9.3' onwards."); } return 0L.Equals(execResult.ExitCode); diff --git a/src/Testcontainers/Builders/Base64Provider.cs b/src/Testcontainers/Builders/Base64Provider.cs index 7998aa5e8..29ed7b142 100644 --- a/src/Testcontainers/Builders/Base64Provider.cs +++ b/src/Testcontainers/Builders/Base64Provider.cs @@ -67,6 +67,17 @@ public IDockerRegistryAuthenticationConfiguration GetAuthConfig(string hostname) return null; } + if (authProperty.Value.TryGetProperty("identitytoken", out var identityToken) && JsonValueKind.String.Equals(identityToken.ValueKind)) + { + var identityTokenValue = identityToken.GetString(); + + if (!string.IsNullOrEmpty(identityTokenValue)) + { + _logger.DockerRegistryCredentialFound(hostname); + return new DockerRegistryAuthenticationConfiguration(authProperty.Name, null, null, identityTokenValue); + } + } + if (!authProperty.Value.TryGetProperty("auth", out var auth)) { return null; diff --git a/tests/Testcontainers.Tests/Unit/Configurations/DockerRegistryAuthenticationProviderTest.cs b/tests/Testcontainers.Tests/Unit/Configurations/DockerRegistryAuthenticationProviderTest.cs index 27beda932..728a083b1 100644 --- a/tests/Testcontainers.Tests/Unit/Configurations/DockerRegistryAuthenticationProviderTest.cs +++ b/tests/Testcontainers.Tests/Unit/Configurations/DockerRegistryAuthenticationProviderTest.cs @@ -93,6 +93,9 @@ public void ResolvePartialDockerRegistry(string jsonDocument) [InlineData("{\"auths\":{\"" + DockerRegistry + "\":{\"auth\":{}}}}", true, "The \"auth\" property value kind for https://index.docker.io/v1/ is invalid: Object")] [InlineData("{\"auths\":{\"" + DockerRegistry + "\":{\"auth\":\"Not_Base64_encoded\"}}}", true, "The \"auth\" property value for https://index.docker.io/v1/ is not a valid Base64 string")] [InlineData("{\"auths\":{\"" + DockerRegistry + "\":{\"auth\":\"dXNlcm5hbWU=\"}}}", true, "The \"auth\" property value for https://index.docker.io/v1/ should contain one colon separating the username and the password (basic authentication)")] + [InlineData("{\"auths\":{\"" + DockerRegistry + "\":{\"identitytoken\":null}}}", true, null)] + [InlineData("{\"auths\":{\"" + DockerRegistry + "\":{\"identitytoken\":\"\"}}}", true, null)] + [InlineData("{\"auths\":{\"" + DockerRegistry + "\":{\"identitytoken\":{}}}}", true, null)] public void ShouldGetNull(string jsonDocument, bool isApplicable, string logMessage) { // Given @@ -116,11 +119,12 @@ public void ShouldGetNull(string jsonDocument, bool isApplicable, string logMess } } - [Fact] - public void ShouldGetAuthConfig() + [Theory] + [InlineData("{\"auths\":{\"" + DockerRegistry + "\":{\"auth\":\"dXNlcm5hbWU6cGFzc3dvcmQ=\"}}}", "username", "password", null)] + [InlineData("{\"auths\":{\"" + DockerRegistry + "\":{\"identitytoken\":\"identitytoken\"}}}", null, null, "identitytoken")] + public void ShouldGetAuthConfig(string jsonDocument, string expectedUsername, string expectedPassword, string expectedIdentityToken) { // Given - const string jsonDocument = "{\"auths\":{\"" + DockerRegistry + "\":{\"auth\":\"dXNlcm5hbWU6cGFzc3dvcmQ=\"}}}"; var jsonElement = JsonDocument.Parse(jsonDocument).RootElement; // When @@ -131,8 +135,9 @@ public void ShouldGetAuthConfig() Assert.True(authenticationProvider.IsApplicable(DockerRegistry)); Assert.NotNull(authConfig); Assert.Equal(DockerRegistry, authConfig.RegistryEndpoint); - Assert.Equal("username", authConfig.Username); - Assert.Equal("password", authConfig.Password); + Assert.Equal(expectedUsername, authConfig.Username); + Assert.Equal(expectedPassword, authConfig.Password); + Assert.Equal(expectedIdentityToken, authConfig.IdentityToken); } } @@ -259,7 +264,7 @@ public void Dispose() private sealed class WarnLogger : ILogger { - private readonly List> _logMessages = new List>(); + private readonly IList> _logMessages = new List>(); public IEnumerable> LogMessages => _logMessages;