Merge branch 'release/v2.8.0'

src/Arkivverket.Arkade.CLI.Tests/Arkivverket.Arkade.CLI.Tests.csproj

@@ -21,10 +21,10 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="FluentAssertions" Version="6.5.1" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
-    <PackageReference Include="xunit" Version="2.4.1" />
-    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
+    <PackageReference Include="FluentAssertions" Version="6.8.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.2" />
+    <PackageReference Include="xunit" Version="2.4.2" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>

src/Arkivverket.Arkade.CLI/Options/ValidateOptions.cs

@@ -10,11 +10,11 @@ public class ValidateOptions
         [Option('i', "item", HelpText = "The file or directory to be validated", Required = true)]
         public string Item { get; set; }
 
-        [Option('f', "format", HelpText = "The format which the file or directory is validated against. Available values: PDF/A", Required = true)]
+        [Option('f', "format", HelpText = "The format which the file or directory is validated against. Available values: PDF/A, DIAS-SIP, DIAS-AIP, DIAS-SIP-Noark5, DIAS-AIP-Noark5", Required = true)]
         public string Format { get; set; }
 
         [Option('o', "output-directory", 
-            HelpText = "Directory to place Arkade output files. Required if -i/--item is a directory.", 
+            HelpText = "Directory to place Arkade output files. Required if -i/--item is a directory and -f/--format is PDF/A.", 
             Required = false)]
         public string OutputDirectory { get; set; }
 
@@ -30,6 +30,14 @@ public static IEnumerable<Example> Examples
                         Item = "/path/to/pdfA-file",
                         Format = "PDF/A"
                     });
+
+                yield return new Example("Validate the file/directory structure of a specified IP against DIAS",
+                    OptionsConfig.FormatStyle,
+                    new ValidateOptions
+                    {
+                        Item = "/path/to/dias-tar-or-directory",
+                        Format = "DIAS SIP"
+                    });
             }
         }
     }

src/Arkivverket.Arkade.CLI/Program.cs

@@ -134,9 +134,9 @@ private static bool ReadyToRun(ValidateOptions validateOptions)
 
             if (Directory.Exists(validateOptions.Item))
             {
-                if (IsNullOrEmpty(validateOptions.OutputDirectory))
+                if (archiveFormat == ArchiveFormat.PdfA && IsNullOrEmpty(validateOptions.OutputDirectory))
                 {
-                    Log.Error(@"The -o/--output-directory argument is required when -i/--item is a directory.");
+                    Log.Error(@"The -o/--output-directory argument is required when -f/--format is PDF/A and -i/--item is a directory.");
                     return false;
                 }
 

src/Arkivverket.Arkade.Core.Tests/Arkivverket.Arkade.Core.Tests.csproj

@@ -51,13 +51,13 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="FluentAssertions" Version="6.7.0" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.2.0" />
-    <PackageReference Include="Moq" Version="4.18.1" />
+    <PackageReference Include="FluentAssertions" Version="6.8.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.2" />
+    <PackageReference Include="Moq" Version="4.18.2" />
     <PackageReference Include="Serilog.Sinks.Observable" Version="2.0.2" />
     <PackageReference Include="SharpZipLib" Version="1.3.3" />
     <PackageReference Include="System.Reactive" Version="5.0.0" />
-    <PackageReference Include="xunit" Version="2.4.1" />
+    <PackageReference Include="xunit" Version="2.4.2" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
@@ -90,6 +90,9 @@
     <None Update="TestData\fagsystem\autodetect\dokumenter\5000001.pdf">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
+    <None Update="TestData\FileTypes\fileTypes.zip">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
     <None Update="TestData\jegerreg-98-dos\arkivuttrekk.xml">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>

src/Arkivverket.Arkade.Core.Tests/Base/Addml/DelimiterFileFormatReaderTest.cs

@@ -4,6 +4,7 @@
 using Arkivverket.Arkade.Core.Base;
 using Arkivverket.Arkade.Core.Base.Addml;
 using Arkivverket.Arkade.Core.Base.Addml.Definitions;
+using Arkivverket.Arkade.Core.Base.Addml.Definitions.DataTypes;
 using Arkivverket.Arkade.Core.Tests.Base.Addml.Builders;
 using FluentAssertions;
 using Xunit;
@@ -223,7 +224,7 @@ public void QuotingCharWithinQuotingCharsAreNotInterpretedAsQuotingChar()
 
                 new AddmlFieldDefinitionBuilder().WithRecordDefinition(recordDefinition).Build();
 
-                var csvData = $"{quotingString}A{quotingString}B{quotingString}";
+                var csvData = $"{quotingString}A{quotingString}{quotingString}B{quotingString}";
 
                 var streamReader = new StreamReader(new MemoryStream(Encoding.UTF8.GetBytes(csvData)));
                 var recordReader = new DelimiterFileFormatReader(new FlatFile(addmlFlatFileDefinition), streamReader);
@@ -262,10 +263,10 @@ public void FieldIsOnlyQuotedWhenStartingAndEndingWithQuotingChar()
                     new AddmlFieldDefinitionBuilder().WithRecordDefinition(recordDefinition).Build();
                     new AddmlFieldDefinitionBuilder().WithRecordDefinition(recordDefinition).Build();
 
-                    string csvData = $"{quotingString}A{quotingString}B{quotingString}{fieldSeparator}" +
-                                     $"C{quotingString}{fieldSeparator}" +
+                    string csvData = $"{quotingString}A{quotingString}{quotingString}B{quotingString}{fieldSeparator}" +
+                                     $"{quotingString}C{quotingString}{quotingString}{quotingString}{fieldSeparator}" +
                                      $"{quotingString}D{quotingString}{quotingString}{fieldSeparator} asd{quotingString}{fieldSeparator}" +
-                                     $"E{quotingString}noko{quotingString}{fieldSeparator}" +
+                                     $"{quotingString}E{quotingString}{quotingString}noko{quotingString}{quotingString}{quotingString}{fieldSeparator}" +
                                      "F";
 
                     var streamReader = new StreamReader(new MemoryStream(Encoding.UTF8.GetBytes(csvData)));
@@ -278,7 +279,7 @@ public void FieldIsOnlyQuotedWhenStartingAndEndingWithQuotingChar()
                     recordReader.Current?.Fields?.Count.Should().Be(5);
                     recordReader.Current?.Fields?[0].Value.Should().Be($"A{quotingString}B");
                     recordReader.Current?.Fields?[1].Value.Should().Be($"C{quotingString}");
-                    recordReader.Current?.Fields?[2].Value.Should().Be($"D{quotingString}{quotingString}{fieldSeparator} asd");
+                    recordReader.Current?.Fields?[2].Value.Should().Be($"D{quotingString}{fieldSeparator} asd");
                     recordReader.Current?.Fields?[3].Value.Should().Be($"E{quotingString}noko{quotingString}");
                     recordReader.Current?.Fields?[4].Value.Should().Be("F");
                 }
@@ -307,5 +308,44 @@ public void QuotingCharEqualToFieldDelimiterShouldThrowException()
 
             actionOfCreatingRecordReader.Should().Throw<ArkadeAddmlDelimiterException>();
         }
+
+        [Fact]
+        public void DateFieldsEncapsulatedByQuotesShouldBeValid()
+        {
+            const string d = ";";
+            const string q = "\"";
+            AddmlFlatFileDefinition addmlFlatFileDefinition = new AddmlFlatFileDefinitionBuilder()
+                .WithRecordSeparator("CRLF")
+                .WithFieldSeparator(d)
+                .WithQuotingChar(q)
+                .Build();
+
+            AddmlRecordDefinition recordDefinition = new AddmlRecordDefinitionBuilder()
+                .WithAddmlFlatFileDefinition(addmlFlatFileDefinition).Build();
+
+            new AddmlFieldDefinitionBuilder()
+                .WithRecordDefinition(recordDefinition)
+                .WithDataType(new DateDataType("YYYY-MM-DD"))
+                .WithName("date")
+                .Build();
+            new AddmlFieldDefinitionBuilder()
+                .WithRecordDefinition(recordDefinition)
+                .WithDataType(new DateDataType("YYYY-MM-DD"))
+                .WithName("date")
+                .Build();
+
+            const string csvData = $"{q}2022-09-06{q}{d}{q}2022-09-05{q}";
+
+            using var streamReader = new StreamReader(new MemoryStream(Encoding.UTF8.GetBytes(csvData)));
+            var recordReader = new DelimiterFileFormatReader(new FlatFile(addmlFlatFileDefinition), streamReader);
+            var actionOfGettingCurrent = (Action)(() => ((Func<object>)(() => recordReader.Current))());
+
+            recordReader.MoveNext();
+
+            actionOfGettingCurrent.Should().NotThrow<Exception>();
+            recordReader.Current?.Fields?.Count.Should().Be(2);
+            recordReader.Current?.Fields?[0].Value.Should().Be("2022-09-06");
+            recordReader.Current?.Fields?[1].Value.Should().Be("2022-09-05");
+        }
     }
 }

src/Arkivverket.Arkade.Core.Tests/Base/Siard/SiardXmlTableReaderTests.cs

@@ -6,6 +6,7 @@
 using Arkivverket.Arkade.Core.Logging;
 using Arkivverket.Arkade.Core.Util.FileFormatIdentification;
 using FluentAssertions;
+using Moq;
 
 namespace Arkivverket.Arkade.Core.Tests.Base.Siard
 {
@@ -15,7 +16,8 @@ public class SiardXmlTableReaderTests
 
         public SiardXmlTableReaderTests()
         {
-            _fileFormatIdentifier = new SiegfriedFileFormatIdentifier(new SiegfriedProcessRunner(new StatusEventHandler()));
+            IStatusEventHandler statusEventHandler = new Mock<IStatusEventHandler>().Object;
+            _fileFormatIdentifier = new SiegfriedFileFormatIdentifier(new SiegfriedProcessRunner(statusEventHandler), statusEventHandler);
         }
 
         [Fact]

src/Arkivverket.Arkade.Core.Tests/Testing/Siard/SiardValidatorRunnerTest.cs

@@ -1,16 +1,24 @@
 using System.Collections.Generic;
-using System.ComponentModel;
 using System.IO;
 using System.Linq;
 using System.Text.RegularExpressions;
+using Arkivverket.Arkade.Core.Logging;
 using Arkivverket.Arkade.Core.Testing.Siard;
 using FluentAssertions;
+using Moq;
 using Xunit;
 
 namespace Arkivverket.Arkade.Core.Tests.Testing.Siard
 {
     public class SiardValidatorRunnerTest
     {
+        private static readonly ISiardValidator Validator;
+
+        static SiardValidatorRunnerTest()
+        {
+            Validator = new SiardValidator(new Mock<IStatusEventHandler>().Object, new Mock<ITestProgressReporter>().Object);
+        }
+
         [Fact]
         [Trait("Category", "Integration")]
         [Trait("Dependency", "JRE"), Trait("Dependency", "DBPTK")]
@@ -19,7 +27,7 @@ public void ShouldGenerateValidationReportFileAtDesignatedDestination()
             string inputFilePath = Path.Combine("TestData", "Siard", "dbptk_produced.siard");
             string reportFilePath = Path.Combine("TestData", "Siard", "testReport.txt");
 
-            _ = SiardValidator.Validate(inputFilePath, reportFilePath);
+            Validator.Validate(inputFilePath, reportFilePath);
 
             File.Exists(reportFilePath).Should().BeTrue();
             // clean up generated files
@@ -36,7 +44,7 @@ public void ShouldReportUnsupportedSiardVersion()
             string inputFilePath = Path.Combine("TestData", "Siard", "siard1_med_blobs.siard");
             string reportFilePath = Path.Combine("TestData", "Siard", "testReport.txt");
 
-            (List<string> results, _) = SiardValidator.Validate(inputFilePath, reportFilePath);
+            List<string> results = Validator.Validate(inputFilePath, reportFilePath).Results;
 
             results.Count.Should().Be(2);
             results[0].Should().Be(Resources.SiardMessages.ErrorMessage);
@@ -57,7 +65,7 @@ public void ShouldValidateExtractProducedBySiardGui()
             string inputFilePath = Path.Combine("TestData", "Siard", "siard2", "siardGui", "external", "siardGui.siard");
             string reportFilePath = Path.Combine("TestData", "Siard", "testReport.txt");
 
-            (_, List<string> errorsAndWarnings) = SiardValidator.Validate(inputFilePath, reportFilePath);
+            List<string> errorsAndWarnings = Validator.Validate(inputFilePath, reportFilePath).Errors;
 
             var errors = new List<string>(errorsAndWarnings.Where(e => e == null || !e.StartsWith("WARN")));
 
@@ -79,7 +87,7 @@ public void ShouldValidateExtractProducedByDbptkDeveloper()
             string inputFilePath = Path.Combine("TestData", "Siard", "siard2", "dbPtk", "external", "dbptk.siard");
             string reportFilePath = Path.Combine("TestData", "Siard", "testReport.txt");
 
-            (_, List<string> errorsAndWarnings) = SiardValidator.Validate(inputFilePath, reportFilePath);
+            List<string> errorsAndWarnings = Validator.Validate(inputFilePath, reportFilePath).Errors;
 
             var errors = new List<string>(errorsAndWarnings.Where(e => e == null || !e.StartsWith("WARN")));
 
@@ -103,7 +111,7 @@ public void ShouldFailToValidateExtractProducedBySpectralCoreFullConvert()
             string inputFilePath = Path.Combine("TestData", "Siard", "siard2", "fullConvert", "external", "scfc.siard");
             string reportFilePath = Path.Combine("TestData", "Siard", "testReport.txt");
 
-            (_, List<string> errorsAndWarnings) = SiardValidator.Validate(inputFilePath, reportFilePath);
+            List<string> errorsAndWarnings = Validator.Validate(inputFilePath, reportFilePath).Errors;
 
             var errors = new List<string>(errorsAndWarnings.Where(e => e == null || !e.StartsWith("WARN")));
 
@@ -129,7 +137,9 @@ public void ShouldReportWarningsWhenExternalLobsAreMissing()
             string inputFilePath = Path.Combine("TestData", "Siard", "siard2", "externalLobsMissing", "dbptk.siard");
             string reportFilePath = Path.Combine("TestData", "Siard", "testReport.txt");
 
-            (List<string> results, List<string> errorsAndWarnings) = SiardValidator.Validate(inputFilePath, reportFilePath);
+            var report = Validator.Validate(inputFilePath, reportFilePath);
+            List<string> results = report.Results;
+            List<string> errorsAndWarnings = report.Errors;
 
             List<string> summary = results.Where(r => r != null && r.Trim().StartsWith("Number of")).ToList();
 

src/Arkivverket.Arkade.Core.Tests/Util/ArchiveFormatValidation/DiasValidation/DiasDirectoryTests.cs

@@ -0,0 +1,69 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using Arkivverket.Arkade.Core.Util.ArchiveFormatValidation;
+using FluentAssertions;
+using Xunit;
+
+namespace Arkivverket.Arkade.Core.Tests.Util.ArchiveFormatValidation.DiasValidation
+{
+    public class DiasDirectoryTests
+    {
+        [Fact]
+        public void GetEntryPathsTest()
+        {
+            var dias =
+                new DiasDirectory("SomeDiasDirectory",
+                    new DiasDirectory("SomeDiasSubDirectory",
+                        new DiasFile("SomeDiasFile.txt")));
+
+            IEnumerable<string> entryPaths = dias.GetEntryPaths("SomeDiasDirectory", recursive: true);
+
+            string expectedEntryPath = Path.Combine("SomeDiasDirectory", "SomeDiasSubDirectory", "SomeDiasFile.txt");
+
+            entryPaths.Should().Contain(e => e.Equals(expectedEntryPath));
+        }
+
+        [Fact]
+        public void GetMissingEntryPathsTest()
+        {
+            DirectoryInfo diasOnDisk = CreateTestDiasDirectory();
+
+            var dias =
+                new DiasDirectory(diasOnDisk.FullName,
+                    new DiasDirectory("SomeDiasSubDirectory",
+                        new DiasFile("SomeDiasFile.txt"),
+                        new DiasFile("SomeOtherDiasFile.txt") // Not on disk
+                    ));
+
+            IEnumerable<string> entryPaths =
+                dias.GetEntryPaths(diasOnDisk.FullName, getNonExistingOnly: true, recursive: true);
+
+            string expectedMissingEntryPath = Path.Combine(
+                AppDomain.CurrentDomain.BaseDirectory,
+                "SomeDiasDirectory",
+                "SomeDiasSubDirectory",
+                "SomeOtherDiasFile.txt"
+            );
+
+            entryPaths.Should().Contain(e => e.Equals(expectedMissingEntryPath));
+
+            diasOnDisk.Delete(true);
+        }
+
+        private static DirectoryInfo CreateTestDiasDirectory()
+        {
+            DirectoryInfo diasRootDirectory = Directory.CreateDirectory(
+                Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "SomeDiasDirectory")
+            );
+
+            DirectoryInfo testSubDirectory = Directory.CreateDirectory(
+                Path.Combine(diasRootDirectory.FullName, "SomeDiasSubDirectory")
+            );
+
+            File.Create(Path.Combine(testSubDirectory.FullName, "SomeDiasFile.txt")).Dispose();
+
+            return diasRootDirectory;
+        }
+    }
+}