add Auto DataCutter

can automatically split some unknown archives.

TextureExtraction tool/Data/DataCutter.cs

@@ -0,0 +1,128 @@
+using AuroraLip.Archives;
+using AuroraLip.Common;
+using System;
+using System.Collections.Generic;
+using System.IO;
+
+namespace DolphinTextureExtraction_tool
+{
+    internal class DataCutter : Archive
+    {
+        protected int maxErr = 15;
+
+        private static readonly List<byte[]> Pattern = new List<byte[]>();
+
+        static DataCutter()
+        {
+            foreach (var item in FormatDictionary.Header)
+            {
+                if (item.Value.Class != null && item.Value.Header.Bytes.Length >= 3)
+                {
+                    Pattern.Add(item.Value.Header.Bytes);
+                }
+            }
+        }
+
+        public DataCutter() { }
+
+        public DataCutter(string filename) : base(filename) { }
+
+        public DataCutter(Stream stream, string filename = null) : base(stream, filename) { }
+
+        protected override void Read(Stream stream)
+            => Read(stream, Pattern);
+
+        protected void Read(Stream stream, IEnumerable<byte[]> pattern)
+        {
+            Root = new ArchiveDirectory() { OwnerArchive = this };
+            int err = 0;
+
+            while (stream.Search(pattern, out byte[] match))
+            {
+                FormatDictionary.Header.TryGetValue(match.ToValidString(), out FormatInfo format);
+                long entrystart = stream.Position - format.Header.Offset;
+
+                if (err > maxErr)
+                    break;
+
+                uint TotalSize = 0;
+                stream.Seek(entrystart, SeekOrigin.Begin);
+                if (!format.IsMatch(stream))
+                {
+                    stream.Seek(entrystart + format.Header.Offset + 1, SeekOrigin.Begin);
+                    err++;
+                    continue;
+                }
+
+                switch (match.ToValidString())
+                {
+                    case "CLZ":
+                    case "AFS":
+                        continue;
+                    case "RSTM":
+                    case "REFT":
+                    case "bres":
+                        uint ByteOrder = BitConverter.ToUInt16(stream.Read(2), 0); //65534 BigEndian
+                        ushort Version = stream.ReadUInt16(Endian.Big);
+                        TotalSize = stream.ReadUInt32(Endian.Big);
+                        if (ByteOrder != 65534 || TotalSize > stream.Length - entrystart)
+                        {
+                            err++;
+                            continue;
+                        }
+                        break;
+                    case "PLT0":
+                    case "MDL0":
+                    case "TEX0":
+                        TotalSize = stream.ReadUInt32(Endian.Big);
+                        if (TotalSize > stream.Length - entrystart)
+                        {
+                            err++;
+                            continue;
+                        }
+                        break;
+                    case " ¯0": //tpl
+                        uint TotalImageCount = stream.ReadUInt32(Endian.Big);
+                        if (TotalImageCount < 1 || TotalImageCount > 1024)
+                        {
+                            err++;
+                            continue;
+                        }
+                        goto default;
+                    case "RTDP":
+                        int EOH = stream.ReadInt32(Endian.Big);
+                        int NrEntries = (int)stream.ReadUInt32(Endian.Big);
+                        TotalSize = stream.ReadUInt32(Endian.Big);
+                        if (NrEntries < 1 || NrEntries > 256 || TotalSize < stream.Length)
+                        {
+                            err++;
+                            continue;
+                        }
+                        break;
+                    default:
+                        stream.Search(pattern, out _);
+                        TotalSize = (uint)(stream.Position - entrystart);
+                        break;
+                }
+
+                ArchiveFile Sub = new ArchiveFile
+                {
+                    Parent = Root,
+                    Name = $"entry_{TotalFileCount + 1}.{format.Extension}",
+                    FileData = new SubStream(stream, TotalSize, entrystart)
+                };
+                Root.Items.Add(Sub.Name, Sub);
+
+                stream.Position = entrystart + TotalSize;
+            }
+
+            if (err > maxErr)
+                Root.Items = null;
+        }
+
+        protected override void Write(Stream ArchiveFile)
+        {
+            throw new NotImplementedException();
+        }
+    }
+}

TextureExtraction tool/Data/TextureExtractor.cs

@@ -567,6 +567,9 @@ private bool TryForce(Stream stream, string subdirectory, FormatInfo FFormat)
                     return true;
                 }
 
+                if (TryCut(stream,subdirectory, FFormat))
+                    return true;
+
                 stream.Position = 0;
                 if (TryBTI(stream, subdirectory))
                     return true;
@@ -605,6 +608,39 @@ private bool TryForce(Stream stream, string subdirectory, FormatInfo FFormat)
             return false;
         }
 
+
+        private Dictionary<FormatInfo, int> badformats = new Dictionary<FormatInfo, int> ();
+        private bool TryCut(Stream stream, string subdirectory, FormatInfo FFormat)
+        {
+            try
+            {
+                foreach (var item in badformats)
+                {
+                    if (item.Key == FFormat)
+                        if (item.Value > 5)
+                            return false;
+                        else
+                            break;
+                }
+                Archive archive = new DataCutter(stream);
+                if (archive.Root.Count > 0)
+                {
+                    badformats.Remove(FFormat);
+                    Scan(archive, subdirectory);
+                    return true;
+                }
+
+                if (badformats.ContainsKey(FFormat))
+                    badformats[FFormat]++;
+                else
+                    badformats.Add(FFormat, 0);
+
+
+            }
+            catch (Exception) {}
+            return false;
+        }
+
         private void AddResultUnsupported(Stream stream, string subdirectory, string Extension, FormatInfo FFormat)
         {
             Log.Write(FileAction.Unsupported, subdirectory + Extension + $" ~{MathEx.SizeSuffix(stream.Length, 2)}", $"Description: {FFormat.GetFullDescription()}");

TextureExtraction tool/DolphinTextureExtraction tool.csproj

@@ -216,6 +216,7 @@
     </Reference>
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="Data\DataCutter.cs" />
     <Compile Include="Data\Cleanup.cs" />
     <Compile Include="Data\ConsoleEx.cs" />
     <Compile Include="Data\Logger.cs" />

TextureExtraction tool/Properties/AssemblyInfo.cs

@@ -31,5 +31,5 @@
 // Sie können alle Werte angeben oder Standardwerte für die Build- und Revisionsnummern verwenden,
 // indem Sie "*" wie unten gezeigt eingeben:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("0.7.6.2")]
-[assembly: AssemblyFileVersion("0.7.6.2")]
+[assembly: AssemblyVersion("0.7.8.0")]
+[assembly: AssemblyFileVersion("0.7.8.0")]

lib/AuroraLip/Common/Extensions/StreamEx.cs

@@ -3,6 +3,7 @@
 using System.Diagnostics;
 using System.IO;
 using System.Text;
+using System.Linq;
 
 namespace AuroraLip.Common
 {
@@ -169,6 +170,42 @@ public static bool Search(this Stream stream, byte[] pattern)
             return false;
         }
 
+        /// <summary>
+        /// searches for a specific pattern in a stream and moves its position until a pattern is found.
+        /// </summary>
+        /// <param name="stream"></param>
+        /// <param name="pattern">a Enumerable of byte[] to search for</param>
+        /// <param name="match">the found byte[] that was found in the stream</param>
+        /// <returns>"true" when the pattern is found</returns>
+        public static bool Search(this Stream stream, IEnumerable<byte[]> pattern, out byte[] match)
+        {
+            int[] i = new int[pattern.Count()];
+            for (int p = 0; p < pattern.Count(); p++)
+                i[p] = 0;
+
+            int readbyte;
+            while ((readbyte = stream.ReadByte()) > -1)
+            {
+                for (int p = 0; p < pattern.Count(); p++)
+                {
+                    if (readbyte == pattern.ElementAt(p)[i[p]])
+                    {
+                        i[p]++;
+                        if (i[p] != pattern.ElementAt(p).Length)
+                            continue;
+
+                        stream.Seek(-pattern.ElementAt(p).Length, SeekOrigin.Current);
+                        match = pattern.ElementAt(p);
+                        return true;
+                    }
+                    else
+                        i[p] = 0;
+                }
+            }
+            match = null;
+            return false;
+        }
+
         /// <summary>
         /// Adds Padding to the Current Position in the provided Stream
         /// </summary>

lib/AuroraLip/Common/Extensions/StreamExString.cs

@@ -81,19 +81,5 @@ public static void WriteString(this Stream FS, string String, byte Terminator)
             FS.WriteByte(Terminator);
         }
 
-        /// <summary>
-        /// Writes a string. String will be NULL terminated
-        /// </summary>
-        /// <param name="FS"></param>
-        /// <param name="String">String to write to the file</param>
-        /// <param name="Encoding"></param>
-        [DebuggerStepThrough]
-        public static void WriteString(this Stream FS, string String, Encoding Encoding)
-        {
-            byte[] Write = Encoding.GetBytes(String);
-            FS.Write(Write, 0, Write.Length);
-            int stride = Encoding.GetMaxByteCount(0);
-            FS.Write(new byte[stride], 0, stride);
-        }
     }
 }

lib/AuroraLip/Common/FormatDictionary_List.cs

@@ -21,7 +21,7 @@ public static partial class FormatDictionary
 
             //Nintendo Textures
             new FormatInfo(".breft","REFT", FormatType.Texture, "Wii Effect Texture", Nin_),
-            new FormatInfo(".TPL", new byte[]{32,175,48}, 1, FormatType.Texture, "Texture Palette Library", Nin_){ Class = typeof(TPL) },
+            new FormatInfo(".TPL", new byte[]{32,175,48}, 1, FormatType.Texture, "Texture Palette Library", Nin_){ Class = typeof(TPL), IsMatch = TPL.Matcher },
             new FormatInfo(".txe", FormatType.Texture, "Dolphin 1 Texture", Nin_){ Class = typeof(TXE), IsMatch = TXE.Matcher },
             new FormatInfo(".bti", FormatType.Texture, "Binary Texture Image", Nin_){ Class = typeof(BTI) },
 

lib/AuroraLip/Common/FormatInfo.cs

@@ -72,7 +72,7 @@ private bool Matcher(Stream stream,in string extension = "")
         {
             if (Header != null)
             {
-                stream.Position = Header.Offset;
+                stream.Position += Header.Offset;
                 return stream.Length >= Header.Bytes.Length && stream.Read(Header.Bytes.Length).ArrayEqual(Header.Bytes);
             }
             return !(extension == "" && new HeaderInfo(stream).Bytes.Length != 0) && Extension.ToLower() == extension.ToLower();

lib/AuroraLip/Texture/Formats/TPL.cs

@@ -16,15 +16,18 @@ namespace AuroraLip.Texture.Formats
     public class TPL : JUTTexture, IFileAccess
     {
 
-        protected readonly byte[] Magic = new byte[4] { 0x00, 0x20, 0xAF, 0x30 };
+        protected static readonly byte[] Magic = new byte[4] { 0x00, 0x20, 0xAF, 0x30 };
 
         public bool CanRead => true;
 
         public bool CanWrite => true;
 
         public bool IsMatch(Stream stream, in string extension = "")
-            => stream.Length > 12 && stream.ReadByte() == Magic[0] && stream.ReadByte() == Magic[1] && stream.ReadByte() == Magic[2] && stream.ReadByte() == Magic[3];
+            => Matcher(stream, extension);
 
+        public static bool Matcher(Stream stream, in string extension = "")
+            => stream.Length > 12 && stream.ReadByte() == Magic[0] && stream.ReadByte() == Magic[1] && stream.ReadByte() == Magic[2] && stream.ReadByte() == Magic[3];
+
         public TPL() : base() { }
         public TPL(string filepath) : base(filepath) { }
         public TPL(Stream stream) => Read(stream);