From ee0c5d99141cf7f3020b349de871541b38b2c3d6 Mon Sep 17 00:00:00 2001
From: Adam <893184+NotAdam@users.noreply.github.com>
Date: Sun, 29 Nov 2020 10:08:24 +1100
Subject: [PATCH] prevent lumina from changing the endianness of a file twice
---
src/Lumina/Data/FileOptionsAttribute.cs | 37 ++++++++++++++++
src/Lumina/Data/Files/Excel/ExcelDataFile.cs | 6 ++-
src/Lumina/Data/Repository.cs | 46 +++++++++++---------
src/Lumina/Excel/ExcelModule.cs | 8 +++-
src/Lumina/Excel/ExcelSheetImpl.cs | 9 ++++
5 files changed, 83 insertions(+), 23 deletions(-)
create mode 100644 src/Lumina/Data/FileOptionsAttribute.cs
diff --git a/src/Lumina/Data/FileOptionsAttribute.cs b/src/Lumina/Data/FileOptionsAttribute.cs
new file mode 100644
index 00000000..19ede42a
--- /dev/null
+++ b/src/Lumina/Data/FileOptionsAttribute.cs
@@ -0,0 +1,37 @@
+using System;
+
+namespace Lumina.Data
+{
+ [AttributeUsage( AttributeTargets.Class )]
+ public class FileOptionsAttribute : Attribute
+ {
+ ///
+ /// The current cache behaviour
+ ///
+ public FileCacheBehaviour CacheBehaviour { get; }
+
+ public FileOptionsAttribute(FileCacheBehaviour cacheBehaviour)
+ {
+ CacheBehaviour = cacheBehaviour;
+ }
+
+ ///
+ /// Changes the behaviour for file caching
+ ///
+ public enum FileCacheBehaviour
+ {
+ ///
+ /// The default lumina option is used from
+ ///
+ None,
+ ///
+ /// The file is always cached
+ ///
+ Always,
+ ///
+ /// The file is never cached
+ ///
+ Never
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Lumina/Data/Files/Excel/ExcelDataFile.cs b/src/Lumina/Data/Files/Excel/ExcelDataFile.cs
index ac71165a..146299f3 100644
--- a/src/Lumina/Data/Files/Excel/ExcelDataFile.cs
+++ b/src/Lumina/Data/Files/Excel/ExcelDataFile.cs
@@ -3,7 +3,6 @@
using System.Collections.Generic;
using System.IO;
using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
using Lumina.Data.Structs.Excel;
using Lumina.Extensions;
@@ -18,6 +17,11 @@ public ExcelDataFile()
public ExcelDataHeader Header { get; protected set; }
public Dictionary< uint, ExcelDataOffset > RowData { get; protected set; }
+
+ ///
+ /// Whether the endianness of the underlying data has been swapped so it doesn't happen twice
+ ///
+ public bool SwappedEndianness { get; internal set; }
public override unsafe void LoadFile()
{
diff --git a/src/Lumina/Data/Repository.cs b/src/Lumina/Data/Repository.cs
index 7752afc0..df51a1f4 100644
--- a/src/Lumina/Data/Repository.cs
+++ b/src/Lumina/Data/Repository.cs
@@ -1,6 +1,5 @@
using System;
using System.Collections.Generic;
-using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
@@ -72,19 +71,19 @@ public T GetFile< T >( string cat, ParsedFilePath path ) where T : FileResource
public T GetFile< T >( byte cat, ParsedFilePath path ) where T : FileResource
{
- if( Categories.TryGetValue( cat, out var categories ) )
+ if( !Categories.TryGetValue( cat, out var categories ) )
{
- foreach( var category in categories )
- {
- var file = category.GetFile< T >( path );
- if( file != null )
- return file;
- }
+ return null;
+ }
+
+ foreach( var category in categories )
+ {
+ return category.GetFile< T >( path );
}
return null;
}
-
+
public SqPackFileInfo? GetFileMetadata( string cat, ParsedFilePath path )
{
if( CategoryNameToIdMap.TryGetValue( cat, out var catId ) )
@@ -97,15 +96,17 @@ public T GetFile< T >( byte cat, ParsedFilePath path ) where T : FileResource
public SqPackFileInfo? GetFileMetadata( byte cat, ParsedFilePath path )
{
- if( Categories.TryGetValue( cat, out var categories ) )
+ if( !Categories.TryGetValue( cat, out var categories ) )
+ {
+ return null;
+ }
+
+ foreach( var category in categories )
{
- foreach( var category in categories )
+ var file = category.GetFileMetadata( path );
+ if( file != null )
{
- var file = category.GetFileMetadata( path );
- if( file != null )
- {
- return file;
- }
+ return file;
}
}
@@ -125,7 +126,10 @@ private void GetExpansionId()
}
catch( FormatException e )
{
- Trace.TraceWarning( "failed to parse expansionid, e: {0}", e.Message );
+ _Lumina.Logger?.Error(
+ "failed to parse expansionid, value: {Value} e: {ExceptionMessage}",
+ e.Message
+ );
}
}
@@ -174,7 +178,7 @@ private void SetupIndexes()
{
continue;
}
-
+
// grab first index from the discovered indexes, this should be index if you have both
// otherwise it _should_ be index2
var file = indexFiles.FirstOrDefault();
@@ -185,7 +189,7 @@ private void SetupIndexes()
var index = new SqPackIndex( file, _Lumina );
- var dat = new Category(
+ var dat = new Category(
cat.Value,
ExpansionId,
chunk,
@@ -225,7 +229,7 @@ public static string BuildDatStr( byte cat, int ex, int chunk, Structs.PlatformI
public List< FileInfo > FindIndexes( byte cat, int ex, int chunk )
{
var files = new List< FileInfo >();
-
+
foreach( var type in new[] { "index", "index2" } )
{
var index = BuildDatStr( cat, ex, chunk, _Lumina.Options.CurrentPlatform, type );
@@ -248,7 +252,7 @@ public bool FileExists( string catName, ParsedFilePath path )
{
return false;
}
-
+
var categories = Categories[ catId ];
foreach( var cat in categories )
diff --git a/src/Lumina/Excel/ExcelModule.cs b/src/Lumina/Excel/ExcelModule.cs
index 8efb15ba..b8619394 100644
--- a/src/Lumina/Excel/ExcelModule.cs
+++ b/src/Lumina/Excel/ExcelModule.cs
@@ -66,7 +66,7 @@ public ExcelModule( Lumina lumina )
///
/// A sheet name
/// An absolute path to an excel header file
- public string BuildExcelHeaderPath( string name )
+ public static string BuildExcelHeaderPath( string name )
{
return $"exd/{name}.exh";
}
@@ -155,6 +155,12 @@ private ExcelSheet< T > CreateNewSheet< T >(
Tuple< Language, ulong > noLangKey
) where T : class, IExcelRow
{
+ _lumina.Logger?.Debug(
+ "sheet {SheetName} not in cache - creating new sheet for language {Language}",
+ name,
+ language
+ );
+
var path = BuildExcelHeaderPath( name );
var headerFile = _lumina.GetFile< ExcelHeaderFile >( path );
diff --git a/src/Lumina/Excel/ExcelSheetImpl.cs b/src/Lumina/Excel/ExcelSheetImpl.cs
index 2b910c00..b9a4c8c7 100644
--- a/src/Lumina/Excel/ExcelSheetImpl.cs
+++ b/src/Lumina/Excel/ExcelSheetImpl.cs
@@ -212,6 +212,7 @@ protected void ProcessDataRow( long offset, MemoryStream ms, BinaryWriter bw, Bi
/// Reverses the endianness of a data file on LE machines so the underlying stream can be copied from as is
///
/// The file to swap endianness for
+ // todo: refactor and move into ExceLDataFile
protected void ProcessDataEndianness( ExcelDataFile file )
{
if( !BitConverter.IsLittleEndian )
@@ -219,6 +220,12 @@ protected void ProcessDataEndianness( ExcelDataFile file )
return;
}
+ if( file.SwappedEndianness )
+ {
+ return;
+ }
+
+
var stream = new MemoryStream( file.Data );
var writer = new BinaryWriter( stream );
var reader = new BinaryReader( stream );
@@ -250,6 +257,8 @@ protected void ProcessDataEndianness( ExcelDataFile file )
ProcessDataRow( offset + 6, stream, writer, reader );
}
}
+
+ file.SwappedEndianness = true;
}
///