From faa158526ede07469a2287ac12d26bf76963e255 Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Sun, 11 Aug 2024 22:56:21 -0700 Subject: [PATCH 01/53] Update dependencies and .net version to 8.0 --- src/Lumina.Cmd/Lumina.Cmd.csproj | 2 +- src/Lumina.Example/Lumina.Example.csproj | 4 ++-- src/Lumina/Lumina.csproj | 13 ++++++------- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/Lumina.Cmd/Lumina.Cmd.csproj b/src/Lumina.Cmd/Lumina.Cmd.csproj index 9fdee00e..47cafbe0 100644 --- a/src/Lumina.Cmd/Lumina.Cmd.csproj +++ b/src/Lumina.Cmd/Lumina.Cmd.csproj @@ -2,7 +2,7 @@ Exe - net6.0 + net8.0 diff --git a/src/Lumina.Example/Lumina.Example.csproj b/src/Lumina.Example/Lumina.Example.csproj index 08b2ba25..82cf9a9b 100644 --- a/src/Lumina.Example/Lumina.Example.csproj +++ b/src/Lumina.Example/Lumina.Example.csproj @@ -1,8 +1,8 @@ - + Exe - net7.0 + net8.0 true diff --git a/src/Lumina/Lumina.csproj b/src/Lumina/Lumina.csproj index 60546230..41f2916b 100644 --- a/src/Lumina/Lumina.csproj +++ b/src/Lumina/Lumina.csproj @@ -1,7 +1,7 @@ - + - net6.0;net7.0;net8.0 + net8.0 latestmajor true NotAdam @@ -13,7 +13,7 @@ Lumina is a small, performant and simple library for interacting with FINAL FANTASY XIV game data. true enable - true + @@ -27,16 +27,15 @@ - - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive - preview + preview.0 From de6a8c3f9e18fbc09bcc57a25e82319c4c481994 Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Mon, 12 Aug 2024 01:25:52 -0700 Subject: [PATCH 02/53] Add new rewritten excel row types --- src/Lumina/Excel/ExcelModule.cs | 388 +++------------ src/Lumina/Excel/ExcelPage.cs | 117 +++-- src/Lumina/Excel/ExcelRow.cs | 41 -- src/Lumina/Excel/ExcelSheet.cs | 398 ++++++++++----- src/Lumina/Excel/ExcelSheetImpl.cs | 284 ----------- ...celSheetColumnChecksumMismatchException.cs | 20 - src/Lumina/Excel/IExcelRow.cs | 31 ++ src/Lumina/Excel/IExcelSheet.cs | 58 +++ src/Lumina/Excel/LazyCollection.cs | 42 ++ src/Lumina/Excel/LazyRow.cs | 150 ------ src/Lumina/Excel/NoCache.cs | 29 -- src/Lumina/Excel/RawExcelSheet.cs | 77 --- src/Lumina/Excel/RowDataCursor.cs | 14 - src/Lumina/Excel/RowParser.cs | 467 ------------------ src/Lumina/Excel/RowRef.cs | 123 +++++ src/Lumina/Excel/SheetAttribute.cs | 39 +- 16 files changed, 708 insertions(+), 1570 deletions(-) delete mode 100644 src/Lumina/Excel/ExcelRow.cs delete mode 100644 src/Lumina/Excel/ExcelSheetImpl.cs delete mode 100644 src/Lumina/Excel/Exceptions/ExcelSheetColumnChecksumMismatchException.cs create mode 100644 src/Lumina/Excel/IExcelRow.cs create mode 100644 src/Lumina/Excel/IExcelSheet.cs create mode 100644 src/Lumina/Excel/LazyCollection.cs delete mode 100644 src/Lumina/Excel/LazyRow.cs delete mode 100644 src/Lumina/Excel/NoCache.cs delete mode 100644 src/Lumina/Excel/RawExcelSheet.cs delete mode 100644 src/Lumina/Excel/RowDataCursor.cs delete mode 100644 src/Lumina/Excel/RowParser.cs create mode 100644 src/Lumina/Excel/RowRef.cs diff --git a/src/Lumina/Excel/ExcelModule.cs b/src/Lumina/Excel/ExcelModule.cs index 5e540be7..81bcbd7a 100644 --- a/src/Lumina/Excel/ExcelModule.cs +++ b/src/Lumina/Excel/ExcelModule.cs @@ -1,343 +1,89 @@ +using Lumina.Data; +using Lumina.Data.Files.Excel; using System; +using System.Collections.Concurrent; using System.Collections.Generic; +using System.ComponentModel; +using System.Diagnostics.CodeAnalysis; using System.IO; -using System.Reflection; -using Lumina.Data; -using Lumina.Data.Files.Excel; -using Lumina.Excel.Exceptions; -using Lumina.Excel.RSV; - -namespace Lumina.Excel -{ - using CacheKeyTuple = Tuple< Language, string, ulong >; - - public class ExcelModule - { - private readonly GameData _gameData; - - /// - /// Mapping between internal IDs used to index sheets loaded at startup to their name. - /// - /// - /// Not actually used for anything in lumina, but kept for reference - /// - private readonly Dictionary< int, string > _immutableIdToSheetMap = new(); - - /// - /// A list of all available sheets, pulled from root.exl - /// - private readonly List< string > _sheetNames = new(); - - private readonly Dictionary< CacheKeyTuple, ExcelSheetImpl > _sheetCache = new(); - - private readonly object _sheetCreateLock = new(); - - /// - /// Allows for lumina to transparently substitute values where they are hidden in sheets by SE. Can be seeded from packets or static data at init time. - /// - public RsvProvider RsvProvider { get; } = new(); - - /// - /// Create a new ExcelModule. This will do all the initial discovery of sheets from the EXL but not load any sheets. - /// - /// The gamedata instance to use to load sheets from - /// Thrown when the root.exl file cannot be found - make sure that an 0a dat is available. - public ExcelModule( GameData gameData ) - { - _gameData = gameData; - - // load all sheet names first - var files = _gameData.GetFile< ExcelListFile >( "exd/root.exl" ); - - if( files == null ) - { - throw new FileNotFoundException( "Unable to load exd/root.exl!" ); - } - - _gameData.Logger?.Information("got {ExltEntryCount} exlt entries", files.ExdMap.Count); - - foreach( var map in files.ExdMap ) - { - _sheetNames.Add( map.Key ); - - if( map.Value == -1 ) - { - continue; - } - - _immutableIdToSheetMap[ map.Value ] = map.Key; - } - } - - /// - /// Generates a path to the header file, given a sheet name. - /// - /// - /// Sheet names must be in the same format as they're in root.exl. You can see all available sheets by iterating . - /// - /// A sheet name - /// An absolute path to an excel header file - public static string BuildExcelHeaderPath( string name ) - { - return $"exd/{name}.exh"; - } - - /// - /// Attempts to load the base excel sheet given it's implementing row parser - /// - /// A class that implements to parse rows - /// An if the sheet exists, null if it does not - public ExcelSheet< T >? GetSheet< T >() where T : ExcelRow - { - return GetSheet< T >( _gameData.Options.DefaultExcelLanguage ); - } - - /// - /// Attempts to load the base excel sheet with a specific language - /// - /// - /// If the language requested doesn't exist for the file, this will silently be ignored and it will return a sheet with the default language: - /// - /// The requested sheet language - /// A class that implements to parse rows - /// An if the sheet exists, null if it does not - public ExcelSheet< T >? GetSheet< T >( Language language ) where T : ExcelRow - { - var attr = typeof( T ).GetCustomAttribute< SheetAttribute >(); - - if( attr == null ) - { - return null; - } - - return GetSheet< T >( attr.Name, language, attr.ColumnHash ); - } - - /// - /// Get a sheet by it's name with a given type. - /// - /// Useful for when a schema is shared (e.g. in the case of quest text sheets) as redefining loads of classes is wasteful. - /// - /// The name of a sheet - /// A class that implements to parse rows - /// An if the sheet exists, null if it does not - public ExcelSheet< T >? GetSheet< T >( string name ) where T : ExcelRow - { - return GetSheet< T >( name, _gameData.Options.DefaultExcelLanguage, null ); - } - - /// - /// Get a sheet by it's name with a given type. - /// - /// Useful for when a schema is shared (e.g. in the case of quest text sheets) as redefining loads of classes is wasteful. - /// - /// The requested sheet language - /// The name of a sheet - /// A class that implements to parse rows - /// An if the sheet exists, null if it does not - public ExcelSheet< T >? GetSheet< T >( Language language, string name ) where T : ExcelRow - { - return GetSheet< T >( name, language, null ); - } - - private ulong BuildTypeIdentifier( Type type ) - { - return (ulong)type.Assembly.GetHashCode() << 32 | (uint)type.MetadataToken; - } - - /// - /// Remove a sheet from the cache completely and free any related resources - /// - /// The sheet type - public void RemoveSheetFromCache< T >() where T : ExcelRow - { - var tid = BuildTypeIdentifier( typeof( T ) ); - - var attr = typeof( T ).GetCustomAttribute< SheetAttribute >(); - - if( attr == null ) - { - return; - } - - RemoveSheetFromCache< T >( attr.Name ); - } - /// - /// Remove a sheet from the cache by name and free any related resources - /// - /// The name of the sheet - /// The sheet type - public void RemoveSheetFromCache< T >( string name ) where T : ExcelRow - { - var tid = BuildTypeIdentifier( typeof( T ) ); - var lowerName = name.ToLowerInvariant(); - - foreach( Language language in Enum.GetValues( typeof( Language ) ) ) - { - var id = Tuple.Create( language, lowerName, tid ); - - _sheetCache.Remove( id ); - } - } - - private ExcelSheet< T >? GetSheet< T >( string name, Language language, uint? expectedHash ) where T : ExcelRow - { - var tid = BuildTypeIdentifier( typeof( T ) ); - var lowerName = name.ToLowerInvariant(); - var idNoLanguage = Tuple.Create( Language.None, lowerName, tid ); - var id = Tuple.Create( language, lowerName, tid ); +namespace Lumina.Excel; - // attempt to get non-localised sheet first, then attempt to fetch a localised sheet from the cache - if( _sheetCache.TryGetValue( idNoLanguage, out var sheet ) ) - { - return sheet as ExcelSheet< T >; - } - - if( _sheetCache.TryGetValue( id, out sheet ) ) - { - if( sheet is ExcelSheet< T > checkedSheet ) - { - return checkedSheet; - } - } +public class ExcelModule +{ + internal GameData GameData { get; } - // create new sheet - lock( _sheetCreateLock ) - { - return CreateNewSheet< T >( name, language, expectedHash, id, idNoLanguage ); - } - } + internal Language Language => GameData.Options.DefaultExcelLanguage; - private ExcelSheet< T >? CreateNewSheet< T >( - string name, - Language language, - uint? expectedHash, - CacheKeyTuple key, - CacheKeyTuple noLangKey - ) where T : ExcelRow - { - _gameData.Logger?.Debug( - "sheet {SheetName} not in cache - creating new sheet for language {Language}", - name, - language - ); - - var path = BuildExcelHeaderPath( name ); - var headerFile = _gameData.GetFile< ExcelHeaderFile >( path ); + private ConcurrentDictionary<(Type sheetType, Language requestedLanguage), IExcelSheet> SheetCache { get; } = []; - if( headerFile == null ) - { - return null; - } + /// + /// Get all available sheets, parsed from root.exl. + /// + public IReadOnlyCollection SheetNames { get; } - // validate checksum if enabled and we have a hash that we expect to find - if( expectedHash.HasValue ) - { - var actualHash = headerFile.GetColumnsHash(); - if( actualHash != expectedHash ) - { - _gameData.Logger?.Warning( - "The sheet impl {SheetImplName} hash doesn't match the hash generated from the header. Expected: {ExpectedHash} actual: {ActualHash}", - typeof( T ).FullName, - expectedHash, - actualHash - ); - - if( _gameData.Options.PanicOnSheetChecksumMismatch ) - { - throw new ExcelSheetColumnChecksumMismatchException( name, expectedHash.Value, actualHash ); - } - } - } + /// + /// Create a new ExcelModule. This will do all the initial discovery of sheets from the EXL but not load any sheets. + /// + /// The instance to load sheets from + /// Thrown when the root.exl file cannot be found - make sure that an 0a dat is available. + public ExcelModule( GameData gameData ) + { + GameData = gameData; - var newSheet = (ExcelSheet< T >?)Activator.CreateInstance( typeof( ExcelSheet< T > ), headerFile, name, language, _gameData ); - newSheet!.GenerateFilePages(); + var files = GameData.GetFile( "exd/root.exl" ) ?? + throw new FileNotFoundException( "Unable to load exd/root.exl!" ); - var id = key; + GameData.Logger?.Information( "got {ExltEntryCount} exlt entries", files.ExdMap.Count ); - // kinda a shit hack but basically this enforces a single language for a sheet that has no localisation - // because it's possible to then load a single sheet many times if someone isn't careful - // so we rewrite the language field on the id so we can hit the cache in the event that this happens - // nb: probably unlikely but I don't want to deal with it later - var langs = newSheet.Languages; - if( langs.Length == 1 && langs[ 0 ] == Language.None ) - { - id = noLangKey; - } + SheetNames = [.. files.ExdMap.Keys]; + } - _sheetCache[ id ] = newSheet; + /// + /// Attempts to load an , optionally with a specific language + /// + /// + /// If the requested language doesn't exist for the file, this will silently be ignored and it will return a sheet with the default language: + /// + /// The requested sheet language. Leave or empty to use the default language. + /// A struct that implements to parse rows + /// An if the sheet exists + /// Thrown when the sheet type is not decorated with a + /// Sheet does not exist or if the column hash has a mismatch + public ExcelSheet GetSheet( Language? language = null ) where T : struct, IExcelRow + { + language ??= Language; - return newSheet; - } + return (ExcelSheet)SheetCache.GetOrAdd( (typeof( T ), language.Value), _ => new ExcelSheet( this, language.Value ) ); + } - /// - /// Returns a raw accessor to an excel sheet allowing you to skip templated row access entirely. - /// - /// Name of the sheet to load - /// A ExcelSheetImpl object, or null if the sheet name was not found. - public RawExcelSheet? GetSheetRaw( string name ) - { - return GetSheetRaw( name, _gameData.Options.DefaultExcelLanguage ); - } + /// + /// Attempts to load an from a reflected , optionally with a specific language + /// + /// + /// Only use this method if you need to create a sheet while using reflection. + /// + /// If the requested language doesn't exist for the file, this will silently be ignored and it will return a sheet with the default language: + /// + /// A that implements to parse rows + /// The requested sheet language. Leave or empty to use the default language. + /// An if the sheet exists + /// Thrown when is not decorated with a + /// Sheet does not exist or if the column hash has a mismatch + [RequiresDynamicCode("Creating a generic sheet from a type requires reflection and dynamic code.")] + [EditorBrowsable(EditorBrowsableState.Never)] + public IExcelSheet GetSheetGeneric(Type rowType, Language? language = null ) + { + if( !rowType.IsValueType ) + throw new ArgumentException( "rowType must be a struct", nameof( rowType ) ); - /// - /// Returns a raw accessor to an excel sheet allowing you to skip templated row access entirely. - /// - /// Name of the sheet to load - /// The requested language to load - /// A ExcelSheetImpl object, or null if the sheet name was not found. - public RawExcelSheet? GetSheetRaw( string name, Language language ) + if (!rowType.IsAssignableTo( typeof( IExcelRow<> ).MakeGenericType( rowType ) ) ) { - // todo: duped code is a bit ass but zzz - - // create new sheet - var path = BuildExcelHeaderPath( name ); - var headerFile = _gameData.GetFile< ExcelHeaderFile >( path ); - - if( headerFile == null ) - { - return null; - } - - var newSheet = new RawExcelSheet( headerFile, name, language, _gameData ); - newSheet.GenerateFilePages(); - - return newSheet; + throw new ArgumentException( "rowType implement IExcelRow", nameof( rowType ) ); } - /// - /// Checks whether a given decorated with a has a column hash that matches a newly created hash - /// of the column data from the . - /// - /// The to check - /// true if the hash matches - /// This function will return false if the is missing or a column hash isn't specified in the attribute - public bool SheetHashMatchesColumnDefinition< T >() where T : ExcelRow - { - var type = typeof( T ); - var attr = type.GetCustomAttribute< SheetAttribute >(); - - if( attr == null ) - { - return false; - } - - var path = BuildExcelHeaderPath( attr.Name ); - var headerFile = _gameData.GetFile< ExcelHeaderFile >( path ); - - if( headerFile == null ) - { - return false; - } - - return attr.ColumnHash == headerFile.GetColumnsHash(); - } + language ??= Language; - /// - /// Get all available sheets, parsed from root.exl. - /// - /// A readonly collection of all available excel sheets - public IReadOnlyCollection< string > GetSheetNames() => _sheetNames; + return SheetCache.GetOrAdd( (rowType, language.Value), _ => (IExcelSheet)Activator.CreateInstance( typeof( ExcelSheet<> ).MakeGenericType( rowType ), this, language.Value )! ); } } \ No newline at end of file diff --git a/src/Lumina/Excel/ExcelPage.cs b/src/Lumina/Excel/ExcelPage.cs index 3be27a9d..4dc2cb97 100644 --- a/src/Lumina/Excel/ExcelPage.cs +++ b/src/Lumina/Excel/ExcelPage.cs @@ -1,34 +1,93 @@ -using System.Collections.Generic; -using Lumina.Data.Files.Excel; -using Lumina.Data.Structs.Excel; +using System.Buffers.Binary; +using System; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using Lumina.Text.ReadOnly; +using System.ComponentModel; -namespace Lumina.Excel +namespace Lumina.Excel; + +/// +/// A single page/file of data from an excel sheet. +/// +/// +/// If you only want to read excel sheets, please refrain from touching this class. This class exists mostly as an implementation detail for parsing excel rows. +/// +[EditorBrowsable( EditorBrowsableState.Advanced )] +public sealed class ExcelPage { - public record ExcelPage + public ExcelModule Module { get; } + + private readonly byte[] data; + private ReadOnlyMemory Data => data; + + private readonly ushort dataOffset; + + internal ExcelPage( ExcelModule module, byte[] pageData, ushort headerDataOffset ) + { + Module = module; + data = pageData; + dataOffset = headerDataOffset; + } + + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + private D Read( nuint offset ) where D : unmanaged => + Unsafe.As( ref Unsafe.AddByteOffset( ref MemoryMarshal.GetArrayDataReference( data ), offset ) ); + + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + private static float ReverseEndianness( float v ) => + Unsafe.BitCast( BinaryPrimitives.ReverseEndianness( Unsafe.BitCast( v ) ) ); + + [MethodImpl( MethodImplOptions.AggressiveInlining )] + public ReadOnlySeString ReadString( nuint offset, nuint structOffset ) { - /// - /// The path to the data file (exd) that contains the rows for the current page - /// - public string FilePath { get; set; } - - /// - /// The start ID of the page - /// - public uint StartId { get; set; } - - /// - /// How many rows are contained in this page - /// - public uint RowCount { get; set; } - - /// - /// An index -> (rowid, offset) list which maps a local row index to where it is inside the current data page - /// - public Dictionary< uint, ExcelDataOffset > RowData => File.RowData; - - /// - /// The underlying data file that contains the sheet data - /// - public ExcelDataFile File { get; set; } + offset = ReadUInt32( offset ) + structOffset + dataOffset; + var data = Data[(int)offset..]; + var stringLength = data.Span.IndexOf( (byte)0 ); + return new ReadOnlySeString( data[..stringLength] ); } + + [MethodImpl( MethodImplOptions.AggressiveInlining )] + public bool ReadBool( nuint offset ) => + Read( offset ); + + [MethodImpl( MethodImplOptions.AggressiveInlining )] + public sbyte ReadInt8( nuint offset ) => + Read( offset ); + + [MethodImpl( MethodImplOptions.AggressiveInlining )] + public byte ReadUInt8( nuint offset ) => + Read( offset ); + + [MethodImpl( MethodImplOptions.AggressiveInlining )] + public short ReadInt16( nuint offset ) => + BinaryPrimitives.ReverseEndianness( Read( offset ) ); + + [MethodImpl( MethodImplOptions.AggressiveInlining )] + public ushort ReadUInt16( nuint offset ) => + BinaryPrimitives.ReverseEndianness( Read( offset ) ); + + [MethodImpl( MethodImplOptions.AggressiveInlining )] + public int ReadInt32( nuint offset ) => + BinaryPrimitives.ReverseEndianness( Read( offset ) ); + + [MethodImpl( MethodImplOptions.AggressiveInlining )] + public uint ReadUInt32( nuint offset ) => + BinaryPrimitives.ReverseEndianness( Read( offset ) ); + + [MethodImpl( MethodImplOptions.AggressiveInlining )] + public float ReadFloat32( nuint offset ) => + ReverseEndianness( Read( offset ) ); + + [MethodImpl( MethodImplOptions.AggressiveInlining )] + public long ReadInt64( nuint offset ) => + BinaryPrimitives.ReverseEndianness( Read( offset ) ); + + [MethodImpl( MethodImplOptions.AggressiveInlining )] + public ulong ReadUInt64( nuint offset ) => + BinaryPrimitives.ReverseEndianness( Read( offset ) ); + + [MethodImpl( MethodImplOptions.AggressiveInlining )] + public bool ReadPackedBool( nuint offset, byte bit ) => + ( Read( offset ) & ( 1 << bit ) ) != 0; } \ No newline at end of file diff --git a/src/Lumina/Excel/ExcelRow.cs b/src/Lumina/Excel/ExcelRow.cs deleted file mode 100644 index 58ffbfa9..00000000 --- a/src/Lumina/Excel/ExcelRow.cs +++ /dev/null @@ -1,41 +0,0 @@ -using Lumina.Data; - -namespace Lumina.Excel -{ - public abstract class ExcelRow - { - public uint RowId { get; set; } - public uint SubRowId { get; set; } - - public string SheetName { get; protected set; } - - public Language SheetLanguage { get; set; } - - protected GameData? _gameData; - - public virtual void PopulateData( RowParser parser, GameData gameData, Language language ) - { - _gameData = gameData; - - RowId = parser.RowId; - SubRowId = parser.SubRowId; - SheetLanguage = language; - - SheetName = parser.Sheet.Name; - } - - /// - /// Implementation shim around what SC calls default columns, allows a sheet impl to provide something more meaningful as a default display value - /// - public virtual object GetDefaultColumnValue() - { - // todo: we can probably just handle subrows in an override instead of doing something funny here because we know statically what are variant 2 sheets - return $"{SheetName}#{RowId}"; - } - - public override string ToString() - { - return $"{SheetName}#{RowId}"; - } - } -} \ No newline at end of file diff --git a/src/Lumina/Excel/ExcelSheet.cs b/src/Lumina/Excel/ExcelSheet.cs index 9d90d9ed..32a3c1f9 100644 --- a/src/Lumina/Excel/ExcelSheet.cs +++ b/src/Lumina/Excel/ExcelSheet.cs @@ -1,150 +1,316 @@ using System; using System.Collections; -using System.Collections.Concurrent; +using System.Collections.Frozen; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Linq; +using System.Reflection; +using System.Runtime.CompilerServices; using Lumina.Data; using Lumina.Data.Files.Excel; using Lumina.Data.Structs.Excel; -namespace Lumina.Excel +namespace Lumina.Excel; + +public sealed class ExcelSheet : IExcelSheet, IReadOnlyList where T : struct, IExcelRow { - public class ExcelSheet< T > : ExcelSheetImpl, IEnumerable< T > where T : ExcelRow + /// + public ExcelModule Module { get; } + + /// + public Language Language { get; } + + private List Pages { get; } + private FrozenDictionary? Rows { get; } + private FrozenDictionary? Subrows { get; } + private ushort SubrowDataOffset { get; } + + /// + /// Whether or not this sheet has subrows, where each row id can have multiple subrows. + /// + [MemberNotNullWhen( true, nameof( Subrows ), nameof( SubrowDataOffset ) )] + [MemberNotNullWhen( false, nameof( Rows ) )] + public bool HasSubrows { get; } + + private static SheetAttribute Attribute => + typeof( T ).GetCustomAttribute() ?? + throw new InvalidOperationException( "T has no SheetAttribute. Use the explicit sheet constructor." ); + + /// + /// The number of rows in this sheet. + /// + /// + /// If this sheet has gaps in row ids, it returns the number of rows that exist, not the highest row id. + /// If this sheet has subrows, this will still return the number of rows and not the total number of subrows. + /// + public int Count => Rows?.Count ?? Subrows!.Count; + + /// + /// Get the th row in this sheet. If this sheet has subrows, it will return the first subrow. + /// + /// This is an indexer helper, but you may want to treat this as a sparse list/matrix since only represents the total number of rows in the sheet, and not the highest row id. + /// The row id of the row you want + /// The row at + /// Throws when the row id does not have a row attached to it. + public T this[int rowId] => GetRow( (uint)rowId ); + + /// + /// Create an instance with the 's default language. + /// + /// The to access sheet data from. + /// does not have a valid + /// parameters were invalid (hash mismatch or invalid sheet name) + public ExcelSheet( ExcelModule module ) : this( module, module.Language ) { - private readonly ConcurrentDictionary< UInt64, T > _rowCache = new(); - public ExcelSheet( ExcelHeaderFile headerFile, string name, Language requestedLanguage, GameData gameData ) : - base( headerFile, name, requestedLanguage, gameData ) - { - } + } - public T? GetRow( uint row ) - { - return GetRow( row, UInt32.MaxValue ); - } + /// + /// Create an instance with a specific . + /// + /// The to access sheet data from. + /// The language to use for this sheet. + /// does not have a valid + /// parameters were invalid (hash mismatch or invalid sheet name) + public ExcelSheet( ExcelModule module, Language requestedLanguage ) : this( module, requestedLanguage, Attribute.Name, Attribute.ColumnHash ) + { + + } + + /// + /// Create an instance with a specific , name, and hash. + /// + /// The to access sheet data from. + /// The language to use for this sheet. + /// The name of the sheet to read from. + /// The hash of the columns in the sheet. If , it will not check the hash. + /// or parameters were invalid (hash mismatch or invalid sheet name) + public ExcelSheet( ExcelModule module, Language requestedLanguage, string sheetName, uint? columnHash = null ) + { + Module = module; - public T? GetRow( uint row, uint subRow ) + var headerFile = module.GameData.GetFile( $"exd/{sheetName}.exh" ) ?? + throw new ArgumentException( "Invalid sheet name", nameof( sheetName ) ); + + if( columnHash is { } hash && headerFile.GetColumnsHash() != hash ) + throw new ArgumentException( "Column hash mismatch", nameof( columnHash ) ); + + HasSubrows = headerFile.Header.Variant == ExcelVariant.Subrows; + + Language = headerFile.Languages.Contains( requestedLanguage ) ? requestedLanguage : Language.None; + + Dictionary? rows = null; + Dictionary? subrows = null; + + if( HasSubrows ) { - return GetRowInternal( row, subRow ); + subrows = new( (int)headerFile.Header.RowCount ); + SubrowDataOffset = headerFile.Header.DataOffset; } + else + rows = new( (int)headerFile.Header.RowCount ); - internal T? GetRowInternal( uint row, uint subRow ) + Pages = new( headerFile.DataPages.Length ); + var pageIdx = 0; + foreach( var pageDef in headerFile.DataPages ) { - var cacheKey = GetCacheKey( row, subRow ); + var filePath = Language == Language.None ? + $"exd/{sheetName}_{pageDef.StartId}.exd" : + $"exd/{sheetName}_{pageDef.StartId}_{LanguageUtil.GetLanguageStr( Language )}.exd"; + var fileData = module.GameData.GetFile( filePath ); + if( fileData == null ) + continue; - if( _rowCache.TryGetValue( cacheKey, out var cachedRow ) ) - { - return cachedRow; - } + var newPage = new ExcelPage( Module, fileData.Data, headerFile.Header.DataOffset ); + Pages.Add( newPage ); - var page = GetPageForRow( row ); - if( page == null ) - { - return null; - } - - var parser = GetRowParser( page, row, subRow ); - if( parser == null ) + foreach( var rowPtr in fileData.RowData.Values ) { - return null; - } - - var rowObj = Activator.CreateInstance< T >(); + var (rowDataSize, subrowCount) = (newPage.ReadUInt32( rowPtr.Offset ), newPage.ReadUInt16( rowPtr.Offset + 4 )); + var rowOffset = rowPtr.Offset + 6; - lock( page.File.ReaderLock ) - { - rowObj.PopulateData( parser, GameData, RequestedLanguage ); - } - - if( !NoCache.IsEnabled ) - { - _rowCache[ cacheKey ] = rowObj; + if( HasSubrows ) + { + if( subrowCount > 0 ) + subrows!.Add( rowPtr.RowId, (pageIdx, rowOffset, subrowCount) ); + } + else + rows!.Add( rowPtr.RowId, (pageIdx, rowOffset) ); } - - return rowObj; + + pageIdx++; } - private T ReadRowObj( RowParser parser, uint rowId ) - { - parser.SeekToRow( rowId ); - - var obj = Activator.CreateInstance< T >(); - - obj.PopulateData( parser, GameData, RequestedLanguage ); + if( HasSubrows ) + Subrows = subrows!.ToFrozenDictionary(); + else + Rows = rows!.ToFrozenDictionary(); + } - return obj; - } + /// + public bool HasRow( uint rowId ) + { + if( HasSubrows ) + return Subrows.ContainsKey( rowId ); - private T ReadSubRowObj( RowParser parser, uint rowId, uint subRowId ) - { - parser.SeekToRow( rowId, subRowId ); - var obj = Activator.CreateInstance< T >(); + return Rows.ContainsKey( rowId ); + } + + /// + public bool HasSubrow( uint rowId, ushort subrowId ) + { + if( !HasSubrows ) + throw new NotSupportedException( "Cannot access subrow in a sheet that doesn't support any." ); + + ref readonly var val = ref Subrows.GetValueRefOrNullRef( rowId ); + if( Unsafe.IsNullRef( in val ) ) + return false; + + return subrowId < val.RowCount; + } + + /// + public ushort? TryGetSubrowCount( uint rowId ) + { + if( !HasSubrows ) + throw new NotSupportedException( "Cannot access subrow in a sheet that doesn't support any." ); + + ref readonly var val = ref Subrows.GetValueRefOrNullRef( rowId ); + if( Unsafe.IsNullRef( in val ) ) + return null; + + return val.RowCount; + } + + /// + public ushort GetSubrowCount( uint rowId ) + { + if( !HasSubrows ) + throw new NotSupportedException( "Cannot access subrow in a sheet that doesn't support any." ); + + ref readonly var val = ref Subrows.GetValueRefOrNullRef( rowId ); + if( Unsafe.IsNullRef( in val ) ) + throw new ArgumentOutOfRangeException( nameof( rowId ), "Row does not exist" ); + + return val.RowCount; + } + + private T CreateRow(uint rowId, in (int PageIdx, uint Offset) val) => + T.Create( Pages[val.PageIdx], val.Offset, rowId ); + + private T CreateSubrow( uint rowId, ushort subrowId, in (int PageIdx, uint Offset, ushort RowCount) val ) => + T.Create( Pages[val.PageIdx], val.Offset + 2 + ( subrowId * ( SubrowDataOffset + 2u ) ), rowId, subrowId ); + + /// + /// Tries to get the th row in this sheet. If this sheet has subrows, it will return the first subrow. + /// + /// The row id to get + /// A nullable row object. Returns null if the row does not exist. + public T? TryGetRow( uint rowId ) + { + if( HasSubrows ) + return TryGetSubrow( rowId, 0 ); + + ref readonly var val = ref Rows.GetValueRefOrNullRef( rowId ); + if( Unsafe.IsNullRef( in val ) ) + return null; + + return CreateRow( rowId, in val ); + } + + + /// + /// Tries to get the th subrow from the th row in this sheet. + /// + /// The row id to get + /// The subrow id to get + /// A nullable row object. Returns null if the subrow does not exist. + /// Thrown if the sheet does not support subrows + public T? TryGetSubrow( uint rowId, ushort subrowId ) + { + if( !HasSubrows ) + throw new NotSupportedException( "Cannot access subrow in a sheet that doesn't support any." ); + + ref readonly var val = ref Subrows.GetValueRefOrNullRef( rowId ); + if( Unsafe.IsNullRef( in val ) ) + return null; + + if( subrowId >= val.RowCount ) + return null; - obj.PopulateData( parser, GameData, RequestedLanguage ); + return CreateSubrow( rowId, subrowId, in val ); + } + + /// + /// Gets the th row in this sheet. If this sheet has subrows, it will return the first subrow. Throws if the row does not exist. + /// + /// The row id to get + /// A row object. + /// Thrown if the sheet does not have a row at that + public T GetRow( uint rowId ) => + TryGetRow( rowId ) ?? + throw new ArgumentOutOfRangeException( nameof( rowId ), "Row does not exist" ); - return obj; + /// + /// Gets the th subrow from the th row in this sheet. Throws if the subrow does not exist. + /// + /// The row id to get + /// The subrow id to get + /// A row object. + /// Thrown if the sheet does not support subrows + /// Thrown if the sheet does not have a row at that + public T GetSubrow( uint rowId, ushort subrowId ) + { + if( !HasSubrows ) + throw new NotSupportedException( "Cannot access subrow in a sheet that doesn't support any." ); + + ref readonly var val = ref Subrows.GetValueRefOrNullRef( rowId ); + if( Unsafe.IsNullRef( in val ) ) + throw new ArgumentOutOfRangeException( nameof( rowId ), "Row does not exist" ); + + ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual( subrowId, val.RowCount ); + + return CreateSubrow( rowId, subrowId, in val ); + } + + /// + /// Returns an enumerator that can be used to iterate over all subrows in all rows in this sheet. + /// + /// An of all subrows in this sheet + /// Thrown if the sheet does not support subrows + public IEnumerator GetSubrowEnumerator() + { + if( !HasSubrows ) + throw new NotSupportedException( "Cannot enumerate subrows in a sheet that doesn't support any." ); + + foreach( var rowData in Subrows ) + { + for( ushort i = 0; i < rowData.Value.RowCount; ++i ) + yield return CreateSubrow( rowData.Key, i, rowData.Value ); } - - public IEnumerator< T > GetEnumerator() + } + + /// + /// Returns an enumerator that can be used to iterate over all rows in this sheet. If this sheet has subrows, it will iterate over the first subrow of every row. + /// + /// An of all rows (or first subrows) in this sheet + public IEnumerator GetEnumerator() + { + if( !HasSubrows ) { - ExcelDataFile file = null!; - RowParser parser = null!; - - foreach( var offset in GetRowDataOffsets() ) - { - var rowPtr = offset.RowOffset; - if( file != offset.SheetPage ) - { - parser = new RowParser( this, offset.SheetPage ); - } - - if( Header.Variant == ExcelVariant.Subrows ) - { - // required to read the row header out and know how many subrows there is - parser.SeekToRow( rowPtr.RowId ); - - // read subrows - for( uint i = 0; i < parser.RowCount; i++ ) - { - var cacheKey = GetCacheKey( rowPtr.RowId, i ); - if( _rowCache.TryGetValue( cacheKey, out var cachedRow ) ) - { - yield return cachedRow; - continue; - } - - var obj = ReadSubRowObj( parser, rowPtr.RowId, i ); - if( !NoCache.IsEnabled ) - { - _rowCache.TryAdd( cacheKey, obj ); - } - - yield return obj; - } - } - else - { - var cacheKey = GetCacheKey( rowPtr.RowId ); - if( _rowCache.TryGetValue( cacheKey, out var cachedRow ) ) - { - yield return cachedRow; - continue; - } - - var obj = ReadRowObj( parser, rowPtr.RowId ); - if( !NoCache.IsEnabled ) - { - _rowCache.TryAdd( cacheKey, obj ); - } - - yield return obj; - } - } + foreach( var rowData in Rows ) + yield return CreateRow( rowData.Key, rowData.Value ); } - - IEnumerator IEnumerable.GetEnumerator() + else { - return GetEnumerator(); + foreach( var rowData in Subrows ) + yield return CreateSubrow( rowData.Key, 0, rowData.Value ); } } -} \ No newline at end of file + + /// + IEnumerator IEnumerable.GetEnumerator() + { + return GetEnumerator(); + } +} diff --git a/src/Lumina/Excel/ExcelSheetImpl.cs b/src/Lumina/Excel/ExcelSheetImpl.cs deleted file mode 100644 index 2bb5de7e..00000000 --- a/src/Lumina/Excel/ExcelSheetImpl.cs +++ /dev/null @@ -1,284 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using Lumina.Data; -using Lumina.Data.Files.Excel; -using Lumina.Data.Structs.Excel; - -namespace Lumina.Excel -{ - public class ExcelSheetImpl - { - internal ExcelSheetImpl( ExcelHeaderFile headerFile, string name, Language requestedLanguage, GameData gameData ) - { - DataPages = new List< ExcelPage >(); - HeaderFile = headerFile; - Name = name; - RequestedLanguage = requestedLanguage; - GameData = gameData; - } - - /// - /// The name of the sheet - /// - public string Name { get; } - - /// - /// The header of the sheet which defines its properties such as total row count, pages and languages - /// - public ExcelHeaderFile HeaderFile { get; } - - /// - /// A quick accessor to the data available in the sheet header - /// - public ExcelHeaderHeader Header => HeaderFile.Header; - - /// - /// The total count of rows irrespective of paging - /// - public uint RowCount => Header.RowCount; - - /// - /// The total number of columns - /// - public uint ColumnCount => Header.ColumnCount; - - /// - /// The kind of sheet - /// - public ExcelVariant Variant => Header.Variant; - - /// - /// The parsed data pages - /// - public readonly List< ExcelPage > DataPages; - - public ExcelColumnDefinition[] Columns => HeaderFile.ColumnDefinitions; - - private Dictionary< ushort, ExcelColumnDefinition >? _columnsByOffset; - - public Dictionary< ushort, ExcelColumnDefinition > ColumnsByOffset - { - get - { - if( _columnsByOffset == null ) - { - _columnsByOffset = Columns.GroupBy( p => p.Offset ).ToDictionary( c => c.Key, c => c.First() ); - } - - return _columnsByOffset; - } - } - - - /// - /// Returns the data pages contained in the Excel header - /// - public ExcelDataPagination[] DataPagination => HeaderFile.DataPages; - - /// - /// The available languages for this sheet. - /// - /// - /// You will need to reload this sheet with a different language if you want to access a single sheet in more than 1 language at a time. - /// - public Language[] Languages => HeaderFile.Languages; - - /// - /// The language that was requested for this sheet when it was loaded - /// - public Language RequestedLanguage { get; protected set; } - - internal readonly GameData GameData; - - /// - /// Generates an absolute path to a data file for a sheet - /// - /// The sheet name - /// The page row start index - /// The requested language - /// An absolute path to the file - protected string GenerateFilePath( string name, uint startId, Language language ) - { - if( language == Language.None ) - { - return $"exd/{name}_{startId}.exd"; - } - - var lang = LanguageUtil.GetLanguageStr( language ); - - return $"exd/{name}_{startId}_{lang}.exd"; - } - - /// - /// Iterates across sheet pagination and creates pages with their associated row counts and breakpoints - /// - internal void GenerateFilePages() - { - var lang = Language.None; - - if( HeaderFile.Languages.Contains( RequestedLanguage ) ) - { - lang = RequestedLanguage; - } - - foreach( var bp in HeaderFile.DataPages ) - { - var filePath = GenerateFilePath( Name, bp.StartId, lang ); - - // ignore languages that don't exist in this client build - if( !GameData.FileExists( filePath ) ) - { - continue; - } - - var segment = new ExcelPage - { - FilePath = filePath, - RowCount = bp.RowCount, - StartId = bp.StartId - }; - - segment.File = GameData.GetFile< ExcelDataFile >( segment.FilePath ); - - DataPages.Add( segment ); - } - } - - /// - /// Gets the corresponding data page for a given row - /// - /// The row id to fetch the parent page for - /// The if found, null otherwise - public ExcelPage? GetPageForRow( uint row ) - { - var page = DataPages.FirstOrDefault( s => row >= s.StartId && row < s.StartId + s.RowCount ); - - if( page?.RowData.ContainsKey( row ) == false ) - { - return null; - } - - return page; - } - - /// - /// Check whether a row exists in a sheet - /// - /// The rowid to check. - /// Subrows in type 2 sheets can't be checked for in this form as the header does not contain an explicit entry for a (row, subrow) pair - /// True if exists, false otherwise - public bool HasRow( uint row ) - { - var page = GetPageForRow( row ); - - if( page == null ) - { - return false; - } - - return page.RowData.Any( x => x.Key == row ); - } - - /// - public bool HasRow( int row ) => HasRow( (uint)row ); - - protected static ulong GetCacheKey( uint rowId, uint subrowId = UInt32.MaxValue ) - { - return (ulong)rowId << 32 | subrowId; - } - - /// - /// Provides direct access to the underlying row parser for any row or subrow - /// - /// The row id to seek to - /// The subrow id to seek to - /// A instance - public RowParser? GetRowParser( uint row, uint subRow = uint.MaxValue ) - { - var page = GetPageForRow( row ); - if( page == null ) - { - return null; - } - - return GetRowParser( page, row, subRow ); - } - - /// - /// Provides direct access to the underlying row parser for any row or subrow - /// - /// The excel page to operate on - /// The row id to seek to - /// The subrow id to seek to - /// A instance - public RowParser? GetRowParser( ExcelPage page, uint row, uint subRow = uint.MaxValue ) - { - RowParser parser = null!; - - if( subRow != uint.MaxValue ) - { - parser = new RowParser( this, page.File, row, subRow ); - } - else - { - parser = new RowParser( this, page.File, row ); - } - - return parser; - } - - /// - /// Iterate across each row data offsets in a sheet - /// - /// A containing each offset to each row or subrow group - public IEnumerable< RowDataCursor > GetRowDataOffsets() - { - foreach( var page in DataPages ) - { - var file = page.File; - var rowPtrs = file.RowData; - - foreach( var rowPtr in rowPtrs.Values ) - { - yield return new() - { - RowOffset = rowPtr, - SheetPage = file - }; - } - } - } - - public IEnumerable< RowParser > GetRowParsers() - { - ExcelDataFile file = null!; - RowParser parser = null!; - - foreach( var offset in GetRowDataOffsets() ) - { - var rowPtr = offset.RowOffset; - if( file != offset.SheetPage ) - { - parser = new RowParser( this, offset.SheetPage ); - } - - parser.SeekToRow( rowPtr.RowId ); - - if( Header.Variant == ExcelVariant.Subrows ) - { - // read subrows - for( uint i = 0; i < parser.RowCount; i++ ) - { - parser.SeekToRow( rowPtr.RowId, i ); - yield return parser; - } - } - else - { - yield return parser; - } - } - } - } -} \ No newline at end of file diff --git a/src/Lumina/Excel/Exceptions/ExcelSheetColumnChecksumMismatchException.cs b/src/Lumina/Excel/Exceptions/ExcelSheetColumnChecksumMismatchException.cs deleted file mode 100644 index 50f11de1..00000000 --- a/src/Lumina/Excel/Exceptions/ExcelSheetColumnChecksumMismatchException.cs +++ /dev/null @@ -1,20 +0,0 @@ -using System; - -namespace Lumina.Excel.Exceptions -{ - public class ExcelSheetColumnChecksumMismatchException : Exception - { - public readonly string SheetName; - public readonly uint ExpectedHash; - public readonly uint ActualHash; - - public ExcelSheetColumnChecksumMismatchException( string name, uint expectedHash, uint actualHash ) - { - SheetName = name; - ExpectedHash = expectedHash; - ActualHash = actualHash; - } - - public override string Message => $"sheet {SheetName} column hash mismatch! expected hash: {ExpectedHash:x8}, actual hash: {ActualHash:x8}!"; - } -} \ No newline at end of file diff --git a/src/Lumina/Excel/IExcelRow.cs b/src/Lumina/Excel/IExcelRow.cs new file mode 100644 index 00000000..bbd6b88e --- /dev/null +++ b/src/Lumina/Excel/IExcelRow.cs @@ -0,0 +1,31 @@ +namespace Lumina.Excel; + +/// +/// Defines a row type/schema for an excel sheet. +/// +/// The type that implements the interface +public interface IExcelRow where T : struct +{ + /// + /// Creates an instance of the current type. Designed only for use within . + /// + /// Only used for sheets that are not using subrows, and will throw otherwise. + /// + /// + /// + /// A newly created row object + /// Thrown when the referenced sheet is using subrows + abstract static T Create( ExcelPage page, uint offset, uint row ); + + /// + /// Creates an instance of the current type. Designed only for use within . + /// + /// Only used for sheets that are using subrows, and will throw otherwise. + /// + /// + /// + /// + /// A newly created subrow object + /// Thrown when the referenced sheet is not using subrows + abstract static T Create( ExcelPage page, uint offset, uint row, ushort subrow ); +} \ No newline at end of file diff --git a/src/Lumina/Excel/IExcelSheet.cs b/src/Lumina/Excel/IExcelSheet.cs new file mode 100644 index 00000000..71b7e432 --- /dev/null +++ b/src/Lumina/Excel/IExcelSheet.cs @@ -0,0 +1,58 @@ +using System.Collections; +using Lumina.Data; + +namespace Lumina.Excel; + +/// +/// A generalized interface that all s implement. +/// +/// This interface exists to assist with more generic and reflection-based operations. +public interface IExcelSheet : IEnumerable +{ + /// + /// The module that this sheet belongs to. + /// + ExcelModule Module { get; } + + /// + /// The language of the rows in this sheet. + /// + /// This can be different from the requested language if it wasn't supported. + Language Language { get; } + + /// + /// Whether or not this sheet has a row with the given . + /// + /// + /// If this sheet has subrows, this will check if the row id has any subrows. + /// + /// The row id to check + /// Whether or not the row exists + bool HasRow( uint rowId ); + + /// + /// Whether or not this sheet has a subrow with the given and . + /// + /// The row id to check + /// The subrow id to check + /// Whether or not the subrow exists + /// Thrown if the sheet does not support subrows + bool HasSubrow( uint rowId, ushort subrowId ); + + /// + /// Tries to get the number of subrows in the th row in this sheet. + /// + /// The row id to get + /// The number of subrows in this row. Returns null if the row does not exist. + /// Thrown if the sheet does not support subrows + ushort? TryGetSubrowCount( uint rowId ); + + /// + /// Gets the number of subrows in the th row in this sheet. Throws if the row does not exist. + /// + /// The row id to get + /// The number of subrows in this row. Returns null if the row does not exist. + /// Thrown if the sheet does not support subrows + /// Thrown if the sheet does not have a row at that + ushort GetSubrowCount( uint rowId ); +} \ No newline at end of file diff --git a/src/Lumina/Excel/LazyCollection.cs b/src/Lumina/Excel/LazyCollection.cs new file mode 100644 index 00000000..ac3f14dd --- /dev/null +++ b/src/Lumina/Excel/LazyCollection.cs @@ -0,0 +1,42 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Runtime.CompilerServices; + +namespace Lumina.Excel; + +/// +/// A collection helper used to layout and structure excel rows. +/// +/// Mostly an implementation detail for reading excel rows. This type does not store or hold any row data, and is therefore lightweight and trivially constructable. +/// A type that wraps a group of fields inside a row. +/// +/// +/// +/// +/// +public readonly struct LazyCollection( ExcelPage page, uint parentOffset, uint offset, Func ctor, int size ) : IReadOnlyList where T : struct +{ + /// + public T this[int index] { + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + get { + ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual( index, size ); + return ctor( page, parentOffset, offset, (uint)index ); + } + } + + /// + public int Count => size; + + /// + public IEnumerator GetEnumerator() + { + for( var i = 0; i < size; ++i ) + yield return this[i]; + } + + /// + IEnumerator IEnumerable.GetEnumerator() => + GetEnumerator(); +} \ No newline at end of file diff --git a/src/Lumina/Excel/LazyRow.cs b/src/Lumina/Excel/LazyRow.cs deleted file mode 100644 index 47b2a434..00000000 --- a/src/Lumina/Excel/LazyRow.cs +++ /dev/null @@ -1,150 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using Lumina.Data; -using Lumina.Data.Files.Excel; -using Lumina.Data.Structs.Excel; - -namespace Lumina.Excel -{ - public interface ILazyRow - { - /// - /// The backing value/row that was passed through when creating the reference - /// - public uint Row { get; } - - /// - /// Checks whether something has loaded successfully. - /// - /// - /// If something fails to load, this will still be false regardless. - /// - public bool IsValueCreated { get; } - - public Language Language { get; } - - public ExcelRow? RawRow { get; } - } - - public class EmptyLazyRow : ILazyRow - { - private static readonly Dictionary< string, HashSet > _ranges = new(); - - public uint Row { get; set; } - public bool IsValueCreated => false; - public Language Language => Language.None; - public ExcelRow? RawRow => null; - - public EmptyLazyRow( uint rowId ) - { - Row = rowId; - } - - public static ILazyRow GetFirstLazyRowOrEmpty( GameData gameData, uint row, params string[] sheetNames ) - { - return GetFirstLazyRowOrEmpty( gameData, row, Language.None, sheetNames ); - } - - public static ILazyRow GetFirstLazyRowOrEmpty( GameData gameData, uint row, Language language, params string[] sheetNames ) - { - foreach( var sheetName in sheetNames ) - { - if( !_ranges.ContainsKey( sheetName ) ) - { - var exh = gameData.Excel.GetSheetRaw( sheetName ); - if( exh == null ) - { - continue; - } - _ranges.Add( sheetName, exh.DataPages.SelectMany(p => p.RowData.Keys).ToHashSet() ); - } - - if( _ranges[ sheetName ].Contains( row ) ) - { - return new LazyRow< ExcelRow >( gameData, row, language ); - } - } - return new EmptyLazyRow( row ); - } - } - - /// - /// Allows for sheet definitions to contain entries which will lazily load the referenced sheet row - /// - /// The row type to load - public class LazyRow< T > : ILazyRow where T : ExcelRow - { - private readonly GameData _gameData; - private readonly uint _row; - private readonly Language _language; - - private T? _value; - - /// - /// The backing value/row that was passed through when creating the reference - /// - public uint Row => _row; - - public Language Language => _language; - - /// - /// Construct a new LazyRow instance - /// - /// The Lumina instance to load from - /// The row id to load if/when the value is fetched - /// The requested language to use when resolving row references - public LazyRow( GameData gameData, uint row, Language language = Language.None ) - { - _gameData = gameData; - _row = row; - _language = language; - } - - /// - /// Construct a new LazyRow instance - /// - /// The Lumina instance to load from - /// The row id to load if/when the value is fetched - /// The language to load the row in - public LazyRow( GameData gameData, int row, Language language = Language.None ) : this( gameData, (uint)row, language ) - { - } - - /// - /// Lazily load the referenced sheet/row, otherwise return the existing row. - /// - public T? Value - { - get - { - if( IsValueCreated ) - { - return _value; - } - - _value = _gameData.GetExcelSheet< T >( _language )?.GetRow( _row ); - - return _value; - } - } - - /// - /// Provides access to the raw row without any fuckery, useful for serialisation and etc. - /// - public ExcelRow? RawRow => Value; - - /// - /// Checks whether something has loaded successfully. - /// - /// - /// If something fails to load, this will still be false regardless. - /// - public bool IsValueCreated => _value != null; - - public override string ToString() - { - return $"{typeof( T ).FullName}#{_row}"; - } - } -} \ No newline at end of file diff --git a/src/Lumina/Excel/NoCache.cs b/src/Lumina/Excel/NoCache.cs deleted file mode 100644 index ad63e6f7..00000000 --- a/src/Lumina/Excel/NoCache.cs +++ /dev/null @@ -1,29 +0,0 @@ -using System; - -namespace Lumina.Excel -{ - /// - /// Class that allows to skip the caching of Excel rows when they are read. - /// - public sealed class NoCache : IDisposable - { - [field: ThreadStatic] - internal static bool IsEnabled { get; private set; } - - /// - /// Disables the caching of Excel rows when they are read. - /// - public NoCache() - { - IsEnabled = true; - } - - /// - /// Re-enables the caching of Excel rows when they are read. - /// - public void Dispose() - { - IsEnabled = false; - } - } -} \ No newline at end of file diff --git a/src/Lumina/Excel/RawExcelSheet.cs b/src/Lumina/Excel/RawExcelSheet.cs deleted file mode 100644 index 1f4c2d77..00000000 --- a/src/Lumina/Excel/RawExcelSheet.cs +++ /dev/null @@ -1,77 +0,0 @@ -using System.Collections; -using System.Collections.Generic; -using Lumina.Data; -using Lumina.Data.Files.Excel; -using Lumina.Data.Structs.Excel; - -namespace Lumina.Excel -{ - public class RawExcelSheet : ExcelSheetImpl, IEnumerable< RowParser > - { - internal RawExcelSheet( ExcelHeaderFile headerFile, string name, Language requestedLanguage, GameData gameData ) : - base( headerFile, name, requestedLanguage, gameData ) - { - } - - public RowParser? GetRow( uint rowId ) - { - var page = GetPageForRow( rowId ); - if( page == null ) - { - return null; - } - - return new RowParser( this, page.File, rowId ); - } - - public RowParser? GetRow( uint rowId, uint subRowId ) - { - var page = GetPageForRow( rowId ); - if( page == null ) - { - return null; - } - - return new RowParser( this, page.File, rowId, subRowId ); - } - - public IEnumerator< RowParser > GetEnumerator() - { - ExcelDataFile file = null!; - RowParser parser = null!; - - foreach( var offset in GetRowDataOffsets() ) - { - var rowPtr = offset.RowOffset; - if( file != offset.SheetPage ) - { - parser = new RowParser( this, offset.SheetPage ); - } - - if( Header.Variant == ExcelVariant.Subrows ) - { - // required to read the row header out and know how many subrows there is - parser.SeekToRow( rowPtr.RowId ); - - // read subrows - for( uint i = 0; i < parser.RowCount; i++ ) - { - parser.SeekToRow( rowPtr.RowId, i ); - yield return parser; - } - } - else - { - parser.SeekToRow( rowPtr.RowId ); - - yield return parser; - } - } - } - - IEnumerator IEnumerable.GetEnumerator() - { - return GetEnumerator(); - } - } -} \ No newline at end of file diff --git a/src/Lumina/Excel/RowDataCursor.cs b/src/Lumina/Excel/RowDataCursor.cs deleted file mode 100644 index c3de20c3..00000000 --- a/src/Lumina/Excel/RowDataCursor.cs +++ /dev/null @@ -1,14 +0,0 @@ -using Lumina.Data.Files.Excel; -using Lumina.Data.Structs.Excel; - -namespace Lumina.Excel -{ - /// - /// A 'cursor' that points to the current row offset and which file the row is in - /// - public record struct RowDataCursor - { - public ExcelDataFile SheetPage; - public ExcelDataOffset RowOffset; - } -} \ No newline at end of file diff --git a/src/Lumina/Excel/RowParser.cs b/src/Lumina/Excel/RowParser.cs deleted file mode 100644 index 4b666ff7..00000000 --- a/src/Lumina/Excel/RowParser.cs +++ /dev/null @@ -1,467 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Runtime.CompilerServices; -using System.Text; -using Lumina.Data; -using Lumina.Data.Files.Excel; -using Lumina.Data.Structs.Excel; -using Lumina.Extensions; -using Lumina.Text; - -namespace Lumina.Excel -{ - public class RowParser - { - private readonly ExcelSheetImpl _sheet; - private readonly Dictionary< uint, ExcelDataOffset > _rowData; - private readonly LuminaBinaryReader _reader; - - private ExcelDataOffset _offset; - private ExcelDataRowHeader _rowHeader; - - private long _rowOffset; - - public uint RowId; - public uint SubRowId; - public uint RowCount => _rowHeader.RowCount; - - - /// - /// Provides access to the base data generated for a sheet - /// - public ExcelSheetImpl Sheet => _sheet; - - public RowParser( ExcelSheetImpl sheet, ExcelDataFile dataFile ) - { - _sheet = sheet; - _rowData = dataFile.RowData; - _reader = new LuminaBinaryReader( dataFile.Data, dataFile.Reader.PlatformId ) { IsLittleEndian = false }; - } - - public RowParser( ExcelSheetImpl sheet, ExcelDataFile dataFile, uint row ) - : this( sheet, dataFile ) - { - SeekToRow( row ); - } - - public RowParser( ExcelSheetImpl sheet, ExcelDataFile dataFile, uint row, uint subRow ) - : this( sheet, dataFile, row ) - { - SeekToRow( row, subRow ); - } - - /// - /// Moves the parser to a row in the current page given its index - /// - /// The row index to seek to - /// /// Given row index was out of bounds - public void SeekToRow( uint row ) - { - if( !TrySeekToRow( row ) ) - { - throw new IndexOutOfRangeException( $"row#{row} could not be found in {_sheet.Name} sheet!" ); - } - } - - /// - /// Moves the parser to a row in the current page given its index - /// - /// The row index to seek to - /// true if the row was seeked to successfully, false if the row wasn't found or otherwise - public bool TrySeekToRow( uint row ) - { - RowId = row; - - if( !_rowData.TryGetValue( RowId, out var offset ) ) - { - return false; - } - - _offset = offset; - - _reader.BaseStream.Position = _offset.Offset; - - _rowHeader = ExcelDataRowHeader.Read( _reader ); - - // header is 6 bytes large, data normally starts here except in the case of variant 2 sheets but we'll keep it anyway - _rowOffset = _offset.Offset + 6; - - return true; - } - - /// - /// Moves the parser to a row + subrow in the current page given their indexes - /// - /// The row index to seek to - /// The subrow index to seek to - /// Given subrow index was out of bounds - public void SeekToRow( uint row, uint subRow ) - { - if( !TrySeekToRow( row, subRow ) ) - { - throw new IndexOutOfRangeException( $"subrow {subRow} > {_rowHeader.RowCount}!" ); - } - } - - /// - /// Moves the parser to a row + subrow in the current page given their indexes - /// - /// The row index to seek to - /// The subrow index to seek to - /// /// true if the row and subrow was seeked to successfully, false if the row or subrow wasn't found or otherwise - public bool TrySeekToRow( uint row, uint subRow ) - { - if( !TrySeekToRow( row ) ) - { - return false; - } - - SubRowId = subRow; - - if( subRow > _rowHeader.RowCount ) - { - return false; - } - - _rowOffset = CalculateSubRowOffset( subRow ); - - return true; - } - - [MethodImpl( MethodImplOptions.AggressiveInlining )] - public long CalculateSubRowOffset( uint subRow ) - { - // +6 is the ExcelDataRowHeader - return _offset.Offset + 6 + ( subRow * _sheet.Header.DataOffset + 2 * ( subRow + 1 ) ); - } - - /// - /// Read n bytes starting from the row offset + offset - /// - /// The offset inside the row - /// The number of bytes to read - /// A copy of the read bytes - public byte[] ReadBytes( int offset, int count ) - { - _reader.BaseStream.Position = _rowOffset + offset; - - return _reader.ReadBytes( count ); - } - - /// - /// Reads a structure from an offset inside the current row - /// - /// The offset to start reading from - /// The type of struct to read out from the row - /// The read structure filled from the row data - public T ReadStructure< T >( int offset ) where T : struct - { - _reader.BaseStream.Position = _rowOffset + offset; - - return _reader.ReadStructure< T >(); - } - - /// - /// Reads structures from an offset inside the current row - /// - /// The offset to start reading from - /// The number of structures to read sequentially - /// The type of struct to read out from the row - /// The read structures filled from the row data - public List< T > ReadStructures< T >( int offset, int count ) where T : struct - { - _reader.BaseStream.Position = _rowOffset + offset; - - return _reader.ReadStructures< T >( count ); - } - - /// - /// Reads structures from an offset inside the current row - /// - /// The offset to start reading from - /// The number of structures to read sequentially - /// The type of struct to read out from the row - /// The read structures filled from the row data - public T[] ReadStructuresAsArray< T >( int offset, int count ) where T : struct - { - _reader.BaseStream.Position = _rowOffset + offset; - - return _reader.ReadStructuresAsArray< T >( count ); - } - - // bit faster way of finding _rsv_ keys without having to convert the whole thing to a string first and shit - private readonly byte[] _rsvMagic = { 0x5f, 0x72, 0x73, 0x76, 0x5f }; - - private SeString? ReplaceRsvKeyWithValue( byte[] originalData ) - { - // so we don't do unnecessary array comparisons - if( !Sheet.GameData.Excel.RsvProvider.HasValues ) - { - return null; - } - - if( originalData.Length < _rsvMagic.Length ) - { - return null; - } - - for( var i = 0; i < _rsvMagic.Length; i++ ) - { - if( originalData[ i ] != _rsvMagic[ i ] ) - { - return null; - } - } - - var key = Encoding.ASCII.GetString( originalData ); - var value = Sheet.GameData.Excel.RsvProvider.GetValue( key ); - - if( value == null ) - { - return null; - } - - return new SeString( value ); - } - - private object ReadFieldInternal( ExcelColumnDataType type ) - { - object? data = null; - - switch( type ) - { - case ExcelColumnDataType.String: - { - var stringOffset = _reader.ReadUInt32(); - var raw = _reader.ReadRawOffsetData( _rowOffset + _sheet.Header.DataOffset + stringOffset ); - - if( Sheet.GameData.Options.ResolveKnownRsvSheetValues ) - { - var replacement = ReplaceRsvKeyWithValue( raw ); - if( replacement != null ) - { - data = replacement; - break; - } - } - - data = new SeString( raw ); - break; - } - case ExcelColumnDataType.Bool: - { - data = _reader.ReadBoolean(); - break; - } - case ExcelColumnDataType.Int8: - { - data = _reader.ReadSByte(); - break; - } - case ExcelColumnDataType.UInt8: - { - data = _reader.ReadByte(); - break; - } - case ExcelColumnDataType.Int16: - { - data = _reader.ReadInt16(); - break; - } - case ExcelColumnDataType.UInt16: - { - data = _reader.ReadUInt16(); - break; - } - case ExcelColumnDataType.Int32: - { - data = _reader.ReadInt32(); - break; - } - case ExcelColumnDataType.UInt32: - { - data = _reader.ReadUInt32(); - break; - } - // case ExcelColumnDataType.Unk: - // break; - case ExcelColumnDataType.Float32: - { - data = _reader.ReadSingle(); - break; - } - case ExcelColumnDataType.Int64: - { - data = _reader.ReadUInt64(); - break; - } - case ExcelColumnDataType.UInt64: - { - data = _reader.ReadUInt64(); - break; - } - // case ExcelColumnDataType.Unk2: - // break; - case ExcelColumnDataType.PackedBool0: - case ExcelColumnDataType.PackedBool1: - case ExcelColumnDataType.PackedBool2: - case ExcelColumnDataType.PackedBool3: - case ExcelColumnDataType.PackedBool4: - case ExcelColumnDataType.PackedBool5: - case ExcelColumnDataType.PackedBool6: - case ExcelColumnDataType.PackedBool7: - { - var shift = (int)type - (int)ExcelColumnDataType.PackedBool0; - var bit = 1 << shift; - - var rawData = _reader.ReadByte(); - - data = ( rawData & bit ) == bit; - - break; - } - default: - throw new ArgumentOutOfRangeException( "type", $"invalid excel column type: {type}" ); - } - - return data; - } - - /// - /// Read a field from the current stream position - /// - /// The sheet type to read - /// The CLR type to store the read data in - /// The read data stored in the provided type - /// An invalid column type was provided - private T? ReadField< T >( ExcelColumnDataType type ) - { - var data = ReadFieldInternal( type ); - - if( _sheet.GameData.Options.ExcelSheetStrictCastingEnabled ) - { - return (T)data; - } - - // todo: this is fucking shit but is a wip fix so that you can still ReadField< string > and get something back because 1am brain can't figure it out rn - if( typeof( T ) == typeof( string ) && data is SeString seString ) - { - // haha fuck you c# - return (T)(object)seString.RawString; - } - - if( data is T castedData ) - { - return castedData; - } - - return default; - } - - /// - /// Given a bitset with 1 flag set, find which index that bit is set at - /// - /// - /// - [MethodImpl( MethodImplOptions.AggressiveInlining )] - private byte GetBitPosition( byte flag ) - { - byte count = 0; - - while( flag != 1 ) - { - flag >>= 1; - count++; - } - - return count; - } - - /// - /// Read a type from an offset in the row - /// - /// The offset to read from - /// Read a specific bit from the underlying position - useful for bools - /// The type to store the data in - /// The read data contained in the provided type - public T? ReadOffset< T >( ushort offset, byte bit = 0 ) - { - _reader.BaseStream.Position = _rowOffset + offset; - - if( bit == 0 ) - { - return ReadField< T >( _sheet.ColumnsByOffset[ offset ].Type ); - } - - var pos = GetBitPosition( bit ); - var flag = ExcelColumnDataType.PackedBool0 + pos; - - return ReadField< T >( flag ); - } - - /// - public T? ReadOffset< T >( int offset, byte bit = 0 ) => ReadOffset< T >( (ushort)offset, bit ); - - /// - public T? ReadOffset< T >( uint offset, byte bit = 0 ) => ReadOffset< T >( (ushort)offset, bit ); - - /// - public T? ReadOffset< T >( short offset, byte bit = 0 ) => ReadOffset< T >( (ushort)offset, bit ); - - /// - /// Read a type from an offset in the row - /// - /// The offset to read from - /// The excel column type to read - /// The read data contained in the provided type - public T? ReadOffset< T >( int offset, ExcelColumnDataType type ) - { - _reader.BaseStream.Position = _rowOffset + offset; - - return ReadField< T >( type ); - } - - /// - /// Read a type from a column index in the row - /// - /// The column index to lookup - /// The type to store the read data in - /// The read data contained in the provided type, or the default value of the type if the column given is out of bounds - public T? ReadColumn< T >( int column ) - { - if( column >= _sheet.ColumnCount ) - { - return default; - } - - var col = _sheet.Columns[ column ]; - - _reader.BaseStream.Position = _rowOffset + col.Offset; - - return ReadField< T >( col.Type ); - } - - /// - /// Grab the raw value from the sheet. - /// - /// - /// This effectively acts as a variant and the object encapsulates it. You'll still need to do a type check or safely cast it to avoid exceptions - /// but this can be useful when you don't need to care about it's type and can use it as is - e.g. ToString and so on - /// - /// The column index to read from - /// An object containing the data from the row, or null if the column given is out of bounds - public object? ReadColumnRaw( int column ) - { - if( column >= _sheet.ColumnCount ) - { - return null; - } - - var col = _sheet.Columns[ column ]; - - _reader.BaseStream.Position = _rowOffset + col.Offset; - - return ReadFieldInternal( col.Type ); - } - } -} \ No newline at end of file diff --git a/src/Lumina/Excel/RowRef.cs b/src/Lumina/Excel/RowRef.cs new file mode 100644 index 00000000..66849b9e --- /dev/null +++ b/src/Lumina/Excel/RowRef.cs @@ -0,0 +1,123 @@ +using System; + +namespace Lumina.Excel; + +/// +/// A helper type to dynamically reference a row in a different excel sheet. +/// +/// The to read sheet data from +/// The referenced row id +/// The referenced row's actual +public readonly struct RowRef( ExcelModule? module, uint rowId, Type? rowType ) +{ + /// + /// The row id of the referenced row. + /// + public uint RowId => rowId; + + /// + /// Whether or not the is untyped. + /// + /// + /// An untyped is one that doesn't know which sheet it links to. + /// + public bool IsUntyped => rowType == null; + + /// + /// Whether or not the reference is of a specific row type. + /// + /// The row type/schema to check against + /// Whether or not this points to a + public bool Is() where T : struct, IExcelRow => + typeof( T ) == rowType; + + /// + /// Tries to get the referenced row as a specific row type. + /// + /// The row type/schema to check against + /// The referenced row type. Returns null if this does not point to a or if the does not exist in its sheet. + public T? TryGetValue() where T : struct, IExcelRow + { + if( !Is() ) + return null; + + return module!.GetSheet().TryGetRow( RowId ); + } + + /// + /// Attempts to create a to a row id of a list of row types, checking with each type in order. + /// + /// The to read sheet data from + /// The referenced row id + /// A list of row types to check against the , in order + /// A to one of the . If the row id does not exist in any of the sheets, an untyped is returned instead. + public static RowRef GetFirstValidRowOrUntyped( ExcelModule module, uint rowId, params Type[] sheetTypes ) + { + foreach( var sheetType in sheetTypes ) + { + if( module.GetSheetGeneric( sheetType ) is { } sheet ) + { + if( sheet.HasRow( rowId ) ) + return new( module, rowId, sheetType ); + } + } + + return CreateUntyped( rowId ); + } + + /// + /// Creates a to a specific row type. + /// + /// The row type referenced by the + /// The to read sheet data from + /// The referenced row id + /// A to a row in a . + public static RowRef Create( ExcelModule module, uint rowId ) where T : struct, IExcelRow => new( module, rowId, typeof( T ) ); + + /// + /// Creates an untyped . + /// + /// The referenced row id + /// An untyped + public static RowRef CreateUntyped( uint rowId ) => new( null, rowId, null ); +} + +/// +/// A helper type to concretely reference a row in a specific excel sheet. +/// +/// The row type referenced by the +/// The to read sheet data from +/// The referenced row id +public readonly struct RowRef( ExcelModule module, uint rowId ) where T : struct, IExcelRow +{ + private readonly ExcelSheet sheet = module.GetSheet(); + + /// + /// The row id of the referenced row. + /// + public uint RowId => rowId; + + /// + /// Whether or not the exists in the sheet. + /// + public bool IsValid => sheet.HasRow( RowId ); + + /// + /// The referenced row value itself. + /// + /// Thrown if is false + public T Value => sheet.GetRow( RowId ); + + /// + /// Attempts to get the referenced row value. Is null if does not exist in the sheet. + /// + public T? ValueNullable => sheet.TryGetRow( RowId ); + + private RowRef ToGeneric() => RowRef.Create( module, rowId ); + + /// + /// Converts a concrete to an generic and dynamically typed . + /// + /// The to convert + public static explicit operator RowRef( RowRef row ) => row.ToGeneric(); +} \ No newline at end of file diff --git a/src/Lumina/Excel/SheetAttribute.cs b/src/Lumina/Excel/SheetAttribute.cs index 4f27ccca..8f8613aa 100644 --- a/src/Lumina/Excel/SheetAttribute.cs +++ b/src/Lumina/Excel/SheetAttribute.cs @@ -1,27 +1,22 @@ using System; -namespace Lumina.Excel -{ - public class SheetAttribute : Attribute - { - /// - /// The sheet name - /// - public readonly string Name; - - /// - /// A column hash - used to warn when a sheet structure has changed - /// - public readonly uint? ColumnHash; +namespace Lumina.Excel; - public SheetAttribute( string name ) - { - Name = name; - } +/// +/// An attribute attached to a schema/struct that represents a sheet in an excel file. +/// +/// The name of the sheet +/// The column hash of the sheet; optionally used to check for schema and sheet changes +[AttributeUsage( AttributeTargets.Struct )] +public class SheetAttribute( string name, uint? columnHash = null ) : Attribute +{ + /// + /// The sheet name + /// + public readonly string Name = name; - public SheetAttribute( string name, uint columnHash ) : this( name ) - { - ColumnHash = columnHash; - } - } + /// + /// A column hash - used to warn when a sheet structure has changed + /// + public readonly uint? ColumnHash = columnHash; } \ No newline at end of file From 66a4d96d658f90d286fe107ce5a342260388de0d Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Mon, 12 Aug 2024 01:48:06 -0700 Subject: [PATCH 03/53] Fix compile errors --- src/Lumina.Cmd/Commands/ExcelUpdate.cs | 8 +++--- src/Lumina.Tests/SeStringBuilderTests.cs | 22 +++++++++++++--- src/Lumina/Excel/ExcelModule.cs | 6 ++--- src/Lumina/Excel/SheetAttribute.cs | 8 +++++- src/Lumina/Extensions/RsvExtensions.cs | 20 +++++++------- src/Lumina/GameData.cs | 33 ++++++++++-------------- 6 files changed, 57 insertions(+), 40 deletions(-) diff --git a/src/Lumina.Cmd/Commands/ExcelUpdate.cs b/src/Lumina.Cmd/Commands/ExcelUpdate.cs index 6c4872e5..a52e8aa9 100644 --- a/src/Lumina.Cmd/Commands/ExcelUpdate.cs +++ b/src/Lumina.Cmd/Commands/ExcelUpdate.cs @@ -24,9 +24,9 @@ public ValueTask ExecuteAsync( IConsole console ) var ol = new GameData( OldPath ); var nl = new GameData( NewPath ); - co.WriteLine( $"old sheets: {ol.Excel.GetSheetNames().Count} new sheets: {nl.Excel.GetSheetNames().Count}" ); + co.WriteLine( $"old sheets: {ol.Excel.SheetNames.Count} new sheets: {nl.Excel.SheetNames.Count}" ); - var removedSheets = ol.Excel.GetSheetNames().Except( nl.Excel.GetSheetNames() ).ToList(); + var removedSheets = ol.Excel.SheetNames.Except( nl.Excel.SheetNames ).ToList(); if( removedSheets.Any() ) { co.WriteLine( $"{removedSheets.Count} sheets removed" ); @@ -37,7 +37,7 @@ public ValueTask ExecuteAsync( IConsole console ) } } - var newSheets = nl.Excel.GetSheetNames().Except( ol.Excel.GetSheetNames() ).ToList(); + var newSheets = nl.Excel.SheetNames.Except( ol.Excel.SheetNames ).ToList(); if( newSheets.Any() ) { co.WriteLine( $"{newSheets.Count} new sheets" ); @@ -49,7 +49,7 @@ public ValueTask ExecuteAsync( IConsole console ) } co.WriteLine( "diffing existing sheets..." ); - var existingSheets = nl.Excel.GetSheetNames().Intersect( ol.Excel.GetSheetNames() ).ToList(); + var existingSheets = nl.Excel.SheetNames.Intersect( ol.Excel.SheetNames ).ToList(); foreach( var eSheet in existingSheets ) { var exhPath = $"exd/{eSheet}.exh"; diff --git a/src/Lumina.Tests/SeStringBuilderTests.cs b/src/Lumina.Tests/SeStringBuilderTests.cs index 55d12fec..aab67490 100644 --- a/src/Lumina.Tests/SeStringBuilderTests.cs +++ b/src/Lumina.Tests/SeStringBuilderTests.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Text; +using Lumina.Excel; using Lumina.Text; using Lumina.Text.Expressions; using Lumina.Text.Payloads; @@ -290,11 +291,25 @@ public void ComplicatedTest() _outputHelper.WriteLine( test.ToString() ); } + [Sheet( "Addon" )] + public readonly struct Addon( ExcelPage page, uint offset, uint row ) : IExcelRow + { + public uint RowId => row; + + public readonly ReadOnlySeString Text => page.ReadString( offset, offset ); + + static Addon IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); + + static Addon IExcelRow.Create( ExcelPage page, uint offset, uint row, ushort subrow ) => + throw new NotSupportedException(); + } + [RequiresGameInstallationFact] public void AddonIsParsedCorrectly() { var gameData = new GameData( @"C:\Program Files (x86)\SquareEnix\FINAL FANTASY XIV - A Realm Reborn\game\sqpack" ); - var addon = gameData.Excel.GetSheetRaw( "Addon" )!; + var addon = gameData.Excel.GetSheet( )!; var ssb = new SeStringBuilder(); var expected = new Dictionary< uint, ReadOnlySeString > { @@ -418,10 +433,9 @@ public void AddonIsParsedCorrectly() }; foreach( var row in addon ) { - var r = row.ReadColumn< SeString >( 0 ).AsReadOnly(); - _outputHelper.WriteLine( $"{row.RowId}\t{r.ExtractText()}\t{r}" ); + _outputHelper.WriteLine( $"{row.RowId}\t{row.Text.ExtractText()}\t{row.Text}" ); if( expected.TryGetValue( row.RowId, out var expectedSeString ) ) - Assert.True( expectedSeString == r, $"{row.RowId} does not match; expected {expectedSeString}" ); + Assert.True( expectedSeString == row.Text, $"{row.RowId} does not match; expected {expectedSeString}" ); } } } \ No newline at end of file diff --git a/src/Lumina/Excel/ExcelModule.cs b/src/Lumina/Excel/ExcelModule.cs index 81bcbd7a..a749d076 100644 --- a/src/Lumina/Excel/ExcelModule.cs +++ b/src/Lumina/Excel/ExcelModule.cs @@ -40,7 +40,7 @@ public ExcelModule( GameData gameData ) } /// - /// Attempts to load an , optionally with a specific language + /// Loads an , optionally with a specific language /// /// /// If the requested language doesn't exist for the file, this will silently be ignored and it will return a sheet with the default language: @@ -58,7 +58,7 @@ public ExcelSheet GetSheet( Language? language = null ) where T : struct, } /// - /// Attempts to load an from a reflected , optionally with a specific language + /// Loads an from a reflected , optionally with a specific language /// /// /// Only use this method if you need to create a sheet while using reflection. @@ -71,7 +71,7 @@ public ExcelSheet GetSheet( Language? language = null ) where T : struct, /// Thrown when is not decorated with a /// Sheet does not exist or if the column hash has a mismatch [RequiresDynamicCode("Creating a generic sheet from a type requires reflection and dynamic code.")] - [EditorBrowsable(EditorBrowsableState.Never)] + [EditorBrowsable(EditorBrowsableState.Advanced)] public IExcelSheet GetSheetGeneric(Type rowType, Language? language = null ) { if( !rowType.IsValueType ) diff --git a/src/Lumina/Excel/SheetAttribute.cs b/src/Lumina/Excel/SheetAttribute.cs index 8f8613aa..e5b94449 100644 --- a/src/Lumina/Excel/SheetAttribute.cs +++ b/src/Lumina/Excel/SheetAttribute.cs @@ -8,7 +8,7 @@ namespace Lumina.Excel; /// The name of the sheet /// The column hash of the sheet; optionally used to check for schema and sheet changes [AttributeUsage( AttributeTargets.Struct )] -public class SheetAttribute( string name, uint? columnHash = null ) : Attribute +public class SheetAttribute( string name, uint columnHash ) : Attribute { /// /// The sheet name @@ -19,4 +19,10 @@ public class SheetAttribute( string name, uint? columnHash = null ) : Attribute /// A column hash - used to warn when a sheet structure has changed /// public readonly uint? ColumnHash = columnHash; + + /// The name of the sheet + public SheetAttribute(string name) : this(name, uint.MaxValue ) + { + ColumnHash = null; + } } \ No newline at end of file diff --git a/src/Lumina/Extensions/RsvExtensions.cs b/src/Lumina/Extensions/RsvExtensions.cs index 8ba873f2..87c69e86 100644 --- a/src/Lumina/Extensions/RsvExtensions.cs +++ b/src/Lumina/Extensions/RsvExtensions.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -14,19 +15,20 @@ public static List< string > GetEmbeddedRsvResources( this Assembly assembly ) public static void RegisterRsvFiles( this Assembly assembly, GameData gameData ) { - var rsv = gameData.Excel.RsvProvider; + throw new NotImplementedException(); + //var rsv = gameData.Excel.RsvProvider; - foreach( var file in GetEmbeddedRsvResources( assembly ) ) - { - gameData.Logger?.Information( "Loading RSV: {RsvFileName}", file ); + //foreach( var file in GetEmbeddedRsvResources( assembly ) ) + //{ + // gameData.Logger?.Information( "Loading RSV: {RsvFileName}", file ); - using var s = assembly.GetManifestResourceStream( file ); - using var sr = new StreamReader( s! ); + // using var s = assembly.GetManifestResourceStream( file ); + // using var sr = new StreamReader( s! ); - var data = sr.ReadToEnd(); + // var data = sr.ReadToEnd(); - rsv.ParseData( data ); - } + // rsv.ParseData( data ); + //} } } } \ No newline at end of file diff --git a/src/Lumina/GameData.cs b/src/Lumina/GameData.cs index 06b02588..797d7257 100644 --- a/src/Lumina/GameData.cs +++ b/src/Lumina/GameData.cs @@ -287,27 +287,22 @@ public static UInt64 GetFileHash( string path ) } /// - /// Attempts to load the base excel sheet given it's implementing row parser + /// Attempts to load an , optionally with a specific language /// - /// A class that implements to parse rows - /// An if the sheet exists, null if it does not - public ExcelSheet< T >? GetExcelSheet< T >() where T : ExcelRow + /// The requested sheet language. Leave or empty to use the default language. + /// A struct that implements to parse rows + /// An if the sheet exists, null if it does not + /// Thrown when is not decorated with a + public ExcelSheet< T >? GetExcelSheet< T >( Language? language = null ) where T : struct, IExcelRow { - return Excel.GetSheet< T >(); - } - - /// - /// Attempts to load the base excel sheet with a specific language - /// - /// - /// If the language requested doesn't exist for the file, this will silently be ignored and it will return a sheet with the default language: - /// - /// The requested sheet language - /// A class that implements to parse rows - /// An if the sheet exists, null if it does not - public ExcelSheet< T >? GetExcelSheet< T >( Language language ) where T : ExcelRow - { - return Excel.GetSheet< T >( language ); + try + { + return Excel.GetSheet(language); + } + catch (ArgumentException) + { + return null; + } } /// From 64efaf8e0d619a5c808b7c465160916d747869fd Mon Sep 17 00:00:00 2001 From: Soreepeong Date: Mon, 12 Aug 2024 22:26:30 +0900 Subject: [PATCH 04/53] Minor doc fixes --- src/Lumina/Excel/ExcelModule.cs | 16 ++++++++-------- src/Lumina/Excel/ExcelSheet.cs | 12 ++++++------ src/Lumina/Excel/IExcelSheet.cs | 10 +++++----- src/Lumina/GameData.cs | 8 ++++---- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/Lumina/Excel/ExcelModule.cs b/src/Lumina/Excel/ExcelModule.cs index a749d076..217dd864 100644 --- a/src/Lumina/Excel/ExcelModule.cs +++ b/src/Lumina/Excel/ExcelModule.cs @@ -40,14 +40,14 @@ public ExcelModule( GameData gameData ) } /// - /// Loads an , optionally with a specific language + /// Loads an , optionally with a specific language /// /// /// If the requested language doesn't exist for the file, this will silently be ignored and it will return a sheet with the default language: /// - /// The requested sheet language. Leave or empty to use the default language. - /// A struct that implements to parse rows - /// An if the sheet exists + /// The requested sheet language. Leave or empty to use the default language. + /// A struct that implements to parse rows + /// An if the sheet exists /// Thrown when the sheet type is not decorated with a /// Sheet does not exist or if the column hash has a mismatch public ExcelSheet GetSheet( Language? language = null ) where T : struct, IExcelRow @@ -58,16 +58,16 @@ public ExcelSheet GetSheet( Language? language = null ) where T : struct, } /// - /// Loads an from a reflected , optionally with a specific language + /// Loads an from a reflected , optionally with a specific language /// /// /// Only use this method if you need to create a sheet while using reflection. /// /// If the requested language doesn't exist for the file, this will silently be ignored and it will return a sheet with the default language: /// - /// A that implements to parse rows - /// The requested sheet language. Leave or empty to use the default language. - /// An if the sheet exists + /// A that implements to parse rows + /// The requested sheet language. Leave or empty to use the default language. + /// An if the sheet exists /// Thrown when is not decorated with a /// Sheet does not exist or if the column hash has a mismatch [RequiresDynamicCode("Creating a generic sheet from a type requires reflection and dynamic code.")] diff --git a/src/Lumina/Excel/ExcelSheet.cs b/src/Lumina/Excel/ExcelSheet.cs index 32a3c1f9..b58c223e 100644 --- a/src/Lumina/Excel/ExcelSheet.cs +++ b/src/Lumina/Excel/ExcelSheet.cs @@ -55,7 +55,7 @@ public sealed class ExcelSheet : IExcelSheet, IReadOnlyList where T : stru public T this[int rowId] => GetRow( (uint)rowId ); /// - /// Create an instance with the 's default language. + /// Create an instance with the 's default language. /// /// The to access sheet data from. /// does not have a valid @@ -66,7 +66,7 @@ public ExcelSheet( ExcelModule module ) : this( module, module.Language ) } /// - /// Create an instance with a specific . + /// Create an instance with a specific . /// /// The to access sheet data from. /// The language to use for this sheet. @@ -78,12 +78,12 @@ public ExcelSheet( ExcelModule module, Language requestedLanguage ) : this( modu } /// - /// Create an instance with a specific , name, and hash. + /// Create an instance with a specific , name, and hash. /// /// The to access sheet data from. /// The language to use for this sheet. /// The name of the sheet to read from. - /// The hash of the columns in the sheet. If , it will not check the hash. + /// The hash of the columns in the sheet. If , it will not check the hash. /// or parameters were invalid (hash mismatch or invalid sheet name) public ExcelSheet( ExcelModule module, Language requestedLanguage, string sheetName, uint? columnHash = null ) { @@ -276,7 +276,7 @@ public T GetSubrow( uint rowId, ushort subrowId ) /// /// Returns an enumerator that can be used to iterate over all subrows in all rows in this sheet. /// - /// An of all subrows in this sheet + /// An of all subrows in this sheet /// Thrown if the sheet does not support subrows public IEnumerator GetSubrowEnumerator() { @@ -293,7 +293,7 @@ public IEnumerator GetSubrowEnumerator() /// /// Returns an enumerator that can be used to iterate over all rows in this sheet. If this sheet has subrows, it will iterate over the first subrow of every row. /// - /// An of all rows (or first subrows) in this sheet + /// An of all rows (or first subrows) in this sheet public IEnumerator GetEnumerator() { if( !HasSubrows ) diff --git a/src/Lumina/Excel/IExcelSheet.cs b/src/Lumina/Excel/IExcelSheet.cs index 71b7e432..96d8a17b 100644 --- a/src/Lumina/Excel/IExcelSheet.cs +++ b/src/Lumina/Excel/IExcelSheet.cs @@ -4,7 +4,7 @@ namespace Lumina.Excel; /// -/// A generalized interface that all s implement. +/// A generalized interface that all s implement. /// /// This interface exists to assist with more generic and reflection-based operations. public interface IExcelSheet : IEnumerable @@ -36,7 +36,7 @@ public interface IExcelSheet : IEnumerable /// The row id to check /// The subrow id to check /// Whether or not the subrow exists - /// Thrown if the sheet does not support subrows + /// Thrown if the sheet does not support subrows bool HasSubrow( uint rowId, ushort subrowId ); /// @@ -44,7 +44,7 @@ public interface IExcelSheet : IEnumerable /// /// The row id to get /// The number of subrows in this row. Returns null if the row does not exist. - /// Thrown if the sheet does not support subrows + /// Thrown if the sheet does not support subrows ushort? TryGetSubrowCount( uint rowId ); /// @@ -52,7 +52,7 @@ public interface IExcelSheet : IEnumerable /// /// The row id to get /// The number of subrows in this row. Returns null if the row does not exist. - /// Thrown if the sheet does not support subrows - /// Thrown if the sheet does not have a row at that + /// Thrown if the sheet does not support subrows + /// Thrown if the sheet does not have a row at that ushort GetSubrowCount( uint rowId ); } \ No newline at end of file diff --git a/src/Lumina/GameData.cs b/src/Lumina/GameData.cs index 797d7257..6f169768 100644 --- a/src/Lumina/GameData.cs +++ b/src/Lumina/GameData.cs @@ -287,11 +287,11 @@ public static UInt64 GetFileHash( string path ) } /// - /// Attempts to load an , optionally with a specific language + /// Attempts to load an , optionally with a specific language /// - /// The requested sheet language. Leave or empty to use the default language. - /// A struct that implements to parse rows - /// An if the sheet exists, null if it does not + /// The requested sheet language. Leave or empty to use the default language. + /// A struct that implements to parse rows + /// An if the sheet exists, null if it does not /// Thrown when is not decorated with a public ExcelSheet< T >? GetExcelSheet< T >( Language? language = null ) where T : struct, IExcelRow { From 27bb2e24d27b1af282c6c47d73849d67c132c8cd Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Mon, 12 Aug 2024 18:35:29 -0700 Subject: [PATCH 05/53] Allow direct indexing of rows and subrows --- src/Lumina/Excel/ExcelSheet.Collection.cs | 139 ++++++++++++++++++++ src/Lumina/Excel/ExcelSheet.cs | 149 ++++++++++------------ 2 files changed, 208 insertions(+), 80 deletions(-) create mode 100644 src/Lumina/Excel/ExcelSheet.Collection.cs diff --git a/src/Lumina/Excel/ExcelSheet.Collection.cs b/src/Lumina/Excel/ExcelSheet.Collection.cs new file mode 100644 index 00000000..2906ca03 --- /dev/null +++ b/src/Lumina/Excel/ExcelSheet.Collection.cs @@ -0,0 +1,139 @@ +using System; +using System.Collections; +using System.Collections.Generic; + +namespace Lumina.Excel; + +public sealed partial class ExcelSheet : IReadOnlyCollection where T : struct, IExcelRow +{ + /// + /// The number of rows in this sheet. + /// + /// + /// If this sheet has gaps in row ids, it returns the number of rows that exist, not the highest row id. + /// If this sheet has subrows, this will still return the number of rows and not the total number of subrows. + /// + public int Count => RowLookup.Count; + + private readonly int subrowCount; + + /// + /// The total number of subrows in this sheet across all rows. + /// + /// Thrown if the sheet does not support subrows + public int SubrowCount => HasSubrows ? subrowCount : throw new NotSupportedException( "This sheet that doesn't support subrows." ); + + /// + /// Returns an enumerator that can be used to iterate over all rows in this sheet. If this sheet has subrows, it will iterate over every subrow of every row. + /// + /// An of all rows or subrows in this sheet + public SheetEnumerator GetEnumerator() => + new( this ); + + /// + IEnumerator IEnumerable.GetEnumerator() => + GetEnumerator(); + + /// + /// Returns an enumerator that can be used to iterate over all rows in this sheet. If this sheet has subrows, it will iterate over the first subrow of every row. + /// + /// An of all rows (or first subrows) in this sheet + public RowEnumerator GetRowEnumerator() => + new( this ); + + IEnumerator IEnumerable.GetEnumerator() => + GetEnumerator(); + + public struct SheetEnumerator( ExcelSheet sheet ) : IEnumerator + { + private int LookupIndex { get; set; } = -1; + private ushort SubrowIndex { get; set; } = 0; + private ushort SubrowCount { get; set; } = 0; + + private readonly bool HasSubrows => sheet.HasSubrows; + private readonly int RowCount => sheet.Count; + + /// + public readonly T Current => !HasSubrows ? sheet.GetRowAt( LookupIndex ) : sheet.GetSubrowAt( LookupIndex, SubrowIndex ); + + /// + readonly object IEnumerator.Current => Current; + + public bool MoveNext() + { + if( !HasSubrows ) + { + if( LookupIndex + 1 < RowCount ) + { + LookupIndex++; + return true; + } + } + else + { + if( SubrowIndex + 1 < SubrowCount && SubrowCount != 0 ) + { + SubrowIndex++; + return true; + } + else if( LookupIndex + 1 < RowCount ) + { + LookupIndex++; + SubrowIndex++; + SubrowCount = sheet.Subrows![LookupIndex].Data.RowCount; + return true; + } + } + return false; + } + + public void Reset() + { + LookupIndex = -1; + SubrowIndex = 0; + SubrowCount = 0; + } + + public readonly void Dispose() + { + + } + } + + public struct RowEnumerator( ExcelSheet sheet ) : IEnumerator, IEnumerable + { + private int LookupIndex { get; set; } = -1; + + private readonly int RowCount => sheet.Count; + + /// + public readonly T Current => sheet.GetRowAt( LookupIndex ); + + /// + readonly object IEnumerator.Current => Current; + + public bool MoveNext() + { + if( LookupIndex + 1 < RowCount ) + { + LookupIndex++; + return true; + } + return false; + } + + public void Reset() + { + LookupIndex = -1; + } + + public readonly void Dispose() + { + + } + + public readonly IEnumerator GetEnumerator() => this; + + readonly IEnumerator IEnumerable.GetEnumerator() => this; + } +} diff --git a/src/Lumina/Excel/ExcelSheet.cs b/src/Lumina/Excel/ExcelSheet.cs index b58c223e..7a5b485b 100644 --- a/src/Lumina/Excel/ExcelSheet.cs +++ b/src/Lumina/Excel/ExcelSheet.cs @@ -1,18 +1,17 @@ +using Lumina.Data; +using Lumina.Data.Files.Excel; +using Lumina.Data.Structs.Excel; using System; -using System.Collections; using System.Collections.Frozen; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Reflection; using System.Runtime.CompilerServices; -using Lumina.Data; -using Lumina.Data.Files.Excel; -using Lumina.Data.Structs.Excel; namespace Lumina.Excel; -public sealed class ExcelSheet : IExcelSheet, IReadOnlyList where T : struct, IExcelRow +public sealed partial class ExcelSheet : IExcelSheet where T : struct, IExcelRow { /// public ExcelModule Module { get; } @@ -20,11 +19,6 @@ public sealed class ExcelSheet : IExcelSheet, IReadOnlyList where T : stru /// public Language Language { get; } - private List Pages { get; } - private FrozenDictionary? Rows { get; } - private FrozenDictionary? Subrows { get; } - private ushort SubrowDataOffset { get; } - /// /// Whether or not this sheet has subrows, where each row id can have multiple subrows. /// @@ -32,27 +26,23 @@ public sealed class ExcelSheet : IExcelSheet, IReadOnlyList where T : stru [MemberNotNullWhen( false, nameof( Rows ) )] public bool HasSubrows { get; } + private List Pages { get; } + private (uint RowId, (int PageIdx, uint Offset) Data)[]? Rows { get; } + private (uint RowId, (int PageIdx, uint Offset, ushort RowCount) Data)[]? Subrows { get; } + private FrozenDictionary RowLookup { get; } + private ushort SubrowDataOffset { get; } + private static SheetAttribute Attribute => typeof( T ).GetCustomAttribute() ?? throw new InvalidOperationException( "T has no SheetAttribute. Use the explicit sheet constructor." ); - /// - /// The number of rows in this sheet. - /// - /// - /// If this sheet has gaps in row ids, it returns the number of rows that exist, not the highest row id. - /// If this sheet has subrows, this will still return the number of rows and not the total number of subrows. - /// - public int Count => Rows?.Count ?? Subrows!.Count; - /// /// Get the th row in this sheet. If this sheet has subrows, it will return the first subrow. /// - /// This is an indexer helper, but you may want to treat this as a sparse list/matrix since only represents the total number of rows in the sheet, and not the highest row id. /// The row id of the row you want /// The row at /// Throws when the row id does not have a row attached to it. - public T this[int rowId] => GetRow( (uint)rowId ); + public T this[uint rowId] => GetRow( rowId ); /// /// Create an instance with the 's default language. @@ -99,8 +89,9 @@ public ExcelSheet( ExcelModule module, Language requestedLanguage, string sheetN Language = headerFile.Languages.Contains( requestedLanguage ) ? requestedLanguage : Language.None; - Dictionary? rows = null; - Dictionary? subrows = null; + List<(uint RowId, (int PageIdx, uint Offset) Data)>? rows = null; + List<(uint RowId, (int PageIdx, uint Offset, ushort RowCount) Data)>? subrows = null; + var totalSubrowCount = 0; if( HasSubrows ) { @@ -126,35 +117,42 @@ public ExcelSheet( ExcelModule module, Language requestedLanguage, string sheetN foreach( var rowPtr in fileData.RowData.Values ) { - var (rowDataSize, subrowCount) = (newPage.ReadUInt32( rowPtr.Offset ), newPage.ReadUInt16( rowPtr.Offset + 4 )); + var subrowCount = newPage.ReadUInt16( rowPtr.Offset + 4 ); var rowOffset = rowPtr.Offset + 6; if( HasSubrows ) { if( subrowCount > 0 ) - subrows!.Add( rowPtr.RowId, (pageIdx, rowOffset, subrowCount) ); + { + subrows!.Add( (rowPtr.RowId, (pageIdx, rowOffset, subrowCount)) ); + totalSubrowCount += subrowCount; + } } else - rows!.Add( rowPtr.RowId, (pageIdx, rowOffset) ); + rows!.Add( (rowPtr.RowId, (pageIdx, rowOffset)) ); } pageIdx++; } if( HasSubrows ) - Subrows = subrows!.ToFrozenDictionary(); + { + Subrows = [.. subrows!]; + int i = 0; + RowLookup = subrows!.ToFrozenDictionary( row => row.RowId, _ => i++ ); + subrowCount = totalSubrowCount; + } else - Rows = rows!.ToFrozenDictionary(); + { + Rows = [.. rows!]; + int i = 0; + RowLookup = rows!.ToFrozenDictionary( row => row.RowId, _ => i++ ); + } } /// - public bool HasRow( uint rowId ) - { - if( HasSubrows ) - return Subrows.ContainsKey( rowId ); - - return Rows.ContainsKey( rowId ); - } + public bool HasRow( uint rowId ) => + RowLookup.ContainsKey( rowId ); /// public bool HasSubrow( uint rowId, ushort subrowId ) @@ -162,11 +160,11 @@ public bool HasSubrow( uint rowId, ushort subrowId ) if( !HasSubrows ) throw new NotSupportedException( "Cannot access subrow in a sheet that doesn't support any." ); - ref readonly var val = ref Subrows.GetValueRefOrNullRef( rowId ); + ref readonly var val = ref RowLookup.GetValueRefOrNullRef( rowId ); if( Unsafe.IsNullRef( in val ) ) return false; - return subrowId < val.RowCount; + return subrowId < Subrows[val].Data.RowCount; } /// @@ -175,11 +173,11 @@ public bool HasSubrow( uint rowId, ushort subrowId ) if( !HasSubrows ) throw new NotSupportedException( "Cannot access subrow in a sheet that doesn't support any." ); - ref readonly var val = ref Subrows.GetValueRefOrNullRef( rowId ); + ref readonly var val = ref RowLookup.GetValueRefOrNullRef( rowId ); if( Unsafe.IsNullRef( in val ) ) return null; - return val.RowCount; + return Subrows[val].Data.RowCount; } /// @@ -188,14 +186,14 @@ public ushort GetSubrowCount( uint rowId ) if( !HasSubrows ) throw new NotSupportedException( "Cannot access subrow in a sheet that doesn't support any." ); - ref readonly var val = ref Subrows.GetValueRefOrNullRef( rowId ); + ref readonly var val = ref RowLookup.GetValueRefOrNullRef( rowId ); if( Unsafe.IsNullRef( in val ) ) throw new ArgumentOutOfRangeException( nameof( rowId ), "Row does not exist" ); - return val.RowCount; + return Subrows[val].Data.RowCount; } - private T CreateRow(uint rowId, in (int PageIdx, uint Offset) val) => + private T CreateRow( uint rowId, in (int PageIdx, uint Offset) val ) => T.Create( Pages[val.PageIdx], val.Offset, rowId ); private T CreateSubrow( uint rowId, ushort subrowId, in (int PageIdx, uint Offset, ushort RowCount) val ) => @@ -211,11 +209,11 @@ private T CreateSubrow( uint rowId, ushort subrowId, in (int PageIdx, uint Offse if( HasSubrows ) return TryGetSubrow( rowId, 0 ); - ref readonly var val = ref Rows.GetValueRefOrNullRef( rowId ); + ref readonly var val = ref RowLookup.GetValueRefOrNullRef( rowId ); if( Unsafe.IsNullRef( in val ) ) return null; - return CreateRow( rowId, in val ); + return CreateRow( rowId, in Rows[val].Data ); } @@ -231,18 +229,18 @@ private T CreateSubrow( uint rowId, ushort subrowId, in (int PageIdx, uint Offse if( !HasSubrows ) throw new NotSupportedException( "Cannot access subrow in a sheet that doesn't support any." ); - ref readonly var val = ref Subrows.GetValueRefOrNullRef( rowId ); + ref readonly var val = ref RowLookup.GetValueRefOrNullRef( rowId ); if( Unsafe.IsNullRef( in val ) ) return null; - if( subrowId >= val.RowCount ) + if( subrowId >= Subrows[val].Data.RowCount ) return null; - return CreateSubrow( rowId, subrowId, in val ); + return CreateSubrow( rowId, subrowId, in Subrows[val].Data ); } /// - /// Gets the th row in this sheet. If this sheet has subrows, it will return the first subrow. Throws if the row does not exist. + /// Gets the row with id in this sheet. If this sheet has subrows, it will return the first subrow with id . Throws if the row does not exist. /// /// The row id to get /// A row object. @@ -252,7 +250,7 @@ public T GetRow( uint rowId ) => throw new ArgumentOutOfRangeException( nameof( rowId ), "Row does not exist" ); /// - /// Gets the th subrow from the th row in this sheet. Throws if the subrow does not exist. + /// Gets the th subrow with row id in this sheet. Throws if the subrow does not exist. /// /// The row id to get /// The subrow id to get @@ -264,53 +262,44 @@ public T GetSubrow( uint rowId, ushort subrowId ) if( !HasSubrows ) throw new NotSupportedException( "Cannot access subrow in a sheet that doesn't support any." ); - ref readonly var val = ref Subrows.GetValueRefOrNullRef( rowId ); + ref readonly var val = ref RowLookup.GetValueRefOrNullRef( rowId ); if( Unsafe.IsNullRef( in val ) ) throw new ArgumentOutOfRangeException( nameof( rowId ), "Row does not exist" ); - ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual( subrowId, val.RowCount ); + ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual( subrowId, Subrows[val].Data.RowCount ); - return CreateSubrow( rowId, subrowId, in val ); + return CreateSubrow( rowId, subrowId, in Subrows[val].Data ); } /// - /// Returns an enumerator that can be used to iterate over all subrows in all rows in this sheet. + /// Gets the th row in this sheet, ordered by row id in ascending order. If this sheet has subrows, it will return the first subrow. /// - /// An of all subrows in this sheet - /// Thrown if the sheet does not support subrows - public IEnumerator GetSubrowEnumerator() + /// If you are looking to find a row by its id, use instead. + /// The zero-based index of this row + /// A row object. + public T GetRowAt( int rowIndex ) { - if( !HasSubrows ) - throw new NotSupportedException( "Cannot enumerate subrows in a sheet that doesn't support any." ); + if( HasSubrows ) + return GetSubrowAt( rowIndex, 0 ); - foreach( var rowData in Subrows ) - { - for( ushort i = 0; i < rowData.Value.RowCount; ++i ) - yield return CreateSubrow( rowData.Key, i, rowData.Value ); - } + var data = Rows[rowIndex]; + return CreateRow( data.RowId, in data.Data ); } /// - /// Returns an enumerator that can be used to iterate over all rows in this sheet. If this sheet has subrows, it will iterate over the first subrow of every row. + /// Gets the th subrow of the th row in this sheet, ordered by row id in ascending order. /// - /// An of all rows (or first subrows) in this sheet - public IEnumerator GetEnumerator() + /// If you are looking to find a subrow by its id, use instead. + /// The zero-based index of this row + /// The subrow id to get + /// A row object. + /// Thrown if the sheet does not support subrows + public T GetSubrowAt( int rowIndex, ushort subrowId ) { if( !HasSubrows ) - { - foreach( var rowData in Rows ) - yield return CreateRow( rowData.Key, rowData.Value ); - } - else - { - foreach( var rowData in Subrows ) - yield return CreateSubrow( rowData.Key, 0, rowData.Value ); - } - } + throw new NotSupportedException( "Cannot access subrow in a sheet that doesn't support any." ); - /// - IEnumerator IEnumerable.GetEnumerator() - { - return GetEnumerator(); + var data = Subrows[rowIndex]; + return CreateSubrow( data.RowId, subrowId, in data.Data ); } } From 0fe084e08225e1dc64d303700f9d2aef70d1b1d9 Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Tue, 13 Aug 2024 21:43:43 -0700 Subject: [PATCH 06/53] Fix iteration bug --- src/Lumina/Excel/ExcelSheet.Collection.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Lumina/Excel/ExcelSheet.Collection.cs b/src/Lumina/Excel/ExcelSheet.Collection.cs index 2906ca03..f5d0b654 100644 --- a/src/Lumina/Excel/ExcelSheet.Collection.cs +++ b/src/Lumina/Excel/ExcelSheet.Collection.cs @@ -79,7 +79,7 @@ public bool MoveNext() else if( LookupIndex + 1 < RowCount ) { LookupIndex++; - SubrowIndex++; + SubrowIndex = 0; SubrowCount = sheet.Subrows![LookupIndex].Data.RowCount; return true; } From 96dceebb57a71f66dedfc21c2c9b274205a78bf2 Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Tue, 13 Aug 2024 21:44:21 -0700 Subject: [PATCH 07/53] docs changes, add more properties to IExcelSheet --- src/Lumina/Excel/ExcelSheet.Collection.cs | 13 ++----------- src/Lumina/Excel/ExcelSheet.cs | 17 +++++------------ src/Lumina/Excel/IExcelSheet.cs | 20 ++++++++++++++++++++ 3 files changed, 27 insertions(+), 23 deletions(-) diff --git a/src/Lumina/Excel/ExcelSheet.Collection.cs b/src/Lumina/Excel/ExcelSheet.Collection.cs index f5d0b654..ead279c1 100644 --- a/src/Lumina/Excel/ExcelSheet.Collection.cs +++ b/src/Lumina/Excel/ExcelSheet.Collection.cs @@ -6,21 +6,12 @@ namespace Lumina.Excel; public sealed partial class ExcelSheet : IReadOnlyCollection where T : struct, IExcelRow { - /// - /// The number of rows in this sheet. - /// - /// - /// If this sheet has gaps in row ids, it returns the number of rows that exist, not the highest row id. - /// If this sheet has subrows, this will still return the number of rows and not the total number of subrows. - /// + /// public int Count => RowLookup.Count; private readonly int subrowCount; - /// - /// The total number of subrows in this sheet across all rows. - /// - /// Thrown if the sheet does not support subrows + /// public int SubrowCount => HasSubrows ? subrowCount : throw new NotSupportedException( "This sheet that doesn't support subrows." ); /// diff --git a/src/Lumina/Excel/ExcelSheet.cs b/src/Lumina/Excel/ExcelSheet.cs index 7a5b485b..c16070df 100644 --- a/src/Lumina/Excel/ExcelSheet.cs +++ b/src/Lumina/Excel/ExcelSheet.cs @@ -19,9 +19,7 @@ public sealed partial class ExcelSheet : IExcelSheet where T : struct, IExcel /// public Language Language { get; } - /// - /// Whether or not this sheet has subrows, where each row id can have multiple subrows. - /// + /// [MemberNotNullWhen( true, nameof( Subrows ), nameof( SubrowDataOffset ) )] [MemberNotNullWhen( false, nameof( Rows ) )] public bool HasSubrows { get; } @@ -36,12 +34,7 @@ public sealed partial class ExcelSheet : IExcelSheet where T : struct, IExcel typeof( T ).GetCustomAttribute() ?? throw new InvalidOperationException( "T has no SheetAttribute. Use the explicit sheet constructor." ); - /// - /// Get the th row in this sheet. If this sheet has subrows, it will return the first subrow. - /// - /// The row id of the row you want - /// The row at - /// Throws when the row id does not have a row attached to it. + /// public T this[uint rowId] => GetRow( rowId ); /// @@ -218,7 +211,7 @@ private T CreateSubrow( uint rowId, ushort subrowId, in (int PageIdx, uint Offse /// - /// Tries to get the th subrow from the th row in this sheet. + /// Tries to get the th subrow with row id in this sheet. /// /// The row id to get /// The subrow id to get @@ -240,11 +233,11 @@ private T CreateSubrow( uint rowId, ushort subrowId, in (int PageIdx, uint Offse } /// - /// Gets the row with id in this sheet. If this sheet has subrows, it will return the first subrow with id . Throws if the row does not exist. + /// Gets the th row in this sheet. If this sheet has subrows, it will return the first subrow. /// /// The row id to get /// A row object. - /// Thrown if the sheet does not have a row at that + /// Throws when the row id does not have a row attached to it. public T GetRow( uint rowId ) => TryGetRow( rowId ) ?? throw new ArgumentOutOfRangeException( nameof( rowId ), "Row does not exist" ); diff --git a/src/Lumina/Excel/IExcelSheet.cs b/src/Lumina/Excel/IExcelSheet.cs index 96d8a17b..01b5c856 100644 --- a/src/Lumina/Excel/IExcelSheet.cs +++ b/src/Lumina/Excel/IExcelSheet.cs @@ -20,6 +20,26 @@ public interface IExcelSheet : IEnumerable /// This can be different from the requested language if it wasn't supported. Language Language { get; } + /// + /// Whether or not this sheet has subrows, where each row id can have multiple subrows. + /// + bool HasSubrows { get; } + + /// + /// The number of rows in this sheet. + /// + /// + /// If this sheet has gaps in row ids, it returns the number of rows that exist, not the highest row id. + /// If this sheet has subrows, this will still return the number of rows and not the total number of subrows. + /// + int Count { get; } + + /// + /// The total number of subrows in this sheet across all rows. + /// + /// Thrown if the sheet does not support subrows + int SubrowCount { get; } + /// /// Whether or not this sheet has a row with the given . /// From 604b7c98f45b162f50dda71e5de8ffef4d3f8cb7 Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Tue, 13 Aug 2024 21:44:38 -0700 Subject: [PATCH 08/53] Add subrow indexer to ExcelSheet --- src/Lumina/Excel/ExcelSheet.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Lumina/Excel/ExcelSheet.cs b/src/Lumina/Excel/ExcelSheet.cs index c16070df..436d447c 100644 --- a/src/Lumina/Excel/ExcelSheet.cs +++ b/src/Lumina/Excel/ExcelSheet.cs @@ -37,6 +37,9 @@ public sealed partial class ExcelSheet : IExcelSheet where T : struct, IExcel /// public T this[uint rowId] => GetRow( rowId ); + /// + public T this[uint rowId, ushort subrowId] => GetSubrow( rowId, subrowId ); + /// /// Create an instance with the 's default language. /// From f688b05b7af79845dc9d822c0fbd1d76842a48f2 Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Tue, 13 Aug 2024 22:35:09 -0700 Subject: [PATCH 09/53] Rewrite RSV resolution --- src/Lumina/Excel/RSV/DictionaryRsvProvider.cs | 24 ++++ src/Lumina/Excel/RSV/IRsvProvider.cs | 36 ++++++ src/Lumina/Excel/RSV/RsvKeyData.cs | 19 --- src/Lumina/Excel/RSV/RsvProvider.cs | 115 ------------------ src/Lumina/Excel/RSV/RsvUtil.cs | 112 +++++------------ 5 files changed, 90 insertions(+), 216 deletions(-) create mode 100644 src/Lumina/Excel/RSV/DictionaryRsvProvider.cs create mode 100644 src/Lumina/Excel/RSV/IRsvProvider.cs delete mode 100644 src/Lumina/Excel/RSV/RsvKeyData.cs delete mode 100644 src/Lumina/Excel/RSV/RsvProvider.cs diff --git a/src/Lumina/Excel/RSV/DictionaryRsvProvider.cs b/src/Lumina/Excel/RSV/DictionaryRsvProvider.cs new file mode 100644 index 00000000..51a53699 --- /dev/null +++ b/src/Lumina/Excel/RSV/DictionaryRsvProvider.cs @@ -0,0 +1,24 @@ +using Lumina.Text.ReadOnly; +using System; +using System.Collections.Generic; + +namespace Lumina.Excel.Rsv; + +public class DictionaryRsvProvider : Dictionary, IRsvProvider +{ + /// + public bool CanResolve( ReadOnlySeString rsvString ) => + rsvString.IsRsv() && ContainsKey( rsvString ); + + /// + public ReadOnlySeString Resolve( ReadOnlySeString rsvString ) => + rsvString.IsRsv() ? this[rsvString] : throw new ArgumentException( "rsvString is not a valid RSV string", nameof( rsvString ) ); + + /// + public ReadOnlySeString ResolveOrSelf( ReadOnlySeString rsvString ) => + ( rsvString.IsRsv() && TryGetValue( rsvString, out var replacedString ) ) ? replacedString : rsvString; + + /// + public ReadOnlySeString? TryResolve( ReadOnlySeString rsvString ) => + ( rsvString.IsRsv() && TryGetValue( rsvString, out var replacedString ) ) ? replacedString : default; +} diff --git a/src/Lumina/Excel/RSV/IRsvProvider.cs b/src/Lumina/Excel/RSV/IRsvProvider.cs new file mode 100644 index 00000000..f834c864 --- /dev/null +++ b/src/Lumina/Excel/RSV/IRsvProvider.cs @@ -0,0 +1,36 @@ +using Lumina.Text.ReadOnly; + +namespace Lumina.Excel.Rsv; + +public interface IRsvProvider +{ + /// + /// Determines if the provider can resolve the given RSV string. + /// + /// The string to check + /// Whether or not the provider contains a resolved string for + bool CanResolve( ReadOnlySeString rsvString ); + + /// + /// Tries to resolve the given RSV string. + /// + /// The string to resolve + /// The newly resolved string. Returns if it could not be resolved. + ReadOnlySeString? TryResolve( ReadOnlySeString rsvString ); + + /// + /// Resolves the given RSV string. + /// + /// The string to resolve + /// The newly resolved string + /// Thrown if the RSV string is not valid. must return true. + /// Thrown if the RSV key is not found in the provider + ReadOnlySeString Resolve( ReadOnlySeString rsvString ); + + /// + /// Tries to resolve the given RSV string. + /// + /// The string to resolve + /// The newly resolved string. Returns if it could not be resolved. + ReadOnlySeString ResolveOrSelf( ReadOnlySeString rsvString ); +} diff --git a/src/Lumina/Excel/RSV/RsvKeyData.cs b/src/Lumina/Excel/RSV/RsvKeyData.cs deleted file mode 100644 index 79d99c51..00000000 --- a/src/Lumina/Excel/RSV/RsvKeyData.cs +++ /dev/null @@ -1,19 +0,0 @@ -using Lumina.Data; - -namespace Lumina.Excel.RSV -{ - public record RsvKeyData - { - // todo: can't use init only because netstandard2.0 is shit - - public uint RowId { get; set; } - - public int SubRowId { get; set; } - - public uint ColumnIndex { get; set; } - - public Language Language { get; set; } - - public string SheetName { get; set; } - } -} \ No newline at end of file diff --git a/src/Lumina/Excel/RSV/RsvProvider.cs b/src/Lumina/Excel/RSV/RsvProvider.cs deleted file mode 100644 index 0e863880..00000000 --- a/src/Lumina/Excel/RSV/RsvProvider.cs +++ /dev/null @@ -1,115 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; - -namespace Lumina.Excel.RSV -{ - using RSVPair = KeyValuePair< string, string >; - - /// - /// Stores a mapping so that Lumina can remap _rsv_* values to their actual values when consuming sheet data. - /// - public class RsvProvider - { - /// - /// Construct an empty . Will need to be seeded with data before this does anything. - /// - public RsvProvider() - { - _rsvEntries = new(); - } - - private readonly Dictionary< string, string > _rsvEntries; - - /// - /// Returns true if there's any RSV values present, otherwise false. - /// - public bool HasValues => _rsvEntries.Any(); - - /// - /// Add a RSV mapping to the collection - /// - /// The RSV key and value - [Obsolete( "use RsvProvider.Add instead" )] - public void Seed( RSVPair rsvEntry ) - { - _rsvEntries[ rsvEntry.Key ] = rsvEntry.Value; - } - - /// - /// Add a RSV mapping to the collection - /// - /// The RSV key and value - public void Add( RSVPair rsvEntry ) - { - _rsvEntries[ rsvEntry.Key ] = rsvEntry.Value; - } - - /// - /// Add a RSV mapping to the collection - /// - /// The RSV key - /// The RSV value (the original string) - public void Add( string key, string value ) - { - _rsvEntries[ key ] = value; - } - - /// - /// Add RSV mappings from a collection of comma separated values [key],[value] - /// - /// Each line from the RSV file - public void ParseLines( IEnumerable< string > lines ) - { - var delim = new[] { ',' }; - - foreach( var entry in lines ) - { - // ignore anything that isn't assumedly an actual rsv key value pair - if( !entry.StartsWith( "_rsv_" ) ) - { - continue; - } - - var data = entry.Split( delim, 2 ); - var rsvKey = RsvUtil.ParseRsvKey( data[ 0 ] ); - if( rsvKey == null ) - { - // todo: log error about invalid key - continue; - } - - Seed( new RSVPair( data[ 0 ], data[ 1 ] ) ); - } - } - - /// - /// Add RSV mappings from a string to the collection - /// - /// - /// These must be delimited by a unix newline (\n) and not a windows newline (\r\n). - /// - /// The data to parse. - public void ParseData( string data ) - { - var lines = data.Split( '\n' ); - - ParseLines( lines ); - } - - /// - /// Attempt to find a value for it's rsv key - /// - /// The key name (from the sheet itself) - /// The resolved value otherwise null if it wasn't found - public string? GetValue( string key ) - { - if( !_rsvEntries.TryGetValue( key, out var value ) ) - { - return null; - } - - return value; - } - } -} \ No newline at end of file diff --git a/src/Lumina/Excel/RSV/RsvUtil.cs b/src/Lumina/Excel/RSV/RsvUtil.cs index 82473a57..00a8a0d3 100644 --- a/src/Lumina/Excel/RSV/RsvUtil.cs +++ b/src/Lumina/Excel/RSV/RsvUtil.cs @@ -1,85 +1,33 @@ +using Lumina.Text.ReadOnly; using System; -using System.Text.RegularExpressions; -using Lumina.Data; -namespace Lumina.Excel.RSV -{ - public class RsvUtil - { - /// - /// Build a key name for a RSV value - /// - /// The sheet row ID - /// The sheet subrow ID or -1 if not subrow variant - /// The column index - /// The language - /// The name of the sheet - /// The RSV key - public static string BuildRsvKeyName( uint rowId, int subRowId, uint columnIdx, Language language, string sheetName ) - { - return $"_rsv_{rowId}_{subRowId}_{(int)language}_C{columnIdx}_0{sheetName}"; - } - - /// - /// Build a key name for a RSV value - /// - /// The sheet row ID - /// The column index - /// The language - /// The name of the sheet - /// The RSV key - public static string BuildRsvKeyName( uint rowId, uint columnIdx, Language language, string sheetName ) - { - return BuildRsvKeyName( rowId, -1, columnIdx, language, sheetName ); - } - - private static readonly Regex RsvKeyRegex = new( - @"_rsv_(\d+)_(-?\d+)_(\d)_C(\d+)_0([\w\d]+)", - RegexOptions.Compiled | RegexOptions.IgnoreCase - ); - - /// - /// Parse an RSV key to extract its information such as row, sheet, etc. - /// - /// The RSV key - /// A object containing its info or null if it failed to parse - public static RsvKeyData? ParseRsvKey( string key ) - { - var results = RsvKeyRegex.Match( key ); - if( results.Groups.Count != 6 ) - { - return null; - } - - if( !uint.TryParse( results.Groups[ 1 ].Value, out var rowId ) ) - { - return null; - } +namespace Lumina.Excel.Rsv; - if( !int.TryParse( results.Groups[ 2 ].Value, out var subRowId ) ) - { - return null; - } - - if( !Enum.TryParse< Language >( results.Groups[ 3 ].Value, out var language ) ) - { - return null; - } - - if( !uint.TryParse( results.Groups[ 4 ].Value, out var columnIndex ) ) - { - return null; - } - - - return new RsvKeyData - { - RowId = rowId, - SubRowId = subRowId, - ColumnIndex = columnIndex, - Language = language, - SheetName = results.Groups[ 5 ].Value - }; - } - } -} \ No newline at end of file +public static class RsvUtil +{ + // RsvPrefix => _rsv_ + private static ReadOnlySpan RsvPrefix => [0x5F, 0x72, 0x73, 0x76, 0x5F]; + + /// + /// Checks if the string is an RSV string and can therefore be resolved. + /// + /// This only checks if the string begins with "_rsv_". + /// The string to check + /// Whether or not the string is an RSV string. + public static bool IsRsv( this ReadOnlySeString rsvString ) => + rsvString.Data.Span.StartsWith( RsvPrefix ); + + /// + public static bool IsRsv( this ReadOnlySeStringSpan rsvString ) => + rsvString.Data.StartsWith( RsvPrefix ); + + /// + /// Attempts to resolve with the given . + /// + /// This is safe to call on strings that are not RSVs, a.k.a. where returns . + /// The string to resolve + /// The provider to check with + /// The newly resolved string. Returns if it could not be resolved. + public static ReadOnlySeString ResolveRsv( this ReadOnlySeString rsvString, IRsvProvider provider ) => + provider.ResolveOrSelf( rsvString ); +} From 6fb77259043e9339b22e22e53ead2140c16efaaf Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Tue, 13 Aug 2024 23:17:09 -0700 Subject: [PATCH 10/53] RowLookup performance change --- src/Lumina/Excel/ExcelSheet.cs | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/Lumina/Excel/ExcelSheet.cs b/src/Lumina/Excel/ExcelSheet.cs index 436d447c..29615f7e 100644 --- a/src/Lumina/Excel/ExcelSheet.cs +++ b/src/Lumina/Excel/ExcelSheet.cs @@ -27,7 +27,9 @@ public sealed partial class ExcelSheet : IExcelSheet where T : struct, IExcel private List Pages { get; } private (uint RowId, (int PageIdx, uint Offset) Data)[]? Rows { get; } private (uint RowId, (int PageIdx, uint Offset, ushort RowCount) Data)[]? Subrows { get; } - private FrozenDictionary RowLookup { get; } + // RowLookup must use int as the key because it benefits from a fast path that removes indirections. + // https://github.com/dotnet/runtime/blob/release/8.0/src/libraries/System.Collections.Immutable/src/System/Collections/Frozen/FrozenDictionary.cs#L140 + private FrozenDictionary RowLookup { get; } private ushort SubrowDataOffset { get; } private static SheetAttribute Attribute => @@ -135,20 +137,20 @@ public ExcelSheet( ExcelModule module, Language requestedLanguage, string sheetN { Subrows = [.. subrows!]; int i = 0; - RowLookup = subrows!.ToFrozenDictionary( row => row.RowId, _ => i++ ); + RowLookup = subrows!.ToFrozenDictionary( row => (int)row.RowId, _ => i++ ); subrowCount = totalSubrowCount; } else { Rows = [.. rows!]; int i = 0; - RowLookup = rows!.ToFrozenDictionary( row => row.RowId, _ => i++ ); + RowLookup = rows!.ToFrozenDictionary( row => (int)row.RowId, _ => i++ ); } } /// public bool HasRow( uint rowId ) => - RowLookup.ContainsKey( rowId ); + RowLookup.ContainsKey( (int)rowId ); /// public bool HasSubrow( uint rowId, ushort subrowId ) @@ -156,7 +158,7 @@ public bool HasSubrow( uint rowId, ushort subrowId ) if( !HasSubrows ) throw new NotSupportedException( "Cannot access subrow in a sheet that doesn't support any." ); - ref readonly var val = ref RowLookup.GetValueRefOrNullRef( rowId ); + ref readonly var val = ref RowLookup.GetValueRefOrNullRef( (int)rowId ); if( Unsafe.IsNullRef( in val ) ) return false; @@ -169,7 +171,7 @@ public bool HasSubrow( uint rowId, ushort subrowId ) if( !HasSubrows ) throw new NotSupportedException( "Cannot access subrow in a sheet that doesn't support any." ); - ref readonly var val = ref RowLookup.GetValueRefOrNullRef( rowId ); + ref readonly var val = ref RowLookup.GetValueRefOrNullRef( (int)rowId ); if( Unsafe.IsNullRef( in val ) ) return null; @@ -182,7 +184,7 @@ public ushort GetSubrowCount( uint rowId ) if( !HasSubrows ) throw new NotSupportedException( "Cannot access subrow in a sheet that doesn't support any." ); - ref readonly var val = ref RowLookup.GetValueRefOrNullRef( rowId ); + ref readonly var val = ref RowLookup.GetValueRefOrNullRef( (int)rowId ); if( Unsafe.IsNullRef( in val ) ) throw new ArgumentOutOfRangeException( nameof( rowId ), "Row does not exist" ); @@ -205,7 +207,7 @@ private T CreateSubrow( uint rowId, ushort subrowId, in (int PageIdx, uint Offse if( HasSubrows ) return TryGetSubrow( rowId, 0 ); - ref readonly var val = ref RowLookup.GetValueRefOrNullRef( rowId ); + ref readonly var val = ref RowLookup.GetValueRefOrNullRef( (int)rowId ); if( Unsafe.IsNullRef( in val ) ) return null; @@ -225,7 +227,7 @@ private T CreateSubrow( uint rowId, ushort subrowId, in (int PageIdx, uint Offse if( !HasSubrows ) throw new NotSupportedException( "Cannot access subrow in a sheet that doesn't support any." ); - ref readonly var val = ref RowLookup.GetValueRefOrNullRef( rowId ); + ref readonly var val = ref RowLookup.GetValueRefOrNullRef( (int)rowId ); if( Unsafe.IsNullRef( in val ) ) return null; @@ -258,7 +260,7 @@ public T GetSubrow( uint rowId, ushort subrowId ) if( !HasSubrows ) throw new NotSupportedException( "Cannot access subrow in a sheet that doesn't support any." ); - ref readonly var val = ref RowLookup.GetValueRefOrNullRef( rowId ); + ref readonly var val = ref RowLookup.GetValueRefOrNullRef( (int)rowId ); if( Unsafe.IsNullRef( in val ) ) throw new ArgumentOutOfRangeException( nameof( rowId ), "Row does not exist" ); From db7e8b278128271324b69ae9192fefaf3a695ea1 Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Tue, 13 Aug 2024 23:28:58 -0700 Subject: [PATCH 11/53] Rename rsv folder --- src/Lumina/Excel/{RSV => Rsv}/DictionaryRsvProvider.cs | 0 src/Lumina/Excel/{RSV => Rsv}/IRsvProvider.cs | 0 src/Lumina/Excel/{RSV => Rsv}/RsvUtil.cs | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename src/Lumina/Excel/{RSV => Rsv}/DictionaryRsvProvider.cs (100%) rename src/Lumina/Excel/{RSV => Rsv}/IRsvProvider.cs (100%) rename src/Lumina/Excel/{RSV => Rsv}/RsvUtil.cs (100%) diff --git a/src/Lumina/Excel/RSV/DictionaryRsvProvider.cs b/src/Lumina/Excel/Rsv/DictionaryRsvProvider.cs similarity index 100% rename from src/Lumina/Excel/RSV/DictionaryRsvProvider.cs rename to src/Lumina/Excel/Rsv/DictionaryRsvProvider.cs diff --git a/src/Lumina/Excel/RSV/IRsvProvider.cs b/src/Lumina/Excel/Rsv/IRsvProvider.cs similarity index 100% rename from src/Lumina/Excel/RSV/IRsvProvider.cs rename to src/Lumina/Excel/Rsv/IRsvProvider.cs diff --git a/src/Lumina/Excel/RSV/RsvUtil.cs b/src/Lumina/Excel/Rsv/RsvUtil.cs similarity index 100% rename from src/Lumina/Excel/RSV/RsvUtil.cs rename to src/Lumina/Excel/Rsv/RsvUtil.cs From 1acb0d605162909d8333b33d8510f25e70580c6b Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Wed, 14 Aug 2024 00:35:05 -0700 Subject: [PATCH 12/53] Revert removal of docs file --- src/Lumina/Lumina.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Lumina/Lumina.csproj b/src/Lumina/Lumina.csproj index 41f2916b..7a4ed33f 100644 --- a/src/Lumina/Lumina.csproj +++ b/src/Lumina/Lumina.csproj @@ -13,7 +13,7 @@ Lumina is a small, performant and simple library for interacting with FINAL FANTASY XIV game data. true enable - + true From 339aea8696ef85d9dcb910498ad463c611dba07c Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Wed, 14 Aug 2024 02:26:34 -0700 Subject: [PATCH 13/53] Change API to be more C# like --- src/Lumina/Excel/ExcelSheet.Collection.cs | 2 +- src/Lumina/Excel/ExcelSheet.cs | 55 ++++++++++++++++++++--- src/Lumina/Excel/IExcelSheet.cs | 5 ++- src/Lumina/Excel/RowRef.cs | 23 ++++++++-- 4 files changed, 72 insertions(+), 13 deletions(-) diff --git a/src/Lumina/Excel/ExcelSheet.Collection.cs b/src/Lumina/Excel/ExcelSheet.Collection.cs index ead279c1..a55aa131 100644 --- a/src/Lumina/Excel/ExcelSheet.Collection.cs +++ b/src/Lumina/Excel/ExcelSheet.Collection.cs @@ -6,7 +6,7 @@ namespace Lumina.Excel; public sealed partial class ExcelSheet : IReadOnlyCollection where T : struct, IExcelRow { - /// + /// public int Count => RowLookup.Count; private readonly int subrowCount; diff --git a/src/Lumina/Excel/ExcelSheet.cs b/src/Lumina/Excel/ExcelSheet.cs index 29615f7e..c79dd692 100644 --- a/src/Lumina/Excel/ExcelSheet.cs +++ b/src/Lumina/Excel/ExcelSheet.cs @@ -166,16 +166,20 @@ public bool HasSubrow( uint rowId, ushort subrowId ) } /// - public ushort? TryGetSubrowCount( uint rowId ) + public bool TryGetSubrowCount( uint rowId, out ushort subrowCount ) { if( !HasSubrows ) throw new NotSupportedException( "Cannot access subrow in a sheet that doesn't support any." ); ref readonly var val = ref RowLookup.GetValueRefOrNullRef( (int)rowId ); if( Unsafe.IsNullRef( in val ) ) - return null; + { + subrowCount = 0; + return false; + } - return Subrows[val].Data.RowCount; + subrowCount = Subrows[val].Data.RowCount; + return true; } /// @@ -202,10 +206,10 @@ private T CreateSubrow( uint rowId, ushort subrowId, in (int PageIdx, uint Offse /// /// The row id to get /// A nullable row object. Returns null if the row does not exist. - public T? TryGetRow( uint rowId ) + public T? GetRowOrDefault( uint rowId ) { if( HasSubrows ) - return TryGetSubrow( rowId, 0 ); + return GetSubrowOrDefault( rowId, 0 ); ref readonly var val = ref RowLookup.GetValueRefOrNullRef( (int)rowId ); if( Unsafe.IsNullRef( in val ) ) @@ -222,7 +226,7 @@ private T CreateSubrow( uint rowId, ushort subrowId, in (int PageIdx, uint Offse /// The subrow id to get /// A nullable row object. Returns null if the subrow does not exist. /// Thrown if the sheet does not support subrows - public T? TryGetSubrow( uint rowId, ushort subrowId ) + public T? GetSubrowOrDefault( uint rowId, ushort subrowId ) { if( !HasSubrows ) throw new NotSupportedException( "Cannot access subrow in a sheet that doesn't support any." ); @@ -237,6 +241,43 @@ private T CreateSubrow( uint rowId, ushort subrowId, in (int PageIdx, uint Offse return CreateSubrow( rowId, subrowId, in Subrows[val].Data ); } + /// + /// Tries to get the th row in this sheet. If this sheet has subrows, it will return the first subrow. + /// + /// The row id to get + /// The output row object. + /// if the row exists and is written to and otherwise. + public bool TryGetRow( uint rowId, out T row ) + { + if( GetRowOrDefault( rowId ) is { } outRow ) + { + row = outRow; + return true; + } + row = default; + return false; + } + + + /// + /// Tries to get the th subrow with row id in this sheet. + /// + /// The row id to get + /// The subrow id to get + /// The output row object. + /// if the row exists and is written to and otherwise. + /// Thrown if the sheet does not support subrows + public bool TryGetSubrow( uint rowId, ushort subrowId, out T subrow ) + { + if (GetSubrowOrDefault(rowId, subrowId) is { } outSubrow) + { + subrow = outSubrow; + return true; + } + subrow = default; + return false; + } + /// /// Gets the th row in this sheet. If this sheet has subrows, it will return the first subrow. /// @@ -244,7 +285,7 @@ private T CreateSubrow( uint rowId, ushort subrowId, in (int PageIdx, uint Offse /// A row object. /// Throws when the row id does not have a row attached to it. public T GetRow( uint rowId ) => - TryGetRow( rowId ) ?? + GetRowOrDefault( rowId ) ?? throw new ArgumentOutOfRangeException( nameof( rowId ), "Row does not exist" ); /// diff --git a/src/Lumina/Excel/IExcelSheet.cs b/src/Lumina/Excel/IExcelSheet.cs index 01b5c856..489c7e64 100644 --- a/src/Lumina/Excel/IExcelSheet.cs +++ b/src/Lumina/Excel/IExcelSheet.cs @@ -63,9 +63,10 @@ public interface IExcelSheet : IEnumerable /// Tries to get the number of subrows in the th row in this sheet. /// /// The row id to get - /// The number of subrows in this row. Returns null if the row does not exist. + /// The number of subrows in this row. + /// if the row exists and is written to and otherwise. /// Thrown if the sheet does not support subrows - ushort? TryGetSubrowCount( uint rowId ); + bool TryGetSubrowCount( uint rowId, out ushort subrowCount ); /// /// Gets the number of subrows in the th row in this sheet. Throws if the row does not exist. diff --git a/src/Lumina/Excel/RowRef.cs b/src/Lumina/Excel/RowRef.cs index 66849b9e..71ff5f0e 100644 --- a/src/Lumina/Excel/RowRef.cs +++ b/src/Lumina/Excel/RowRef.cs @@ -36,12 +36,29 @@ public bool Is() where T : struct, IExcelRow => /// /// The row type/schema to check against /// The referenced row type. Returns null if this does not point to a or if the does not exist in its sheet. - public T? TryGetValue() where T : struct, IExcelRow + public T? GetValueOrDefault() where T : struct, IExcelRow { if( !Is() ) return null; - return module!.GetSheet().TryGetRow( RowId ); + return module!.GetSheet().GetRowOrDefault( RowId ); + } + + /// + /// Tries to get the referenced row as a specific row type. + /// + /// The row type/schema to check against + /// The output row object + /// if the type is valid, the row exists, and is written to, and otherwise. + public bool TryGetValue(out T row) where T : struct, IExcelRow + { + if( !Is() ) + { + row = default; + return false; + } + + return module!.GetSheet().TryGetRow( RowId, out row ); } /// @@ -111,7 +128,7 @@ public readonly struct RowRef( ExcelModule module, uint rowId ) where T : str /// /// Attempts to get the referenced row value. Is null if does not exist in the sheet. /// - public T? ValueNullable => sheet.TryGetRow( RowId ); + public T? ValueNullable => sheet.GetRowOrDefault( RowId ); private RowRef ToGeneric() => RowRef.Create( module, rowId ); From 271cfde921694620dfade22e1bbab43746e3750f Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Wed, 14 Aug 2024 14:15:14 -0700 Subject: [PATCH 14/53] Add ExcelPage documentation --- src/Lumina/Excel/ExcelPage.cs | 65 +++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/src/Lumina/Excel/ExcelPage.cs b/src/Lumina/Excel/ExcelPage.cs index 4dc2cb97..1df20de1 100644 --- a/src/Lumina/Excel/ExcelPage.cs +++ b/src/Lumina/Excel/ExcelPage.cs @@ -16,6 +16,9 @@ namespace Lumina.Excel; [EditorBrowsable( EditorBrowsableState.Advanced )] public sealed class ExcelPage { + /// + /// The module that this page belongs to. + /// public ExcelModule Module { get; } private readonly byte[] data; @@ -38,6 +41,12 @@ private D Read( nuint offset ) where D : unmanaged => private static float ReverseEndianness( float v ) => Unsafe.BitCast( BinaryPrimitives.ReverseEndianness( Unsafe.BitCast( v ) ) ); + /// + /// Reads a from the page data at . + /// + /// Offset of the field inside the page. + /// Offset of the row inside the page. + /// The . [MethodImpl( MethodImplOptions.AggressiveInlining )] public ReadOnlySeString ReadString( nuint offset, nuint structOffset ) { @@ -47,46 +56,102 @@ public ReadOnlySeString ReadString( nuint offset, nuint structOffset ) return new ReadOnlySeString( data[..stringLength] ); } + /// + /// Reads a from the page data at . + /// + /// Offset of the field inside the page. + /// The . [MethodImpl( MethodImplOptions.AggressiveInlining )] public bool ReadBool( nuint offset ) => Read( offset ); + /// + /// Reads a from the page data at . + /// + /// Offset of the field inside the page. + /// The . [MethodImpl( MethodImplOptions.AggressiveInlining )] public sbyte ReadInt8( nuint offset ) => Read( offset ); + /// + /// Reads a from the page data at . + /// + /// Offset of the field inside the page. + /// The . [MethodImpl( MethodImplOptions.AggressiveInlining )] public byte ReadUInt8( nuint offset ) => Read( offset ); + /// + /// Reads a from the page data at . + /// + /// Offset of the field inside the page. + /// The . [MethodImpl( MethodImplOptions.AggressiveInlining )] public short ReadInt16( nuint offset ) => BinaryPrimitives.ReverseEndianness( Read( offset ) ); + /// + /// Reads a from the page data at . + /// + /// Offset of the field inside the page. + /// The . [MethodImpl( MethodImplOptions.AggressiveInlining )] public ushort ReadUInt16( nuint offset ) => BinaryPrimitives.ReverseEndianness( Read( offset ) ); + /// + /// Reads a from the page data at . + /// + /// Offset of the field inside the page. + /// The . [MethodImpl( MethodImplOptions.AggressiveInlining )] public int ReadInt32( nuint offset ) => BinaryPrimitives.ReverseEndianness( Read( offset ) ); + /// + /// Reads a from the page data at . + /// + /// Offset of the field inside the page. + /// The . [MethodImpl( MethodImplOptions.AggressiveInlining )] public uint ReadUInt32( nuint offset ) => BinaryPrimitives.ReverseEndianness( Read( offset ) ); + /// + /// Reads a from the page data at . + /// + /// Offset of the field inside the page. + /// The . [MethodImpl( MethodImplOptions.AggressiveInlining )] public float ReadFloat32( nuint offset ) => ReverseEndianness( Read( offset ) ); + /// + /// Reads a from the page data at . + /// + /// Offset of the field inside the page. + /// The . [MethodImpl( MethodImplOptions.AggressiveInlining )] public long ReadInt64( nuint offset ) => BinaryPrimitives.ReverseEndianness( Read( offset ) ); + /// + /// Reads a from the page data at . + /// + /// Offset of the field inside the page. + /// The . [MethodImpl( MethodImplOptions.AggressiveInlining )] public ulong ReadUInt64( nuint offset ) => BinaryPrimitives.ReverseEndianness( Read( offset ) ); + /// + /// Reads a from the page data at at bit offset . + /// + /// Byte offset of the field inside the page. + /// Bit offset of the field inside the byte. (0 - 7) + /// The . [MethodImpl( MethodImplOptions.AggressiveInlining )] public bool ReadPackedBool( nuint offset, byte bit ) => ( Read( offset ) & ( 1 << bit ) ) != 0; From a092322c74a443e7863f9634039fc776ee622461 Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Wed, 14 Aug 2024 14:24:58 -0700 Subject: [PATCH 15/53] Publicize column information --- src/Lumina/Excel/ExcelSheet.cs | 9 +++++++++ src/Lumina/Excel/IExcelSheet.cs | 15 +++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/src/Lumina/Excel/ExcelSheet.cs b/src/Lumina/Excel/ExcelSheet.cs index c79dd692..c726de55 100644 --- a/src/Lumina/Excel/ExcelSheet.cs +++ b/src/Lumina/Excel/ExcelSheet.cs @@ -24,6 +24,9 @@ public sealed partial class ExcelSheet : IExcelSheet where T : struct, IExcel [MemberNotNullWhen( false, nameof( Rows ) )] public bool HasSubrows { get; } + /// + public IReadOnlyList Columns { get; } + private List Pages { get; } private (uint RowId, (int PageIdx, uint Offset) Data)[]? Rows { get; } private (uint RowId, (int PageIdx, uint Offset, ushort RowCount) Data)[]? Subrows { get; } @@ -87,6 +90,8 @@ public ExcelSheet( ExcelModule module, Language requestedLanguage, string sheetN Language = headerFile.Languages.Contains( requestedLanguage ) ? requestedLanguage : Language.None; + Columns = headerFile.ColumnDefinitions; + List<(uint RowId, (int PageIdx, uint Offset) Data)>? rows = null; List<(uint RowId, (int PageIdx, uint Offset, ushort RowCount) Data)>? subrows = null; var totalSubrowCount = 0; @@ -148,6 +153,10 @@ public ExcelSheet( ExcelModule module, Language requestedLanguage, string sheetN } } + /// + public ushort GetColumnOffset( int columnIdx ) => + Columns[columnIdx].Offset; + /// public bool HasRow( uint rowId ) => RowLookup.ContainsKey( (int)rowId ); diff --git a/src/Lumina/Excel/IExcelSheet.cs b/src/Lumina/Excel/IExcelSheet.cs index 489c7e64..499cd44b 100644 --- a/src/Lumina/Excel/IExcelSheet.cs +++ b/src/Lumina/Excel/IExcelSheet.cs @@ -1,5 +1,7 @@ using System.Collections; +using System.Collections.Generic; using Lumina.Data; +using Lumina.Data.Structs.Excel; namespace Lumina.Excel; @@ -25,6 +27,11 @@ public interface IExcelSheet : IEnumerable /// bool HasSubrows { get; } + /// + /// Contains information on the columns in this sheet. + /// + IReadOnlyList Columns { get; } + /// /// The number of rows in this sheet. /// @@ -40,6 +47,14 @@ public interface IExcelSheet : IEnumerable /// Thrown if the sheet does not support subrows int SubrowCount { get; } + /// + /// Gets the offset of the column at in the row data. + /// + /// The index of the column. + /// The offset of the column. + /// Thrown when the column index is invalid. It must be less than .Count. + ushort GetColumnOffset( int columnIdx ); + /// /// Whether or not this sheet has a row with the given . /// From c9b9bb1a55231de28674e7471a2300ab3ec56e98 Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Wed, 14 Aug 2024 15:33:37 -0700 Subject: [PATCH 16/53] Formatting changes --- src/Lumina/Excel/ExcelModule.cs | 37 +++++++++------- src/Lumina/Excel/ExcelPage.cs | 6 +-- src/Lumina/Excel/ExcelSheet.Collection.cs | 4 +- src/Lumina/Excel/ExcelSheet.cs | 6 ++- src/Lumina/Excel/IExcelRow.cs | 10 ++--- src/Lumina/Excel/IExcelSheet.cs | 28 ++++++------ src/Lumina/Excel/RowRef.cs | 44 +++++++++---------- src/Lumina/Excel/Rsv/DictionaryRsvProvider.cs | 3 ++ src/Lumina/Excel/Rsv/IRsvProvider.cs | 17 ++++--- src/Lumina/Excel/Rsv/RsvUtil.cs | 11 +++-- src/Lumina/Excel/SheetAttribute.cs | 15 ++++--- 11 files changed, 100 insertions(+), 81 deletions(-) diff --git a/src/Lumina/Excel/ExcelModule.cs b/src/Lumina/Excel/ExcelModule.cs index 217dd864..3e947eff 100644 --- a/src/Lumina/Excel/ExcelModule.cs +++ b/src/Lumina/Excel/ExcelModule.cs @@ -9,6 +9,9 @@ namespace Lumina.Excel; +/// +/// Represents a module for working with excel sheets for a instance. +/// public class ExcelModule { internal GameData GameData { get; } @@ -18,7 +21,7 @@ public class ExcelModule private ConcurrentDictionary<(Type sheetType, Language requestedLanguage), IExcelSheet> SheetCache { get; } = []; /// - /// Get all available sheets, parsed from root.exl. + /// Get the names of all available sheets, parsed from root.exl. /// public IReadOnlyCollection SheetNames { get; } @@ -26,7 +29,7 @@ public class ExcelModule /// Create a new ExcelModule. This will do all the initial discovery of sheets from the EXL but not load any sheets. /// /// The instance to load sheets from - /// Thrown when the root.exl file cannot be found - make sure that an 0a dat is available. + /// Thrown when the root.exl file cannot be found - make sure that a 0a dat is available. public ExcelModule( GameData gameData ) { GameData = gameData; @@ -40,16 +43,16 @@ public ExcelModule( GameData gameData ) } /// - /// Loads an , optionally with a specific language + /// Loads an , optionally with a specific language. /// /// - /// If the requested language doesn't exist for the file, this will silently be ignored and it will return a sheet with the default language: + /// If the requested language doesn't exist for the file, this will silently be ignored and it will return a sheet with the default language: . /// + /// A struct that implements to parse rows. /// The requested sheet language. Leave or empty to use the default language. - /// A struct that implements to parse rows /// An if the sheet exists - /// Thrown when the sheet type is not decorated with a - /// Sheet does not exist or if the column hash has a mismatch + /// Thrown when is not decorated with a . + /// Sheet does not exist or if the column hash has a mismatch. public ExcelSheet GetSheet( Language? language = null ) where T : struct, IExcelRow { language ??= Language; @@ -58,26 +61,26 @@ public ExcelSheet GetSheet( Language? language = null ) where T : struct, } /// - /// Loads an from a reflected , optionally with a specific language + /// Loads an from a reflected , optionally with a specific language. /// /// /// Only use this method if you need to create a sheet while using reflection. /// - /// If the requested language doesn't exist for the file, this will silently be ignored and it will return a sheet with the default language: + /// If the requested language doesn't exist for the file, this will silently be ignored and it will return a sheet with the default language: . /// - /// A that implements to parse rows + /// A that implements to parse rows. /// The requested sheet language. Leave or empty to use the default language. - /// An if the sheet exists - /// Thrown when is not decorated with a - /// Sheet does not exist or if the column hash has a mismatch - [RequiresDynamicCode("Creating a generic sheet from a type requires reflection and dynamic code.")] - [EditorBrowsable(EditorBrowsableState.Advanced)] - public IExcelSheet GetSheetGeneric(Type rowType, Language? language = null ) + /// An if the sheet exists. + /// Thrown when is not decorated with a . + /// Sheet does not exist or if the column hash has a mismatch. + [RequiresDynamicCode( "Creating a generic sheet from a type requires reflection and dynamic code." )] + [EditorBrowsable( EditorBrowsableState.Advanced )] + public IExcelSheet GetSheetGeneric( Type rowType, Language? language = null ) { if( !rowType.IsValueType ) throw new ArgumentException( "rowType must be a struct", nameof( rowType ) ); - if (!rowType.IsAssignableTo( typeof( IExcelRow<> ).MakeGenericType( rowType ) ) ) + if( !rowType.IsAssignableTo( typeof( IExcelRow<> ).MakeGenericType( rowType ) ) ) { throw new ArgumentException( "rowType implement IExcelRow", nameof( rowType ) ); } diff --git a/src/Lumina/Excel/ExcelPage.cs b/src/Lumina/Excel/ExcelPage.cs index 1df20de1..a2945335 100644 --- a/src/Lumina/Excel/ExcelPage.cs +++ b/src/Lumina/Excel/ExcelPage.cs @@ -1,9 +1,9 @@ -using System.Buffers.Binary; +using Lumina.Text.ReadOnly; using System; +using System.Buffers.Binary; +using System.ComponentModel; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; -using Lumina.Text.ReadOnly; -using System.ComponentModel; namespace Lumina.Excel; diff --git a/src/Lumina/Excel/ExcelSheet.Collection.cs b/src/Lumina/Excel/ExcelSheet.Collection.cs index a55aa131..375ab103 100644 --- a/src/Lumina/Excel/ExcelSheet.Collection.cs +++ b/src/Lumina/Excel/ExcelSheet.Collection.cs @@ -17,7 +17,7 @@ public sealed partial class ExcelSheet : IReadOnlyCollection where T : str /// /// Returns an enumerator that can be used to iterate over all rows in this sheet. If this sheet has subrows, it will iterate over every subrow of every row. /// - /// An of all rows or subrows in this sheet + /// An of all rows or subrows in this sheet. public SheetEnumerator GetEnumerator() => new( this ); @@ -28,7 +28,7 @@ IEnumerator IEnumerable.GetEnumerator() => /// /// Returns an enumerator that can be used to iterate over all rows in this sheet. If this sheet has subrows, it will iterate over the first subrow of every row. /// - /// An of all rows (or first subrows) in this sheet + /// An of all rows (or first subrows) in this sheet. public RowEnumerator GetRowEnumerator() => new( this ); diff --git a/src/Lumina/Excel/ExcelSheet.cs b/src/Lumina/Excel/ExcelSheet.cs index c726de55..c125d71b 100644 --- a/src/Lumina/Excel/ExcelSheet.cs +++ b/src/Lumina/Excel/ExcelSheet.cs @@ -11,6 +11,10 @@ namespace Lumina.Excel; +/// +/// A strongly-typed wrapper around an excel sheet. +/// +/// The row type. public sealed partial class ExcelSheet : IExcelSheet where T : struct, IExcelRow { /// @@ -278,7 +282,7 @@ public bool TryGetRow( uint rowId, out T row ) /// Thrown if the sheet does not support subrows public bool TryGetSubrow( uint rowId, ushort subrowId, out T subrow ) { - if (GetSubrowOrDefault(rowId, subrowId) is { } outSubrow) + if( GetSubrowOrDefault( rowId, subrowId ) is { } outSubrow ) { subrow = outSubrow; return true; diff --git a/src/Lumina/Excel/IExcelRow.cs b/src/Lumina/Excel/IExcelRow.cs index bbd6b88e..011a799f 100644 --- a/src/Lumina/Excel/IExcelRow.cs +++ b/src/Lumina/Excel/IExcelRow.cs @@ -3,7 +3,7 @@ namespace Lumina.Excel; /// /// Defines a row type/schema for an excel sheet. /// -/// The type that implements the interface +/// The type that implements the interface. public interface IExcelRow where T : struct { /// @@ -13,8 +13,8 @@ public interface IExcelRow where T : struct /// /// /// - /// A newly created row object - /// Thrown when the referenced sheet is using subrows + /// A newly created row object. + /// Thrown when the referenced sheet is using subrows. abstract static T Create( ExcelPage page, uint offset, uint row ); /// @@ -25,7 +25,7 @@ public interface IExcelRow where T : struct /// /// /// - /// A newly created subrow object - /// Thrown when the referenced sheet is not using subrows + /// A newly created subrow object. + /// Thrown when the referenced sheet is not using subrows. abstract static T Create( ExcelPage page, uint offset, uint row, ushort subrow ); } \ No newline at end of file diff --git a/src/Lumina/Excel/IExcelSheet.cs b/src/Lumina/Excel/IExcelSheet.cs index 499cd44b..0e99fd32 100644 --- a/src/Lumina/Excel/IExcelSheet.cs +++ b/src/Lumina/Excel/IExcelSheet.cs @@ -1,7 +1,7 @@ -using System.Collections; -using System.Collections.Generic; using Lumina.Data; using Lumina.Data.Structs.Excel; +using System.Collections; +using System.Collections.Generic; namespace Lumina.Excel; @@ -44,7 +44,7 @@ public interface IExcelSheet : IEnumerable /// /// The total number of subrows in this sheet across all rows. /// - /// Thrown if the sheet does not support subrows + /// Thrown if the sheet does not support subrows. int SubrowCount { get; } /// @@ -61,34 +61,34 @@ public interface IExcelSheet : IEnumerable /// /// If this sheet has subrows, this will check if the row id has any subrows. /// - /// The row id to check - /// Whether or not the row exists + /// The row id to check. + /// Whether or not the row exists. bool HasRow( uint rowId ); /// /// Whether or not this sheet has a subrow with the given and . /// - /// The row id to check - /// The subrow id to check - /// Whether or not the subrow exists - /// Thrown if the sheet does not support subrows + /// The row id to check. + /// The subrow id to check. + /// Whether or not the subrow exists. + /// Thrown if the sheet does not support subrows. bool HasSubrow( uint rowId, ushort subrowId ); /// /// Tries to get the number of subrows in the th row in this sheet. /// - /// The row id to get + /// The row id to get. /// The number of subrows in this row. /// if the row exists and is written to and otherwise. - /// Thrown if the sheet does not support subrows + /// Thrown if the sheet does not support subrows. bool TryGetSubrowCount( uint rowId, out ushort subrowCount ); /// /// Gets the number of subrows in the th row in this sheet. Throws if the row does not exist. /// - /// The row id to get + /// The row id to get. /// The number of subrows in this row. Returns null if the row does not exist. - /// Thrown if the sheet does not support subrows - /// Thrown if the sheet does not have a row at that + /// Thrown if the sheet does not support subrows. + /// Thrown if the sheet does not have a row at that . ushort GetSubrowCount( uint rowId ); } \ No newline at end of file diff --git a/src/Lumina/Excel/RowRef.cs b/src/Lumina/Excel/RowRef.cs index 71ff5f0e..4d1ae5b5 100644 --- a/src/Lumina/Excel/RowRef.cs +++ b/src/Lumina/Excel/RowRef.cs @@ -5,9 +5,9 @@ namespace Lumina.Excel; /// /// A helper type to dynamically reference a row in a different excel sheet. /// -/// The to read sheet data from -/// The referenced row id -/// The referenced row's actual +/// The to read sheet data from. +/// The referenced row id. +/// The referenced row's actual . public readonly struct RowRef( ExcelModule? module, uint rowId, Type? rowType ) { /// @@ -26,15 +26,15 @@ public readonly struct RowRef( ExcelModule? module, uint rowId, Type? rowType ) /// /// Whether or not the reference is of a specific row type. /// - /// The row type/schema to check against - /// Whether or not this points to a + /// The row type/schema to check against. + /// Whether or not this points to a . public bool Is() where T : struct, IExcelRow => typeof( T ) == rowType; /// /// Tries to get the referenced row as a specific row type. /// - /// The row type/schema to check against + /// The row type/schema to check against. /// The referenced row type. Returns null if this does not point to a or if the does not exist in its sheet. public T? GetValueOrDefault() where T : struct, IExcelRow { @@ -47,10 +47,10 @@ public bool Is() where T : struct, IExcelRow => /// /// Tries to get the referenced row as a specific row type. /// - /// The row type/schema to check against - /// The output row object + /// The row type/schema to check against. + /// The output row object. /// if the type is valid, the row exists, and is written to, and otherwise. - public bool TryGetValue(out T row) where T : struct, IExcelRow + public bool TryGetValue( out T row ) where T : struct, IExcelRow { if( !Is() ) { @@ -64,9 +64,9 @@ public bool TryGetValue(out T row) where T : struct, IExcelRow /// /// Attempts to create a to a row id of a list of row types, checking with each type in order. /// - /// The to read sheet data from - /// The referenced row id - /// A list of row types to check against the , in order + /// The to read sheet data from. + /// The referenced row id. + /// A list of row types to check against the , in order. /// A to one of the . If the row id does not exist in any of the sheets, an untyped is returned instead. public static RowRef GetFirstValidRowOrUntyped( ExcelModule module, uint rowId, params Type[] sheetTypes ) { @@ -85,26 +85,26 @@ public static RowRef GetFirstValidRowOrUntyped( ExcelModule module, uint rowId, /// /// Creates a to a specific row type. /// - /// The row type referenced by the - /// The to read sheet data from - /// The referenced row id + /// The row type referenced by the . + /// The to read sheet data from. + /// The referenced row id. /// A to a row in a . public static RowRef Create( ExcelModule module, uint rowId ) where T : struct, IExcelRow => new( module, rowId, typeof( T ) ); /// /// Creates an untyped . /// - /// The referenced row id - /// An untyped + /// The referenced row id. + /// An untyped . public static RowRef CreateUntyped( uint rowId ) => new( null, rowId, null ); } /// /// A helper type to concretely reference a row in a specific excel sheet. /// -/// The row type referenced by the -/// The to read sheet data from -/// The referenced row id +/// The row type referenced by the . +/// The to read sheet data from. +/// The referenced row id. public readonly struct RowRef( ExcelModule module, uint rowId ) where T : struct, IExcelRow { private readonly ExcelSheet sheet = module.GetSheet(); @@ -122,7 +122,7 @@ public readonly struct RowRef( ExcelModule module, uint rowId ) where T : str /// /// The referenced row value itself. /// - /// Thrown if is false + /// Thrown if is false. public T Value => sheet.GetRow( RowId ); /// @@ -135,6 +135,6 @@ public readonly struct RowRef( ExcelModule module, uint rowId ) where T : str /// /// Converts a concrete to an generic and dynamically typed . /// - /// The to convert + /// The to convert. public static explicit operator RowRef( RowRef row ) => row.ToGeneric(); } \ No newline at end of file diff --git a/src/Lumina/Excel/Rsv/DictionaryRsvProvider.cs b/src/Lumina/Excel/Rsv/DictionaryRsvProvider.cs index 51a53699..243bbde2 100644 --- a/src/Lumina/Excel/Rsv/DictionaryRsvProvider.cs +++ b/src/Lumina/Excel/Rsv/DictionaryRsvProvider.cs @@ -4,6 +4,9 @@ namespace Lumina.Excel.Rsv; +/// +/// Represents a dictionary-based RSV provider. +/// public class DictionaryRsvProvider : Dictionary, IRsvProvider { /// diff --git a/src/Lumina/Excel/Rsv/IRsvProvider.cs b/src/Lumina/Excel/Rsv/IRsvProvider.cs index f834c864..60978c3b 100644 --- a/src/Lumina/Excel/Rsv/IRsvProvider.cs +++ b/src/Lumina/Excel/Rsv/IRsvProvider.cs @@ -2,35 +2,38 @@ namespace Lumina.Excel.Rsv; +/// +/// Represents a provider for resolving RSV strings. +/// public interface IRsvProvider { /// /// Determines if the provider can resolve the given RSV string. /// - /// The string to check - /// Whether or not the provider contains a resolved string for + /// The string to check. + /// Whether or not the provider contains a resolved string for . bool CanResolve( ReadOnlySeString rsvString ); /// /// Tries to resolve the given RSV string. /// - /// The string to resolve + /// The string to resolve. /// The newly resolved string. Returns if it could not be resolved. ReadOnlySeString? TryResolve( ReadOnlySeString rsvString ); /// /// Resolves the given RSV string. /// - /// The string to resolve - /// The newly resolved string + /// The string to resolve. + /// The newly resolved string. /// Thrown if the RSV string is not valid. must return true. - /// Thrown if the RSV key is not found in the provider + /// Thrown if the RSV key is not found in the provider. ReadOnlySeString Resolve( ReadOnlySeString rsvString ); /// /// Tries to resolve the given RSV string. /// - /// The string to resolve + /// The string to resolve. /// The newly resolved string. Returns if it could not be resolved. ReadOnlySeString ResolveOrSelf( ReadOnlySeString rsvString ); } diff --git a/src/Lumina/Excel/Rsv/RsvUtil.cs b/src/Lumina/Excel/Rsv/RsvUtil.cs index 00a8a0d3..633ae331 100644 --- a/src/Lumina/Excel/Rsv/RsvUtil.cs +++ b/src/Lumina/Excel/Rsv/RsvUtil.cs @@ -3,6 +3,9 @@ namespace Lumina.Excel.Rsv; +/// +/// Utility class for RSV string operations. +/// public static class RsvUtil { // RsvPrefix => _rsv_ @@ -12,7 +15,7 @@ public static class RsvUtil /// Checks if the string is an RSV string and can therefore be resolved. /// /// This only checks if the string begins with "_rsv_". - /// The string to check + /// The string to check. /// Whether or not the string is an RSV string. public static bool IsRsv( this ReadOnlySeString rsvString ) => rsvString.Data.Span.StartsWith( RsvPrefix ); @@ -24,9 +27,9 @@ public static bool IsRsv( this ReadOnlySeStringSpan rsvString ) => /// /// Attempts to resolve with the given . /// - /// This is safe to call on strings that are not RSVs, a.k.a. where returns . - /// The string to resolve - /// The provider to check with + /// This method is safe to call on strings that are not RSVs, i.e., where returns . + /// The string to resolve. + /// The provider to check with. /// The newly resolved string. Returns if it could not be resolved. public static ReadOnlySeString ResolveRsv( this ReadOnlySeString rsvString, IRsvProvider provider ) => provider.ResolveOrSelf( rsvString ); diff --git a/src/Lumina/Excel/SheetAttribute.cs b/src/Lumina/Excel/SheetAttribute.cs index e5b94449..c045dd6c 100644 --- a/src/Lumina/Excel/SheetAttribute.cs +++ b/src/Lumina/Excel/SheetAttribute.cs @@ -5,23 +5,26 @@ namespace Lumina.Excel; /// /// An attribute attached to a schema/struct that represents a sheet in an excel file. /// -/// The name of the sheet -/// The column hash of the sheet; optionally used to check for schema and sheet changes +/// The name of the sheet. +/// The column hash of the sheet; optionally used to check for schema and sheet changes. [AttributeUsage( AttributeTargets.Struct )] public class SheetAttribute( string name, uint columnHash ) : Attribute { /// - /// The sheet name + /// The name of the sheet. /// public readonly string Name = name; /// - /// A column hash - used to warn when a sheet structure has changed + /// Gets the column hash of the sheet; optionally used to check for schema and sheet changes. /// public readonly uint? ColumnHash = columnHash; - /// The name of the sheet - public SheetAttribute(string name) : this(name, uint.MaxValue ) + /// + /// Creates a new instance of the class. + /// + /// The name of the sheet. + public SheetAttribute( string name ) : this( name, uint.MaxValue ) { ColumnHash = null; } From fc7fb2e1f5df16aa853bba921297a17e89a5e917 Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Thu, 15 Aug 2024 13:47:54 -0700 Subject: [PATCH 17/53] Refactor RSV resolution Use a plain delegate instead and resolve by default --- src/Lumina/Excel/ExcelModule.cs | 14 ++++++ src/Lumina/Excel/ExcelPage.cs | 12 ++++- src/Lumina/Excel/ExcelSheet.Collection.cs | 31 ++++++++++-- src/Lumina/Excel/ExcelSheet.cs | 48 +++++++++--------- src/Lumina/Excel/Rsv/DictionaryRsvProvider.cs | 27 ---------- src/Lumina/Excel/Rsv/IRsvProvider.cs | 39 --------------- src/Lumina/Excel/Rsv/RsvUtil.cs | 36 ------------- src/Lumina/Extensions/RsvExtensions.cs | 50 ++++++++----------- src/Lumina/LuminaOptions.cs | 10 ++-- 9 files changed, 100 insertions(+), 167 deletions(-) delete mode 100644 src/Lumina/Excel/Rsv/DictionaryRsvProvider.cs delete mode 100644 src/Lumina/Excel/Rsv/IRsvProvider.cs delete mode 100644 src/Lumina/Excel/Rsv/RsvUtil.cs diff --git a/src/Lumina/Excel/ExcelModule.cs b/src/Lumina/Excel/ExcelModule.cs index 3e947eff..4928beac 100644 --- a/src/Lumina/Excel/ExcelModule.cs +++ b/src/Lumina/Excel/ExcelModule.cs @@ -1,5 +1,6 @@ using Lumina.Data; using Lumina.Data.Files.Excel; +using Lumina.Text.ReadOnly; using System; using System.Collections.Concurrent; using System.Collections.Generic; @@ -18,13 +19,26 @@ public class ExcelModule internal Language Language => GameData.Options.DefaultExcelLanguage; + internal bool VerifySheetChecksums => GameData.Options.PanicOnSheetChecksumMismatch; + + internal ResolveRsvDelegate? RsvResolver => GameData.Options.RsvResolver; + private ConcurrentDictionary<(Type sheetType, Language requestedLanguage), IExcelSheet> SheetCache { get; } = []; + /// + /// A delegate provided by the user to resolve RSV strings. + /// + /// The string to resolve. It is guaranteed that this string it begins with _rsv_. + /// The output resolved string. + /// if resolved and is written to and otherwise. + public delegate bool ResolveRsvDelegate(ReadOnlySeString rsvString, out ReadOnlySeString resolvedString); + /// /// Get the names of all available sheets, parsed from root.exl. /// public IReadOnlyCollection SheetNames { get; } + /// /// Create a new ExcelModule. This will do all the initial discovery of sheets from the EXL but not load any sheets. /// diff --git a/src/Lumina/Excel/ExcelPage.cs b/src/Lumina/Excel/ExcelPage.cs index a2945335..20c50345 100644 --- a/src/Lumina/Excel/ExcelPage.cs +++ b/src/Lumina/Excel/ExcelPage.cs @@ -1,3 +1,4 @@ +using Lumina.Extensions; using Lumina.Text.ReadOnly; using System; using System.Buffers.Binary; @@ -44,6 +45,9 @@ private static float ReverseEndianness( float v ) => /// /// Reads a from the page data at . /// + /// + /// If the string is a valid RSV string and is set, the resolved string will be returned if it exists. + /// /// Offset of the field inside the page. /// Offset of the row inside the page. /// The . @@ -53,7 +57,13 @@ public ReadOnlySeString ReadString( nuint offset, nuint structOffset ) offset = ReadUInt32( offset ) + structOffset + dataOffset; var data = Data[(int)offset..]; var stringLength = data.Span.IndexOf( (byte)0 ); - return new ReadOnlySeString( data[..stringLength] ); + var ret = new ReadOnlySeString( data[..stringLength] ); + if( ret.IsRsv() && Module.RsvResolver != null ) + { + if( Module.RsvResolver.Invoke( ret, out var resolvedString ) ) + return resolvedString; + } + return ret; } /// diff --git a/src/Lumina/Excel/ExcelSheet.Collection.cs b/src/Lumina/Excel/ExcelSheet.Collection.cs index 375ab103..ad0521e0 100644 --- a/src/Lumina/Excel/ExcelSheet.Collection.cs +++ b/src/Lumina/Excel/ExcelSheet.Collection.cs @@ -21,6 +21,10 @@ public sealed partial class ExcelSheet : IReadOnlyCollection where T : str public SheetEnumerator GetEnumerator() => new( this ); + /// + IEnumerator IEnumerable.GetEnumerator() => + GetEnumerator(); + /// IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); @@ -32,10 +36,11 @@ IEnumerator IEnumerable.GetEnumerator() => public RowEnumerator GetRowEnumerator() => new( this ); - IEnumerator IEnumerable.GetEnumerator() => - GetEnumerator(); - - public struct SheetEnumerator( ExcelSheet sheet ) : IEnumerator + /// + /// Represents an enumerator that iterates over all rows/subrows in a . + /// + /// The sheet to iterate over. + public struct SheetEnumerator( ExcelSheet sheet ) : IEnumerator, IEnumerable { private int LookupIndex { get; set; } = -1; private ushort SubrowIndex { get; set; } = 0; @@ -50,6 +55,7 @@ public struct SheetEnumerator( ExcelSheet sheet ) : IEnumerator /// readonly object IEnumerator.Current => Current; + /// public bool MoveNext() { if( !HasSubrows ) @@ -78,6 +84,7 @@ public bool MoveNext() return false; } + /// public void Reset() { LookupIndex = -1; @@ -85,12 +92,23 @@ public void Reset() SubrowCount = 0; } + /// public readonly void Dispose() { } + + /// + public readonly IEnumerator GetEnumerator() => this; + + /// + readonly IEnumerator IEnumerable.GetEnumerator() => this; } + /// + /// Represents an enumerator that iterates over all rows or the first subrow of every row in a . + /// + /// The sheet to iterate over. public struct RowEnumerator( ExcelSheet sheet ) : IEnumerator, IEnumerable { private int LookupIndex { get; set; } = -1; @@ -103,6 +121,7 @@ public struct RowEnumerator( ExcelSheet sheet ) : IEnumerator, IEnumerable /// readonly object IEnumerator.Current => Current; + /// public bool MoveNext() { if( LookupIndex + 1 < RowCount ) @@ -113,18 +132,22 @@ public bool MoveNext() return false; } + /// public void Reset() { LookupIndex = -1; } + /// public readonly void Dispose() { } + /// public readonly IEnumerator GetEnumerator() => this; + /// readonly IEnumerator IEnumerable.GetEnumerator() => this; } } diff --git a/src/Lumina/Excel/ExcelSheet.cs b/src/Lumina/Excel/ExcelSheet.cs index c125d71b..c8df0ebf 100644 --- a/src/Lumina/Excel/ExcelSheet.cs +++ b/src/Lumina/Excel/ExcelSheet.cs @@ -53,8 +53,8 @@ public sealed partial class ExcelSheet : IExcelSheet where T : struct, IExcel /// Create an instance with the 's default language. /// /// The to access sheet data from. - /// does not have a valid - /// parameters were invalid (hash mismatch or invalid sheet name) + /// does not have a valid . + /// parameters were invalid (hash mismatch or invalid sheet name). public ExcelSheet( ExcelModule module ) : this( module, module.Language ) { @@ -65,8 +65,8 @@ public ExcelSheet( ExcelModule module ) : this( module, module.Language ) /// /// The to access sheet data from. /// The language to use for this sheet. - /// does not have a valid - /// parameters were invalid (hash mismatch or invalid sheet name) + /// does not have a valid . + /// parameters were invalid (hash mismatch or invalid sheet name). public ExcelSheet( ExcelModule module, Language requestedLanguage ) : this( module, requestedLanguage, Attribute.Name, Attribute.ColumnHash ) { @@ -79,7 +79,7 @@ public ExcelSheet( ExcelModule module, Language requestedLanguage ) : this( modu /// The language to use for this sheet. /// The name of the sheet to read from. /// The hash of the columns in the sheet. If , it will not check the hash. - /// or parameters were invalid (hash mismatch or invalid sheet name) + /// or parameters were invalid (hash mismatch or invalid sheet name). public ExcelSheet( ExcelModule module, Language requestedLanguage, string sheetName, uint? columnHash = null ) { Module = module; @@ -87,7 +87,7 @@ public ExcelSheet( ExcelModule module, Language requestedLanguage, string sheetN var headerFile = module.GameData.GetFile( $"exd/{sheetName}.exh" ) ?? throw new ArgumentException( "Invalid sheet name", nameof( sheetName ) ); - if( columnHash is { } hash && headerFile.GetColumnsHash() != hash ) + if( module.VerifySheetChecksums && columnHash is { } hash && headerFile.GetColumnsHash() != hash ) throw new ArgumentException( "Column hash mismatch", nameof( columnHash ) ); HasSubrows = headerFile.Header.Variant == ExcelVariant.Subrows; @@ -217,7 +217,7 @@ private T CreateSubrow( uint rowId, ushort subrowId, in (int PageIdx, uint Offse /// /// Tries to get the th row in this sheet. If this sheet has subrows, it will return the first subrow. /// - /// The row id to get + /// The row id to get. /// A nullable row object. Returns null if the row does not exist. public T? GetRowOrDefault( uint rowId ) { @@ -235,10 +235,10 @@ private T CreateSubrow( uint rowId, ushort subrowId, in (int PageIdx, uint Offse /// /// Tries to get the th subrow with row id in this sheet. /// - /// The row id to get - /// The subrow id to get + /// The row id to get. + /// The subrow id to get. /// A nullable row object. Returns null if the subrow does not exist. - /// Thrown if the sheet does not support subrows + /// Thrown if the sheet does not support subrows. public T? GetSubrowOrDefault( uint rowId, ushort subrowId ) { if( !HasSubrows ) @@ -257,7 +257,7 @@ private T CreateSubrow( uint rowId, ushort subrowId, in (int PageIdx, uint Offse /// /// Tries to get the th row in this sheet. If this sheet has subrows, it will return the first subrow. /// - /// The row id to get + /// The row id to get. /// The output row object. /// if the row exists and is written to and otherwise. public bool TryGetRow( uint rowId, out T row ) @@ -275,11 +275,11 @@ public bool TryGetRow( uint rowId, out T row ) /// /// Tries to get the th subrow with row id in this sheet. /// - /// The row id to get - /// The subrow id to get + /// The row id to get. + /// The subrow id to get. /// The output row object. - /// if the row exists and is written to and otherwise. - /// Thrown if the sheet does not support subrows + /// if the subrow exists and is written to and otherwise. + /// Thrown if the sheet does not support subrows. public bool TryGetSubrow( uint rowId, ushort subrowId, out T subrow ) { if( GetSubrowOrDefault( rowId, subrowId ) is { } outSubrow ) @@ -294,7 +294,7 @@ public bool TryGetSubrow( uint rowId, ushort subrowId, out T subrow ) /// /// Gets the th row in this sheet. If this sheet has subrows, it will return the first subrow. /// - /// The row id to get + /// The row id to get. /// A row object. /// Throws when the row id does not have a row attached to it. public T GetRow( uint rowId ) => @@ -304,11 +304,11 @@ public T GetRow( uint rowId ) => /// /// Gets the th subrow with row id in this sheet. Throws if the subrow does not exist. /// - /// The row id to get - /// The subrow id to get + /// The row id to get. + /// The subrow id to get. /// A row object. - /// Thrown if the sheet does not support subrows - /// Thrown if the sheet does not have a row at that + /// Thrown if the sheet does not support subrows. + /// Thrown if the sheet does not have a row at that . public T GetSubrow( uint rowId, ushort subrowId ) { if( !HasSubrows ) @@ -327,7 +327,7 @@ public T GetSubrow( uint rowId, ushort subrowId ) /// Gets the th row in this sheet, ordered by row id in ascending order. If this sheet has subrows, it will return the first subrow. /// /// If you are looking to find a row by its id, use instead. - /// The zero-based index of this row + /// The zero-based index of this row. /// A row object. public T GetRowAt( int rowIndex ) { @@ -342,10 +342,10 @@ public T GetRowAt( int rowIndex ) /// Gets the th subrow of the th row in this sheet, ordered by row id in ascending order. /// /// If you are looking to find a subrow by its id, use instead. - /// The zero-based index of this row - /// The subrow id to get + /// The zero-based index of this row. + /// The subrow id to get. /// A row object. - /// Thrown if the sheet does not support subrows + /// Thrown if the sheet does not support subrows. public T GetSubrowAt( int rowIndex, ushort subrowId ) { if( !HasSubrows ) diff --git a/src/Lumina/Excel/Rsv/DictionaryRsvProvider.cs b/src/Lumina/Excel/Rsv/DictionaryRsvProvider.cs deleted file mode 100644 index 243bbde2..00000000 --- a/src/Lumina/Excel/Rsv/DictionaryRsvProvider.cs +++ /dev/null @@ -1,27 +0,0 @@ -using Lumina.Text.ReadOnly; -using System; -using System.Collections.Generic; - -namespace Lumina.Excel.Rsv; - -/// -/// Represents a dictionary-based RSV provider. -/// -public class DictionaryRsvProvider : Dictionary, IRsvProvider -{ - /// - public bool CanResolve( ReadOnlySeString rsvString ) => - rsvString.IsRsv() && ContainsKey( rsvString ); - - /// - public ReadOnlySeString Resolve( ReadOnlySeString rsvString ) => - rsvString.IsRsv() ? this[rsvString] : throw new ArgumentException( "rsvString is not a valid RSV string", nameof( rsvString ) ); - - /// - public ReadOnlySeString ResolveOrSelf( ReadOnlySeString rsvString ) => - ( rsvString.IsRsv() && TryGetValue( rsvString, out var replacedString ) ) ? replacedString : rsvString; - - /// - public ReadOnlySeString? TryResolve( ReadOnlySeString rsvString ) => - ( rsvString.IsRsv() && TryGetValue( rsvString, out var replacedString ) ) ? replacedString : default; -} diff --git a/src/Lumina/Excel/Rsv/IRsvProvider.cs b/src/Lumina/Excel/Rsv/IRsvProvider.cs deleted file mode 100644 index 60978c3b..00000000 --- a/src/Lumina/Excel/Rsv/IRsvProvider.cs +++ /dev/null @@ -1,39 +0,0 @@ -using Lumina.Text.ReadOnly; - -namespace Lumina.Excel.Rsv; - -/// -/// Represents a provider for resolving RSV strings. -/// -public interface IRsvProvider -{ - /// - /// Determines if the provider can resolve the given RSV string. - /// - /// The string to check. - /// Whether or not the provider contains a resolved string for . - bool CanResolve( ReadOnlySeString rsvString ); - - /// - /// Tries to resolve the given RSV string. - /// - /// The string to resolve. - /// The newly resolved string. Returns if it could not be resolved. - ReadOnlySeString? TryResolve( ReadOnlySeString rsvString ); - - /// - /// Resolves the given RSV string. - /// - /// The string to resolve. - /// The newly resolved string. - /// Thrown if the RSV string is not valid. must return true. - /// Thrown if the RSV key is not found in the provider. - ReadOnlySeString Resolve( ReadOnlySeString rsvString ); - - /// - /// Tries to resolve the given RSV string. - /// - /// The string to resolve. - /// The newly resolved string. Returns if it could not be resolved. - ReadOnlySeString ResolveOrSelf( ReadOnlySeString rsvString ); -} diff --git a/src/Lumina/Excel/Rsv/RsvUtil.cs b/src/Lumina/Excel/Rsv/RsvUtil.cs deleted file mode 100644 index 633ae331..00000000 --- a/src/Lumina/Excel/Rsv/RsvUtil.cs +++ /dev/null @@ -1,36 +0,0 @@ -using Lumina.Text.ReadOnly; -using System; - -namespace Lumina.Excel.Rsv; - -/// -/// Utility class for RSV string operations. -/// -public static class RsvUtil -{ - // RsvPrefix => _rsv_ - private static ReadOnlySpan RsvPrefix => [0x5F, 0x72, 0x73, 0x76, 0x5F]; - - /// - /// Checks if the string is an RSV string and can therefore be resolved. - /// - /// This only checks if the string begins with "_rsv_". - /// The string to check. - /// Whether or not the string is an RSV string. - public static bool IsRsv( this ReadOnlySeString rsvString ) => - rsvString.Data.Span.StartsWith( RsvPrefix ); - - /// - public static bool IsRsv( this ReadOnlySeStringSpan rsvString ) => - rsvString.Data.StartsWith( RsvPrefix ); - - /// - /// Attempts to resolve with the given . - /// - /// This method is safe to call on strings that are not RSVs, i.e., where returns . - /// The string to resolve. - /// The provider to check with. - /// The newly resolved string. Returns if it could not be resolved. - public static ReadOnlySeString ResolveRsv( this ReadOnlySeString rsvString, IRsvProvider provider ) => - provider.ResolveOrSelf( rsvString ); -} diff --git a/src/Lumina/Extensions/RsvExtensions.cs b/src/Lumina/Extensions/RsvExtensions.cs index 87c69e86..92bde4ed 100644 --- a/src/Lumina/Extensions/RsvExtensions.cs +++ b/src/Lumina/Extensions/RsvExtensions.cs @@ -1,34 +1,26 @@ +using Lumina.Text.ReadOnly; using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Reflection; +using System.Runtime.CompilerServices; -namespace Lumina.Extensions -{ - public static class RsvExtensions - { - public static List< string > GetEmbeddedRsvResources( this Assembly assembly ) - { - return assembly.GetManifestResourceNames().Where( x => x.EndsWith( ".rsv" ) ).ToList(); - } - - public static void RegisterRsvFiles( this Assembly assembly, GameData gameData ) - { - throw new NotImplementedException(); - //var rsv = gameData.Excel.RsvProvider; - - //foreach( var file in GetEmbeddedRsvResources( assembly ) ) - //{ - // gameData.Logger?.Information( "Loading RSV: {RsvFileName}", file ); +namespace Lumina.Extensions; - // using var s = assembly.GetManifestResourceStream( file ); - // using var sr = new StreamReader( s! ); - - // var data = sr.ReadToEnd(); +/// +/// Utility class for RSV string operations. +/// +public static class RsvExtensions +{ + /// + /// Checks if the string is an RSV string and can therefore be resolved. + /// + /// This only checks if the string begins with _rsv_. + /// The string to check. + /// Whether or not the string is an RSV string. + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + public static bool IsRsv( this ReadOnlySeString rsvString ) => + rsvString.Data.Span.StartsWith( "_rsv_"u8 ); - // rsv.ParseData( data ); - //} - } - } + /// + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + public static bool IsRsv( this ReadOnlySeStringSpan rsvString ) => + rsvString.Data.StartsWith( "_rsv_"u8 ); } \ No newline at end of file diff --git a/src/Lumina/LuminaOptions.cs b/src/Lumina/LuminaOptions.cs index 15da396a..0594782f 100644 --- a/src/Lumina/LuminaOptions.cs +++ b/src/Lumina/LuminaOptions.cs @@ -1,4 +1,5 @@ using Lumina.Data; +using Lumina.Excel; namespace Lumina { @@ -31,14 +32,9 @@ public class LuminaOptions public bool PanicOnSheetChecksumMismatch { get; set; } = true; /// - /// If enabled, when a cast fails in an excel sheet, an InvalidCastException will be thrown instead of the types default value being inserted instead. + /// The resolver delegate to use when resolving RSV strings. Leave if you don't need it. /// - public bool ExcelSheetStrictCastingEnabled { get; set; } = false; - - /// - /// Whether or not known RSV values in sheets should be resolved when loading sheets - /// - public bool ResolveKnownRsvSheetValues { get; set; } = true; + public ExcelModule.ResolveRsvDelegate? RsvResolver { get; set; } /// /// If enabled, resources will be loaded using multiple threads. From 2f075cc87d8e8a28655547ba88566d3f9ec11dfc Mon Sep 17 00:00:00 2001 From: Soreepeong Date: Fri, 16 Aug 2024 20:12:46 +0900 Subject: [PATCH 18/53] Some speedups * `IExcelRow.RowId` and `IExcelRow.SubrowId` exist to implement `ICollection.Contains`. * `System.Collections.Generic.EnumerableHelpers.ToArray` and alike in LINQ has an optimization for `ICollection`. As it requires `Contains` to be implemented, exposing `RowId` and `SubrowId` in a generic way will make it possible to implement that in O(1). * `ExcelSheet` constructor now preallocates lookup lookup tables. * `.exh` file comes with information on how many rows are there, so we know the exact number of items that needs to be allocated. * Using an array directly bypassing list wrappers may provide an additional speed boost. * In case `.exh` file contains a wrong information on number of rows, which is an unlikely case, `Array.Resize` is used to reallocate the array. * `ExcelSheet.UnsafeCreateRow/Subrow/At` has been added. * These functions assume that boundary checks are done by callers. * As enumerators always work inside the boundary, especially when the collection is immutable, `IEnumerator{T}.Current` can skip boundary checks. * `DefaultExcelSheet` and `SubrowsExcelSheet` has been added. * As sheets are usually not meaningful without knowing what is in it in the first place, it would be better to specialize for each variants. * This effectively hides subrow operations from sheets of default variants. * This removes `HasSubrows` check from getter functions. * Added `SubrowsExcelSheet.Try/GetRow/OrDefault` variants that returns `SubrowCollection` instead. * This makes it convenient to iterate over subrows under one row ID. * This makes it faster to access multiple subrows under one row ID, as lookup operation is done on obtaining the collection. Once the collection is constructed, accessing subrows is an O(1) operation. * `ExcelModule.GetSheet` uses static lambda in place of `SheetCache.GetOrAdd`. * This will avoid heap allocation if a corresponding sheet is already loaded. * Named value tuples in `ExcelSheet` are replaced with `record struct`. * This reduces the size of each lookup element from 16 bytes to 12 bytes. --- src/Lumina.Tests/SeStringBuilderTests.cs | 6 +- src/Lumina/Excel/DefaultExcelSheet.cs | 155 ++++++ .../ExcelLanguageNotSupportedException.cs | 30 ++ src/Lumina/Excel/ExcelModule.cs | 148 ++++-- src/Lumina/Excel/ExcelSheet.Collection.cs | 153 ------ src/Lumina/Excel/ExcelSheet.cs | 469 +++++++----------- src/Lumina/Excel/IExcelRow.cs | 15 +- src/Lumina/Excel/IExcelSheet.cs | 94 ---- src/Lumina/Excel/RowRef.cs | 60 ++- src/Lumina/Excel/SubrowCollection.cs | 141 ++++++ src/Lumina/Excel/SubrowsExcelSheet.cs | 365 ++++++++++++++ src/Lumina/GameData.cs | 34 +- 12 files changed, 1056 insertions(+), 614 deletions(-) create mode 100644 src/Lumina/Excel/DefaultExcelSheet.cs create mode 100644 src/Lumina/Excel/ExcelLanguageNotSupportedException.cs delete mode 100644 src/Lumina/Excel/ExcelSheet.Collection.cs delete mode 100644 src/Lumina/Excel/IExcelSheet.cs create mode 100644 src/Lumina/Excel/SubrowCollection.cs create mode 100644 src/Lumina/Excel/SubrowsExcelSheet.cs diff --git a/src/Lumina.Tests/SeStringBuilderTests.cs b/src/Lumina.Tests/SeStringBuilderTests.cs index aab67490..c2f1ef96 100644 --- a/src/Lumina.Tests/SeStringBuilderTests.cs +++ b/src/Lumina.Tests/SeStringBuilderTests.cs @@ -296,7 +296,9 @@ public readonly struct Addon( ExcelPage page, uint offset, uint row ) : IExcelRo { public uint RowId => row; - public readonly ReadOnlySeString Text => page.ReadString( offset, offset ); + ushort IExcelRow< Addon >.SubrowId => throw new NotSupportedException(); + + public ReadOnlySeString Text => page.ReadString( offset, offset ); static Addon IExcelRow.Create( ExcelPage page, uint offset, uint row ) => new( page, offset, row ); @@ -309,7 +311,7 @@ static Addon IExcelRow.Create( ExcelPage page, uint offset, uint row, ush public void AddonIsParsedCorrectly() { var gameData = new GameData( @"C:\Program Files (x86)\SquareEnix\FINAL FANTASY XIV - A Realm Reborn\game\sqpack" ); - var addon = gameData.Excel.GetSheet( )!; + var addon = gameData.Excel.GetDefaultSheet< Addon >(); var ssb = new SeStringBuilder(); var expected = new Dictionary< uint, ReadOnlySeString > { diff --git a/src/Lumina/Excel/DefaultExcelSheet.cs b/src/Lumina/Excel/DefaultExcelSheet.cs new file mode 100644 index 00000000..4b7948d2 --- /dev/null +++ b/src/Lumina/Excel/DefaultExcelSheet.cs @@ -0,0 +1,155 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using Lumina.Data; +using Lumina.Data.Files.Excel; +using Lumina.Data.Structs.Excel; + +namespace Lumina.Excel; + +/// An excel sheet of variant. +/// Type of the rows contained within. +public sealed class DefaultExcelSheet< T > : ExcelSheet, ICollection, IReadOnlyCollection where T : struct, IExcelRow< T > +{ + internal DefaultExcelSheet( ExcelModule module, ExcelHeaderFile headerFile, Language requestedLanguage, string sheetName ) + : base( module, headerFile, requestedLanguage, sheetName ) + { } + + /// + bool ICollection< T >.IsReadOnly => true; + + /// + public T this[ uint rowId ] { + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + get => GetRow( rowId ); + } + + /// + /// Tries to get the th row in this sheet. If this sheet has subrows, it will return the first subrow. + /// + /// The row id to get. + /// A nullable row object. Returns null if the row does not exist. + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + public T? GetRowOrDefault( uint rowId ) + { + ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); + return Unsafe.IsNullRef(in lookup) ? null : UnsafeCreateRow< T >( in lookup ); + } + + /// + /// Tries to get the th row in this sheet. If this sheet has subrows, it will return the first subrow. + /// + /// The row id to get. + /// The output row object. + /// if the row exists and is written to and otherwise. + public bool TryGetRow( uint rowId, out T row ) + { + ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); + if( Unsafe.IsNullRef(in lookup) ) + { + row = default; + return false; + } + + row = UnsafeCreateRow< T >( in lookup ); + return true; + } + + /// + /// Gets the th row in this sheet. If this sheet has subrows, it will return the first subrow. + /// + /// The row id to get. + /// A row object. + /// Throws when the row id does not have a row attached to it. + public T GetRow( uint rowId ) + { + ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); + return Unsafe.IsNullRef(in lookup) ? throw new ArgumentOutOfRangeException( nameof( rowId ), rowId, null ) : UnsafeCreateRow< T >( in lookup ); + } + + /// + /// Gets the th row in this sheet, ordered by row id in ascending order. If this sheet has subrows, it will return the first subrow. + /// + /// If you are looking to find a row by its id, use instead. + /// The zero-based index of this row. + /// A row object. + public T GetRowAt( int rowIndex ) + { + ArgumentOutOfRangeException.ThrowIfNegative( rowIndex ); + ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual( rowIndex, OffsetLookupTable.Length ); + + return UnsafeCreateRowAt< T >( rowIndex ); + } + + /// + public Enumerator GetEnumerator() => new( this ); + + /// + public bool Contains( T item ) => TryGetRow( item.RowId, out var row ) && EqualityComparer< T >.Default.Equals( item, row ); + + /// + public void CopyTo( T[] array, int arrayIndex ) + { + ArgumentNullException.ThrowIfNull( array ); + ArgumentOutOfRangeException.ThrowIfNegative( arrayIndex ); + if( Count > array.Length - arrayIndex ) + throw new ArgumentException( "The number of elements in the source list is greater than the available space." ); + foreach (var lookup in OffsetLookupTable) + array[ arrayIndex++ ] = UnsafeCreateRow( lookup ); + } + + /// + void ICollection< T >.Add( T item ) => throw new NotSupportedException(); + + /// + void ICollection< T >.Clear() => throw new NotSupportedException(); + + /// + bool ICollection< T >.Remove( T item ) => throw new NotSupportedException(); + + /// + IEnumerator< T > IEnumerable< T >.GetEnumerator() => GetEnumerator(); + + /// + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + + /// Represents an enumerator that iterates over all rows in a . + /// The sheet to iterate over. + public struct Enumerator( DefaultExcelSheet< T > sheet ) : IEnumerator< T > + { + private int _index = -1; + + /// + public readonly T Current { + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + get => sheet.UnsafeCreateRowAt< T >( _index ); + } + + /// + readonly object IEnumerator.Current { + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + get => Current; + } + + /// + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + public bool MoveNext() + { + if( ++_index < sheet.Count ) + return true; + + --_index; + return false; + } + + /// + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + public void Reset() => _index = -1; + + /// + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + public readonly void Dispose() + { } + } +} \ No newline at end of file diff --git a/src/Lumina/Excel/ExcelLanguageNotSupportedException.cs b/src/Lumina/Excel/ExcelLanguageNotSupportedException.cs new file mode 100644 index 00000000..cd6af860 --- /dev/null +++ b/src/Lumina/Excel/ExcelLanguageNotSupportedException.cs @@ -0,0 +1,30 @@ +using System; + +namespace Lumina.Excel; + +/// Exception indicating that the requested language is not supported by the requested sheet. +public sealed class ExcelLanguageNotSupportedException : ArgumentOutOfRangeException +{ + private const string ErrorMessage = "Specified excel language is not supported for the sheet."; + + /// + public ExcelLanguageNotSupportedException() : base( null, ErrorMessage ) + { } + + /// + public ExcelLanguageNotSupportedException( string? paramName ) : this( ErrorMessage, paramName ) + { } + + /// + public ExcelLanguageNotSupportedException( string? message, Exception? innerException ) : base( message ?? ErrorMessage, innerException ) + { } + + /// + public ExcelLanguageNotSupportedException( string? paramName, object? actualValue, string? message ) + : base( paramName, actualValue, message ?? ErrorMessage ) + { } + + /// + public ExcelLanguageNotSupportedException( string? paramName, string? message ) : base( paramName, message ?? ErrorMessage ) + { } +} \ No newline at end of file diff --git a/src/Lumina/Excel/ExcelModule.cs b/src/Lumina/Excel/ExcelModule.cs index 4928beac..6334a0b0 100644 --- a/src/Lumina/Excel/ExcelModule.cs +++ b/src/Lumina/Excel/ExcelModule.cs @@ -7,6 +7,9 @@ using System.ComponentModel; using System.Diagnostics.CodeAnalysis; using System.IO; +using System.Reflection; +using System.Runtime.CompilerServices; +using Lumina.Data.Structs.Excel; namespace Lumina.Excel; @@ -23,7 +26,7 @@ public class ExcelModule internal ResolveRsvDelegate? RsvResolver => GameData.Options.RsvResolver; - private ConcurrentDictionary<(Type sheetType, Language requestedLanguage), IExcelSheet> SheetCache { get; } = []; + private ConcurrentDictionary< (Type Type, Language Language), ExcelSheet > SheetCache { get; } = []; /// /// A delegate provided by the user to resolve RSV strings. @@ -31,12 +34,12 @@ public class ExcelModule /// The string to resolve. It is guaranteed that this string it begins with _rsv_. /// The output resolved string. /// if resolved and is written to and otherwise. - public delegate bool ResolveRsvDelegate(ReadOnlySeString rsvString, out ReadOnlySeString resolvedString); + public delegate bool ResolveRsvDelegate( ReadOnlySeString rsvString, out ReadOnlySeString resolvedString ); /// /// Get the names of all available sheets, parsed from root.exl. /// - public IReadOnlyCollection SheetNames { get; } + public IReadOnlyCollection< string > SheetNames { get; } /// @@ -48,7 +51,7 @@ public ExcelModule( GameData gameData ) { GameData = gameData; - var files = GameData.GetFile( "exd/root.exl" ) ?? + var files = GameData.GetFile< ExcelListFile >( "exd/root.exl" ) ?? throw new FileNotFoundException( "Unable to load exd/root.exl!" ); GameData.Logger?.Information( "got {ExltEntryCount} exlt entries", files.ExdMap.Count ); @@ -56,51 +59,128 @@ public ExcelModule( GameData gameData ) SheetNames = [.. files.ExdMap.Keys]; } - /// - /// Loads an , optionally with a specific language. - /// + /// Loads an . + /// The requested sheet language. Leave or empty to use the default language. + /// An instance of corresponding to and that may be created anew or + /// reused from a previous invocation of this method. /// - /// If the requested language doesn't exist for the file, this will silently be ignored and it will return a sheet with the default language: . + /// If the requested language doesn't exist for the file where is not , the language-neutral + /// sheet using will be loaded instead. If the language-neutral sheet does not exist, then the function will fail with + /// . /// - /// A struct that implements to parse rows. - /// The requested sheet language. Leave or empty to use the default language. - /// An if the sheet exists - /// Thrown when is not decorated with a . /// Sheet does not exist or if the column hash has a mismatch. - public ExcelSheet GetSheet( Language? language = null ) where T : struct, IExcelRow - { - language ??= Language; - - return (ExcelSheet)SheetCache.GetOrAdd( (typeof( T ), language.Value), _ => new ExcelSheet( this, language.Value ) ); - } + /// Sheet does not support nor . + /// Sheet is not of the variant . + public DefaultExcelSheet< T > GetDefaultSheet< T >( Language? language = null ) where T : struct, IExcelRow< T > => + (DefaultExcelSheet< T >) GetSheet< T >( language ); - /// - /// Loads an from a reflected , optionally with a specific language. - /// + /// Loads an . + /// The requested sheet language. Leave or empty to use the default language. + /// An instance of corresponding to and that may be created anew or + /// reused from a previous invocation of this method. /// - /// Only use this method if you need to create a sheet while using reflection. - /// - /// If the requested language doesn't exist for the file, this will silently be ignored and it will return a sheet with the default language: . + /// If the requested language doesn't exist for the file where is not , the language-neutral + /// sheet using will be loaded instead. If the language-neutral sheet does not exist, then the function will fail with + /// . /// - /// A that implements to parse rows. + /// Sheet does not exist or if the column hash has a mismatch. + /// Sheet does not support nor . + /// Sheet is not of the variant . + public SubrowsExcelSheet< T > GetSubrowsSheet< T >( Language? language = null ) where T : struct, IExcelRow< T > => + (SubrowsExcelSheet< T >) GetSheet< T >( language ); + + /// Loads an . + /// Type of the rows in the sheet. /// The requested sheet language. Leave or empty to use the default language. - /// An if the sheet exists. - /// Thrown when is not decorated with a . + /// An instance of corresponding to and that may be created anew or + /// reused from a previous invocation of this method. + /// + /// Only use this method if you need to create a sheet while using reflection. + /// If the requested language doesn't exist for the file where is not , the language-neutral + /// sheet using will be loaded instead. If the language-neutral sheet does not exist, then the function will fail with + /// . + /// The returned instance of should be cast to or + /// before accessing its rows. + /// /// Sheet does not exist or if the column hash has a mismatch. + /// Sheet does not support nor . [RequiresDynamicCode( "Creating a generic sheet from a type requires reflection and dynamic code." )] [EditorBrowsable( EditorBrowsableState.Advanced )] - public IExcelSheet GetSheetGeneric( Type rowType, Language? language = null ) + public ExcelSheet GetSheet( Type rowType, Language? language = null ) { if( !rowType.IsValueType ) - throw new ArgumentException( "rowType must be a struct", nameof( rowType ) ); + throw new ArgumentException( $"{nameof( rowType )} must be a struct.", nameof( rowType ) ); if( !rowType.IsAssignableTo( typeof( IExcelRow<> ).MakeGenericType( rowType ) ) ) - { - throw new ArgumentException( "rowType implement IExcelRow", nameof( rowType ) ); - } + throw new ArgumentException( $"{nameof( rowType )} must implement {typeof( IExcelRow<> ).Name}.", nameof( rowType ) ); + + var sheet = SheetCache.GetOrAdd( + ( rowType, language ?? Language ), + static ( key, module ) => { + var m = typeof( ExcelSheet ) + .GetMethod( nameof( ExcelSheet.From ), BindingFlags.Static | BindingFlags.Public )! + .MakeGenericMethod( key.Type ); + try + { + return m.Invoke( null, [module, key.Language] ) as ExcelSheet ?? throw new InvalidOperationException( "Something went wrong" ); + } + catch( ExcelLanguageNotSupportedException ) + { + return LanguageNotSupportedPlaceholder.Instance; + } + }, + this ); + + if( sheet != LanguageNotSupportedPlaceholder.Instance ) + return sheet; + if( language == Language.None ) + throw new ExcelLanguageNotSupportedException( nameof( language ), language, null ); + return GetSheet( rowType, Language.None ); + } + + /// Loads an . + /// The requested sheet language. Leave or empty to use the default language. + /// An instance of corresponding to and that may be created anew or + /// reused from a previous invocation of this method. + /// + /// If the requested language doesn't exist for the file where is not , the language-neutral + /// sheet using will be loaded instead. If the language-neutral sheet does not exist, then the function will fail with + /// . + /// The returned instance of should be cast to or + /// before accessing its rows. + /// + /// Sheet does not exist or if the column hash has a mismatch. + /// Sheet does not support nor . + [EditorBrowsable( EditorBrowsableState.Advanced )] + public ExcelSheet GetSheet< T >( Language? language = null ) where T : struct, IExcelRow< T > + { + var sheet = SheetCache.GetOrAdd( + ( typeof( T ), language ?? Language ), + static ( key, module ) => { + try + { + return ExcelSheet.From< T >( module, key.Language ); + } + catch( ExcelLanguageNotSupportedException ) + { + return LanguageNotSupportedPlaceholder.Instance; + } + }, + this ); + + if( sheet != LanguageNotSupportedPlaceholder.Instance ) + return sheet; + if( language == Language.None ) + throw new ExcelLanguageNotSupportedException( nameof( language ), language, null ); + return GetSheet< T >( Language.None ); + } - language ??= Language; + private sealed class LanguageNotSupportedPlaceholder : ExcelSheet + { + public static readonly ExcelSheet Instance = (ExcelSheet) RuntimeHelpers.GetUninitializedObject( typeof( LanguageNotSupportedPlaceholder ) ); - return SheetCache.GetOrAdd( (rowType, language.Value), _ => (IExcelSheet)Activator.CreateInstance( typeof( ExcelSheet<> ).MakeGenericType( rowType ), this, language.Value )! ); + internal LanguageNotSupportedPlaceholder( ExcelModule module, ExcelHeaderFile headerFile, Language requestedLanguage, string sheetName ) + : base( module, headerFile, requestedLanguage, sheetName ) + { } } } \ No newline at end of file diff --git a/src/Lumina/Excel/ExcelSheet.Collection.cs b/src/Lumina/Excel/ExcelSheet.Collection.cs deleted file mode 100644 index ad0521e0..00000000 --- a/src/Lumina/Excel/ExcelSheet.Collection.cs +++ /dev/null @@ -1,153 +0,0 @@ -using System; -using System.Collections; -using System.Collections.Generic; - -namespace Lumina.Excel; - -public sealed partial class ExcelSheet : IReadOnlyCollection where T : struct, IExcelRow -{ - /// - public int Count => RowLookup.Count; - - private readonly int subrowCount; - - /// - public int SubrowCount => HasSubrows ? subrowCount : throw new NotSupportedException( "This sheet that doesn't support subrows." ); - - /// - /// Returns an enumerator that can be used to iterate over all rows in this sheet. If this sheet has subrows, it will iterate over every subrow of every row. - /// - /// An of all rows or subrows in this sheet. - public SheetEnumerator GetEnumerator() => - new( this ); - - /// - IEnumerator IEnumerable.GetEnumerator() => - GetEnumerator(); - - /// - IEnumerator IEnumerable.GetEnumerator() => - GetEnumerator(); - - /// - /// Returns an enumerator that can be used to iterate over all rows in this sheet. If this sheet has subrows, it will iterate over the first subrow of every row. - /// - /// An of all rows (or first subrows) in this sheet. - public RowEnumerator GetRowEnumerator() => - new( this ); - - /// - /// Represents an enumerator that iterates over all rows/subrows in a . - /// - /// The sheet to iterate over. - public struct SheetEnumerator( ExcelSheet sheet ) : IEnumerator, IEnumerable - { - private int LookupIndex { get; set; } = -1; - private ushort SubrowIndex { get; set; } = 0; - private ushort SubrowCount { get; set; } = 0; - - private readonly bool HasSubrows => sheet.HasSubrows; - private readonly int RowCount => sheet.Count; - - /// - public readonly T Current => !HasSubrows ? sheet.GetRowAt( LookupIndex ) : sheet.GetSubrowAt( LookupIndex, SubrowIndex ); - - /// - readonly object IEnumerator.Current => Current; - - /// - public bool MoveNext() - { - if( !HasSubrows ) - { - if( LookupIndex + 1 < RowCount ) - { - LookupIndex++; - return true; - } - } - else - { - if( SubrowIndex + 1 < SubrowCount && SubrowCount != 0 ) - { - SubrowIndex++; - return true; - } - else if( LookupIndex + 1 < RowCount ) - { - LookupIndex++; - SubrowIndex = 0; - SubrowCount = sheet.Subrows![LookupIndex].Data.RowCount; - return true; - } - } - return false; - } - - /// - public void Reset() - { - LookupIndex = -1; - SubrowIndex = 0; - SubrowCount = 0; - } - - /// - public readonly void Dispose() - { - - } - - /// - public readonly IEnumerator GetEnumerator() => this; - - /// - readonly IEnumerator IEnumerable.GetEnumerator() => this; - } - - /// - /// Represents an enumerator that iterates over all rows or the first subrow of every row in a . - /// - /// The sheet to iterate over. - public struct RowEnumerator( ExcelSheet sheet ) : IEnumerator, IEnumerable - { - private int LookupIndex { get; set; } = -1; - - private readonly int RowCount => sheet.Count; - - /// - public readonly T Current => sheet.GetRowAt( LookupIndex ); - - /// - readonly object IEnumerator.Current => Current; - - /// - public bool MoveNext() - { - if( LookupIndex + 1 < RowCount ) - { - LookupIndex++; - return true; - } - return false; - } - - /// - public void Reset() - { - LookupIndex = -1; - } - - /// - public readonly void Dispose() - { - - } - - /// - public readonly IEnumerator GetEnumerator() => this; - - /// - readonly IEnumerator IEnumerable.GetEnumerator() => this; - } -} diff --git a/src/Lumina/Excel/ExcelSheet.cs b/src/Lumina/Excel/ExcelSheet.cs index c8df0ebf..025c81ee 100644 --- a/src/Lumina/Excel/ExcelSheet.cs +++ b/src/Lumina/Excel/ExcelSheet.cs @@ -4,354 +4,233 @@ using System; using System.Collections.Frozen; using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Reflection; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; namespace Lumina.Excel; -/// -/// A strongly-typed wrapper around an excel sheet. -/// -/// The row type. -public sealed partial class ExcelSheet : IExcelSheet where T : struct, IExcelRow +/// A wrapper around an excel sheet. +public abstract class ExcelSheet { - /// - public ExcelModule Module { get; } - - /// - public Language Language { get; } + private readonly ExcelPage[] _pages; + private readonly RowOffsetLookup[] _rowOffsetLookupTable; + private readonly ushort _subrowDataOffset; - /// - [MemberNotNullWhen( true, nameof( Subrows ), nameof( SubrowDataOffset ) )] - [MemberNotNullWhen( false, nameof( Rows ) )] - public bool HasSubrows { get; } - - /// - public IReadOnlyList Columns { get; } - - private List Pages { get; } - private (uint RowId, (int PageIdx, uint Offset) Data)[]? Rows { get; } - private (uint RowId, (int PageIdx, uint Offset, ushort RowCount) Data)[]? Subrows { get; } // RowLookup must use int as the key because it benefits from a fast path that removes indirections. // https://github.com/dotnet/runtime/blob/release/8.0/src/libraries/System.Collections.Immutable/src/System/Collections/Frozen/FrozenDictionary.cs#L140 - private FrozenDictionary RowLookup { get; } - private ushort SubrowDataOffset { get; } - - private static SheetAttribute Attribute => - typeof( T ).GetCustomAttribute() ?? - throw new InvalidOperationException( "T has no SheetAttribute. Use the explicit sheet constructor." ); + private readonly FrozenDictionary< int, int > _rowIndexLookupTable; - /// - public T this[uint rowId] => GetRow( rowId ); - - /// - public T this[uint rowId, ushort subrowId] => GetSubrow( rowId, subrowId ); + /// The module that this sheet belongs to. + public ExcelModule Module { get; } - /// - /// Create an instance with the 's default language. - /// - /// The to access sheet data from. - /// does not have a valid . - /// parameters were invalid (hash mismatch or invalid sheet name). - public ExcelSheet( ExcelModule module ) : this( module, module.Language ) - { + /// The language of the rows in this sheet. + /// This can be different from the requested language if it wasn't supported. + public Language Language { get; } - } + /// Contains information on the columns in this sheet. + public IReadOnlyList< ExcelColumnDefinition > Columns { get; } - /// - /// Create an instance with a specific . - /// - /// The to access sheet data from. - /// The language to use for this sheet. - /// does not have a valid . - /// parameters were invalid (hash mismatch or invalid sheet name). - public ExcelSheet( ExcelModule module, Language requestedLanguage ) : this( module, requestedLanguage, Attribute.Name, Attribute.ColumnHash ) + private protected ExcelSheet( ExcelModule module, ExcelHeaderFile headerFile, Language requestedLanguage, string sheetName ) { + var hasSubrows = headerFile.Header.Variant == ExcelVariant.Subrows; - } - - /// - /// Create an instance with a specific , name, and hash. - /// - /// The to access sheet data from. - /// The language to use for this sheet. - /// The name of the sheet to read from. - /// The hash of the columns in the sheet. If , it will not check the hash. - /// or parameters were invalid (hash mismatch or invalid sheet name). - public ExcelSheet( ExcelModule module, Language requestedLanguage, string sheetName, uint? columnHash = null ) - { Module = module; - - var headerFile = module.GameData.GetFile( $"exd/{sheetName}.exh" ) ?? - throw new ArgumentException( "Invalid sheet name", nameof( sheetName ) ); - - if( module.VerifySheetChecksums && columnHash is { } hash && headerFile.GetColumnsHash() != hash ) - throw new ArgumentException( "Column hash mismatch", nameof( columnHash ) ); - - HasSubrows = headerFile.Header.Variant == ExcelVariant.Subrows; - Language = headerFile.Languages.Contains( requestedLanguage ) ? requestedLanguage : Language.None; - Columns = headerFile.ColumnDefinitions; + _subrowDataOffset = hasSubrows ? headerFile.Header.DataOffset : (ushort) 0; + _pages = new ExcelPage[headerFile.DataPages.Length]; + _rowOffsetLookupTable = new RowOffsetLookup[headerFile.Header.RowCount]; - List<(uint RowId, (int PageIdx, uint Offset) Data)>? rows = null; - List<(uint RowId, (int PageIdx, uint Offset, ushort RowCount) Data)>? subrows = null; - var totalSubrowCount = 0; - - if( HasSubrows ) - { - subrows = new( (int)headerFile.Header.RowCount ); - SubrowDataOffset = headerFile.Header.DataOffset; - } - else - rows = new( (int)headerFile.Header.RowCount ); - - Pages = new( headerFile.DataPages.Length ); - var pageIdx = 0; - foreach( var pageDef in headerFile.DataPages ) + var i = 0; + for( var pageIdx = (ushort) 0; pageIdx < headerFile.DataPages.Length; pageIdx++ ) { - var filePath = Language == Language.None ? - $"exd/{sheetName}_{pageDef.StartId}.exd" : - $"exd/{sheetName}_{pageDef.StartId}_{LanguageUtil.GetLanguageStr( Language )}.exd"; - var fileData = module.GameData.GetFile( filePath ); + var pageDef = headerFile.DataPages[ pageIdx ]; + var filePath = Language == Language.None + ? $"exd/{sheetName}_{pageDef.StartId}.exd" + : $"exd/{sheetName}_{pageDef.StartId}_{LanguageUtil.GetLanguageStr( Language )}.exd"; + var fileData = module.GameData.GetFile< ExcelDataFile >( filePath ); if( fileData == null ) continue; - var newPage = new ExcelPage( Module, fileData.Data, headerFile.Header.DataOffset ); - Pages.Add( newPage ); + var newPage = _pages[ pageIdx ] = new( Module, fileData.Data, headerFile.Header.DataOffset ); + + // If row count information from exh file is incorrect, cope with it. + if( i + fileData.RowData.Count > _rowOffsetLookupTable.Length ) + Array.Resize( ref _rowOffsetLookupTable, i + fileData.RowData.Count ); foreach( var rowPtr in fileData.RowData.Values ) { - var subrowCount = newPage.ReadUInt16( rowPtr.Offset + 4 ); + var subrowCount = hasSubrows ? newPage.ReadUInt16( rowPtr.Offset + 4 ) : (ushort) 1; var rowOffset = rowPtr.Offset + 6; - - if( HasSubrows ) - { - if( subrowCount > 0 ) - { - subrows!.Add( (rowPtr.RowId, (pageIdx, rowOffset, subrowCount)) ); - totalSubrowCount += subrowCount; - } - } - else - rows!.Add( (rowPtr.RowId, (pageIdx, rowOffset)) ); + _rowOffsetLookupTable[ i++ ] = new( rowPtr.RowId, rowOffset, pageIdx, subrowCount ); } - - pageIdx++; - } - - if( HasSubrows ) - { - Subrows = [.. subrows!]; - int i = 0; - RowLookup = subrows!.ToFrozenDictionary( row => (int)row.RowId, _ => i++ ); - subrowCount = totalSubrowCount; } - else - { - Rows = [.. rows!]; - int i = 0; - RowLookup = rows!.ToFrozenDictionary( row => (int)row.RowId, _ => i++ ); - } - } - - /// - public ushort GetColumnOffset( int columnIdx ) => - Columns[columnIdx].Offset; - /// - public bool HasRow( uint rowId ) => - RowLookup.ContainsKey( (int)rowId ); + // If row count information from exh file is incorrect, cope with it. (2) + if( i != _rowOffsetLookupTable.Length ) + Array.Resize( ref _rowOffsetLookupTable, i ); - /// - public bool HasSubrow( uint rowId, ushort subrowId ) - { - if( !HasSubrows ) - throw new NotSupportedException( "Cannot access subrow in a sheet that doesn't support any." ); - - ref readonly var val = ref RowLookup.GetValueRefOrNullRef( (int)rowId ); - if( Unsafe.IsNullRef( in val ) ) - return false; - - return subrowId < Subrows[val].Data.RowCount; - } - - /// - public bool TryGetSubrowCount( uint rowId, out ushort subrowCount ) - { - if( !HasSubrows ) - throw new NotSupportedException( "Cannot access subrow in a sheet that doesn't support any." ); - - ref readonly var val = ref RowLookup.GetValueRefOrNullRef( (int)rowId ); - if( Unsafe.IsNullRef( in val ) ) - { - subrowCount = 0; - return false; - } - - subrowCount = Subrows[val].Data.RowCount; - return true; + i = 0; + _rowIndexLookupTable = _rowOffsetLookupTable.ToFrozenDictionary( static row => (int) row.RowId, _ => i++ ); } - /// - public ushort GetSubrowCount( uint rowId ) - { - if( !HasSubrows ) - throw new NotSupportedException( "Cannot access subrow in a sheet that doesn't support any." ); - - ref readonly var val = ref RowLookup.GetValueRefOrNullRef( (int)rowId ); - if( Unsafe.IsNullRef( in val ) ) - throw new ArgumentOutOfRangeException( nameof( rowId ), "Row does not exist" ); - - return Subrows[val].Data.RowCount; - } - - private T CreateRow( uint rowId, in (int PageIdx, uint Offset) val ) => - T.Create( Pages[val.PageIdx], val.Offset, rowId ); - - private T CreateSubrow( uint rowId, ushort subrowId, in (int PageIdx, uint Offset, ushort RowCount) val ) => - T.Create( Pages[val.PageIdx], val.Offset + 2 + ( subrowId * ( SubrowDataOffset + 2u ) ), rowId, subrowId ); + /// Creates a new instance of with the 's default language, deducing sheet names and column + /// hashes from . + /// The to access sheet data from. + /// does not have a valid . + /// parameters were invalid (hash mismatch or invalid sheet name). + /// A new instance of that should be cast to or + /// before further use. + public static ExcelSheet From< T >( ExcelModule module ) where T : struct, IExcelRow< T > => + From< T >( module, module.Language ); - /// - /// Tries to get the th row in this sheet. If this sheet has subrows, it will return the first subrow. - /// - /// The row id to get. - /// A nullable row object. Returns null if the row does not exist. - public T? GetRowOrDefault( uint rowId ) + /// Creates a new instance of , deducing sheet names and column hashes from . + /// The to access sheet data from. + /// The language to use for this sheet. + /// does not have a valid . + /// parameters were invalid (hash mismatch or invalid sheet name). + /// A new instance of that should be cast to or + /// before further use. + public static ExcelSheet From< T >( ExcelModule module, Language language ) where T : struct, IExcelRow< T > { - if( HasSubrows ) - return GetSubrowOrDefault( rowId, 0 ); - - ref readonly var val = ref RowLookup.GetValueRefOrNullRef( (int)rowId ); - if( Unsafe.IsNullRef( in val ) ) - return null; + var attribute = typeof( T ).GetCustomAttribute< SheetAttribute >() ?? + throw new InvalidOperationException( $"{nameof( T )} has no {nameof( SheetAttribute )}. Use the overload of {nameof( From )} with 4 parameters." ); - return CreateRow( rowId, in Rows[val].Data ); + return From< T >( module, language, attribute.Name, attribute.ColumnHash ); } - - /// - /// Tries to get the th subrow with row id in this sheet. - /// - /// The row id to get. - /// The subrow id to get. - /// A nullable row object. Returns null if the subrow does not exist. - /// Thrown if the sheet does not support subrows. - public T? GetSubrowOrDefault( uint rowId, ushort subrowId ) + /// Creates a new instance of . + /// The to access sheet data from. + /// The language to use for this sheet. + /// The name of the sheet to read from. + /// The hash of the columns in the sheet. If , it will not check the hash. + /// or parameters were invalid (hash mismatch or invalid sheet name). + /// Header file had a value that is not supported. + /// Sheet is of variant, but does not implement it. + /// + /// A new instance of that should be cast to or + /// before further use. + public static ExcelSheet From< T >( ExcelModule module, Language language, string sheetName, uint? columnHash = null ) + where T : struct, IExcelRow< T > { - if( !HasSubrows ) - throw new NotSupportedException( "Cannot access subrow in a sheet that doesn't support any." ); - - ref readonly var val = ref RowLookup.GetValueRefOrNullRef( (int)rowId ); - if( Unsafe.IsNullRef( in val ) ) - return null; + var headerFile = module.GameData.GetFile< ExcelHeaderFile >( $"exd/{sheetName}.exh" ) ?? + throw new ArgumentException( "Invalid sheet name", nameof( sheetName ) ); - if( subrowId >= Subrows[val].Data.RowCount ) - return null; + if( module.VerifySheetChecksums && columnHash is { } hash && headerFile.GetColumnsHash() != hash ) + throw new ArgumentException( "Column hash mismatch", nameof( columnHash ) ); - return CreateSubrow( rowId, subrowId, in Subrows[val].Data ); - } + if( !headerFile.Languages.Contains( language ) ) + throw new ExcelLanguageNotSupportedException(); - /// - /// Tries to get the th row in this sheet. If this sheet has subrows, it will return the first subrow. - /// - /// The row id to get. - /// The output row object. - /// if the row exists and is written to and otherwise. - public bool TryGetRow( uint rowId, out T row ) - { - if( GetRowOrDefault( rowId ) is { } outRow ) + switch( headerFile.Header.Variant ) { - row = outRow; - return true; + case ExcelVariant.Default: + return new DefaultExcelSheet< T >( module, headerFile, language, sheetName ); + case ExcelVariant.Subrows: + return new SubrowsExcelSheet< T >( module, headerFile, language, sheetName ); + case ExcelVariant.Unknown: + default: + throw new NotSupportedException( $"Specified sheet variant {headerFile.Header.Variant} is not supported." ); } - row = default; - return false; } - - /// - /// Tries to get the th subrow with row id in this sheet. - /// - /// The row id to get. - /// The subrow id to get. - /// The output row object. - /// if the subrow exists and is written to and otherwise. - /// Thrown if the sheet does not support subrows. - public bool TryGetSubrow( uint rowId, ushort subrowId, out T subrow ) - { - if( GetSubrowOrDefault( rowId, subrowId ) is { } outSubrow ) - { - subrow = outSubrow; - return true; - } - subrow = default; - return false; + /// The number of rows in this sheet. + /// + /// If this sheet has gaps in row ids, it returns the number of rows that exist, not the highest row id. + /// If this sheet has subrows, this will still return the number of rows and not the total number of subrows. + /// + public int Count { + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + get => _rowOffsetLookupTable.Length; } - /// - /// Gets the th row in this sheet. If this sheet has subrows, it will return the first subrow. - /// - /// The row id to get. - /// A row object. - /// Throws when the row id does not have a row attached to it. - public T GetRow( uint rowId ) => - GetRowOrDefault( rowId ) ?? - throw new ArgumentOutOfRangeException( nameof( rowId ), "Row does not exist" ); - - /// - /// Gets the th subrow with row id in this sheet. Throws if the subrow does not exist. - /// - /// The row id to get. - /// The subrow id to get. - /// A row object. - /// Thrown if the sheet does not support subrows. - /// Thrown if the sheet does not have a row at that . - public T GetSubrow( uint rowId, ushort subrowId ) - { - if( !HasSubrows ) - throw new NotSupportedException( "Cannot access subrow in a sheet that doesn't support any." ); - - ref readonly var val = ref RowLookup.GetValueRefOrNullRef( (int)rowId ); - if( Unsafe.IsNullRef( in val ) ) - throw new ArgumentOutOfRangeException( nameof( rowId ), "Row does not exist" ); - - ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual( subrowId, Subrows[val].Data.RowCount ); - - return CreateSubrow( rowId, subrowId, in Subrows[val].Data ); + /// Gets the offset lookup table. + private protected ReadOnlySpan< RowOffsetLookup > OffsetLookupTable { + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + get => _rowOffsetLookupTable; } - /// - /// Gets the th row in this sheet, ordered by row id in ascending order. If this sheet has subrows, it will return the first subrow. - /// - /// If you are looking to find a row by its id, use instead. - /// The zero-based index of this row. - /// A row object. - public T GetRowAt( int rowIndex ) + /// Gets the offset of the column at in the row data. + /// The index of the column. + /// The offset of the column. + /// Thrown when the column index is invalid. It must be less than .Count. + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + public ushort GetColumnOffset( int columnIdx ) => + Columns[ columnIdx ].Offset; + + /// Whether this sheet has a row with the given . + /// If this sheet has subrows, this will check if the row id has any subrows. + /// The row id to check. + /// Whether the row exists. + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + public bool HasRow( uint rowId ) { - if( HasSubrows ) - return GetSubrowAt( rowIndex, 0 ); - - var data = Rows[rowIndex]; - return CreateRow( data.RowId, in data.Data ); + ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); + return !Unsafe.IsNullRef( in lookup ) && lookup.SubrowCount > 0; } - /// - /// Gets the th subrow of the th row in this sheet, ordered by row id in ascending order. - /// - /// If you are looking to find a subrow by its id, use instead. - /// The zero-based index of this row. - /// The subrow id to get. - /// A row object. - /// Thrown if the sheet does not support subrows. - public T GetSubrowAt( int rowIndex, ushort subrowId ) + /// Gets a row lookup at the given index, if possible. + /// Index of the desired row. + /// Lookup data for the desired row, or a null reference if no corresponding row exists. + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + internal ref readonly RowOffsetLookup GetRowLookupOrNullRef( uint rowId ) { - if( !HasSubrows ) - throw new NotSupportedException( "Cannot access subrow in a sheet that doesn't support any." ); - - var data = Subrows[rowIndex]; - return CreateSubrow( data.RowId, subrowId, in data.Data ); + ref readonly var rowIndexRef = ref _rowIndexLookupTable.GetValueRefOrNullRef( (int) rowId ); + if( Unsafe.IsNullRef( in rowIndexRef ) ) + return ref Unsafe.NullRef(); + return ref UnsafeGetRowLookupAt( rowIndexRef ); } -} + + /// Gets a row lookup at the given index, without checking for bounds or preconditions. + /// Index of the desired row. + /// Lookup data for the desired row. + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + internal ref readonly RowOffsetLookup UnsafeGetRowLookupAt( int rowIndex ) => + ref Unsafe.Add( ref MemoryMarshal.GetArrayDataReference( _rowOffsetLookupTable ), rowIndex ); + + /// Creates a row at the given index, without checking for bounds or preconditions. + /// Index of the desired row. + /// A new instance of . + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + internal T UnsafeCreateRowAt< T >( int rowIndex ) where T : struct, IExcelRow< T > => + UnsafeCreateRow< T >( in UnsafeGetRowLookupAt( rowIndex ) ); + + /// Creates a subrow at the given index, without checking for bounds or preconditions. + /// Index of the desired row. + /// Index of the desired subrow. + /// A new instance of . + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + internal T UnsafeCreateSubrowAt< T >( int rowIndex, ushort subrowId ) where T : struct, IExcelRow< T > => + UnsafeCreateSubrow< T >( in UnsafeGetRowLookupAt( rowIndex ), subrowId ); + + /// Creates a row using the given lookup data, without checking for bounds or preconditions. + /// Lookup data for the desired row. + /// A new instance of . + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + internal T UnsafeCreateRow< T >( scoped in RowOffsetLookup lookup ) where T : struct, IExcelRow< T > => + T.Create( + Unsafe.Add( ref MemoryMarshal.GetArrayDataReference( _pages ), lookup.PageIndex ), + lookup.Offset, + lookup.RowId ); + + /// Creates a subrow using the given lookup data, without checking for bounds or preconditions. + /// Lookup data for the desired row. + /// Index of the desired subrow. + /// A new instance of . + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + internal T UnsafeCreateSubrow< T >( scoped in RowOffsetLookup lookup, ushort subrowId ) where T : struct, IExcelRow< T > => + T.Create( + Unsafe.Add( ref MemoryMarshal.GetArrayDataReference( _pages ), lookup.PageIndex ), + lookup.Offset + 2 + subrowId * ( _subrowDataOffset + 2u ), + lookup.RowId, + subrowId ); + + /// Lookup data for locating backing data for a row. + /// ID of the row. This is separate from the row indices. + /// Byte offset of the row, relative to the beginning of an exd file. + /// Index of the page that contains the data for this row. + /// Number of subrows in the row, or 1 if the sheet does not support subrows. + internal readonly record struct RowOffsetLookup( uint RowId, uint Offset, ushort PageIndex, ushort SubrowCount ); +} \ No newline at end of file diff --git a/src/Lumina/Excel/IExcelRow.cs b/src/Lumina/Excel/IExcelRow.cs index 011a799f..9a823a13 100644 --- a/src/Lumina/Excel/IExcelRow.cs +++ b/src/Lumina/Excel/IExcelRow.cs @@ -1,3 +1,5 @@ +using System; + namespace Lumina.Excel; /// @@ -14,7 +16,7 @@ public interface IExcelRow where T : struct /// /// /// A newly created row object. - /// Thrown when the referenced sheet is using subrows. + /// Thrown when the referenced sheet is using subrows. abstract static T Create( ExcelPage page, uint offset, uint row ); /// @@ -26,6 +28,13 @@ public interface IExcelRow where T : struct /// /// /// A newly created subrow object. - /// Thrown when the referenced sheet is not using subrows. + /// Thrown when the referenced sheet is not using subrows. abstract static T Create( ExcelPage page, uint offset, uint row, ushort subrow ); -} \ No newline at end of file + + /// Gets the row ID. + public uint RowId { get; } + + /// Gets the subrow ID. + /// Thrown when the referenced sheet is not using subrows. + public ushort SubrowId { get; } +} diff --git a/src/Lumina/Excel/IExcelSheet.cs b/src/Lumina/Excel/IExcelSheet.cs deleted file mode 100644 index 0e99fd32..00000000 --- a/src/Lumina/Excel/IExcelSheet.cs +++ /dev/null @@ -1,94 +0,0 @@ -using Lumina.Data; -using Lumina.Data.Structs.Excel; -using System.Collections; -using System.Collections.Generic; - -namespace Lumina.Excel; - -/// -/// A generalized interface that all s implement. -/// -/// This interface exists to assist with more generic and reflection-based operations. -public interface IExcelSheet : IEnumerable -{ - /// - /// The module that this sheet belongs to. - /// - ExcelModule Module { get; } - - /// - /// The language of the rows in this sheet. - /// - /// This can be different from the requested language if it wasn't supported. - Language Language { get; } - - /// - /// Whether or not this sheet has subrows, where each row id can have multiple subrows. - /// - bool HasSubrows { get; } - - /// - /// Contains information on the columns in this sheet. - /// - IReadOnlyList Columns { get; } - - /// - /// The number of rows in this sheet. - /// - /// - /// If this sheet has gaps in row ids, it returns the number of rows that exist, not the highest row id. - /// If this sheet has subrows, this will still return the number of rows and not the total number of subrows. - /// - int Count { get; } - - /// - /// The total number of subrows in this sheet across all rows. - /// - /// Thrown if the sheet does not support subrows. - int SubrowCount { get; } - - /// - /// Gets the offset of the column at in the row data. - /// - /// The index of the column. - /// The offset of the column. - /// Thrown when the column index is invalid. It must be less than .Count. - ushort GetColumnOffset( int columnIdx ); - - /// - /// Whether or not this sheet has a row with the given . - /// - /// - /// If this sheet has subrows, this will check if the row id has any subrows. - /// - /// The row id to check. - /// Whether or not the row exists. - bool HasRow( uint rowId ); - - /// - /// Whether or not this sheet has a subrow with the given and . - /// - /// The row id to check. - /// The subrow id to check. - /// Whether or not the subrow exists. - /// Thrown if the sheet does not support subrows. - bool HasSubrow( uint rowId, ushort subrowId ); - - /// - /// Tries to get the number of subrows in the th row in this sheet. - /// - /// The row id to get. - /// The number of subrows in this row. - /// if the row exists and is written to and otherwise. - /// Thrown if the sheet does not support subrows. - bool TryGetSubrowCount( uint rowId, out ushort subrowCount ); - - /// - /// Gets the number of subrows in the th row in this sheet. Throws if the row does not exist. - /// - /// The row id to get. - /// The number of subrows in this row. Returns null if the row does not exist. - /// Thrown if the sheet does not support subrows. - /// Thrown if the sheet does not have a row at that . - ushort GetSubrowCount( uint rowId ); -} \ No newline at end of file diff --git a/src/Lumina/Excel/RowRef.cs b/src/Lumina/Excel/RowRef.cs index 4d1ae5b5..bed8870d 100644 --- a/src/Lumina/Excel/RowRef.cs +++ b/src/Lumina/Excel/RowRef.cs @@ -1,4 +1,5 @@ using System; +using System.Runtime.CompilerServices; namespace Lumina.Excel; @@ -16,7 +17,7 @@ public readonly struct RowRef( ExcelModule? module, uint rowId, Type? rowType ) public uint RowId => rowId; /// - /// Whether or not the is untyped. + /// Whether the is untyped. /// /// /// An untyped is one that doesn't know which sheet it links to. @@ -24,11 +25,11 @@ public readonly struct RowRef( ExcelModule? module, uint rowId, Type? rowType ) public bool IsUntyped => rowType == null; /// - /// Whether or not the reference is of a specific row type. + /// Whether the reference is of a specific row type. /// /// The row type/schema to check against. - /// Whether or not this points to a . - public bool Is() where T : struct, IExcelRow => + /// Whether this points to a . + public bool Is< T >() where T : struct, IExcelRow< T > => typeof( T ) == rowType; /// @@ -36,12 +37,12 @@ public bool Is() where T : struct, IExcelRow => /// /// The row type/schema to check against. /// The referenced row type. Returns null if this does not point to a or if the does not exist in its sheet. - public T? GetValueOrDefault() where T : struct, IExcelRow + public T? GetValueOrDefault< T >() where T : struct, IExcelRow< T > { - if( !Is() ) + if( !Is< T >() || module is null ) return null; - return module!.GetSheet().GetRowOrDefault( RowId ); + return new RowRef< T >( module, rowId ).ValueNullable; } /// @@ -50,15 +51,16 @@ public bool Is() where T : struct, IExcelRow => /// The row type/schema to check against. /// The output row object. /// if the type is valid, the row exists, and is written to, and otherwise. - public bool TryGetValue( out T row ) where T : struct, IExcelRow + public bool TryGetValue< T >( out T row ) where T : struct, IExcelRow< T > { - if( !Is() ) + if( new RowRef< T >( module, rowId ).ValueNullable is { } v ) { - row = default; - return false; + row = v; + return true; } - return module!.GetSheet().TryGetRow( RowId, out row ); + row = default; + return false; } /// @@ -72,7 +74,7 @@ public static RowRef GetFirstValidRowOrUntyped( ExcelModule module, uint rowId, { foreach( var sheetType in sheetTypes ) { - if( module.GetSheetGeneric( sheetType ) is { } sheet ) + if( module.GetSheet( sheetType ) is { } sheet ) { if( sheet.HasRow( rowId ) ) return new( module, rowId, sheetType ); @@ -88,8 +90,8 @@ public static RowRef GetFirstValidRowOrUntyped( ExcelModule module, uint rowId, /// The row type referenced by the . /// The to read sheet data from. /// The referenced row id. - /// A to a row in a . - public static RowRef Create( ExcelModule module, uint rowId ) where T : struct, IExcelRow => new( module, rowId, typeof( T ) ); + /// A to a row in a . + public static RowRef Create< T >( ExcelModule? module, uint rowId ) where T : struct, IExcelRow< T > => new( module, rowId, typeof( T ) ); /// /// Creates an untyped . @@ -105,9 +107,9 @@ public static RowRef GetFirstValidRowOrUntyped( ExcelModule module, uint rowId, /// The row type referenced by the . /// The to read sheet data from. /// The referenced row id. -public readonly struct RowRef( ExcelModule module, uint rowId ) where T : struct, IExcelRow +public readonly struct RowRef< T >( ExcelModule? module, uint rowId ) where T : struct, IExcelRow< T > { - private readonly ExcelSheet sheet = module.GetSheet(); + private readonly ExcelSheet? _sheet = module?.GetSheet< T >(); /// /// The row id of the referenced row. @@ -115,26 +117,34 @@ public readonly struct RowRef( ExcelModule module, uint rowId ) where T : str public uint RowId => rowId; /// - /// Whether or not the exists in the sheet. + /// Whether the exists in the sheet. /// - public bool IsValid => sheet.HasRow( RowId ); + public bool IsValid => _sheet?.HasRow( RowId ) ?? false; /// /// The referenced row value itself. /// - /// Thrown if is false. - public T Value => sheet.GetRow( RowId ); + /// Thrown if is false. + public T Value => ValueNullable ?? throw new InvalidOperationException(); /// /// Attempts to get the referenced row value. Is null if does not exist in the sheet. /// - public T? ValueNullable => sheet.GetRowOrDefault( RowId ); + public T? ValueNullable { + get { + if( _sheet is null ) + return null; - private RowRef ToGeneric() => RowRef.Create( module, rowId ); + ref readonly var lookup = ref _sheet.GetRowLookupOrNullRef( rowId ); + return Unsafe.IsNullRef( in lookup ) ? null : _sheet.UnsafeCreateRow< T >( lookup ); + } + } + + private RowRef ToGeneric() => RowRef.Create< T >( module, rowId ); /// - /// Converts a concrete to an generic and dynamically typed . + /// Converts a concrete to a generic and dynamically typed . /// /// The to convert. - public static explicit operator RowRef( RowRef row ) => row.ToGeneric(); + public static explicit operator RowRef( RowRef< T > row ) => row.ToGeneric(); } \ No newline at end of file diff --git a/src/Lumina/Excel/SubrowCollection.cs b/src/Lumina/Excel/SubrowCollection.cs new file mode 100644 index 00000000..457ab543 --- /dev/null +++ b/src/Lumina/Excel/SubrowCollection.cs @@ -0,0 +1,141 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Runtime.CompilerServices; + +namespace Lumina.Excel; + +/// Collection of subrows under one row. +/// Type of the row. +public readonly struct SubrowCollection< T > : IList< T >, IReadOnlyList< T > + where T : struct, IExcelRow< T > +{ + private readonly ExcelSheet.RowOffsetLookup _lookup; + + internal SubrowCollection( SubrowsExcelSheet< T > sheet, scoped in ExcelSheet.RowOffsetLookup lookup ) + { + Sheet = sheet; + _lookup = lookup; + } + + /// Gets the associated sheet. + public SubrowsExcelSheet< T > Sheet { get; } + + /// Gets the Row ID of the subrows contained within. + public uint RowId { + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + get => _lookup.RowId; + } + + /// + public int Count { + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + get => _lookup.SubrowCount; + } + + /// + bool ICollection< T >.IsReadOnly => true; + + /// + public T this[ int index ] { + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + get { + ArgumentOutOfRangeException.ThrowIfNegative( index ); + ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual( index, Count ); + return Sheet.UnsafeCreateSubrow< T >( _lookup, unchecked( (ushort) index ) ); + } + } + + /// + T IList< T >.this[ int index ] { + get => this[ index ]; + set => throw new NotSupportedException(); + } + + /// + public int IndexOf( T item ) + { + if( item.RowId != RowId || item.SubrowId >= Count ) + return -1; + + var row = Sheet.UnsafeCreateSubrow< T >( _lookup, item.SubrowId ); + return EqualityComparer< T >.Default.Equals( item, row ) ? item.SubrowId : -1; + } + + /// + public bool Contains( T item ) => IndexOf( item ) != -1; + + /// + public void CopyTo( T[] array, int arrayIndex ) + { + ArgumentNullException.ThrowIfNull( array ); + ArgumentOutOfRangeException.ThrowIfNegative( arrayIndex ); + if( Count > array.Length - arrayIndex ) + throw new ArgumentException( "The number of elements in the source list is greater than the available space." ); + for( var i = 0; i < Count; i++ ) + array[ arrayIndex++ ] = Sheet.UnsafeCreateSubrow< T >( _lookup, unchecked( (ushort) i ) ); + } + + /// + void IList< T >.Insert( int index, T item ) => throw new NotSupportedException(); + + /// + void IList< T >.RemoveAt( int index ) => throw new NotSupportedException(); + + /// + void ICollection< T >.Add( T item ) => throw new NotSupportedException(); + + /// + void ICollection< T >.Clear() => throw new NotSupportedException(); + + /// + bool ICollection< T >.Remove( T item ) => throw new NotSupportedException(); + + /// + public Enumerator GetEnumerator() => new( this ); + + /// + IEnumerator< T > IEnumerable< T >.GetEnumerator() => GetEnumerator(); + + /// + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + + /// Enumerator that enumerates over subrows under one row. + /// Subrow collection to iterate over. + public struct Enumerator( SubrowCollection< T > subrowCollection ) : IEnumerator< T > + { + private int _index = -1; + + /// + public readonly T Current { + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + get => subrowCollection.Sheet.UnsafeCreateSubrow< T >( subrowCollection._lookup, unchecked( (ushort) _index ) ); + } + + /// + readonly object IEnumerator.Current { + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + get => Current; + } + + /// + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + public bool MoveNext() + { + if( ++_index < subrowCollection.Count ) + return true; + + --_index; + return false; + } + + /// + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + public void Reset() => _index = -1; + + /// + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + public readonly void Dispose() + { } + } +} \ No newline at end of file diff --git a/src/Lumina/Excel/SubrowsExcelSheet.cs b/src/Lumina/Excel/SubrowsExcelSheet.cs new file mode 100644 index 00000000..524b7b30 --- /dev/null +++ b/src/Lumina/Excel/SubrowsExcelSheet.cs @@ -0,0 +1,365 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using Lumina.Data; +using Lumina.Data.Files.Excel; +using Lumina.Data.Structs.Excel; + +namespace Lumina.Excel; + +/// An excel sheet of variant. +public abstract class SubrowsExcelSheet : ExcelSheet +{ + private protected SubrowsExcelSheet( ExcelModule module, ExcelHeaderFile headerFile, Language requestedLanguage, string sheetName ) + : base( module, headerFile, requestedLanguage, sheetName ) + { + foreach( var f in OffsetLookupTable ) + TotalSubrowCount += f.SubrowCount; + } + + /// + /// The total number of subrows in this sheet across all rows. + /// + public int TotalSubrowCount { get; } + + /// + /// Whether this sheet has a subrow with the given and . + /// + /// The row id to check. + /// The subrow id to check. + /// Whether the subrow exists. + public bool HasSubrow( uint rowId, ushort subrowId ) + { + ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); + return !Unsafe.IsNullRef( in lookup ) && subrowId < lookup.SubrowCount; + } + + /// + /// Tries to get the number of subrows in the th row in this sheet. + /// + /// The row id to get. + /// The number of subrows in this row. + /// if the row exists and is written to and otherwise. + public bool TryGetSubrowCount( uint rowId, out ushort subrowCount ) + { + ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); + if( Unsafe.IsNullRef( in lookup ) ) + { + subrowCount = 0; + return false; + } + + subrowCount = lookup.SubrowCount; + return true; + } + + /// + /// Gets the number of subrows in the th row in this sheet. Throws if the row does not exist. + /// + /// The row id to get. + /// The number of subrows in this row. Returns null if the row does not exist. + /// Thrown if the sheet does not have a row at that . + public ushort GetSubrowCount( uint rowId ) + { + ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); + return Unsafe.IsNullRef( in lookup ) ? throw new ArgumentOutOfRangeException( nameof( rowId ), rowId, null ) : lookup.SubrowCount; + } +} + +/// An excel sheet of variant. +/// Type of the rows contained within. +public sealed class SubrowsExcelSheet< T > + : SubrowsExcelSheet, ICollection< SubrowCollection< T > >, IReadOnlyCollection< SubrowCollection< T > > + where T : struct, IExcelRow< T > +{ + internal SubrowsExcelSheet( ExcelModule module, ExcelHeaderFile headerFile, Language requestedLanguage, string sheetName ) + : base( module, headerFile, requestedLanguage, sheetName ) + { + } + + /// + bool ICollection< SubrowCollection< T > >.IsReadOnly => true; + + /// + public SubrowCollection< T > this[ uint rowId ] { + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + get => GetRow( rowId ); + } + + /// + public T this[ uint rowId, ushort subrowId ] { + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + get => GetSubrow( rowId, subrowId ); + } + + /// + /// Tries to get the subrow collection with row id in this sheet. + /// + /// The row id to get. + /// A nullable subrow collection object. Returns null if the row does not exist. + public SubrowCollection< T >? GetRowOrDefault( uint rowId ) + { + ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); + return Unsafe.IsNullRef( in lookup ) ? null : new( this, lookup ); + } + + /// + /// Tries to get the subrow collection with row id in this sheet. + /// + /// The row id to get. + /// The output row object. + /// if the row exists and is written to and otherwise. + public bool TryGetRow( uint rowId, out SubrowCollection< T > row ) + { + ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); + if( Unsafe.IsNullRef( in lookup ) ) + { + row = default; + return false; + } + + row = new( this, lookup ); + return true; + } + + /// + /// Gets the subrow collection with row id in this sheet. Throws if the row does not exist. + /// + /// The row id to get. + /// A row object. + /// Thrown if the sheet does not have a row at that . + public SubrowCollection< T > GetRow( uint rowId ) + { + ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); + return Unsafe.IsNullRef( in lookup ) ? throw new ArgumentOutOfRangeException( nameof( rowId ), rowId, null ) : new( this, lookup ); + } + + /// + /// Gets the subrow collection of the th row in this sheet, ordered by row id in ascending order. + /// + /// If you are looking to find a row by its id, use instead. + /// The zero-based index of this row. + /// A row object. + public SubrowCollection< T > GetRowAt( int rowIndex ) + { + ArgumentOutOfRangeException.ThrowIfNegative( rowIndex ); + ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual( rowIndex, OffsetLookupTable.Length ); + + return new( this, UnsafeGetRowLookupAt( rowIndex ) ); + } + + /// + /// Tries to get the th subrow with row id in this sheet. + /// + /// The row id to get. + /// The subrow id to get. + /// A nullable row object. Returns null if the subrow does not exist. + public T? GetSubrowOrDefault( uint rowId, ushort subrowId ) + { + ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); + return Unsafe.IsNullRef( in lookup ) || subrowId >= lookup.SubrowCount ? null : UnsafeCreateSubrow< T >( in lookup, subrowId ); + } + + /// + /// Tries to get the th subrow with row id in this sheet. + /// + /// The row id to get. + /// The subrow id to get. + /// The output row object. + /// if the subrow exists and is written to and otherwise. + public bool TryGetSubrow( uint rowId, ushort subrowId, out T subrow ) + { + ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); + if( Unsafe.IsNullRef( in lookup ) || subrowId >= lookup.SubrowCount ) + { + subrow = default; + return false; + } + + subrow = UnsafeCreateSubrow< T >( in lookup, subrowId ); + return true; + } + + /// + /// Gets the th subrow with row id in this sheet. Throws if the subrow does not exist. + /// + /// The row id to get. + /// The subrow id to get. + /// A row object. + /// Thrown if the sheet does not have a row at that . + public T GetSubrow( uint rowId, ushort subrowId ) + { + ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); + if( Unsafe.IsNullRef( in lookup ) ) + throw new ArgumentOutOfRangeException( nameof( rowId ), rowId, null ); + + ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual( subrowId, lookup.SubrowCount ); + + return UnsafeCreateSubrow< T >( in lookup, subrowId ); + } + + /// + /// Gets the th subrow of the th row in this sheet, ordered by row id in ascending order. + /// + /// If you are looking to find a subrow by its id, use instead. + /// The zero-based index of this row. + /// The subrow id to get. + /// A row object. + public T GetSubrowAt( int rowIndex, ushort subrowId ) + { + var offsetLookupTable = OffsetLookupTable; + ArgumentOutOfRangeException.ThrowIfNegative( rowIndex ); + ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual( rowIndex, offsetLookupTable.Length ); + + ref readonly var lookup = ref UnsafeGetRowLookupAt( rowIndex ); + ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual( subrowId, lookup.SubrowCount ); + + return UnsafeCreateSubrow< T >( in lookup, subrowId ); + } + + /// + public bool Contains( SubrowCollection< T > item ) => ReferenceEquals( item.Sheet, this ) && HasRow( item.RowId ); + + /// + public void CopyTo( SubrowCollection< T >[] array, int arrayIndex ) + { + ArgumentNullException.ThrowIfNull( array ); + ArgumentOutOfRangeException.ThrowIfNegative( arrayIndex ); + if( Count > array.Length - arrayIndex ) + throw new ArgumentException( "The number of elements in the source list is greater than the available space." ); + foreach( var lookup in OffsetLookupTable ) + array[ arrayIndex++ ] = new( this, lookup ); + } + + /// + void ICollection< SubrowCollection< T > >.Add( SubrowCollection< T > item ) => throw new NotSupportedException(); + + /// + void ICollection< SubrowCollection< T > >.Clear() => throw new NotSupportedException(); + + /// + bool ICollection< SubrowCollection< T > >.Remove( SubrowCollection< T > item ) => throw new NotSupportedException(); + + /// Gets an enumerator that enumerates over all subrows. + /// A new enumerator. + public FlatEnumerator Flatten() => new( this ); + + /// + public Enumerator GetEnumerator() => new( this ); + + /// + IEnumerator< SubrowCollection< T > > IEnumerable< SubrowCollection< T > >.GetEnumerator() => GetEnumerator(); + + /// + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + + /// Represents an enumerator that iterates over all rows in a . + /// The sheet to iterate over. + public struct Enumerator( SubrowsExcelSheet< T > sheet ) : IEnumerator< SubrowCollection< T > > + { + private int _index = -1; + + /// + public readonly SubrowCollection< T > Current { + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + get => new( sheet, sheet.UnsafeGetRowLookupAt( _index ) ); + } + + /// + readonly object IEnumerator.Current { + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + get => Current; + } + + /// + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + public bool MoveNext() + { + if( ++_index < sheet.Count ) + return true; + + --_index; + return false; + } + + /// + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + public void Reset() => _index = -1; + + /// + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + public readonly void Dispose() + { } + } + + /// Represents an enumerator that iterates over all subrows in a . + /// The sheet to iterate over. + public struct FlatEnumerator( SubrowsExcelSheet< T > sheet ) : IEnumerator< T >, IEnumerable< T > + { + private int _index = -1; + private ushort _subrowIndex = ushort.MaxValue; + private ushort _subrowCount; + + /// + public readonly T Current { + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + get => sheet.UnsafeCreateSubrowAt< T >( _index, _subrowIndex ); + } + + /// + readonly object IEnumerator.Current { + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + get => Current; + } + + /// + public bool MoveNext() + { + if( ++_subrowIndex >= _subrowCount ) + { + while (true) + { + if( ++_index >= sheet.Count ) + { + --_subrowIndex; + --_index; + return false; + } + + _subrowCount = sheet.UnsafeGetRowLookupAt( _index ).SubrowCount; + if( _subrowCount == 0 ) + continue; + _subrowIndex = 0; + return true; + } + } + + return true; + } + + /// + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + public void Reset() + { + _index = -1; + _subrowIndex = ushort.MaxValue; + _subrowCount = 0; + } + + /// + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + public readonly void Dispose() + { } + + /// + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + public FlatEnumerator GetEnumerator() => new( sheet ); + + /// + IEnumerator< T > IEnumerable< T >.GetEnumerator() => GetEnumerator(); + + /// + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } +} \ No newline at end of file diff --git a/src/Lumina/GameData.cs b/src/Lumina/GameData.cs index 6f169768..7cbf7f6f 100644 --- a/src/Lumina/GameData.cs +++ b/src/Lumina/GameData.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using Lumina.Data; using Lumina.Data.Structs; +using Lumina.Data.Structs.Excel; using Lumina.Excel; using Lumina.Misc; @@ -286,20 +287,37 @@ public static UInt64 GetFileHash( string path ) return (UInt64) Crc32.Get( folder ) << 32 | Crc32.Get( filename ); } - /// - /// Attempts to load an , optionally with a specific language - /// + /// Loads an . + /// The requested sheet language. Leave or empty to use the default language. + /// An instance of corresponding to and that may be created anew or + /// reused from a previous invocation of this method, or if a corresponding sheet could not be loaded. + /// Thrown when is not decorated with a + /// Sheet is not of the variant . + public DefaultExcelSheet< T >? GetDefaultExcelSheet< T >( Language? language = null ) where T : struct, IExcelRow< T > + { + try + { + return Excel.GetDefaultSheet< T >( language ); + } + catch( ArgumentException ) + { + return null; + } + } + + /// Loads an . /// The requested sheet language. Leave or empty to use the default language. - /// A struct that implements to parse rows - /// An if the sheet exists, null if it does not + /// An instance of corresponding to and that may be created anew or + /// reused from a previous invocation of this method, or if a corresponding sheet could not be loaded. /// Thrown when is not decorated with a - public ExcelSheet< T >? GetExcelSheet< T >( Language? language = null ) where T : struct, IExcelRow + /// Sheet is not of the variant . + public SubrowsExcelSheet< T >? GetSubrowsExcelSheet< T >( Language? language = null ) where T : struct, IExcelRow< T > { try { - return Excel.GetSheet(language); + return Excel.GetSubrowsSheet< T >( language ); } - catch (ArgumentException) + catch( ArgumentException ) { return null; } From 79c90a587e0f7c0bf8d8997ec658dbe0865d94c1 Mon Sep 17 00:00:00 2001 From: Soreepeong Date: Fri, 16 Aug 2024 23:30:34 +0900 Subject: [PATCH 19/53] Add row index lookup array Most of sheets do not have large gaps across items. That fact can be used to make a lookup array instead of lookup dictionary, which will even further reduce the time spent translating row ID to row index. --- src/Lumina/Excel/ExcelSheet.cs | 112 ++++++++++++++++++++++++++++++--- 1 file changed, 104 insertions(+), 8 deletions(-) diff --git a/src/Lumina/Excel/ExcelSheet.cs b/src/Lumina/Excel/ExcelSheet.cs index 025c81ee..a770102e 100644 --- a/src/Lumina/Excel/ExcelSheet.cs +++ b/src/Lumina/Excel/ExcelSheet.cs @@ -14,13 +14,56 @@ namespace Lumina.Excel; /// A wrapper around an excel sheet. public abstract class ExcelSheet { + /// Number of items in that may resolve to no entry. + // 7.05h: across 7292 sheets that exist and are referenced from exlt file, following ratio can be represented solely using lookup array of certain sizes. + // Max Gap, Coverage, Net Wasted + // 1024, 99.15%, 38KB + // 2048, 99.25%, 82KB + // 3072, 99.29%, 109KB + // 4096, 99.36%, 183KB + // 5120, 99.40%, 239KB + // 6144, 99.41%, 259KB + // 9216, 99.42%, 295KB + // 10240, 99.47%, 410KB + // 14336, 99.48%, 463KB + // 16384, 99.49%, 525KB + // 19456, 99.51%, 599KB + // 24576, 99.52%, 692KB + // 26624, 99.53%, 793KB + // 28672, 99.56%, 1011KB + // 29696, 99.57%, 1127KB + // 30720, 99.59%, 1244KB + // 33792, 99.63%, 1633KB + // 34816, 99.64%, 1765KB + // 41984, 99.67%, 2089KB + // 43008, 99.68%, 2255KB + // 44032, 99.71%, 2594KB + // 50176, 99.73%, 2789KB + // 64512, 99.74%, 3041KB + // 65536, 99.75%, 3293KB + // 70656, 99.84%, 4941KB + // 71680, 99.88%, 5773KB + // 89088, 99.89%, 6118KB + // 720896, 99.90%, 8934KB + // 721920, 99.92%, 11754KB + // 1049600, 99.93%, 15853KB + // 1507328, 99.95%, 21741KB + // 2001920, 99.96%, 29559KB + // 2990080, 99.97%, 41236KB + // 9832448, 99.99%, 79643KB + // 10146816, 100.00%, 119276KB + // We're allowing up to 65536 lookup items in _rowOffsetLookupTable, at cost of up to 3293KB of lookup items that resolve to nonexistence per language. + private const int MaxUnusedLookupItemCount = 65536; + private readonly ExcelPage[] _pages; private readonly RowOffsetLookup[] _rowOffsetLookupTable; private readonly ushort _subrowDataOffset; // RowLookup must use int as the key because it benefits from a fast path that removes indirections. // https://github.com/dotnet/runtime/blob/release/8.0/src/libraries/System.Collections.Immutable/src/System/Collections/Frozen/FrozenDictionary.cs#L140 - private readonly FrozenDictionary< int, int > _rowIndexLookupTable; + private readonly FrozenDictionary< int, int > _rowIndexLookupDict; + + private readonly int[] _rowIndexLookupArray; /// The module that this sheet belongs to. public ExcelModule Module { get; } @@ -72,8 +115,55 @@ private protected ExcelSheet( ExcelModule module, ExcelHeaderFile headerFile, La if( i != _rowOffsetLookupTable.Length ) Array.Resize( ref _rowOffsetLookupTable, i ); - i = 0; - _rowIndexLookupTable = _rowOffsetLookupTable.ToFrozenDictionary( static row => (int) row.RowId, _ => i++ ); + // A lot of sheets do not have large gap between row IDs. If total number of gaps is less than a threshold, then make a lookup array. + if( _rowOffsetLookupTable.Length > 0 ) + { + var firstId = _rowOffsetLookupTable[ 0 ].RowId; + var numSlots = _rowOffsetLookupTable[ ^1 ].RowId - firstId + 1; + var numUnused = numSlots - headerFile.Header.RowCount; + if( numUnused <= MaxUnusedLookupItemCount ) + { + _rowIndexLookupArray = new int[ numSlots ]; + _rowIndexLookupArray.AsSpan().Fill( -1 ); + for (i = 0; i < _rowOffsetLookupTable.Length; i++) + _rowIndexLookupArray[_rowOffsetLookupTable[ i ].RowId - firstId] = i; + + // All items can be looked up from _rowIndexLookupArray. Dictionary is unnecessary. + _rowIndexLookupDict = FrozenDictionary< int, int >.Empty; + } + else + { + _rowIndexLookupArray = new int[MaxUnusedLookupItemCount]; + _rowIndexLookupArray.AsSpan().Fill( -1 ); + + var lastLookupArrayRowId = uint.MaxValue; + for( i = 0; i < _rowOffsetLookupTable.Length; i++ ) + { + var offsetRowId = _rowOffsetLookupTable[ i ].RowId - firstId; + if( offsetRowId >= MaxUnusedLookupItemCount ) + { + // Discard the unused entries. + Array.Resize( ref _rowIndexLookupArray, unchecked( (int) ( lastLookupArrayRowId + 1 ) ) ); + break; + } + + _rowIndexLookupArray[ offsetRowId ] = i; + lastLookupArrayRowId = offsetRowId; + } + + // Skip the items that can be looked up from _rowIndexLookupArray. + _rowIndexLookupDict = _rowOffsetLookupTable.Skip( i ).ToFrozenDictionary( static row => (int) row.RowId, _ => i++ ); + } + + Count = _rowOffsetLookupTable.Length; + } + else + { + _rowIndexLookupDict = FrozenDictionary< int, int >.Empty; + _rowIndexLookupArray = []; + _rowOffsetLookupTable = [default]; // so that MemoryMarshal.GetArrayDataReference( _rowOffsetLookupTable ) is always valid. + Count = 0; + } } /// Creates a new instance of with the 's default language, deducing sheet names and column @@ -141,10 +231,7 @@ public static ExcelSheet From< T >( ExcelModule module, Language language, strin /// If this sheet has gaps in row ids, it returns the number of rows that exist, not the highest row id. /// If this sheet has subrows, this will still return the number of rows and not the total number of subrows. /// - public int Count { - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] - get => _rowOffsetLookupTable.Length; - } + public int Count { get; } /// Gets the offset lookup table. private protected ReadOnlySpan< RowOffsetLookup > OffsetLookupTable { @@ -177,7 +264,16 @@ public bool HasRow( uint rowId ) [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] internal ref readonly RowOffsetLookup GetRowLookupOrNullRef( uint rowId ) { - ref readonly var rowIndexRef = ref _rowIndexLookupTable.GetValueRefOrNullRef( (int) rowId ); + var lookupArrayIndex = unchecked( rowId - MemoryMarshal.GetArrayDataReference( _rowOffsetLookupTable ).RowId ); + if( lookupArrayIndex < _rowIndexLookupArray.Length ) + { + var rowIndex = Unsafe.Add( ref MemoryMarshal.GetArrayDataReference( _rowIndexLookupArray ), lookupArrayIndex ); + if (rowIndex == -1) + return ref Unsafe.NullRef(); + return ref UnsafeGetRowLookupAt( rowIndex ); + } + + ref readonly var rowIndexRef = ref _rowIndexLookupDict.GetValueRefOrNullRef( (int) rowId ); if( Unsafe.IsNullRef( in rowIndexRef ) ) return ref Unsafe.NullRef(); return ref UnsafeGetRowLookupAt( rowIndexRef ); From 105fd19ec477ccafa2f076160c967efdab968e3a Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Sat, 17 Aug 2024 03:22:02 -0700 Subject: [PATCH 20/53] Additional changes --- src/Lumina.Tests/SeStringBuilderTests.cs | 2 +- src/Lumina/Excel/BaseExcelSheet.cs | 323 +++++++++++++++ src/Lumina/Excel/BaseSubrowExcelSheet.cs | 66 +++ src/Lumina/Excel/Collection.cs | 98 +++++ src/Lumina/Excel/DefaultExcelSheet.cs | 155 ------- .../ExcelLanguageNotSupportedException.cs | 30 -- src/Lumina/Excel/ExcelModule.cs | 143 +++---- src/Lumina/Excel/ExcelPage.cs | 6 +- src/Lumina/Excel/ExcelSheet.cs | 392 +++++------------- src/Lumina/Excel/LazyCollection.cs | 42 -- .../Excel/MismatchedColumnHashException.cs | 11 + src/Lumina/Excel/RowRef.cs | 6 +- src/Lumina/Excel/SubrowCollection.cs | 64 +-- ...browsExcelSheet.cs => SubrowExcelSheet.cs} | 190 +++------ .../Excel/UnsupportedLanguageException.cs | 30 ++ src/Lumina/Extensions/SpanExtensions.cs | 47 ++- src/Lumina/GameData.cs | 47 ++- 17 files changed, 824 insertions(+), 828 deletions(-) create mode 100644 src/Lumina/Excel/BaseExcelSheet.cs create mode 100644 src/Lumina/Excel/BaseSubrowExcelSheet.cs create mode 100644 src/Lumina/Excel/Collection.cs delete mode 100644 src/Lumina/Excel/DefaultExcelSheet.cs delete mode 100644 src/Lumina/Excel/ExcelLanguageNotSupportedException.cs delete mode 100644 src/Lumina/Excel/LazyCollection.cs create mode 100644 src/Lumina/Excel/MismatchedColumnHashException.cs rename src/Lumina/Excel/{SubrowsExcelSheet.cs => SubrowExcelSheet.cs} (52%) create mode 100644 src/Lumina/Excel/UnsupportedLanguageException.cs diff --git a/src/Lumina.Tests/SeStringBuilderTests.cs b/src/Lumina.Tests/SeStringBuilderTests.cs index c2f1ef96..6ce67166 100644 --- a/src/Lumina.Tests/SeStringBuilderTests.cs +++ b/src/Lumina.Tests/SeStringBuilderTests.cs @@ -311,7 +311,7 @@ static Addon IExcelRow.Create( ExcelPage page, uint offset, uint row, ush public void AddonIsParsedCorrectly() { var gameData = new GameData( @"C:\Program Files (x86)\SquareEnix\FINAL FANTASY XIV - A Realm Reborn\game\sqpack" ); - var addon = gameData.Excel.GetDefaultSheet< Addon >(); + var addon = gameData.Excel.GetSheet< Addon >(); var ssb = new SeStringBuilder(); var expected = new Dictionary< uint, ReadOnlySeString > { diff --git a/src/Lumina/Excel/BaseExcelSheet.cs b/src/Lumina/Excel/BaseExcelSheet.cs new file mode 100644 index 00000000..4d5a1a68 --- /dev/null +++ b/src/Lumina/Excel/BaseExcelSheet.cs @@ -0,0 +1,323 @@ +using Lumina.Data; +using Lumina.Data.Files.Excel; +using Lumina.Data.Structs.Excel; +using Lumina.Extensions; +using System; +using System.Collections.Frozen; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Runtime.CompilerServices; + +namespace Lumina.Excel; + +/// A wrapper around an excel sheet. +public abstract class BaseExcelSheet +{ + /// Number of items in that may resolve to no entry. + // 7.05h: across 7292 sheets that exist and are referenced from exlt file, following ratio can be represented solely using lookup array of certain sizes. + // Max Gap, Coverage, Net Wasted + // 1024, 99.15%, 38KB + // 2048, 99.25%, 82KB + // 3072, 99.29%, 109KB + // 4096, 99.36%, 183KB + // 5120, 99.40%, 239KB + // 6144, 99.41%, 259KB + // 9216, 99.42%, 295KB + // 10240, 99.47%, 410KB + // 14336, 99.48%, 463KB + // 16384, 99.49%, 525KB + // 19456, 99.51%, 599KB + // 24576, 99.52%, 692KB + // 26624, 99.53%, 793KB + // 28672, 99.56%, 1011KB + // 29696, 99.57%, 1127KB + // 30720, 99.59%, 1244KB + // 33792, 99.63%, 1633KB + // 34816, 99.64%, 1765KB + // 41984, 99.67%, 2089KB + // 43008, 99.68%, 2255KB + // 44032, 99.71%, 2594KB + // 50176, 99.73%, 2789KB + // 64512, 99.74%, 3041KB + // 65536, 99.75%, 3293KB + // 70656, 99.84%, 4941KB + // 71680, 99.88%, 5773KB + // 89088, 99.89%, 6118KB + // 720896, 99.90%, 8934KB + // 721920, 99.92%, 11754KB + // 1049600, 99.93%, 15853KB + // 1507328, 99.95%, 21741KB + // 2001920, 99.96%, 29559KB + // 2990080, 99.97%, 41236KB + // 9832448, 99.99%, 79643KB + // 10146816, 100.00%, 119276KB + // We're allowing up to 65536 lookup items in _rowOffsetLookupTable, at cost of up to 3293KB of lookup items that resolve to nonexistence per language. + private const int MaxUnusedLookupItemCount = 65536; + + private readonly ExcelPage[] _pages; + private readonly RowOffsetLookup[] _rowOffsetLookupTable; + private readonly ushort _subrowDataOffset; + + // RowLookup must use int as the key because it benefits from a fast path that removes indirections. + // https://github.com/dotnet/runtime/blob/release/8.0/src/libraries/System.Collections.Immutable/src/System/Collections/Frozen/FrozenDictionary.cs#L140 + private readonly FrozenDictionary< int, int > _rowIndexLookupDict; + + private readonly int[] _rowIndexLookupArray; + + /// The module that this sheet belongs to. + public ExcelModule Module { get; } + + /// The language of the rows in this sheet. + /// This can be different from the requested language if it wasn't supported. + public Language Language { get; } + + /// Contains information on the columns in this sheet. + public IReadOnlyList< ExcelColumnDefinition > Columns { get; } + + private protected BaseExcelSheet( ExcelModule module, ExcelHeaderFile headerFile, Language requestedLanguage, string sheetName ) + { + var hasSubrows = headerFile.Header.Variant == ExcelVariant.Subrows; + + Module = module; + Language = headerFile.Languages.Contains( requestedLanguage ) ? requestedLanguage : Language.None; + Columns = headerFile.ColumnDefinitions; + _subrowDataOffset = hasSubrows ? headerFile.Header.DataOffset : (ushort) 0; + _pages = new ExcelPage[headerFile.DataPages.Length]; + _rowOffsetLookupTable = new RowOffsetLookup[headerFile.Header.RowCount]; + + var i = 0; + for( ushort pageIdx = 0; pageIdx < headerFile.DataPages.Length; pageIdx++ ) + { + var pageDef = headerFile.DataPages[ pageIdx ]; + var filePath = Language == Language.None + ? $"exd/{sheetName}_{pageDef.StartId}.exd" + : $"exd/{sheetName}_{pageDef.StartId}_{LanguageUtil.GetLanguageStr( Language )}.exd"; + var fileData = module.GameData.GetFile< ExcelDataFile >( filePath ); + if( fileData == null ) + continue; + + var newPage = _pages[ pageIdx ] = new( Module, fileData.Data, headerFile.Header.DataOffset ); + + // If row count information from exh file is incorrect, cope with it. + if( i + fileData.RowData.Count > _rowOffsetLookupTable.Length ) + Array.Resize( ref _rowOffsetLookupTable, i + fileData.RowData.Count ); + + foreach( var rowPtr in fileData.RowData.Values ) + { + var subrowCount = hasSubrows ? newPage.ReadUInt16( rowPtr.Offset + 4 ) : (ushort) 1; + var rowOffset = rowPtr.Offset + 6; + _rowOffsetLookupTable[ i++ ] = new( rowPtr.RowId, rowOffset, pageIdx, subrowCount ); + } + } + + // If row count information from exh file is incorrect, cope with it. (2) + if( i != _rowOffsetLookupTable.Length ) + Array.Resize( ref _rowOffsetLookupTable, i ); + + // A lot of sheets do not have large gap between row IDs. If total number of gaps is less than a threshold, then make a lookup array. + if( _rowOffsetLookupTable.Length > 0 ) + { + var firstId = _rowOffsetLookupTable[ 0 ].RowId; + var numSlots = _rowOffsetLookupTable[ ^1 ].RowId - firstId + 1; + var numUnused = numSlots - headerFile.Header.RowCount; + if( numUnused <= MaxUnusedLookupItemCount ) + { + _rowIndexLookupArray = new int[ numSlots ]; + _rowIndexLookupArray.AsSpan().Fill( -1 ); + for (i = 0; i < _rowOffsetLookupTable.Length; i++) + _rowIndexLookupArray[_rowOffsetLookupTable[ i ].RowId - firstId] = i; + + // All items can be looked up from _rowIndexLookupArray. Dictionary is unnecessary. + _rowIndexLookupDict = FrozenDictionary< int, int >.Empty; + } + else + { + _rowIndexLookupArray = new int[MaxUnusedLookupItemCount]; + _rowIndexLookupArray.AsSpan().Fill( -1 ); + + var lastLookupArrayRowId = uint.MaxValue; + for( i = 0; i < _rowOffsetLookupTable.Length; i++ ) + { + var offsetRowId = _rowOffsetLookupTable[ i ].RowId - firstId; + if( offsetRowId >= MaxUnusedLookupItemCount ) + { + // Discard the unused entries. + Array.Resize( ref _rowIndexLookupArray, unchecked( (int) ( lastLookupArrayRowId + 1 ) ) ); + break; + } + + _rowIndexLookupArray[ offsetRowId ] = i; + lastLookupArrayRowId = offsetRowId; + } + + // Skip the items that can be looked up from _rowIndexLookupArray. + _rowIndexLookupDict = _rowOffsetLookupTable.Skip( i ).ToFrozenDictionary( static row => (int) row.RowId, _ => i++ ); + } + + Count = _rowOffsetLookupTable.Length; + } + else + { + _rowIndexLookupDict = FrozenDictionary< int, int >.Empty; + _rowIndexLookupArray = []; + _rowOffsetLookupTable = [default]; // so that _rowOffsetLookupTable.UnsafeAt(0) is always valid. + Count = 0; + } + } + + /// Creates a new instance of with the 's default language, deducing sheet names and column + /// hashes from . + /// The to access sheet data from. + /// does not have a valid . + /// was invalid (invalid sheet name). + /// was invalid (hash mismatch). + /// Sheet had an unsupported language. + /// Header file had a value that is not supported. + /// A new instance of that should be cast to or + /// before further use. + public static BaseExcelSheet From< T >( ExcelModule module ) where T : struct, IExcelRow< T > => + From< T >( module, module.Language ); + + /// Creates a new instance of , deducing sheet names and column hashes from . + /// The to access sheet data from. + /// The language to use for this sheet. + /// does not have a valid . + /// was invalid (invalid sheet name). + /// was invalid (hash mismatch). + /// Sheet had an unsupported language. + /// Header file had a value that is not supported. + /// A new instance of that should be cast to or + /// before further use. + public static BaseExcelSheet From< T >( ExcelModule module, Language language ) where T : struct, IExcelRow< T > + { + var attribute = typeof( T ).GetCustomAttribute< SheetAttribute >() ?? + throw new InvalidOperationException( $"{nameof( T )} has no {nameof( SheetAttribute )}. Use the overload of {nameof( From )} with 4 parameters." ); + + return From< T >( module, language, attribute.Name, attribute.ColumnHash ); + } + + /// Creates a new instance of . + /// The to access sheet data from. + /// The language to use for this sheet. + /// The name of the sheet to read from. + /// The hash of the columns in the sheet. If , it will not check the hash. + /// was invalid (invalid sheet name). + /// was invalid (hash mismatch). + /// Sheet had an unsupported language. + /// Header file had a value that is not supported. + /// A new instance of that should be cast to or + /// before further use. + public static BaseExcelSheet From< T >( ExcelModule module, Language language, string sheetName, uint? columnHash = null ) + where T : struct, IExcelRow< T > + { + var headerFile = module.GameData.GetFile< ExcelHeaderFile >( $"exd/{sheetName}.exh" ) ?? + throw new ArgumentException( "Invalid sheet name", nameof( sheetName ) ); + + if( module.VerifySheetChecksums && columnHash is { } hash && headerFile.GetColumnsHash() != hash ) + throw new MismatchedColumnHashException(hash, headerFile.GetColumnsHash(), nameof(columnHash) ); + + if( !headerFile.Languages.Contains( language ) ) + throw new UnsupportedLanguageException(); + + return headerFile.Header.Variant switch + { + ExcelVariant.Default => new ExcelSheet( module, headerFile, language, sheetName ), + ExcelVariant.Subrows => new SubrowExcelSheet( module, headerFile, language, sheetName ), + _ => throw new NotSupportedException( $"Specified sheet variant {headerFile.Header.Variant} is not supported." ), + }; + } + + /// The number of rows in this sheet. + /// + /// If this sheet has gaps in row ids, it returns the number of rows that exist, not the highest row id. + /// If this sheet has subrows, this will still return the number of rows and not the total number of subrows. + /// + public int Count { get; } + + /// Gets the offset lookup table. + private protected ReadOnlySpan< RowOffsetLookup > OffsetLookupTable => _rowOffsetLookupTable; + + /// Gets the offset of the column at in the row data. + /// The index of the column. + /// The offset of the column. + /// Thrown when the column index is invalid. It must be less than .Count. + public ushort GetColumnOffset( int columnIdx ) => Columns[ columnIdx ].Offset; + + /// Whether this sheet has a row with the given . + /// If this sheet has subrows, this will check if the row id has any subrows. + /// The row id to check. + /// Whether the row exists. + public bool HasRow( uint rowId ) + { + ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); + return !Unsafe.IsNullRef( in lookup ) && lookup.SubrowCount > 0; + } + + /// Gets a row lookup at the given index, if possible. + /// Index of the desired row. + /// Lookup data for the desired row, or a null reference if no corresponding row exists. + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + internal ref readonly RowOffsetLookup GetRowLookupOrNullRef( uint rowId ) + { + var lookupArrayIndex = unchecked( rowId - _rowOffsetLookupTable.UnsafeAt(0).RowId ); + if( lookupArrayIndex < _rowIndexLookupArray.Length ) + { + var rowIndex = _rowIndexLookupArray.UnsafeAt( (int) lookupArrayIndex ); + if (rowIndex == -1) + return ref Unsafe.NullRef(); + return ref UnsafeGetRowLookupAt( rowIndex ); + } + + ref readonly var rowIndexRef = ref _rowIndexLookupDict.GetValueRefOrNullRef( (int) rowId ); + if( Unsafe.IsNullRef( in rowIndexRef ) ) + return ref Unsafe.NullRef(); + return ref UnsafeGetRowLookupAt( rowIndexRef ); + } + + /// Gets a row lookup at the given index, without checking for bounds or preconditions. + /// Index of the desired row. + /// Lookup data for the desired row. + internal ref readonly RowOffsetLookup UnsafeGetRowLookupAt( int rowIndex ) => + ref _rowOffsetLookupTable.UnsafeAt(rowIndex); + + /// Creates a row at the given index, without checking for bounds or preconditions. + /// Index of the desired row. + /// A new instance of . + internal T UnsafeCreateRowAt< T >( int rowIndex ) where T : struct, IExcelRow< T > => + UnsafeCreateRow< T >( in UnsafeGetRowLookupAt( rowIndex ) ); + + /// Creates a subrow at the given index, without checking for bounds or preconditions. + /// Index of the desired row. + /// Index of the desired subrow. + /// A new instance of . + internal T UnsafeCreateSubrowAt< T >( int rowIndex, ushort subrowId ) where T : struct, IExcelRow< T > => + UnsafeCreateSubrow< T >( in UnsafeGetRowLookupAt( rowIndex ), subrowId ); + + /// Creates a row using the given lookup data, without checking for bounds or preconditions. + /// Lookup data for the desired row. + /// A new instance of . + internal T UnsafeCreateRow< T >( scoped ref readonly RowOffsetLookup lookup ) where T : struct, IExcelRow< T > => + T.Create( + _pages.UnsafeAt(lookup.PageIndex), + lookup.Offset, + lookup.RowId ); + + /// Creates a subrow using the given lookup data, without checking for bounds or preconditions. + /// Lookup data for the desired row. + /// Index of the desired subrow. + /// A new instance of . + internal T UnsafeCreateSubrow< T >( scoped ref readonly RowOffsetLookup lookup, ushort subrowId ) where T : struct, IExcelRow< T > => + T.Create( + _pages.UnsafeAt(lookup.PageIndex), + lookup.Offset + 2 + subrowId * ( _subrowDataOffset + 2u ), + lookup.RowId, + subrowId ); + + /// Lookup data for locating backing data for a row. + /// ID of the row. This is separate from the row indices. + /// Byte offset of the row, relative to the beginning of an exd file. + /// Index of the page that contains the data for this row. + /// Number of subrows in the row, or 1 if the sheet does not support subrows. + internal readonly record struct RowOffsetLookup( uint RowId, uint Offset, ushort PageIndex, ushort SubrowCount ); +} \ No newline at end of file diff --git a/src/Lumina/Excel/BaseSubrowExcelSheet.cs b/src/Lumina/Excel/BaseSubrowExcelSheet.cs new file mode 100644 index 00000000..dcc6fc81 --- /dev/null +++ b/src/Lumina/Excel/BaseSubrowExcelSheet.cs @@ -0,0 +1,66 @@ +using System; +using System.Runtime.CompilerServices; +using Lumina.Data; +using Lumina.Data.Files.Excel; +using Lumina.Data.Structs.Excel; + +namespace Lumina.Excel; + +/// An excel sheet of variant. +public abstract class BaseSubrowExcelSheet : BaseExcelSheet +{ + private protected BaseSubrowExcelSheet( ExcelModule module, ExcelHeaderFile headerFile, Language requestedLanguage, string sheetName ) + : base( module, headerFile, requestedLanguage, sheetName ) + { + foreach( var f in OffsetLookupTable ) + TotalSubrowCount += f.SubrowCount; + } + + /// + /// The total number of subrows in this sheet across all rows. + /// + public int TotalSubrowCount { get; } + + /// + /// Whether this sheet has a subrow with the given and . + /// + /// The row id to check. + /// The subrow id to check. + /// Whether the subrow exists. + public bool HasSubrow( uint rowId, ushort subrowId ) + { + ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); + return !Unsafe.IsNullRef( in lookup ) && subrowId < lookup.SubrowCount; + } + + /// + /// Tries to get the number of subrows in the th row in this sheet. + /// + /// The row id to get. + /// The number of subrows in this row. + /// if the row exists and is written to and otherwise. + public bool TryGetSubrowCount( uint rowId, out ushort subrowCount ) + { + ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); + if( Unsafe.IsNullRef( in lookup ) ) + { + subrowCount = 0; + return false; + } + + subrowCount = lookup.SubrowCount; + return true; + } + + /// + /// Gets the number of subrows in the th row in this sheet. Throws if the row does not exist. + /// + /// The row id to get. + /// The number of subrows in this row. Returns null if the row does not exist. + /// Thrown if the sheet does not have a row at that . + public ushort GetSubrowCount( uint rowId ) + { + ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); + return Unsafe.IsNullRef( in lookup ) ? throw new ArgumentOutOfRangeException( nameof( rowId ), rowId, null ) : lookup.SubrowCount; + } +} \ No newline at end of file diff --git a/src/Lumina/Excel/Collection.cs b/src/Lumina/Excel/Collection.cs new file mode 100644 index 00000000..e58b89db --- /dev/null +++ b/src/Lumina/Excel/Collection.cs @@ -0,0 +1,98 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Runtime.CompilerServices; + +namespace Lumina.Excel; + +/// +/// A collection helper used to layout and structure excel rows. +/// +/// Mostly an implementation detail for reading excel rows. This type does not store or hold any row data, and is therefore lightweight and trivially constructable. +/// A type that wraps a group of fields inside a row. +/// +/// +/// +/// +/// +public readonly struct Collection( ExcelPage page, uint parentOffset, uint offset, Func ctor, int size ) : IReadOnlyList, ICollection where T : struct +{ + /// + public T this[int index] { + [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + get { + ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual( index, size ); + return ctor( page, parentOffset, offset, (uint)index ); + } + } + + /// + public int Count => size; + + bool ICollection.IsReadOnly => true; + + void ICollection.Add( T item ) => throw new NotSupportedException(); + + void ICollection.Clear() => throw new NotSupportedException(); + + bool ICollection.Remove( T item ) => throw new NotSupportedException(); + + /// + public bool Contains( T item ) + { + var comparer = EqualityComparer.Default; + foreach (var element in this ) + { + if( comparer.Equals( item, element ) ) + return true; + } + return false; + } + + /// + public void CopyTo( T[] array, int arrayIndex ) + { + ArgumentNullException.ThrowIfNull( array ); + ArgumentOutOfRangeException.ThrowIfNegative( arrayIndex ); + if( Count > array.Length - arrayIndex ) + throw new ArgumentException( "The number of elements in the source list is greater than the available space." ); + for (var i = 0; i < Count; i++ ) + array[ arrayIndex++ ] = this[i]; + } + + /// + public Enumerator GetEnumerator() => new( this ); + + readonly IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + + readonly IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + + /// Enumerator that enumerates over the different items. + /// Collection to iterate over. + public struct Enumerator( Collection collection ) : IEnumerator + { + private int _index = -1; + + /// + public readonly T Current => collection[_index]; + + readonly object IEnumerator.Current => Current; + + /// + public bool MoveNext() + { + if( ++_index < collection.Count ) + return true; + + --_index; + return false; + } + + /// + public void Reset() => _index = -1; + + /// + public readonly void Dispose() + { } + } +} \ No newline at end of file diff --git a/src/Lumina/Excel/DefaultExcelSheet.cs b/src/Lumina/Excel/DefaultExcelSheet.cs deleted file mode 100644 index 4b7948d2..00000000 --- a/src/Lumina/Excel/DefaultExcelSheet.cs +++ /dev/null @@ -1,155 +0,0 @@ -using System; -using System.Collections; -using System.Collections.Generic; -using System.Runtime.CompilerServices; -using Lumina.Data; -using Lumina.Data.Files.Excel; -using Lumina.Data.Structs.Excel; - -namespace Lumina.Excel; - -/// An excel sheet of variant. -/// Type of the rows contained within. -public sealed class DefaultExcelSheet< T > : ExcelSheet, ICollection, IReadOnlyCollection where T : struct, IExcelRow< T > -{ - internal DefaultExcelSheet( ExcelModule module, ExcelHeaderFile headerFile, Language requestedLanguage, string sheetName ) - : base( module, headerFile, requestedLanguage, sheetName ) - { } - - /// - bool ICollection< T >.IsReadOnly => true; - - /// - public T this[ uint rowId ] { - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] - get => GetRow( rowId ); - } - - /// - /// Tries to get the th row in this sheet. If this sheet has subrows, it will return the first subrow. - /// - /// The row id to get. - /// A nullable row object. Returns null if the row does not exist. - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] - public T? GetRowOrDefault( uint rowId ) - { - ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); - return Unsafe.IsNullRef(in lookup) ? null : UnsafeCreateRow< T >( in lookup ); - } - - /// - /// Tries to get the th row in this sheet. If this sheet has subrows, it will return the first subrow. - /// - /// The row id to get. - /// The output row object. - /// if the row exists and is written to and otherwise. - public bool TryGetRow( uint rowId, out T row ) - { - ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); - if( Unsafe.IsNullRef(in lookup) ) - { - row = default; - return false; - } - - row = UnsafeCreateRow< T >( in lookup ); - return true; - } - - /// - /// Gets the th row in this sheet. If this sheet has subrows, it will return the first subrow. - /// - /// The row id to get. - /// A row object. - /// Throws when the row id does not have a row attached to it. - public T GetRow( uint rowId ) - { - ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); - return Unsafe.IsNullRef(in lookup) ? throw new ArgumentOutOfRangeException( nameof( rowId ), rowId, null ) : UnsafeCreateRow< T >( in lookup ); - } - - /// - /// Gets the th row in this sheet, ordered by row id in ascending order. If this sheet has subrows, it will return the first subrow. - /// - /// If you are looking to find a row by its id, use instead. - /// The zero-based index of this row. - /// A row object. - public T GetRowAt( int rowIndex ) - { - ArgumentOutOfRangeException.ThrowIfNegative( rowIndex ); - ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual( rowIndex, OffsetLookupTable.Length ); - - return UnsafeCreateRowAt< T >( rowIndex ); - } - - /// - public Enumerator GetEnumerator() => new( this ); - - /// - public bool Contains( T item ) => TryGetRow( item.RowId, out var row ) && EqualityComparer< T >.Default.Equals( item, row ); - - /// - public void CopyTo( T[] array, int arrayIndex ) - { - ArgumentNullException.ThrowIfNull( array ); - ArgumentOutOfRangeException.ThrowIfNegative( arrayIndex ); - if( Count > array.Length - arrayIndex ) - throw new ArgumentException( "The number of elements in the source list is greater than the available space." ); - foreach (var lookup in OffsetLookupTable) - array[ arrayIndex++ ] = UnsafeCreateRow( lookup ); - } - - /// - void ICollection< T >.Add( T item ) => throw new NotSupportedException(); - - /// - void ICollection< T >.Clear() => throw new NotSupportedException(); - - /// - bool ICollection< T >.Remove( T item ) => throw new NotSupportedException(); - - /// - IEnumerator< T > IEnumerable< T >.GetEnumerator() => GetEnumerator(); - - /// - IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); - - /// Represents an enumerator that iterates over all rows in a . - /// The sheet to iterate over. - public struct Enumerator( DefaultExcelSheet< T > sheet ) : IEnumerator< T > - { - private int _index = -1; - - /// - public readonly T Current { - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] - get => sheet.UnsafeCreateRowAt< T >( _index ); - } - - /// - readonly object IEnumerator.Current { - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] - get => Current; - } - - /// - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] - public bool MoveNext() - { - if( ++_index < sheet.Count ) - return true; - - --_index; - return false; - } - - /// - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] - public void Reset() => _index = -1; - - /// - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] - public readonly void Dispose() - { } - } -} \ No newline at end of file diff --git a/src/Lumina/Excel/ExcelLanguageNotSupportedException.cs b/src/Lumina/Excel/ExcelLanguageNotSupportedException.cs deleted file mode 100644 index cd6af860..00000000 --- a/src/Lumina/Excel/ExcelLanguageNotSupportedException.cs +++ /dev/null @@ -1,30 +0,0 @@ -using System; - -namespace Lumina.Excel; - -/// Exception indicating that the requested language is not supported by the requested sheet. -public sealed class ExcelLanguageNotSupportedException : ArgumentOutOfRangeException -{ - private const string ErrorMessage = "Specified excel language is not supported for the sheet."; - - /// - public ExcelLanguageNotSupportedException() : base( null, ErrorMessage ) - { } - - /// - public ExcelLanguageNotSupportedException( string? paramName ) : this( ErrorMessage, paramName ) - { } - - /// - public ExcelLanguageNotSupportedException( string? message, Exception? innerException ) : base( message ?? ErrorMessage, innerException ) - { } - - /// - public ExcelLanguageNotSupportedException( string? paramName, object? actualValue, string? message ) - : base( paramName, actualValue, message ?? ErrorMessage ) - { } - - /// - public ExcelLanguageNotSupportedException( string? paramName, string? message ) : base( paramName, message ?? ErrorMessage ) - { } -} \ No newline at end of file diff --git a/src/Lumina/Excel/ExcelModule.cs b/src/Lumina/Excel/ExcelModule.cs index 6334a0b0..ef94e35c 100644 --- a/src/Lumina/Excel/ExcelModule.cs +++ b/src/Lumina/Excel/ExcelModule.cs @@ -26,7 +26,7 @@ public class ExcelModule internal ResolveRsvDelegate? RsvResolver => GameData.Options.RsvResolver; - private ConcurrentDictionary< (Type Type, Language Language), ExcelSheet > SheetCache { get; } = []; + private ConcurrentDictionary< (Type Type, Language Language), BaseExcelSheet > SheetCache { get; } = []; /// /// A delegate provided by the user to resolve RSV strings. @@ -59,54 +59,59 @@ public ExcelModule( GameData gameData ) SheetNames = [.. files.ExdMap.Keys]; } - /// Loads an . - /// The requested sheet language. Leave or empty to use the default language. - /// An instance of corresponding to and that may be created anew or - /// reused from a previous invocation of this method. + /// Loads an . + /// Sheet is not of the variant . + /// + public ExcelSheet< T > GetSheet< T >( Language? language = null ) where T : struct, IExcelRow< T > => + (ExcelSheet< T >) GetBaseSheet< T >( language ); + + /// Loads an . /// - /// If the requested language doesn't exist for the file where is not , the language-neutral + /// If the requested language doesn't exist for the file where is not , the language-neutral /// sheet using will be loaded instead. If the language-neutral sheet does not exist, then the function will fail with - /// . + /// . /// - /// Sheet does not exist or if the column hash has a mismatch. - /// Sheet does not support nor . - /// Sheet is not of the variant . - public DefaultExcelSheet< T > GetDefaultSheet< T >( Language? language = null ) where T : struct, IExcelRow< T > => - (DefaultExcelSheet< T >) GetSheet< T >( language ); + /// Sheet is not of the variant . + /// + public SubrowExcelSheet< T > GetSubrowSheet< T >( Language? language = null ) where T : struct, IExcelRow< T > => + (SubrowExcelSheet< T >) GetBaseSheet< T >( language ); - /// Loads an . - /// The requested sheet language. Leave or empty to use the default language. - /// An instance of corresponding to and that may be created anew or + /// An excel sheet corresponding to and that may be created anew or /// reused from a previous invocation of this method. /// - /// If the requested language doesn't exist for the file where is not , the language-neutral + /// If the requested language doesn't exist for the file where is not , the language-neutral /// sheet using will be loaded instead. If the language-neutral sheet does not exist, then the function will fail with - /// . + /// . + /// The returned instance of should be cast to or + /// before accessing its rows. /// - /// Sheet does not exist or if the column hash has a mismatch. - /// Sheet does not support nor . - /// Sheet is not of the variant . - public SubrowsExcelSheet< T > GetSubrowsSheet< T >( Language? language = null ) where T : struct, IExcelRow< T > => - (SubrowsExcelSheet< T >) GetSheet< T >( language ); + /// does not have a valid . + /// + [EditorBrowsable( EditorBrowsableState.Advanced )] + public BaseExcelSheet GetBaseSheet( Language? language = null ) where T : struct, IExcelRow => + GetBaseSheet( typeof( T ), language ); - /// Loads an . + /// Loads an . /// Type of the rows in the sheet. /// The requested sheet language. Leave or empty to use the default language. - /// An instance of corresponding to and that may be created anew or + /// An excel sheet corresponding to and that may be created anew or /// reused from a previous invocation of this method. /// /// Only use this method if you need to create a sheet while using reflection. /// If the requested language doesn't exist for the file where is not , the language-neutral /// sheet using will be loaded instead. If the language-neutral sheet does not exist, then the function will fail with - /// . - /// The returned instance of should be cast to or + /// . + /// The returned instance of should be cast to or /// before accessing its rows. /// - /// Sheet does not exist or if the column hash has a mismatch. - /// Sheet does not support nor . + /// does not have a valid . + /// Sheet does not exist. + /// Sheet had a mismatched column hash. + /// Sheet does not support nor . + /// Sheet had an unsupported . [RequiresDynamicCode( "Creating a generic sheet from a type requires reflection and dynamic code." )] [EditorBrowsable( EditorBrowsableState.Advanced )] - public ExcelSheet GetSheet( Type rowType, Language? language = null ) + public BaseExcelSheet GetBaseSheet( Type rowType, Language? language = null ) { if( !rowType.IsValueType ) throw new ArgumentException( $"{nameof( rowType )} must be a struct.", nameof( rowType ) ); @@ -117,70 +122,46 @@ public ExcelSheet GetSheet( Type rowType, Language? language = null ) var sheet = SheetCache.GetOrAdd( ( rowType, language ?? Language ), static ( key, module ) => { - var m = typeof( ExcelSheet ) - .GetMethod( nameof( ExcelSheet.From ), BindingFlags.Static | BindingFlags.Public )! + var m = typeof( BaseExcelSheet ) + .GetMethod( nameof( BaseExcelSheet.From ), BindingFlags.Static | BindingFlags.Public )! .MakeGenericMethod( key.Type ); try { - return m.Invoke( null, [module, key.Language] ) as ExcelSheet ?? throw new InvalidOperationException( "Something went wrong" ); + return m.Invoke( null, [module, key.Language] ) as BaseExcelSheet ?? throw new InvalidOperationException( "Something went wrong" ); } - catch( ExcelLanguageNotSupportedException ) + catch( Exception e ) { - return LanguageNotSupportedPlaceholder.Instance; + return InvalidSheet.Create( e ); } }, this ); - if( sheet != LanguageNotSupportedPlaceholder.Instance ) + if( sheet is not InvalidSheet { Exception: var e } ) return sheet; - if( language == Language.None ) - throw new ExcelLanguageNotSupportedException( nameof( language ), language, null ); - return GetSheet( rowType, Language.None ); + if( e is UnsupportedLanguageException ) + { + if( language == Language.None ) + throw new UnsupportedLanguageException( nameof( language ), language, null ); + return GetBaseSheet( rowType, Language.None ); + } + throw e; } - /// Loads an . - /// The requested sheet language. Leave or empty to use the default language. - /// An instance of corresponding to and that may be created anew or - /// reused from a previous invocation of this method. - /// - /// If the requested language doesn't exist for the file where is not , the language-neutral - /// sheet using will be loaded instead. If the language-neutral sheet does not exist, then the function will fail with - /// . - /// The returned instance of should be cast to or - /// before accessing its rows. - /// - /// Sheet does not exist or if the column hash has a mismatch. - /// Sheet does not support nor . - [EditorBrowsable( EditorBrowsableState.Advanced )] - public ExcelSheet GetSheet< T >( Language? language = null ) where T : struct, IExcelRow< T > + private sealed class InvalidSheet : BaseExcelSheet { - var sheet = SheetCache.GetOrAdd( - ( typeof( T ), language ?? Language ), - static ( key, module ) => { - try - { - return ExcelSheet.From< T >( module, key.Language ); - } - catch( ExcelLanguageNotSupportedException ) - { - return LanguageNotSupportedPlaceholder.Instance; - } - }, - this ); - - if( sheet != LanguageNotSupportedPlaceholder.Instance ) - return sheet; - if( language == Language.None ) - throw new ExcelLanguageNotSupportedException( nameof( language ), language, null ); - return GetSheet< T >( Language.None ); - } - - private sealed class LanguageNotSupportedPlaceholder : ExcelSheet - { - public static readonly ExcelSheet Instance = (ExcelSheet) RuntimeHelpers.GetUninitializedObject( typeof( LanguageNotSupportedPlaceholder ) ); - - internal LanguageNotSupportedPlaceholder( ExcelModule module, ExcelHeaderFile headerFile, Language requestedLanguage, string sheetName ) - : base( module, headerFile, requestedLanguage, sheetName ) - { } + public Exception Exception { get; private set; } + + // never actually called + private InvalidSheet( ExcelModule module, ExcelHeaderFile headerFile, Language requestedLanguage, string sheetName ) : base(module, headerFile, requestedLanguage, sheetName) + { + Exception = null!; + } + + public static InvalidSheet Create(Exception exception ) + { + var ret = (InvalidSheet)RuntimeHelpers.GetUninitializedObject( typeof( InvalidSheet ) ); + ret.Exception = exception; + return ret; + } } } \ No newline at end of file diff --git a/src/Lumina/Excel/ExcelPage.cs b/src/Lumina/Excel/ExcelPage.cs index 20c50345..7f1f9ec8 100644 --- a/src/Lumina/Excel/ExcelPage.cs +++ b/src/Lumina/Excel/ExcelPage.cs @@ -34,11 +34,13 @@ internal ExcelPage( ExcelModule module, byte[] pageData, ushort headerDataOffset dataOffset = headerDataOffset; } - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + // Ignores bounds checks to speed up reading data. + // https://t.ly/EmR4n (Sharplab link) + [MethodImpl( MethodImplOptions.AggressiveInlining )] private D Read( nuint offset ) where D : unmanaged => Unsafe.As( ref Unsafe.AddByteOffset( ref MemoryMarshal.GetArrayDataReference( data ), offset ) ); - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] + [MethodImpl( MethodImplOptions.AggressiveInlining )] private static float ReverseEndianness( float v ) => Unsafe.BitCast( BinaryPrimitives.ReverseEndianness( Unsafe.BitCast( v ) ) ); diff --git a/src/Lumina/Excel/ExcelSheet.cs b/src/Lumina/Excel/ExcelSheet.cs index a770102e..901e01e4 100644 --- a/src/Lumina/Excel/ExcelSheet.cs +++ b/src/Lumina/Excel/ExcelSheet.cs @@ -1,332 +1,136 @@ -using Lumina.Data; -using Lumina.Data.Files.Excel; -using Lumina.Data.Structs.Excel; using System; -using System.Collections.Frozen; +using System.Collections; using System.Collections.Generic; -using System.Linq; -using System.Reflection; using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; +using Lumina.Data; +using Lumina.Data.Files.Excel; +using Lumina.Data.Structs.Excel; namespace Lumina.Excel; -/// A wrapper around an excel sheet. -public abstract class ExcelSheet +/// An excel sheet of variant. +/// Type of the rows contained within. +public sealed class ExcelSheet< T > : BaseExcelSheet, ICollection, IReadOnlyCollection where T : struct, IExcelRow< T > { - /// Number of items in that may resolve to no entry. - // 7.05h: across 7292 sheets that exist and are referenced from exlt file, following ratio can be represented solely using lookup array of certain sizes. - // Max Gap, Coverage, Net Wasted - // 1024, 99.15%, 38KB - // 2048, 99.25%, 82KB - // 3072, 99.29%, 109KB - // 4096, 99.36%, 183KB - // 5120, 99.40%, 239KB - // 6144, 99.41%, 259KB - // 9216, 99.42%, 295KB - // 10240, 99.47%, 410KB - // 14336, 99.48%, 463KB - // 16384, 99.49%, 525KB - // 19456, 99.51%, 599KB - // 24576, 99.52%, 692KB - // 26624, 99.53%, 793KB - // 28672, 99.56%, 1011KB - // 29696, 99.57%, 1127KB - // 30720, 99.59%, 1244KB - // 33792, 99.63%, 1633KB - // 34816, 99.64%, 1765KB - // 41984, 99.67%, 2089KB - // 43008, 99.68%, 2255KB - // 44032, 99.71%, 2594KB - // 50176, 99.73%, 2789KB - // 64512, 99.74%, 3041KB - // 65536, 99.75%, 3293KB - // 70656, 99.84%, 4941KB - // 71680, 99.88%, 5773KB - // 89088, 99.89%, 6118KB - // 720896, 99.90%, 8934KB - // 721920, 99.92%, 11754KB - // 1049600, 99.93%, 15853KB - // 1507328, 99.95%, 21741KB - // 2001920, 99.96%, 29559KB - // 2990080, 99.97%, 41236KB - // 9832448, 99.99%, 79643KB - // 10146816, 100.00%, 119276KB - // We're allowing up to 65536 lookup items in _rowOffsetLookupTable, at cost of up to 3293KB of lookup items that resolve to nonexistence per language. - private const int MaxUnusedLookupItemCount = 65536; + internal ExcelSheet( ExcelModule module, ExcelHeaderFile headerFile, Language requestedLanguage, string sheetName ) + : base( module, headerFile, requestedLanguage, sheetName ) + { } - private readonly ExcelPage[] _pages; - private readonly RowOffsetLookup[] _rowOffsetLookupTable; - private readonly ushort _subrowDataOffset; + bool ICollection< T >.IsReadOnly => true; - // RowLookup must use int as the key because it benefits from a fast path that removes indirections. - // https://github.com/dotnet/runtime/blob/release/8.0/src/libraries/System.Collections.Immutable/src/System/Collections/Frozen/FrozenDictionary.cs#L140 - private readonly FrozenDictionary< int, int > _rowIndexLookupDict; + /// + public T this[ uint rowId ] => GetRow( rowId ); - private readonly int[] _rowIndexLookupArray; - - /// The module that this sheet belongs to. - public ExcelModule Module { get; } - - /// The language of the rows in this sheet. - /// This can be different from the requested language if it wasn't supported. - public Language Language { get; } - - /// Contains information on the columns in this sheet. - public IReadOnlyList< ExcelColumnDefinition > Columns { get; } - - private protected ExcelSheet( ExcelModule module, ExcelHeaderFile headerFile, Language requestedLanguage, string sheetName ) + /// + /// Tries to get the th row in this sheet. + /// + /// The row id to get. + /// A nullable row object. Returns if the row does not exist. + public T? GetRowOrDefault( uint rowId ) { - var hasSubrows = headerFile.Header.Variant == ExcelVariant.Subrows; - - Module = module; - Language = headerFile.Languages.Contains( requestedLanguage ) ? requestedLanguage : Language.None; - Columns = headerFile.ColumnDefinitions; - _subrowDataOffset = hasSubrows ? headerFile.Header.DataOffset : (ushort) 0; - _pages = new ExcelPage[headerFile.DataPages.Length]; - _rowOffsetLookupTable = new RowOffsetLookup[headerFile.Header.RowCount]; + ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); + return Unsafe.IsNullRef(in lookup) ? null : UnsafeCreateRow< T >( in lookup ); + } - var i = 0; - for( var pageIdx = (ushort) 0; pageIdx < headerFile.DataPages.Length; pageIdx++ ) + /// + /// Tries to get the th row in this sheet. + /// + /// The row id to get. + /// The output row object. + /// if the row exists and is written to and otherwise. + public bool TryGetRow( uint rowId, out T row ) + { + ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); + if( Unsafe.IsNullRef(in lookup) ) { - var pageDef = headerFile.DataPages[ pageIdx ]; - var filePath = Language == Language.None - ? $"exd/{sheetName}_{pageDef.StartId}.exd" - : $"exd/{sheetName}_{pageDef.StartId}_{LanguageUtil.GetLanguageStr( Language )}.exd"; - var fileData = module.GameData.GetFile< ExcelDataFile >( filePath ); - if( fileData == null ) - continue; - - var newPage = _pages[ pageIdx ] = new( Module, fileData.Data, headerFile.Header.DataOffset ); - - // If row count information from exh file is incorrect, cope with it. - if( i + fileData.RowData.Count > _rowOffsetLookupTable.Length ) - Array.Resize( ref _rowOffsetLookupTable, i + fileData.RowData.Count ); - - foreach( var rowPtr in fileData.RowData.Values ) - { - var subrowCount = hasSubrows ? newPage.ReadUInt16( rowPtr.Offset + 4 ) : (ushort) 1; - var rowOffset = rowPtr.Offset + 6; - _rowOffsetLookupTable[ i++ ] = new( rowPtr.RowId, rowOffset, pageIdx, subrowCount ); - } + row = default; + return false; } - // If row count information from exh file is incorrect, cope with it. (2) - if( i != _rowOffsetLookupTable.Length ) - Array.Resize( ref _rowOffsetLookupTable, i ); - - // A lot of sheets do not have large gap between row IDs. If total number of gaps is less than a threshold, then make a lookup array. - if( _rowOffsetLookupTable.Length > 0 ) - { - var firstId = _rowOffsetLookupTable[ 0 ].RowId; - var numSlots = _rowOffsetLookupTable[ ^1 ].RowId - firstId + 1; - var numUnused = numSlots - headerFile.Header.RowCount; - if( numUnused <= MaxUnusedLookupItemCount ) - { - _rowIndexLookupArray = new int[ numSlots ]; - _rowIndexLookupArray.AsSpan().Fill( -1 ); - for (i = 0; i < _rowOffsetLookupTable.Length; i++) - _rowIndexLookupArray[_rowOffsetLookupTable[ i ].RowId - firstId] = i; - - // All items can be looked up from _rowIndexLookupArray. Dictionary is unnecessary. - _rowIndexLookupDict = FrozenDictionary< int, int >.Empty; - } - else - { - _rowIndexLookupArray = new int[MaxUnusedLookupItemCount]; - _rowIndexLookupArray.AsSpan().Fill( -1 ); - - var lastLookupArrayRowId = uint.MaxValue; - for( i = 0; i < _rowOffsetLookupTable.Length; i++ ) - { - var offsetRowId = _rowOffsetLookupTable[ i ].RowId - firstId; - if( offsetRowId >= MaxUnusedLookupItemCount ) - { - // Discard the unused entries. - Array.Resize( ref _rowIndexLookupArray, unchecked( (int) ( lastLookupArrayRowId + 1 ) ) ); - break; - } - - _rowIndexLookupArray[ offsetRowId ] = i; - lastLookupArrayRowId = offsetRowId; - } - - // Skip the items that can be looked up from _rowIndexLookupArray. - _rowIndexLookupDict = _rowOffsetLookupTable.Skip( i ).ToFrozenDictionary( static row => (int) row.RowId, _ => i++ ); - } - - Count = _rowOffsetLookupTable.Length; - } - else - { - _rowIndexLookupDict = FrozenDictionary< int, int >.Empty; - _rowIndexLookupArray = []; - _rowOffsetLookupTable = [default]; // so that MemoryMarshal.GetArrayDataReference( _rowOffsetLookupTable ) is always valid. - Count = 0; - } + row = UnsafeCreateRow< T >( in lookup ); + return true; } - /// Creates a new instance of with the 's default language, deducing sheet names and column - /// hashes from . - /// The to access sheet data from. - /// does not have a valid . - /// parameters were invalid (hash mismatch or invalid sheet name). - /// A new instance of that should be cast to or - /// before further use. - public static ExcelSheet From< T >( ExcelModule module ) where T : struct, IExcelRow< T > => - From< T >( module, module.Language ); - - /// Creates a new instance of , deducing sheet names and column hashes from . - /// The to access sheet data from. - /// The language to use for this sheet. - /// does not have a valid . - /// parameters were invalid (hash mismatch or invalid sheet name). - /// A new instance of that should be cast to or - /// before further use. - public static ExcelSheet From< T >( ExcelModule module, Language language ) where T : struct, IExcelRow< T > + /// + /// Gets the th row in this sheet. + /// + /// The row id to get. + /// A row object. + /// Throws when the row id does not have a row attached to it. + public T GetRow( uint rowId ) { - var attribute = typeof( T ).GetCustomAttribute< SheetAttribute >() ?? - throw new InvalidOperationException( $"{nameof( T )} has no {nameof( SheetAttribute )}. Use the overload of {nameof( From )} with 4 parameters." ); - - return From< T >( module, language, attribute.Name, attribute.ColumnHash ); + ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); + return Unsafe.IsNullRef(in lookup) ? throw new ArgumentOutOfRangeException( nameof( rowId ), rowId, null ) : UnsafeCreateRow< T >( in lookup ); } - /// Creates a new instance of . - /// The to access sheet data from. - /// The language to use for this sheet. - /// The name of the sheet to read from. - /// The hash of the columns in the sheet. If , it will not check the hash. - /// or parameters were invalid (hash mismatch or invalid sheet name). - /// Header file had a value that is not supported. - /// Sheet is of variant, but does not implement it. - /// - /// A new instance of that should be cast to or - /// before further use. - public static ExcelSheet From< T >( ExcelModule module, Language language, string sheetName, uint? columnHash = null ) - where T : struct, IExcelRow< T > + /// + /// Gets the th row in this sheet, ordered by row id in ascending order. + /// + /// If you are looking to find a row by its id, use instead. + /// The zero-based index of this row. + /// A row object. + public T GetRowAt( int rowIndex ) { - var headerFile = module.GameData.GetFile< ExcelHeaderFile >( $"exd/{sheetName}.exh" ) ?? - throw new ArgumentException( "Invalid sheet name", nameof( sheetName ) ); + ArgumentOutOfRangeException.ThrowIfNegative( rowIndex ); + ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual( rowIndex, OffsetLookupTable.Length ); - if( module.VerifySheetChecksums && columnHash is { } hash && headerFile.GetColumnsHash() != hash ) - throw new ArgumentException( "Column hash mismatch", nameof( columnHash ) ); + return UnsafeCreateRowAt< T >( rowIndex ); + } - if( !headerFile.Languages.Contains( language ) ) - throw new ExcelLanguageNotSupportedException(); + /// + public bool Contains( T item ) => TryGetRow( item.RowId, out var row ) && EqualityComparer< T >.Default.Equals( item, row ); - switch( headerFile.Header.Variant ) - { - case ExcelVariant.Default: - return new DefaultExcelSheet< T >( module, headerFile, language, sheetName ); - case ExcelVariant.Subrows: - return new SubrowsExcelSheet< T >( module, headerFile, language, sheetName ); - case ExcelVariant.Unknown: - default: - throw new NotSupportedException( $"Specified sheet variant {headerFile.Header.Variant} is not supported." ); - } + /// + public void CopyTo( T[] array, int arrayIndex ) + { + ArgumentNullException.ThrowIfNull( array ); + ArgumentOutOfRangeException.ThrowIfNegative( arrayIndex ); + if( Count > array.Length - arrayIndex ) + throw new ArgumentException( "The number of elements in the source list is greater than the available space." ); + foreach (var lookup in OffsetLookupTable) + array[ arrayIndex++ ] = UnsafeCreateRow( in lookup ); } - /// The number of rows in this sheet. - /// - /// If this sheet has gaps in row ids, it returns the number of rows that exist, not the highest row id. - /// If this sheet has subrows, this will still return the number of rows and not the total number of subrows. - /// - public int Count { get; } + void ICollection< T >.Add( T item ) => throw new NotSupportedException(); - /// Gets the offset lookup table. - private protected ReadOnlySpan< RowOffsetLookup > OffsetLookupTable { - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] - get => _rowOffsetLookupTable; - } + void ICollection< T >.Clear() => throw new NotSupportedException(); - /// Gets the offset of the column at in the row data. - /// The index of the column. - /// The offset of the column. - /// Thrown when the column index is invalid. It must be less than .Count. - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] - public ushort GetColumnOffset( int columnIdx ) => - Columns[ columnIdx ].Offset; + bool ICollection< T >.Remove( T item ) => throw new NotSupportedException(); - /// Whether this sheet has a row with the given . - /// If this sheet has subrows, this will check if the row id has any subrows. - /// The row id to check. - /// Whether the row exists. - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] - public bool HasRow( uint rowId ) - { - ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); - return !Unsafe.IsNullRef( in lookup ) && lookup.SubrowCount > 0; - } + /// + public Enumerator GetEnumerator() => new( this ); - /// Gets a row lookup at the given index, if possible. - /// Index of the desired row. - /// Lookup data for the desired row, or a null reference if no corresponding row exists. - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] - internal ref readonly RowOffsetLookup GetRowLookupOrNullRef( uint rowId ) - { - var lookupArrayIndex = unchecked( rowId - MemoryMarshal.GetArrayDataReference( _rowOffsetLookupTable ).RowId ); - if( lookupArrayIndex < _rowIndexLookupArray.Length ) - { - var rowIndex = Unsafe.Add( ref MemoryMarshal.GetArrayDataReference( _rowIndexLookupArray ), lookupArrayIndex ); - if (rowIndex == -1) - return ref Unsafe.NullRef(); - return ref UnsafeGetRowLookupAt( rowIndex ); - } + IEnumerator< T > IEnumerable< T >.GetEnumerator() => GetEnumerator(); - ref readonly var rowIndexRef = ref _rowIndexLookupDict.GetValueRefOrNullRef( (int) rowId ); - if( Unsafe.IsNullRef( in rowIndexRef ) ) - return ref Unsafe.NullRef(); - return ref UnsafeGetRowLookupAt( rowIndexRef ); - } + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + + /// Represents an enumerator that iterates over all rows in a . + /// The sheet to iterate over. + public struct Enumerator( ExcelSheet< T > sheet ) : IEnumerator< T > + { + private int _index = -1; - /// Gets a row lookup at the given index, without checking for bounds or preconditions. - /// Index of the desired row. - /// Lookup data for the desired row. - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] - internal ref readonly RowOffsetLookup UnsafeGetRowLookupAt( int rowIndex ) => - ref Unsafe.Add( ref MemoryMarshal.GetArrayDataReference( _rowOffsetLookupTable ), rowIndex ); + /// + public readonly T Current => sheet.UnsafeCreateRowAt< T >( _index ); - /// Creates a row at the given index, without checking for bounds or preconditions. - /// Index of the desired row. - /// A new instance of . - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] - internal T UnsafeCreateRowAt< T >( int rowIndex ) where T : struct, IExcelRow< T > => - UnsafeCreateRow< T >( in UnsafeGetRowLookupAt( rowIndex ) ); + readonly object IEnumerator.Current => Current; - /// Creates a subrow at the given index, without checking for bounds or preconditions. - /// Index of the desired row. - /// Index of the desired subrow. - /// A new instance of . - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] - internal T UnsafeCreateSubrowAt< T >( int rowIndex, ushort subrowId ) where T : struct, IExcelRow< T > => - UnsafeCreateSubrow< T >( in UnsafeGetRowLookupAt( rowIndex ), subrowId ); + /// + public bool MoveNext() + { + if( ++_index < sheet.Count ) + return true; - /// Creates a row using the given lookup data, without checking for bounds or preconditions. - /// Lookup data for the desired row. - /// A new instance of . - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] - internal T UnsafeCreateRow< T >( scoped in RowOffsetLookup lookup ) where T : struct, IExcelRow< T > => - T.Create( - Unsafe.Add( ref MemoryMarshal.GetArrayDataReference( _pages ), lookup.PageIndex ), - lookup.Offset, - lookup.RowId ); + --_index; + return false; + } - /// Creates a subrow using the given lookup data, without checking for bounds or preconditions. - /// Lookup data for the desired row. - /// Index of the desired subrow. - /// A new instance of . - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] - internal T UnsafeCreateSubrow< T >( scoped in RowOffsetLookup lookup, ushort subrowId ) where T : struct, IExcelRow< T > => - T.Create( - Unsafe.Add( ref MemoryMarshal.GetArrayDataReference( _pages ), lookup.PageIndex ), - lookup.Offset + 2 + subrowId * ( _subrowDataOffset + 2u ), - lookup.RowId, - subrowId ); + /// + public void Reset() => + _index = -1; - /// Lookup data for locating backing data for a row. - /// ID of the row. This is separate from the row indices. - /// Byte offset of the row, relative to the beginning of an exd file. - /// Index of the page that contains the data for this row. - /// Number of subrows in the row, or 1 if the sheet does not support subrows. - internal readonly record struct RowOffsetLookup( uint RowId, uint Offset, ushort PageIndex, ushort SubrowCount ); + /// + public readonly void Dispose() + { } + } } \ No newline at end of file diff --git a/src/Lumina/Excel/LazyCollection.cs b/src/Lumina/Excel/LazyCollection.cs deleted file mode 100644 index ac3f14dd..00000000 --- a/src/Lumina/Excel/LazyCollection.cs +++ /dev/null @@ -1,42 +0,0 @@ -using System; -using System.Collections; -using System.Collections.Generic; -using System.Runtime.CompilerServices; - -namespace Lumina.Excel; - -/// -/// A collection helper used to layout and structure excel rows. -/// -/// Mostly an implementation detail for reading excel rows. This type does not store or hold any row data, and is therefore lightweight and trivially constructable. -/// A type that wraps a group of fields inside a row. -/// -/// -/// -/// -/// -public readonly struct LazyCollection( ExcelPage page, uint parentOffset, uint offset, Func ctor, int size ) : IReadOnlyList where T : struct -{ - /// - public T this[int index] { - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] - get { - ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual( index, size ); - return ctor( page, parentOffset, offset, (uint)index ); - } - } - - /// - public int Count => size; - - /// - public IEnumerator GetEnumerator() - { - for( var i = 0; i < size; ++i ) - yield return this[i]; - } - - /// - IEnumerator IEnumerable.GetEnumerator() => - GetEnumerator(); -} \ No newline at end of file diff --git a/src/Lumina/Excel/MismatchedColumnHashException.cs b/src/Lumina/Excel/MismatchedColumnHashException.cs new file mode 100644 index 00000000..155385e2 --- /dev/null +++ b/src/Lumina/Excel/MismatchedColumnHashException.cs @@ -0,0 +1,11 @@ +using System; + +namespace Lumina.Excel; + +/// Exception indicating that the requested row type's column hash is different from game data. +/// +public sealed class MismatchedColumnHashException( uint typeHash, uint gameHash, string? paramName ) : + ArgumentException( + $"The requested row type has a column hash that is different from game data. (Type: {typeHash:X08}, Game: {gameHash:X08})", + paramName ) +{ } \ No newline at end of file diff --git a/src/Lumina/Excel/RowRef.cs b/src/Lumina/Excel/RowRef.cs index bed8870d..0df16726 100644 --- a/src/Lumina/Excel/RowRef.cs +++ b/src/Lumina/Excel/RowRef.cs @@ -74,7 +74,7 @@ public static RowRef GetFirstValidRowOrUntyped( ExcelModule module, uint rowId, { foreach( var sheetType in sheetTypes ) { - if( module.GetSheet( sheetType ) is { } sheet ) + if( module.GetBaseSheet( sheetType ) is { } sheet ) { if( sheet.HasRow( rowId ) ) return new( module, rowId, sheetType ); @@ -90,7 +90,7 @@ public static RowRef GetFirstValidRowOrUntyped( ExcelModule module, uint rowId, /// The row type referenced by the . /// The to read sheet data from. /// The referenced row id. - /// A to a row in a . + /// A to a row in a . public static RowRef Create< T >( ExcelModule? module, uint rowId ) where T : struct, IExcelRow< T > => new( module, rowId, typeof( T ) ); /// @@ -109,7 +109,7 @@ public static RowRef GetFirstValidRowOrUntyped( ExcelModule module, uint rowId, /// The referenced row id. public readonly struct RowRef< T >( ExcelModule? module, uint rowId ) where T : struct, IExcelRow< T > { - private readonly ExcelSheet? _sheet = module?.GetSheet< T >(); + private readonly BaseExcelSheet? _sheet = module?.GetBaseSheet< T >(); /// /// The row id of the referenced row. diff --git a/src/Lumina/Excel/SubrowCollection.cs b/src/Lumina/Excel/SubrowCollection.cs index 457ab543..610e48c6 100644 --- a/src/Lumina/Excel/SubrowCollection.cs +++ b/src/Lumina/Excel/SubrowCollection.cs @@ -10,30 +10,23 @@ namespace Lumina.Excel; public readonly struct SubrowCollection< T > : IList< T >, IReadOnlyList< T > where T : struct, IExcelRow< T > { - private readonly ExcelSheet.RowOffsetLookup _lookup; + private readonly BaseExcelSheet.RowOffsetLookup _lookup; - internal SubrowCollection( SubrowsExcelSheet< T > sheet, scoped in ExcelSheet.RowOffsetLookup lookup ) + internal SubrowCollection( SubrowExcelSheet< T > sheet, scoped ref readonly BaseExcelSheet.RowOffsetLookup lookup ) { Sheet = sheet; _lookup = lookup; } /// Gets the associated sheet. - public SubrowsExcelSheet< T > Sheet { get; } + public SubrowExcelSheet< T > Sheet { get; } /// Gets the Row ID of the subrows contained within. - public uint RowId { - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] - get => _lookup.RowId; - } + public uint RowId => _lookup.RowId; /// - public int Count { - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] - get => _lookup.SubrowCount; - } + public int Count => _lookup.SubrowCount; - /// bool ICollection< T >.IsReadOnly => true; /// @@ -42,7 +35,7 @@ public T this[ int index ] { get { ArgumentOutOfRangeException.ThrowIfNegative( index ); ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual( index, Count ); - return Sheet.UnsafeCreateSubrow< T >( _lookup, unchecked( (ushort) index ) ); + return Sheet.UnsafeCreateSubrow< T >( in _lookup, unchecked( (ushort) index ) ); } } @@ -52,13 +45,23 @@ T IList< T >.this[ int index ] { set => throw new NotSupportedException(); } + void IList.Insert( int index, T item ) => throw new NotSupportedException(); + + void IList.RemoveAt( int index ) => throw new NotSupportedException(); + + void ICollection.Add( T item ) => throw new NotSupportedException(); + + void ICollection.Clear() => throw new NotSupportedException(); + + bool ICollection.Remove( T item ) => throw new NotSupportedException(); + /// public int IndexOf( T item ) { if( item.RowId != RowId || item.SubrowId >= Count ) return -1; - var row = Sheet.UnsafeCreateSubrow< T >( _lookup, item.SubrowId ); + var row = Sheet.UnsafeCreateSubrow< T >( in _lookup, item.SubrowId ); return EqualityComparer< T >.Default.Equals( item, row ) ? item.SubrowId : -1; } @@ -73,31 +76,14 @@ public void CopyTo( T[] array, int arrayIndex ) if( Count > array.Length - arrayIndex ) throw new ArgumentException( "The number of elements in the source list is greater than the available space." ); for( var i = 0; i < Count; i++ ) - array[ arrayIndex++ ] = Sheet.UnsafeCreateSubrow< T >( _lookup, unchecked( (ushort) i ) ); + array[ arrayIndex++ ] = Sheet.UnsafeCreateSubrow< T >( in _lookup, unchecked( (ushort) i ) ); } - /// - void IList< T >.Insert( int index, T item ) => throw new NotSupportedException(); - - /// - void IList< T >.RemoveAt( int index ) => throw new NotSupportedException(); - - /// - void ICollection< T >.Add( T item ) => throw new NotSupportedException(); - - /// - void ICollection< T >.Clear() => throw new NotSupportedException(); - - /// - bool ICollection< T >.Remove( T item ) => throw new NotSupportedException(); - /// public Enumerator GetEnumerator() => new( this ); - /// IEnumerator< T > IEnumerable< T >.GetEnumerator() => GetEnumerator(); - /// IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); /// Enumerator that enumerates over subrows under one row. @@ -107,19 +93,11 @@ public struct Enumerator( SubrowCollection< T > subrowCollection ) : IEnumerator private int _index = -1; /// - public readonly T Current { - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] - get => subrowCollection.Sheet.UnsafeCreateSubrow< T >( subrowCollection._lookup, unchecked( (ushort) _index ) ); - } + public readonly T Current => subrowCollection.Sheet.UnsafeCreateSubrow< T >( in subrowCollection._lookup, unchecked( (ushort) _index ) ); - /// - readonly object IEnumerator.Current { - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] - get => Current; - } + readonly object IEnumerator.Current => Current; /// - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] public bool MoveNext() { if( ++_index < subrowCollection.Count ) @@ -130,11 +108,9 @@ public bool MoveNext() } /// - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] public void Reset() => _index = -1; /// - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] public readonly void Dispose() { } } diff --git a/src/Lumina/Excel/SubrowsExcelSheet.cs b/src/Lumina/Excel/SubrowExcelSheet.cs similarity index 52% rename from src/Lumina/Excel/SubrowsExcelSheet.cs rename to src/Lumina/Excel/SubrowExcelSheet.cs index 524b7b30..9a2fd604 100644 --- a/src/Lumina/Excel/SubrowsExcelSheet.cs +++ b/src/Lumina/Excel/SubrowExcelSheet.cs @@ -1,116 +1,49 @@ +using Lumina.Data.Files.Excel; +using Lumina.Data; using System; using System.Collections; using System.Collections.Generic; using System.Runtime.CompilerServices; -using Lumina.Data; -using Lumina.Data.Files.Excel; -using Lumina.Data.Structs.Excel; namespace Lumina.Excel; -/// An excel sheet of variant. -public abstract class SubrowsExcelSheet : ExcelSheet -{ - private protected SubrowsExcelSheet( ExcelModule module, ExcelHeaderFile headerFile, Language requestedLanguage, string sheetName ) - : base( module, headerFile, requestedLanguage, sheetName ) - { - foreach( var f in OffsetLookupTable ) - TotalSubrowCount += f.SubrowCount; - } - - /// - /// The total number of subrows in this sheet across all rows. - /// - public int TotalSubrowCount { get; } - - /// - /// Whether this sheet has a subrow with the given and . - /// - /// The row id to check. - /// The subrow id to check. - /// Whether the subrow exists. - public bool HasSubrow( uint rowId, ushort subrowId ) - { - ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); - return !Unsafe.IsNullRef( in lookup ) && subrowId < lookup.SubrowCount; - } - - /// - /// Tries to get the number of subrows in the th row in this sheet. - /// - /// The row id to get. - /// The number of subrows in this row. - /// if the row exists and is written to and otherwise. - public bool TryGetSubrowCount( uint rowId, out ushort subrowCount ) - { - ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); - if( Unsafe.IsNullRef( in lookup ) ) - { - subrowCount = 0; - return false; - } - - subrowCount = lookup.SubrowCount; - return true; - } - - /// - /// Gets the number of subrows in the th row in this sheet. Throws if the row does not exist. - /// - /// The row id to get. - /// The number of subrows in this row. Returns null if the row does not exist. - /// Thrown if the sheet does not have a row at that . - public ushort GetSubrowCount( uint rowId ) - { - ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); - return Unsafe.IsNullRef( in lookup ) ? throw new ArgumentOutOfRangeException( nameof( rowId ), rowId, null ) : lookup.SubrowCount; - } -} - -/// An excel sheet of variant. /// Type of the rows contained within. -public sealed class SubrowsExcelSheet< T > - : SubrowsExcelSheet, ICollection< SubrowCollection< T > >, IReadOnlyCollection< SubrowCollection< T > > - where T : struct, IExcelRow< T > +/// +public sealed class SubrowExcelSheet + : BaseSubrowExcelSheet, ICollection>, IReadOnlyCollection> + where T : struct, IExcelRow { - internal SubrowsExcelSheet( ExcelModule module, ExcelHeaderFile headerFile, Language requestedLanguage, string sheetName ) + internal SubrowExcelSheet( ExcelModule module, ExcelHeaderFile headerFile, Language requestedLanguage, string sheetName ) : base( module, headerFile, requestedLanguage, sheetName ) - { - } + { } /// - bool ICollection< SubrowCollection< T > >.IsReadOnly => true; + bool ICollection>.IsReadOnly => true; /// - public SubrowCollection< T > this[ uint rowId ] { - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] - get => GetRow( rowId ); - } + public SubrowCollection this[uint rowId] => GetRow( rowId ); /// - public T this[ uint rowId, ushort subrowId ] { - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] - get => GetSubrow( rowId, subrowId ); - } + public T this[uint rowId, ushort subrowId] => GetSubrow( rowId, subrowId ); /// /// Tries to get the subrow collection with row id in this sheet. /// /// The row id to get. - /// A nullable subrow collection object. Returns null if the row does not exist. - public SubrowCollection< T >? GetRowOrDefault( uint rowId ) + /// A nullable subrow collection object. Returns if the row does not exist. + public SubrowCollection? GetRowOrDefault( uint rowId ) { ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); - return Unsafe.IsNullRef( in lookup ) ? null : new( this, lookup ); + return Unsafe.IsNullRef( in lookup ) ? null : new( this, in lookup ); } /// /// Tries to get the subrow collection with row id in this sheet. /// /// The row id to get. - /// The output row object. + /// The output subrow collection object. /// if the row exists and is written to and otherwise. - public bool TryGetRow( uint rowId, out SubrowCollection< T > row ) + public bool TryGetRow( uint rowId, out SubrowCollection row ) { ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); if( Unsafe.IsNullRef( in lookup ) ) @@ -119,7 +52,7 @@ public bool TryGetRow( uint rowId, out SubrowCollection< T > row ) return false; } - row = new( this, lookup ); + row = new( this, in lookup ); return true; } @@ -127,12 +60,12 @@ public bool TryGetRow( uint rowId, out SubrowCollection< T > row ) /// Gets the subrow collection with row id in this sheet. Throws if the row does not exist. /// /// The row id to get. - /// A row object. + /// A subrow collection object. /// Thrown if the sheet does not have a row at that . - public SubrowCollection< T > GetRow( uint rowId ) + public SubrowCollection GetRow( uint rowId ) { ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); - return Unsafe.IsNullRef( in lookup ) ? throw new ArgumentOutOfRangeException( nameof( rowId ), rowId, null ) : new( this, lookup ); + return Unsafe.IsNullRef( in lookup ) ? throw new ArgumentOutOfRangeException( nameof( rowId ), rowId, null ) : new( this, in lookup ); } /// @@ -140,13 +73,13 @@ public SubrowCollection< T > GetRow( uint rowId ) /// /// If you are looking to find a row by its id, use instead. /// The zero-based index of this row. - /// A row object. - public SubrowCollection< T > GetRowAt( int rowIndex ) + /// A subrow collection object. + public SubrowCollection GetRowAt( int rowIndex ) { ArgumentOutOfRangeException.ThrowIfNegative( rowIndex ); ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual( rowIndex, OffsetLookupTable.Length ); - return new( this, UnsafeGetRowLookupAt( rowIndex ) ); + return new( this, in UnsafeGetRowLookupAt( rowIndex ) ); } /// @@ -158,7 +91,7 @@ public SubrowCollection< T > GetRowAt( int rowIndex ) public T? GetSubrowOrDefault( uint rowId, ushort subrowId ) { ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); - return Unsafe.IsNullRef( in lookup ) || subrowId >= lookup.SubrowCount ? null : UnsafeCreateSubrow< T >( in lookup, subrowId ); + return Unsafe.IsNullRef( in lookup ) || subrowId >= lookup.SubrowCount ? null : UnsafeCreateSubrow( in lookup, subrowId ); } /// @@ -177,7 +110,7 @@ public bool TryGetSubrow( uint rowId, ushort subrowId, out T subrow ) return false; } - subrow = UnsafeCreateSubrow< T >( in lookup, subrowId ); + subrow = UnsafeCreateSubrow( in lookup, subrowId ); return true; } @@ -196,7 +129,7 @@ public T GetSubrow( uint rowId, ushort subrowId ) ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual( subrowId, lookup.SubrowCount ); - return UnsafeCreateSubrow< T >( in lookup, subrowId ); + return UnsafeCreateSubrow( in lookup, subrowId ); } /// @@ -215,31 +148,28 @@ public T GetSubrowAt( int rowIndex, ushort subrowId ) ref readonly var lookup = ref UnsafeGetRowLookupAt( rowIndex ); ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual( subrowId, lookup.SubrowCount ); - return UnsafeCreateSubrow< T >( in lookup, subrowId ); + return UnsafeCreateSubrow( in lookup, subrowId ); } /// - public bool Contains( SubrowCollection< T > item ) => ReferenceEquals( item.Sheet, this ) && HasRow( item.RowId ); + public bool Contains( SubrowCollection item ) => ReferenceEquals( item.Sheet, this ) && HasRow( item.RowId ); /// - public void CopyTo( SubrowCollection< T >[] array, int arrayIndex ) + public void CopyTo( SubrowCollection[] array, int arrayIndex ) { ArgumentNullException.ThrowIfNull( array ); ArgumentOutOfRangeException.ThrowIfNegative( arrayIndex ); if( Count > array.Length - arrayIndex ) throw new ArgumentException( "The number of elements in the source list is greater than the available space." ); foreach( var lookup in OffsetLookupTable ) - array[ arrayIndex++ ] = new( this, lookup ); + array[arrayIndex++] = new( this, in lookup ); } - /// - void ICollection< SubrowCollection< T > >.Add( SubrowCollection< T > item ) => throw new NotSupportedException(); + void ICollection>.Add( SubrowCollection item ) => throw new NotSupportedException(); - /// - void ICollection< SubrowCollection< T > >.Clear() => throw new NotSupportedException(); + void ICollection>.Clear() => throw new NotSupportedException(); - /// - bool ICollection< SubrowCollection< T > >.Remove( SubrowCollection< T > item ) => throw new NotSupportedException(); + bool ICollection>.Remove( SubrowCollection item ) => throw new NotSupportedException(); /// Gets an enumerator that enumerates over all subrows. /// A new enumerator. @@ -248,32 +178,23 @@ public void CopyTo( SubrowCollection< T >[] array, int arrayIndex ) /// public Enumerator GetEnumerator() => new( this ); - /// - IEnumerator< SubrowCollection< T > > IEnumerable< SubrowCollection< T > >.GetEnumerator() => GetEnumerator(); + IEnumerator> IEnumerable>.GetEnumerator() => GetEnumerator(); - /// IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); - /// Represents an enumerator that iterates over all rows in a . + /// Represents an enumerator that iterates over all rows in a . /// The sheet to iterate over. - public struct Enumerator( SubrowsExcelSheet< T > sheet ) : IEnumerator< SubrowCollection< T > > + public struct Enumerator( SubrowExcelSheet sheet ) : IEnumerator> { private int _index = -1; /// - public readonly SubrowCollection< T > Current { - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] - get => new( sheet, sheet.UnsafeGetRowLookupAt( _index ) ); - } + public readonly SubrowCollection Current => new( sheet, in sheet.UnsafeGetRowLookupAt( _index ) ); - /// - readonly object IEnumerator.Current { - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] - get => Current; - } + readonly object IEnumerator.Current => + Current; /// - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] public bool MoveNext() { if( ++_index < sheet.Count ) @@ -284,41 +205,33 @@ public bool MoveNext() } /// - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] - public void Reset() => _index = -1; + public void Reset() => + _index = -1; /// - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] public readonly void Dispose() { } } - /// Represents an enumerator that iterates over all subrows in a . + /// Represents an enumerator that iterates over all subrows in a . /// The sheet to iterate over. - public struct FlatEnumerator( SubrowsExcelSheet< T > sheet ) : IEnumerator< T >, IEnumerable< T > + public struct FlatEnumerator( SubrowExcelSheet sheet ) : IEnumerator, IEnumerable { private int _index = -1; private ushort _subrowIndex = ushort.MaxValue; private ushort _subrowCount; /// - public readonly T Current { - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] - get => sheet.UnsafeCreateSubrowAt< T >( _index, _subrowIndex ); - } + public readonly T Current => sheet.UnsafeCreateSubrowAt( _index, _subrowIndex ); - /// - readonly object IEnumerator.Current { - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] - get => Current; - } + readonly object IEnumerator.Current => Current; /// public bool MoveNext() { if( ++_subrowIndex >= _subrowCount ) { - while (true) + while( true ) { if( ++_index >= sheet.Count ) { @@ -339,7 +252,6 @@ public bool MoveNext() } /// - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] public void Reset() { _index = -1; @@ -348,18 +260,14 @@ public void Reset() } /// - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] public readonly void Dispose() { } /// - [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] - public FlatEnumerator GetEnumerator() => new( sheet ); + public readonly FlatEnumerator GetEnumerator() => new( sheet ); - /// - IEnumerator< T > IEnumerable< T >.GetEnumerator() => GetEnumerator(); + readonly IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); - /// - IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + readonly IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); } } \ No newline at end of file diff --git a/src/Lumina/Excel/UnsupportedLanguageException.cs b/src/Lumina/Excel/UnsupportedLanguageException.cs new file mode 100644 index 00000000..b68958eb --- /dev/null +++ b/src/Lumina/Excel/UnsupportedLanguageException.cs @@ -0,0 +1,30 @@ +using System; + +namespace Lumina.Excel; + +/// Exception indicating that the requested language is not supported by the requested sheet. +public sealed class UnsupportedLanguageException : ArgumentOutOfRangeException +{ + private const string ErrorMessage = "Specified excel language is not supported for the sheet."; + + /// + public UnsupportedLanguageException() : base( null, ErrorMessage ) + { } + + /// + public UnsupportedLanguageException( string? paramName ) : this( ErrorMessage, paramName ) + { } + + /// + public UnsupportedLanguageException( string? message, Exception? innerException ) : base( message ?? ErrorMessage, innerException ) + { } + + /// + public UnsupportedLanguageException( string? paramName, object? actualValue, string? message ) + : base( paramName, actualValue, message ?? ErrorMessage ) + { } + + /// + public UnsupportedLanguageException( string? paramName, string? message ) : base( paramName, message ?? ErrorMessage ) + { } +} \ No newline at end of file diff --git a/src/Lumina/Extensions/SpanExtensions.cs b/src/Lumina/Extensions/SpanExtensions.cs index c0f71ac0..5acd8147 100644 --- a/src/Lumina/Extensions/SpanExtensions.cs +++ b/src/Lumina/Extensions/SpanExtensions.cs @@ -1,34 +1,41 @@ using System; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; -namespace Lumina.Extensions +namespace Lumina.Extensions; + +public static class SpanExtensions { - public static class SpanExtensions + public static unsafe T ReadStructure< T >( this Span< byte > span ) where T : struct { - - public static unsafe T ReadStructure< T >( this Span< byte > span ) where T : struct - { #if NETSTANDARD - fixed( byte* bp = &span.GetPinnableReference() ) - { - return Marshal.PtrToStructure< T >( new IntPtr( bp ) ); - } + fixed( byte* bp = &span.GetPinnableReference() ) + { + return Marshal.PtrToStructure< T >( new IntPtr( bp ) ); + } #else - return MemoryMarshal.Read< T >( span ); + return MemoryMarshal.Read< T >( span ); #endif - } + } - public static unsafe T ReadStructure< T >( this Span< byte > span, int offset ) where T : struct - { + public static unsafe T ReadStructure< T >( this Span< byte > span, int offset ) where T : struct + { #if NETSTANDARD - fixed( byte* bp = &span.GetPinnableReference() ) - { - return Marshal.PtrToStructure< T >( new IntPtr( bp + offset ) ); - } + fixed( byte* bp = &span.GetPinnableReference() ) + { + return Marshal.PtrToStructure< T >( new IntPtr( bp + offset ) ); + } #else - return MemoryMarshal.Read< T >( span.Slice( offset ) ); + return MemoryMarshal.Read< T >( span.Slice( offset ) ); #endif - - } } + + public static unsafe ref T UnsafeAt( this Span span, int index ) => + ref Unsafe.Add( ref MemoryMarshal.GetReference( span ), index ); + + public static unsafe ref readonly T UnsafeAt( this ReadOnlySpan span, int index ) => + ref Unsafe.Add( ref MemoryMarshal.GetReference( span ), index ); + + public static unsafe ref readonly T UnsafeAt(this T[] array, int index) => + ref Unsafe.Add( ref MemoryMarshal.GetArrayDataReference( array ), index ); } \ No newline at end of file diff --git a/src/Lumina/GameData.cs b/src/Lumina/GameData.cs index 7cbf7f6f..7e312c5a 100644 --- a/src/Lumina/GameData.cs +++ b/src/Lumina/GameData.cs @@ -43,8 +43,7 @@ public class GameData : IDisposable /// /// Provides access to EXD/EXH data, internally called Excel. /// - /// Loaded by default on init unless you opt not to load it. Can be loaded at a later time by calling Lumina.InitExcelModule or optionally - /// constructing your own Excel.ExcelModule. + /// Loaded by default on init unless you opt not to load it. /// public ExcelModule Excel { get; private set; } @@ -57,7 +56,7 @@ public class GameData : IDisposable /// /// Provides a pool for file streams for .dat files. /// - /// The pool will be disposed when is called. + /// The pool will be disposed when is called. public SqPackStreamPool? StreamPool { get; set; } internal ILogger? Logger { get; private set; } @@ -287,40 +286,58 @@ public static UInt64 GetFileHash( string path ) return (UInt64) Crc32.Get( folder ) << 32 | Crc32.Get( filename ); } - /// Loads an . + /// Loads an . Returns if the sheet does not exist, has an invalid column hash or unsupported variant, or was requested with an unsupported language. /// The requested sheet language. Leave or empty to use the default language. - /// An instance of corresponding to and that may be created anew or - /// reused from a previous invocation of this method, or if a corresponding sheet could not be loaded. - /// Thrown when is not decorated with a + /// An excel sheet corresponding to and that may be created anew or + /// reused from a previous invocation of this method. + /// + /// If the requested language doesn't exist for the file where is not , the language-neutral + /// sheet using will be loaded instead. If the language-neutral sheet does not exist, then the function will fail with + /// . + /// /// Sheet is not of the variant . - public DefaultExcelSheet< T >? GetDefaultExcelSheet< T >( Language? language = null ) where T : struct, IExcelRow< T > + /// does not have a valid . + public ExcelSheet< T >? GetExcelSheet< T >( Language? language = null ) where T : struct, IExcelRow< T > { try { - return Excel.GetDefaultSheet< T >( language ); + return Excel.GetSheet< T >( language ); } catch( ArgumentException ) { return null; } + catch( NotSupportedException ) + { + return null; + } } - /// Loads an . + /// Loads an . Returns if the sheet does not exist, has an invalid column hash or unsupported variant, or was requested with an unsupported language. /// The requested sheet language. Leave or empty to use the default language. - /// An instance of corresponding to and that may be created anew or - /// reused from a previous invocation of this method, or if a corresponding sheet could not be loaded. - /// Thrown when is not decorated with a + /// An excel sheet corresponding to and that may be created anew or + /// reused from a previous invocation of this method. + /// + /// If the requested language doesn't exist for the file where is not , the language-neutral + /// sheet using will be loaded instead. If the language-neutral sheet does not exist, then the function will fail with + /// . + /// /// Sheet is not of the variant . - public SubrowsExcelSheet< T >? GetSubrowsExcelSheet< T >( Language? language = null ) where T : struct, IExcelRow< T > + /// does not have a valid . + public SubrowExcelSheet< T >? GetSubrowExcelSheet< T >( Language? language = null ) where T : struct, IExcelRow< T > { try { - return Excel.GetSubrowsSheet< T >( language ); + return Excel.GetSubrowSheet< T >( language ); } catch( ArgumentException ) { return null; } + catch ( NotSupportedException ) + { + return null; + } } /// From 0956b1ff467d8d2aa38a2e5981554106386f0558 Mon Sep 17 00:00:00 2001 From: Soreepeong Date: Sat, 17 Aug 2024 19:54:47 +0900 Subject: [PATCH 21/53] Reformat code --- src/Lumina/Excel/BaseExcelSheet.cs | 26 ++++++------- src/Lumina/Excel/Collection.cs | 30 +++++++------- src/Lumina/Excel/ExcelModule.cs | 10 +++-- src/Lumina/Excel/ExcelPage.cs | 37 +++++++++--------- src/Lumina/Excel/ExcelSheet.cs | 12 +++--- src/Lumina/Excel/IExcelRow.cs | 4 +- src/Lumina/Excel/SubrowCollection.cs | 10 ++--- src/Lumina/Excel/SubrowExcelSheet.cs | 52 ++++++++++++------------- src/Lumina/Extensions/SpanExtensions.cs | 6 +-- 9 files changed, 96 insertions(+), 91 deletions(-) diff --git a/src/Lumina/Excel/BaseExcelSheet.cs b/src/Lumina/Excel/BaseExcelSheet.cs index 4d5a1a68..46b47d58 100644 --- a/src/Lumina/Excel/BaseExcelSheet.cs +++ b/src/Lumina/Excel/BaseExcelSheet.cs @@ -123,10 +123,10 @@ private protected BaseExcelSheet( ExcelModule module, ExcelHeaderFile headerFile var numUnused = numSlots - headerFile.Header.RowCount; if( numUnused <= MaxUnusedLookupItemCount ) { - _rowIndexLookupArray = new int[ numSlots ]; + _rowIndexLookupArray = new int[numSlots]; _rowIndexLookupArray.AsSpan().Fill( -1 ); - for (i = 0; i < _rowOffsetLookupTable.Length; i++) - _rowIndexLookupArray[_rowOffsetLookupTable[ i ].RowId - firstId] = i; + for( i = 0; i < _rowOffsetLookupTable.Length; i++ ) + _rowIndexLookupArray[ _rowOffsetLookupTable[ i ].RowId - firstId ] = i; // All items can be looked up from _rowIndexLookupArray. Dictionary is unnecessary. _rowIndexLookupDict = FrozenDictionary< int, int >.Empty; @@ -215,15 +215,15 @@ public static BaseExcelSheet From< T >( ExcelModule module, Language language, s throw new ArgumentException( "Invalid sheet name", nameof( sheetName ) ); if( module.VerifySheetChecksums && columnHash is { } hash && headerFile.GetColumnsHash() != hash ) - throw new MismatchedColumnHashException(hash, headerFile.GetColumnsHash(), nameof(columnHash) ); + throw new MismatchedColumnHashException( hash, headerFile.GetColumnsHash(), nameof( columnHash ) ); if( !headerFile.Languages.Contains( language ) ) throw new UnsupportedLanguageException(); return headerFile.Header.Variant switch { - ExcelVariant.Default => new ExcelSheet( module, headerFile, language, sheetName ), - ExcelVariant.Subrows => new SubrowExcelSheet( module, headerFile, language, sheetName ), + ExcelVariant.Default => new ExcelSheet< T >( module, headerFile, language, sheetName ), + ExcelVariant.Subrows => new SubrowExcelSheet< T >( module, headerFile, language, sheetName ), _ => throw new NotSupportedException( $"Specified sheet variant {headerFile.Header.Variant} is not supported." ), }; } @@ -260,18 +260,18 @@ public bool HasRow( uint rowId ) [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] internal ref readonly RowOffsetLookup GetRowLookupOrNullRef( uint rowId ) { - var lookupArrayIndex = unchecked( rowId - _rowOffsetLookupTable.UnsafeAt(0).RowId ); + var lookupArrayIndex = unchecked( rowId - _rowOffsetLookupTable.UnsafeAt( 0 ).RowId ); if( lookupArrayIndex < _rowIndexLookupArray.Length ) { var rowIndex = _rowIndexLookupArray.UnsafeAt( (int) lookupArrayIndex ); - if (rowIndex == -1) - return ref Unsafe.NullRef(); + if( rowIndex == -1 ) + return ref Unsafe.NullRef< RowOffsetLookup >(); return ref UnsafeGetRowLookupAt( rowIndex ); } ref readonly var rowIndexRef = ref _rowIndexLookupDict.GetValueRefOrNullRef( (int) rowId ); if( Unsafe.IsNullRef( in rowIndexRef ) ) - return ref Unsafe.NullRef(); + return ref Unsafe.NullRef< RowOffsetLookup >(); return ref UnsafeGetRowLookupAt( rowIndexRef ); } @@ -279,7 +279,7 @@ internal ref readonly RowOffsetLookup GetRowLookupOrNullRef( uint rowId ) /// Index of the desired row. /// Lookup data for the desired row. internal ref readonly RowOffsetLookup UnsafeGetRowLookupAt( int rowIndex ) => - ref _rowOffsetLookupTable.UnsafeAt(rowIndex); + ref _rowOffsetLookupTable.UnsafeAt( rowIndex ); /// Creates a row at the given index, without checking for bounds or preconditions. /// Index of the desired row. @@ -299,7 +299,7 @@ internal T UnsafeCreateSubrowAt< T >( int rowIndex, ushort subrowId ) where T : /// A new instance of . internal T UnsafeCreateRow< T >( scoped ref readonly RowOffsetLookup lookup ) where T : struct, IExcelRow< T > => T.Create( - _pages.UnsafeAt(lookup.PageIndex), + _pages.UnsafeAt( lookup.PageIndex ), lookup.Offset, lookup.RowId ); @@ -309,7 +309,7 @@ internal T UnsafeCreateRow< T >( scoped ref readonly RowOffsetLookup lookup ) wh /// A new instance of . internal T UnsafeCreateSubrow< T >( scoped ref readonly RowOffsetLookup lookup, ushort subrowId ) where T : struct, IExcelRow< T > => T.Create( - _pages.UnsafeAt(lookup.PageIndex), + _pages.UnsafeAt( lookup.PageIndex ), lookup.Offset + 2 + subrowId * ( _subrowDataOffset + 2u ), lookup.RowId, subrowId ); diff --git a/src/Lumina/Excel/Collection.cs b/src/Lumina/Excel/Collection.cs index e58b89db..2a65e5e1 100644 --- a/src/Lumina/Excel/Collection.cs +++ b/src/Lumina/Excel/Collection.cs @@ -15,37 +15,39 @@ namespace Lumina.Excel; /// /// /// -public readonly struct Collection( ExcelPage page, uint parentOffset, uint offset, Func ctor, int size ) : IReadOnlyList, ICollection where T : struct +public readonly struct Collection< T >( ExcelPage page, uint parentOffset, uint offset, Func< ExcelPage, uint, uint, uint, T > ctor, int size ) + : IReadOnlyList< T >, ICollection< T > where T : struct { /// - public T this[int index] { + public T this[ int index ] { [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] get { ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual( index, size ); - return ctor( page, parentOffset, offset, (uint)index ); + return ctor( page, parentOffset, offset, (uint) index ); } } /// public int Count => size; - bool ICollection.IsReadOnly => true; + bool ICollection< T >.IsReadOnly => true; - void ICollection.Add( T item ) => throw new NotSupportedException(); + void ICollection< T >.Add( T item ) => throw new NotSupportedException(); - void ICollection.Clear() => throw new NotSupportedException(); + void ICollection< T >.Clear() => throw new NotSupportedException(); - bool ICollection.Remove( T item ) => throw new NotSupportedException(); + bool ICollection< T >.Remove( T item ) => throw new NotSupportedException(); /// public bool Contains( T item ) { - var comparer = EqualityComparer.Default; - foreach (var element in this ) + var comparer = EqualityComparer< T >.Default; + foreach( var element in this ) { if( comparer.Equals( item, element ) ) return true; } + return false; } @@ -56,25 +58,25 @@ public void CopyTo( T[] array, int arrayIndex ) ArgumentOutOfRangeException.ThrowIfNegative( arrayIndex ); if( Count > array.Length - arrayIndex ) throw new ArgumentException( "The number of elements in the source list is greater than the available space." ); - for (var i = 0; i < Count; i++ ) - array[ arrayIndex++ ] = this[i]; + for( var i = 0; i < Count; i++ ) + array[ arrayIndex++ ] = this[ i ]; } /// public Enumerator GetEnumerator() => new( this ); - readonly IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + readonly IEnumerator< T > IEnumerable< T >.GetEnumerator() => GetEnumerator(); readonly IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); /// Enumerator that enumerates over the different items. /// Collection to iterate over. - public struct Enumerator( Collection collection ) : IEnumerator + public struct Enumerator( Collection< T > collection ) : IEnumerator< T > { private int _index = -1; /// - public readonly T Current => collection[_index]; + public readonly T Current => collection[ _index ]; readonly object IEnumerator.Current => Current; diff --git a/src/Lumina/Excel/ExcelModule.cs b/src/Lumina/Excel/ExcelModule.cs index ef94e35c..a92207d8 100644 --- a/src/Lumina/Excel/ExcelModule.cs +++ b/src/Lumina/Excel/ExcelModule.cs @@ -88,7 +88,7 @@ public SubrowExcelSheet< T > GetSubrowSheet< T >( Language? language = null ) wh /// does not have a valid . /// [EditorBrowsable( EditorBrowsableState.Advanced )] - public BaseExcelSheet GetBaseSheet( Language? language = null ) where T : struct, IExcelRow => + public BaseExcelSheet GetBaseSheet< T >( Language? language = null ) where T : struct, IExcelRow< T > => GetBaseSheet( typeof( T ), language ); /// Loads an . @@ -144,6 +144,7 @@ public BaseExcelSheet GetBaseSheet( Type rowType, Language? language = null ) throw new UnsupportedLanguageException( nameof( language ), language, null ); return GetBaseSheet( rowType, Language.None ); } + throw e; } @@ -152,14 +153,15 @@ private sealed class InvalidSheet : BaseExcelSheet public Exception Exception { get; private set; } // never actually called - private InvalidSheet( ExcelModule module, ExcelHeaderFile headerFile, Language requestedLanguage, string sheetName ) : base(module, headerFile, requestedLanguage, sheetName) + private InvalidSheet( ExcelModule module, ExcelHeaderFile headerFile, Language requestedLanguage, string sheetName ) : base( module, headerFile, + requestedLanguage, sheetName ) { Exception = null!; } - public static InvalidSheet Create(Exception exception ) + public static InvalidSheet Create( Exception exception ) { - var ret = (InvalidSheet)RuntimeHelpers.GetUninitializedObject( typeof( InvalidSheet ) ); + var ret = (InvalidSheet) RuntimeHelpers.GetUninitializedObject( typeof( InvalidSheet ) ); ret.Exception = exception; return ret; } diff --git a/src/Lumina/Excel/ExcelPage.cs b/src/Lumina/Excel/ExcelPage.cs index 7f1f9ec8..956e1e58 100644 --- a/src/Lumina/Excel/ExcelPage.cs +++ b/src/Lumina/Excel/ExcelPage.cs @@ -23,7 +23,7 @@ public sealed class ExcelPage public ExcelModule Module { get; } private readonly byte[] data; - private ReadOnlyMemory Data => data; + private ReadOnlyMemory< byte > Data => data; private readonly ushort dataOffset; @@ -37,12 +37,12 @@ internal ExcelPage( ExcelModule module, byte[] pageData, ushort headerDataOffset // Ignores bounds checks to speed up reading data. // https://t.ly/EmR4n (Sharplab link) [MethodImpl( MethodImplOptions.AggressiveInlining )] - private D Read( nuint offset ) where D : unmanaged => - Unsafe.As( ref Unsafe.AddByteOffset( ref MemoryMarshal.GetArrayDataReference( data ), offset ) ); + private D Read< D >( nuint offset ) where D : unmanaged => + Unsafe.As< byte, D >( ref Unsafe.AddByteOffset( ref MemoryMarshal.GetArrayDataReference( data ), offset ) ); [MethodImpl( MethodImplOptions.AggressiveInlining )] private static float ReverseEndianness( float v ) => - Unsafe.BitCast( BinaryPrimitives.ReverseEndianness( Unsafe.BitCast( v ) ) ); + Unsafe.BitCast< uint, float >( BinaryPrimitives.ReverseEndianness( Unsafe.BitCast< float, uint >( v ) ) ); /// /// Reads a from the page data at . @@ -57,14 +57,15 @@ private static float ReverseEndianness( float v ) => public ReadOnlySeString ReadString( nuint offset, nuint structOffset ) { offset = ReadUInt32( offset ) + structOffset + dataOffset; - var data = Data[(int)offset..]; - var stringLength = data.Span.IndexOf( (byte)0 ); - var ret = new ReadOnlySeString( data[..stringLength] ); + var data = Data[ (int) offset.. ]; + var stringLength = data.Span.IndexOf( (byte) 0 ); + var ret = new ReadOnlySeString( data[ ..stringLength ] ); if( ret.IsRsv() && Module.RsvResolver != null ) { if( Module.RsvResolver.Invoke( ret, out var resolvedString ) ) return resolvedString; } + return ret; } @@ -75,7 +76,7 @@ public ReadOnlySeString ReadString( nuint offset, nuint structOffset ) /// The . [MethodImpl( MethodImplOptions.AggressiveInlining )] public bool ReadBool( nuint offset ) => - Read( offset ); + Read< bool >( offset ); /// /// Reads a from the page data at . @@ -84,7 +85,7 @@ public bool ReadBool( nuint offset ) => /// The . [MethodImpl( MethodImplOptions.AggressiveInlining )] public sbyte ReadInt8( nuint offset ) => - Read( offset ); + Read< sbyte >( offset ); /// /// Reads a from the page data at . @@ -93,7 +94,7 @@ public sbyte ReadInt8( nuint offset ) => /// The . [MethodImpl( MethodImplOptions.AggressiveInlining )] public byte ReadUInt8( nuint offset ) => - Read( offset ); + Read< byte >( offset ); /// /// Reads a from the page data at . @@ -102,7 +103,7 @@ public byte ReadUInt8( nuint offset ) => /// The . [MethodImpl( MethodImplOptions.AggressiveInlining )] public short ReadInt16( nuint offset ) => - BinaryPrimitives.ReverseEndianness( Read( offset ) ); + BinaryPrimitives.ReverseEndianness( Read< short >( offset ) ); /// /// Reads a from the page data at . @@ -111,7 +112,7 @@ public short ReadInt16( nuint offset ) => /// The . [MethodImpl( MethodImplOptions.AggressiveInlining )] public ushort ReadUInt16( nuint offset ) => - BinaryPrimitives.ReverseEndianness( Read( offset ) ); + BinaryPrimitives.ReverseEndianness( Read< ushort >( offset ) ); /// /// Reads a from the page data at . @@ -120,7 +121,7 @@ public ushort ReadUInt16( nuint offset ) => /// The . [MethodImpl( MethodImplOptions.AggressiveInlining )] public int ReadInt32( nuint offset ) => - BinaryPrimitives.ReverseEndianness( Read( offset ) ); + BinaryPrimitives.ReverseEndianness( Read< int >( offset ) ); /// /// Reads a from the page data at . @@ -129,7 +130,7 @@ public int ReadInt32( nuint offset ) => /// The . [MethodImpl( MethodImplOptions.AggressiveInlining )] public uint ReadUInt32( nuint offset ) => - BinaryPrimitives.ReverseEndianness( Read( offset ) ); + BinaryPrimitives.ReverseEndianness( Read< uint >( offset ) ); /// /// Reads a from the page data at . @@ -138,7 +139,7 @@ public uint ReadUInt32( nuint offset ) => /// The . [MethodImpl( MethodImplOptions.AggressiveInlining )] public float ReadFloat32( nuint offset ) => - ReverseEndianness( Read( offset ) ); + ReverseEndianness( Read< float >( offset ) ); /// /// Reads a from the page data at . @@ -147,7 +148,7 @@ public float ReadFloat32( nuint offset ) => /// The . [MethodImpl( MethodImplOptions.AggressiveInlining )] public long ReadInt64( nuint offset ) => - BinaryPrimitives.ReverseEndianness( Read( offset ) ); + BinaryPrimitives.ReverseEndianness( Read< long >( offset ) ); /// /// Reads a from the page data at . @@ -156,7 +157,7 @@ public long ReadInt64( nuint offset ) => /// The . [MethodImpl( MethodImplOptions.AggressiveInlining )] public ulong ReadUInt64( nuint offset ) => - BinaryPrimitives.ReverseEndianness( Read( offset ) ); + BinaryPrimitives.ReverseEndianness( Read< ulong >( offset ) ); /// /// Reads a from the page data at at bit offset . @@ -166,5 +167,5 @@ public ulong ReadUInt64( nuint offset ) => /// The . [MethodImpl( MethodImplOptions.AggressiveInlining )] public bool ReadPackedBool( nuint offset, byte bit ) => - ( Read( offset ) & ( 1 << bit ) ) != 0; + ( Read< byte >( offset ) & ( 1 << bit ) ) != 0; } \ No newline at end of file diff --git a/src/Lumina/Excel/ExcelSheet.cs b/src/Lumina/Excel/ExcelSheet.cs index 901e01e4..b5053e0f 100644 --- a/src/Lumina/Excel/ExcelSheet.cs +++ b/src/Lumina/Excel/ExcelSheet.cs @@ -10,7 +10,7 @@ namespace Lumina.Excel; /// An excel sheet of variant. /// Type of the rows contained within. -public sealed class ExcelSheet< T > : BaseExcelSheet, ICollection, IReadOnlyCollection where T : struct, IExcelRow< T > +public sealed class ExcelSheet< T > : BaseExcelSheet, ICollection< T >, IReadOnlyCollection< T > where T : struct, IExcelRow< T > { internal ExcelSheet( ExcelModule module, ExcelHeaderFile headerFile, Language requestedLanguage, string sheetName ) : base( module, headerFile, requestedLanguage, sheetName ) @@ -29,7 +29,7 @@ internal ExcelSheet( ExcelModule module, ExcelHeaderFile headerFile, Language re public T? GetRowOrDefault( uint rowId ) { ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); - return Unsafe.IsNullRef(in lookup) ? null : UnsafeCreateRow< T >( in lookup ); + return Unsafe.IsNullRef( in lookup ) ? null : UnsafeCreateRow< T >( in lookup ); } /// @@ -41,7 +41,7 @@ internal ExcelSheet( ExcelModule module, ExcelHeaderFile headerFile, Language re public bool TryGetRow( uint rowId, out T row ) { ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); - if( Unsafe.IsNullRef(in lookup) ) + if( Unsafe.IsNullRef( in lookup ) ) { row = default; return false; @@ -60,7 +60,7 @@ public bool TryGetRow( uint rowId, out T row ) public T GetRow( uint rowId ) { ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); - return Unsafe.IsNullRef(in lookup) ? throw new ArgumentOutOfRangeException( nameof( rowId ), rowId, null ) : UnsafeCreateRow< T >( in lookup ); + return Unsafe.IsNullRef( in lookup ) ? throw new ArgumentOutOfRangeException( nameof( rowId ), rowId, null ) : UnsafeCreateRow< T >( in lookup ); } /// @@ -87,8 +87,8 @@ public void CopyTo( T[] array, int arrayIndex ) ArgumentOutOfRangeException.ThrowIfNegative( arrayIndex ); if( Count > array.Length - arrayIndex ) throw new ArgumentException( "The number of elements in the source list is greater than the available space." ); - foreach (var lookup in OffsetLookupTable) - array[ arrayIndex++ ] = UnsafeCreateRow( in lookup ); + foreach( var lookup in OffsetLookupTable ) + array[ arrayIndex++ ] = UnsafeCreateRow< T >( in lookup ); } void ICollection< T >.Add( T item ) => throw new NotSupportedException(); diff --git a/src/Lumina/Excel/IExcelRow.cs b/src/Lumina/Excel/IExcelRow.cs index 9a823a13..36a67c30 100644 --- a/src/Lumina/Excel/IExcelRow.cs +++ b/src/Lumina/Excel/IExcelRow.cs @@ -6,7 +6,7 @@ namespace Lumina.Excel; /// Defines a row type/schema for an excel sheet. /// /// The type that implements the interface. -public interface IExcelRow where T : struct +public interface IExcelRow< T > where T : struct { /// /// Creates an instance of the current type. Designed only for use within . @@ -37,4 +37,4 @@ public interface IExcelRow where T : struct /// Gets the subrow ID. /// Thrown when the referenced sheet is not using subrows. public ushort SubrowId { get; } -} +} \ No newline at end of file diff --git a/src/Lumina/Excel/SubrowCollection.cs b/src/Lumina/Excel/SubrowCollection.cs index 610e48c6..4e1ed293 100644 --- a/src/Lumina/Excel/SubrowCollection.cs +++ b/src/Lumina/Excel/SubrowCollection.cs @@ -45,15 +45,15 @@ T IList< T >.this[ int index ] { set => throw new NotSupportedException(); } - void IList.Insert( int index, T item ) => throw new NotSupportedException(); + void IList< T >.Insert( int index, T item ) => throw new NotSupportedException(); - void IList.RemoveAt( int index ) => throw new NotSupportedException(); + void IList< T >.RemoveAt( int index ) => throw new NotSupportedException(); - void ICollection.Add( T item ) => throw new NotSupportedException(); + void ICollection< T >.Add( T item ) => throw new NotSupportedException(); - void ICollection.Clear() => throw new NotSupportedException(); + void ICollection< T >.Clear() => throw new NotSupportedException(); - bool ICollection.Remove( T item ) => throw new NotSupportedException(); + bool ICollection< T >.Remove( T item ) => throw new NotSupportedException(); /// public int IndexOf( T item ) diff --git a/src/Lumina/Excel/SubrowExcelSheet.cs b/src/Lumina/Excel/SubrowExcelSheet.cs index 9a2fd604..dd0a3be4 100644 --- a/src/Lumina/Excel/SubrowExcelSheet.cs +++ b/src/Lumina/Excel/SubrowExcelSheet.cs @@ -9,29 +9,29 @@ namespace Lumina.Excel; /// Type of the rows contained within. /// -public sealed class SubrowExcelSheet - : BaseSubrowExcelSheet, ICollection>, IReadOnlyCollection> - where T : struct, IExcelRow +public sealed class SubrowExcelSheet< T > + : BaseSubrowExcelSheet, ICollection< SubrowCollection< T > >, IReadOnlyCollection< SubrowCollection< T > > + where T : struct, IExcelRow< T > { internal SubrowExcelSheet( ExcelModule module, ExcelHeaderFile headerFile, Language requestedLanguage, string sheetName ) : base( module, headerFile, requestedLanguage, sheetName ) { } /// - bool ICollection>.IsReadOnly => true; + bool ICollection< SubrowCollection< T > >.IsReadOnly => true; /// - public SubrowCollection this[uint rowId] => GetRow( rowId ); + public SubrowCollection< T > this[ uint rowId ] => GetRow( rowId ); /// - public T this[uint rowId, ushort subrowId] => GetSubrow( rowId, subrowId ); + public T this[ uint rowId, ushort subrowId ] => GetSubrow( rowId, subrowId ); /// /// Tries to get the subrow collection with row id in this sheet. /// /// The row id to get. /// A nullable subrow collection object. Returns if the row does not exist. - public SubrowCollection? GetRowOrDefault( uint rowId ) + public SubrowCollection< T >? GetRowOrDefault( uint rowId ) { ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); return Unsafe.IsNullRef( in lookup ) ? null : new( this, in lookup ); @@ -43,7 +43,7 @@ internal SubrowExcelSheet( ExcelModule module, ExcelHeaderFile headerFile, Langu /// The row id to get. /// The output subrow collection object. /// if the row exists and is written to and otherwise. - public bool TryGetRow( uint rowId, out SubrowCollection row ) + public bool TryGetRow( uint rowId, out SubrowCollection< T > row ) { ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); if( Unsafe.IsNullRef( in lookup ) ) @@ -62,7 +62,7 @@ public bool TryGetRow( uint rowId, out SubrowCollection row ) /// The row id to get. /// A subrow collection object. /// Thrown if the sheet does not have a row at that . - public SubrowCollection GetRow( uint rowId ) + public SubrowCollection< T > GetRow( uint rowId ) { ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); return Unsafe.IsNullRef( in lookup ) ? throw new ArgumentOutOfRangeException( nameof( rowId ), rowId, null ) : new( this, in lookup ); @@ -74,7 +74,7 @@ public SubrowCollection GetRow( uint rowId ) /// If you are looking to find a row by its id, use instead. /// The zero-based index of this row. /// A subrow collection object. - public SubrowCollection GetRowAt( int rowIndex ) + public SubrowCollection< T > GetRowAt( int rowIndex ) { ArgumentOutOfRangeException.ThrowIfNegative( rowIndex ); ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual( rowIndex, OffsetLookupTable.Length ); @@ -91,7 +91,7 @@ public SubrowCollection GetRowAt( int rowIndex ) public T? GetSubrowOrDefault( uint rowId, ushort subrowId ) { ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); - return Unsafe.IsNullRef( in lookup ) || subrowId >= lookup.SubrowCount ? null : UnsafeCreateSubrow( in lookup, subrowId ); + return Unsafe.IsNullRef( in lookup ) || subrowId >= lookup.SubrowCount ? null : UnsafeCreateSubrow< T >( in lookup, subrowId ); } /// @@ -110,7 +110,7 @@ public bool TryGetSubrow( uint rowId, ushort subrowId, out T subrow ) return false; } - subrow = UnsafeCreateSubrow( in lookup, subrowId ); + subrow = UnsafeCreateSubrow< T >( in lookup, subrowId ); return true; } @@ -129,7 +129,7 @@ public T GetSubrow( uint rowId, ushort subrowId ) ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual( subrowId, lookup.SubrowCount ); - return UnsafeCreateSubrow( in lookup, subrowId ); + return UnsafeCreateSubrow< T >( in lookup, subrowId ); } /// @@ -148,28 +148,28 @@ public T GetSubrowAt( int rowIndex, ushort subrowId ) ref readonly var lookup = ref UnsafeGetRowLookupAt( rowIndex ); ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual( subrowId, lookup.SubrowCount ); - return UnsafeCreateSubrow( in lookup, subrowId ); + return UnsafeCreateSubrow< T >( in lookup, subrowId ); } /// - public bool Contains( SubrowCollection item ) => ReferenceEquals( item.Sheet, this ) && HasRow( item.RowId ); + public bool Contains( SubrowCollection< T > item ) => ReferenceEquals( item.Sheet, this ) && HasRow( item.RowId ); /// - public void CopyTo( SubrowCollection[] array, int arrayIndex ) + public void CopyTo( SubrowCollection< T >[] array, int arrayIndex ) { ArgumentNullException.ThrowIfNull( array ); ArgumentOutOfRangeException.ThrowIfNegative( arrayIndex ); if( Count > array.Length - arrayIndex ) throw new ArgumentException( "The number of elements in the source list is greater than the available space." ); foreach( var lookup in OffsetLookupTable ) - array[arrayIndex++] = new( this, in lookup ); + array[ arrayIndex++ ] = new( this, in lookup ); } - void ICollection>.Add( SubrowCollection item ) => throw new NotSupportedException(); + void ICollection< SubrowCollection< T > >.Add( SubrowCollection< T > item ) => throw new NotSupportedException(); - void ICollection>.Clear() => throw new NotSupportedException(); + void ICollection< SubrowCollection< T > >.Clear() => throw new NotSupportedException(); - bool ICollection>.Remove( SubrowCollection item ) => throw new NotSupportedException(); + bool ICollection< SubrowCollection< T > >.Remove( SubrowCollection< T > item ) => throw new NotSupportedException(); /// Gets an enumerator that enumerates over all subrows. /// A new enumerator. @@ -178,18 +178,18 @@ public void CopyTo( SubrowCollection[] array, int arrayIndex ) /// public Enumerator GetEnumerator() => new( this ); - IEnumerator> IEnumerable>.GetEnumerator() => GetEnumerator(); + IEnumerator< SubrowCollection< T > > IEnumerable< SubrowCollection< T > >.GetEnumerator() => GetEnumerator(); IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); /// Represents an enumerator that iterates over all rows in a . /// The sheet to iterate over. - public struct Enumerator( SubrowExcelSheet sheet ) : IEnumerator> + public struct Enumerator( SubrowExcelSheet< T > sheet ) : IEnumerator< SubrowCollection< T > > { private int _index = -1; /// - public readonly SubrowCollection Current => new( sheet, in sheet.UnsafeGetRowLookupAt( _index ) ); + public readonly SubrowCollection< T > Current => new( sheet, in sheet.UnsafeGetRowLookupAt( _index ) ); readonly object IEnumerator.Current => Current; @@ -215,14 +215,14 @@ public readonly void Dispose() /// Represents an enumerator that iterates over all subrows in a . /// The sheet to iterate over. - public struct FlatEnumerator( SubrowExcelSheet sheet ) : IEnumerator, IEnumerable + public struct FlatEnumerator( SubrowExcelSheet< T > sheet ) : IEnumerator< T >, IEnumerable< T > { private int _index = -1; private ushort _subrowIndex = ushort.MaxValue; private ushort _subrowCount; /// - public readonly T Current => sheet.UnsafeCreateSubrowAt( _index, _subrowIndex ); + public readonly T Current => sheet.UnsafeCreateSubrowAt< T >( _index, _subrowIndex ); readonly object IEnumerator.Current => Current; @@ -266,7 +266,7 @@ public readonly void Dispose() /// public readonly FlatEnumerator GetEnumerator() => new( sheet ); - readonly IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + readonly IEnumerator< T > IEnumerable< T >.GetEnumerator() => GetEnumerator(); readonly IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); } diff --git a/src/Lumina/Extensions/SpanExtensions.cs b/src/Lumina/Extensions/SpanExtensions.cs index 5acd8147..324608df 100644 --- a/src/Lumina/Extensions/SpanExtensions.cs +++ b/src/Lumina/Extensions/SpanExtensions.cs @@ -30,12 +30,12 @@ public static unsafe T ReadStructure< T >( this Span< byte > span, int offset ) #endif } - public static unsafe ref T UnsafeAt( this Span span, int index ) => + public static unsafe ref T UnsafeAt< T >( this Span< T > span, int index ) => ref Unsafe.Add( ref MemoryMarshal.GetReference( span ), index ); - public static unsafe ref readonly T UnsafeAt( this ReadOnlySpan span, int index ) => + public static unsafe ref readonly T UnsafeAt< T >( this ReadOnlySpan< T > span, int index ) => ref Unsafe.Add( ref MemoryMarshal.GetReference( span ), index ); - public static unsafe ref readonly T UnsafeAt(this T[] array, int index) => + public static unsafe ref readonly T UnsafeAt< T >( this T[] array, int index ) => ref Unsafe.Add( ref MemoryMarshal.GetArrayDataReference( array ), index ); } \ No newline at end of file From 5af0b02a611c3c1948b9763cc007090ca511abd3 Mon Sep 17 00:00:00 2001 From: Soreepeong Date: Sat, 17 Aug 2024 20:30:36 +0900 Subject: [PATCH 22/53] Correctness and documents * `MethodInfo.Invoke` throws `TargetInvocationException` if the method throws an exception; changed to handle that. * Added comments for some functions. --- src/Lumina/Excel/BaseExcelSheet.cs | 17 +++++++---- src/Lumina/Excel/ExcelModule.cs | 40 +++++++++++++++++-------- src/Lumina/Excel/IExcelRow.cs | 6 ++-- src/Lumina/Extensions/SpanExtensions.cs | 19 ++++++++++-- src/Lumina/GameData.cs | 12 ++++---- 5 files changed, 64 insertions(+), 30 deletions(-) diff --git a/src/Lumina/Excel/BaseExcelSheet.cs b/src/Lumina/Excel/BaseExcelSheet.cs index 46b47d58..92c4e5c4 100644 --- a/src/Lumina/Excel/BaseExcelSheet.cs +++ b/src/Lumina/Excel/BaseExcelSheet.cs @@ -64,6 +64,7 @@ public abstract class BaseExcelSheet private readonly FrozenDictionary< int, int > _rowIndexLookupDict; private readonly int[] _rowIndexLookupArray; + private readonly uint _rowIndexLookupArrayOffset; /// The module that this sheet belongs to. public ExcelModule Module { get; } @@ -118,15 +119,15 @@ private protected BaseExcelSheet( ExcelModule module, ExcelHeaderFile headerFile // A lot of sheets do not have large gap between row IDs. If total number of gaps is less than a threshold, then make a lookup array. if( _rowOffsetLookupTable.Length > 0 ) { - var firstId = _rowOffsetLookupTable[ 0 ].RowId; - var numSlots = _rowOffsetLookupTable[ ^1 ].RowId - firstId + 1; + _rowIndexLookupArrayOffset = _rowOffsetLookupTable[ 0 ].RowId; + var numSlots = _rowOffsetLookupTable[ ^1 ].RowId - _rowIndexLookupArrayOffset + 1; var numUnused = numSlots - headerFile.Header.RowCount; if( numUnused <= MaxUnusedLookupItemCount ) { _rowIndexLookupArray = new int[numSlots]; _rowIndexLookupArray.AsSpan().Fill( -1 ); for( i = 0; i < _rowOffsetLookupTable.Length; i++ ) - _rowIndexLookupArray[ _rowOffsetLookupTable[ i ].RowId - firstId ] = i; + _rowIndexLookupArray[ _rowOffsetLookupTable[ i ].RowId - _rowIndexLookupArrayOffset ] = i; // All items can be looked up from _rowIndexLookupArray. Dictionary is unnecessary. _rowIndexLookupDict = FrozenDictionary< int, int >.Empty; @@ -139,7 +140,7 @@ private protected BaseExcelSheet( ExcelModule module, ExcelHeaderFile headerFile var lastLookupArrayRowId = uint.MaxValue; for( i = 0; i < _rowOffsetLookupTable.Length; i++ ) { - var offsetRowId = _rowOffsetLookupTable[ i ].RowId - firstId; + var offsetRowId = _rowOffsetLookupTable[ i ].RowId - _rowIndexLookupArrayOffset; if( offsetRowId >= MaxUnusedLookupItemCount ) { // Discard the unused entries. @@ -161,6 +162,7 @@ private protected BaseExcelSheet( ExcelModule module, ExcelHeaderFile headerFile { _rowIndexLookupDict = FrozenDictionary< int, int >.Empty; _rowIndexLookupArray = []; + _rowIndexLookupArrayOffset = 0; _rowOffsetLookupTable = [default]; // so that _rowOffsetLookupTable.UnsafeAt(0) is always valid. Count = 0; } @@ -168,6 +170,7 @@ private protected BaseExcelSheet( ExcelModule module, ExcelHeaderFile headerFile /// Creates a new instance of with the 's default language, deducing sheet names and column /// hashes from . + /// Type of each row. /// The to access sheet data from. /// does not have a valid . /// was invalid (invalid sheet name). @@ -180,6 +183,7 @@ public static BaseExcelSheet From< T >( ExcelModule module ) where T : struct, I From< T >( module, module.Language ); /// Creates a new instance of , deducing sheet names and column hashes from . + /// Type of each row. /// The to access sheet data from. /// The language to use for this sheet. /// does not have a valid . @@ -198,6 +202,7 @@ public static BaseExcelSheet From< T >( ExcelModule module, Language language ) } /// Creates a new instance of . + /// Type of each row. /// The to access sheet data from. /// The language to use for this sheet. /// The name of the sheet to read from. @@ -218,7 +223,7 @@ public static BaseExcelSheet From< T >( ExcelModule module, Language language, s throw new MismatchedColumnHashException( hash, headerFile.GetColumnsHash(), nameof( columnHash ) ); if( !headerFile.Languages.Contains( language ) ) - throw new UnsupportedLanguageException(); + throw new UnsupportedLanguageException( nameof( language ), language, null ); return headerFile.Header.Variant switch { @@ -260,7 +265,7 @@ public bool HasRow( uint rowId ) [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] internal ref readonly RowOffsetLookup GetRowLookupOrNullRef( uint rowId ) { - var lookupArrayIndex = unchecked( rowId - _rowOffsetLookupTable.UnsafeAt( 0 ).RowId ); + var lookupArrayIndex = unchecked( rowId - _rowIndexLookupArrayOffset ); if( lookupArrayIndex < _rowIndexLookupArray.Length ) { var rowIndex = _rowIndexLookupArray.UnsafeAt( (int) lookupArrayIndex ); diff --git a/src/Lumina/Excel/ExcelModule.cs b/src/Lumina/Excel/ExcelModule.cs index a92207d8..9b466835 100644 --- a/src/Lumina/Excel/ExcelModule.cs +++ b/src/Lumina/Excel/ExcelModule.cs @@ -61,7 +61,7 @@ public ExcelModule( GameData gameData ) /// Loads an . /// Sheet is not of the variant . - /// + /// public ExcelSheet< T > GetSheet< T >( Language? language = null ) where T : struct, IExcelRow< T > => (ExcelSheet< T >) GetBaseSheet< T >( language ); @@ -72,7 +72,7 @@ public ExcelSheet< T > GetSheet< T >( Language? language = null ) where T : stru /// . /// /// Sheet is not of the variant . - /// + /// public SubrowExcelSheet< T > GetSubrowSheet< T >( Language? language = null ) where T : struct, IExcelRow< T > => (SubrowExcelSheet< T >) GetBaseSheet< T >( language ); @@ -86,7 +86,7 @@ public SubrowExcelSheet< T > GetSubrowSheet< T >( Language? language = null ) wh /// before accessing its rows. /// /// does not have a valid . - /// + /// [EditorBrowsable( EditorBrowsableState.Advanced )] public BaseExcelSheet GetBaseSheet< T >( Language? language = null ) where T : struct, IExcelRow< T > => GetBaseSheet( typeof( T ), language ); @@ -113,22 +113,38 @@ public BaseExcelSheet GetBaseSheet< T >( Language? language = null ) where T : s [EditorBrowsable( EditorBrowsableState.Advanced )] public BaseExcelSheet GetBaseSheet( Type rowType, Language? language = null ) { - if( !rowType.IsValueType ) - throw new ArgumentException( $"{nameof( rowType )} must be a struct.", nameof( rowType ) ); - - if( !rowType.IsAssignableTo( typeof( IExcelRow<> ).MakeGenericType( rowType ) ) ) - throw new ArgumentException( $"{nameof( rowType )} must implement {typeof( IExcelRow<> ).Name}.", nameof( rowType ) ); - var sheet = SheetCache.GetOrAdd( ( rowType, language ?? Language ), static ( key, module ) => { - var m = typeof( BaseExcelSheet ) - .GetMethod( nameof( BaseExcelSheet.From ), BindingFlags.Static | BindingFlags.Public )! - .MakeGenericMethod( key.Type ); + MethodInfo m; + try + { + // As BaseExcelSheet.From has a constraint that T : IExcelRow, it is implicitly required that T is also a struct. + // MakeGenericMethod will check for constraints, and throw ArgumentException if constraints aren't met. + m = typeof( BaseExcelSheet ) + .GetMethod( + nameof( BaseExcelSheet.From ), + BindingFlags.Static | BindingFlags.Public, + [typeof( ExcelModule ), typeof( Language )] )! + .MakeGenericMethod( key.Type ); + } + catch( ArgumentException e ) + { + // Exception thrown here will propagate outside ConcurrentDictionary<>.GetOrAdd without touching the data stored inside dictionary. + throw new ArgumentException( + $"{key.Type.Name} must implement {typeof( IExcelRow<> ).Name.Split( '`', 2 )[ 0 ]}<{key.Type.Name}>.", + nameof( rowType ), + e ); + } + try { return m.Invoke( null, [module, key.Language] ) as BaseExcelSheet ?? throw new InvalidOperationException( "Something went wrong" ); } + catch( TargetInvocationException e ) + { + return InvalidSheet.Create( e.InnerException ?? e ); + } catch( Exception e ) { return InvalidSheet.Create( e ); diff --git a/src/Lumina/Excel/IExcelRow.cs b/src/Lumina/Excel/IExcelRow.cs index 36a67c30..f006fcb9 100644 --- a/src/Lumina/Excel/IExcelRow.cs +++ b/src/Lumina/Excel/IExcelRow.cs @@ -6,7 +6,7 @@ namespace Lumina.Excel; /// Defines a row type/schema for an excel sheet. /// /// The type that implements the interface. -public interface IExcelRow< T > where T : struct +public interface IExcelRow< out T > where T : struct { /// /// Creates an instance of the current type. Designed only for use within . @@ -32,9 +32,9 @@ public interface IExcelRow< T > where T : struct abstract static T Create( ExcelPage page, uint offset, uint row, ushort subrow ); /// Gets the row ID. - public uint RowId { get; } + uint RowId { get; } /// Gets the subrow ID. /// Thrown when the referenced sheet is not using subrows. - public ushort SubrowId { get; } + ushort SubrowId { get; } } \ No newline at end of file diff --git a/src/Lumina/Extensions/SpanExtensions.cs b/src/Lumina/Extensions/SpanExtensions.cs index 324608df..0acda0c3 100644 --- a/src/Lumina/Extensions/SpanExtensions.cs +++ b/src/Lumina/Extensions/SpanExtensions.cs @@ -30,12 +30,25 @@ public static unsafe T ReadStructure< T >( this Span< byte > span, int offset ) #endif } - public static unsafe ref T UnsafeAt< T >( this Span< T > span, int index ) => + /// Gets the reference to the item at the given index in the given span, without boundary checks. + /// Span to get an item reference from. + /// Index of the item that should be at least 0 and at most span.Length - 1. + /// Type of elements. + /// Reference to the item at the given index in the span. + /// If is negative or greater than or equal to span.Length, then the behavior is undefined. + public static ref T UnsafeAt< T >( this Span< T > span, int index ) => ref Unsafe.Add( ref MemoryMarshal.GetReference( span ), index ); - public static unsafe ref readonly T UnsafeAt< T >( this ReadOnlySpan< T > span, int index ) => + /// + public static ref readonly T UnsafeAt< T >( this ReadOnlySpan< T > span, int index ) => ref Unsafe.Add( ref MemoryMarshal.GetReference( span ), index ); - public static unsafe ref readonly T UnsafeAt< T >( this T[] array, int index ) => + /// Gets the reference to the item at the given index in the given array, without boundary checks. + /// Array to get an item reference from. + /// Index of the item that should be at least 0 and at most array.Length - 1. + /// Type of elements. + /// Reference to the item at the given index in the array. + /// If is negative or greater than or equal to array.Length, then the behavior is undefined. + public static ref T UnsafeAt< T >( this T[] array, int index ) => ref Unsafe.Add( ref MemoryMarshal.GetArrayDataReference( array ), index ); } \ No newline at end of file diff --git a/src/Lumina/GameData.cs b/src/Lumina/GameData.cs index 7e312c5a..283c6f5d 100644 --- a/src/Lumina/GameData.cs +++ b/src/Lumina/GameData.cs @@ -291,9 +291,9 @@ public static UInt64 GetFileHash( string path ) /// An excel sheet corresponding to and that may be created anew or /// reused from a previous invocation of this method. /// - /// If the requested language doesn't exist for the file where is not , the language-neutral - /// sheet using will be loaded instead. If the language-neutral sheet does not exist, then the function will fail with - /// . + /// If the requested language doesn't exist for the file where is not , the + /// language-neutral sheet using will be loaded instead. If the language-neutral sheet does not exist, then the function + /// will return . /// /// Sheet is not of the variant . /// does not have a valid . @@ -318,9 +318,9 @@ public static UInt64 GetFileHash( string path ) /// An excel sheet corresponding to and that may be created anew or /// reused from a previous invocation of this method. /// - /// If the requested language doesn't exist for the file where is not , the language-neutral - /// sheet using will be loaded instead. If the language-neutral sheet does not exist, then the function will fail with - /// . + /// If the requested language doesn't exist for the file where is not , the + /// language-neutral sheet using will be loaded instead. If the language-neutral sheet does not exist, then the function + /// will return . /// /// Sheet is not of the variant . /// does not have a valid . From e1c7d7833663a5969567df8c665c0e3613ceb9aa Mon Sep 17 00:00:00 2001 From: Soreepeong Date: Sat, 17 Aug 2024 20:47:34 +0900 Subject: [PATCH 23/53] Set `IEnumerator.Current` on `MoveNext` Making `IEnumerator.Current` evaluate on demand can let an invalid value get passed to UnsafeCreate functions. Creating them on `MoveNext` will guarantee that UnsafeCreate functions are called only from the context where the preconditions are met. --- src/Lumina/Excel/Collection.cs | 50 +++++++++++++++++++++------- src/Lumina/Excel/ExcelSheet.cs | 11 ++++-- src/Lumina/Excel/SubrowCollection.cs | 8 ++++- src/Lumina/Excel/SubrowExcelSheet.cs | 20 ++++++++--- 4 files changed, 68 insertions(+), 21 deletions(-) diff --git a/src/Lumina/Excel/Collection.cs b/src/Lumina/Excel/Collection.cs index 2a65e5e1..3be64629 100644 --- a/src/Lumina/Excel/Collection.cs +++ b/src/Lumina/Excel/Collection.cs @@ -16,21 +16,31 @@ namespace Lumina.Excel; /// /// public readonly struct Collection< T >( ExcelPage page, uint parentOffset, uint offset, Func< ExcelPage, uint, uint, uint, T > ctor, int size ) - : IReadOnlyList< T >, ICollection< T > where T : struct + : IList< T >, IReadOnlyList< T > where T : struct { + /// + public int Count => size; + + bool ICollection< T >.IsReadOnly => true; + /// public T this[ int index ] { [MethodImpl( MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization )] get { ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual( index, size ); - return ctor( page, parentOffset, offset, (uint) index ); + return UnsafeCreateAt( index ); } } /// - public int Count => size; + T IList< T >.this[ int index ] { + get => this[ index ]; + set => throw new NotSupportedException(); + } - bool ICollection< T >.IsReadOnly => true; + void IList< T >.Insert( int index, T item ) => throw new NotSupportedException(); + + void IList< T >.RemoveAt( int index ) => throw new NotSupportedException(); void ICollection< T >.Add( T item ) => throw new NotSupportedException(); @@ -39,18 +49,23 @@ public T this[ int index ] { bool ICollection< T >.Remove( T item ) => throw new NotSupportedException(); /// - public bool Contains( T item ) + public int IndexOf( T item ) { + var i = 0; var comparer = EqualityComparer< T >.Default; foreach( var element in this ) { if( comparer.Equals( item, element ) ) - return true; + return i; + ++i; } - return false; + return -1; } + /// + public bool Contains( T item ) => IndexOf( item ) != -1; + /// public void CopyTo( T[] array, int arrayIndex ) { @@ -58,16 +73,21 @@ public void CopyTo( T[] array, int arrayIndex ) ArgumentOutOfRangeException.ThrowIfNegative( arrayIndex ); if( Count > array.Length - arrayIndex ) throw new ArgumentException( "The number of elements in the source list is greater than the available space." ); - for( var i = 0; i < Count; i++ ) - array[ arrayIndex++ ] = this[ i ]; + foreach( var e in this ) + array[ arrayIndex++ ] = e; } /// public Enumerator GetEnumerator() => new( this ); - readonly IEnumerator< T > IEnumerable< T >.GetEnumerator() => GetEnumerator(); + IEnumerator< T > IEnumerable< T >.GetEnumerator() => GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); - readonly IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + /// Creates an item at the given index, without checking for boundaries. + /// Index of the item. + /// Newly created item. + private T UnsafeCreateAt( int index ) => ctor( page, parentOffset, offset, unchecked( (uint) index ) ); /// Enumerator that enumerates over the different items. /// Collection to iterate over. @@ -76,7 +96,7 @@ public struct Enumerator( Collection< T > collection ) : IEnumerator< T > private int _index = -1; /// - public readonly T Current => collection[ _index ]; + public T Current { get; private set; } readonly object IEnumerator.Current => Current; @@ -84,7 +104,13 @@ public struct Enumerator( Collection< T > collection ) : IEnumerator< T > public bool MoveNext() { if( ++_index < collection.Count ) + { + // UnsafeCreateAt must be called only when the preconditions are validated. + // If it is to be called on-demand from get_Current, then it may end up being called with invalid parameters, + // so we create the instance in advance here. + Current = collection.UnsafeCreateAt( _index ); return true; + } --_index; return false; diff --git a/src/Lumina/Excel/ExcelSheet.cs b/src/Lumina/Excel/ExcelSheet.cs index b5053e0f..a28a43c8 100644 --- a/src/Lumina/Excel/ExcelSheet.cs +++ b/src/Lumina/Excel/ExcelSheet.cs @@ -111,7 +111,7 @@ public struct Enumerator( ExcelSheet< T > sheet ) : IEnumerator< T > private int _index = -1; /// - public readonly T Current => sheet.UnsafeCreateRowAt< T >( _index ); + public T Current { get; private set; } readonly object IEnumerator.Current => Current; @@ -119,15 +119,20 @@ public struct Enumerator( ExcelSheet< T > sheet ) : IEnumerator< T > public bool MoveNext() { if( ++_index < sheet.Count ) + { + // UnsafeCreateRowAt must be called only when the preconditions are validated. + // If it is to be called on-demand from get_Current, then it may end up being called with invalid parameters, + // so we create the instance in advance here. + Current = sheet.UnsafeCreateRowAt< T >( _index ); return true; + } --_index; return false; } /// - public void Reset() => - _index = -1; + public void Reset() => _index = -1; /// public readonly void Dispose() diff --git a/src/Lumina/Excel/SubrowCollection.cs b/src/Lumina/Excel/SubrowCollection.cs index 4e1ed293..551d4e72 100644 --- a/src/Lumina/Excel/SubrowCollection.cs +++ b/src/Lumina/Excel/SubrowCollection.cs @@ -93,7 +93,7 @@ public struct Enumerator( SubrowCollection< T > subrowCollection ) : IEnumerator private int _index = -1; /// - public readonly T Current => subrowCollection.Sheet.UnsafeCreateSubrow< T >( in subrowCollection._lookup, unchecked( (ushort) _index ) ); + public T Current { get; private set; } readonly object IEnumerator.Current => Current; @@ -101,7 +101,13 @@ public struct Enumerator( SubrowCollection< T > subrowCollection ) : IEnumerator public bool MoveNext() { if( ++_index < subrowCollection.Count ) + { + // UnsafeCreateSubrow must be called only when the preconditions are validated. + // If it is to be called on-demand from get_Current, then it may end up being called with invalid parameters, + // so we create the instance in advance here. + Current = subrowCollection.Sheet.UnsafeCreateSubrow< T >( in subrowCollection._lookup, unchecked( (ushort) _index ) ); return true; + } --_index; return false; diff --git a/src/Lumina/Excel/SubrowExcelSheet.cs b/src/Lumina/Excel/SubrowExcelSheet.cs index dd0a3be4..ebfabb8e 100644 --- a/src/Lumina/Excel/SubrowExcelSheet.cs +++ b/src/Lumina/Excel/SubrowExcelSheet.cs @@ -189,16 +189,21 @@ public struct Enumerator( SubrowExcelSheet< T > sheet ) : IEnumerator< SubrowCol private int _index = -1; /// - public readonly SubrowCollection< T > Current => new( sheet, in sheet.UnsafeGetRowLookupAt( _index ) ); + public SubrowCollection< T > Current { get; private set; } - readonly object IEnumerator.Current => - Current; + readonly object IEnumerator.Current => Current; /// public bool MoveNext() { if( ++_index < sheet.Count ) + { + // UnsafeGetRowLookupAt must be called only when the preconditions are validated. + // If it is to be called on-demand from get_Current, then it may end up being called with invalid parameters, + // so we create the instance in advance here. + Current = new( sheet, in sheet.UnsafeGetRowLookupAt( _index ) ); return true; + } --_index; return false; @@ -222,7 +227,7 @@ public struct FlatEnumerator( SubrowExcelSheet< T > sheet ) : IEnumerator< T >, private ushort _subrowCount; /// - public readonly T Current => sheet.UnsafeCreateSubrowAt< T >( _index, _subrowIndex ); + public T Current { get; private set; } readonly object IEnumerator.Current => Current; @@ -243,11 +248,16 @@ public bool MoveNext() _subrowCount = sheet.UnsafeGetRowLookupAt( _index ).SubrowCount; if( _subrowCount == 0 ) continue; + _subrowIndex = 0; - return true; + break; } } + // UnsafeCreateSubrowAt must be called only when the preconditions are validated. + // If it is to be called on-demand from get_Current, then it may end up being called with invalid parameters, + // so we create the instance in advance here. + Current = sheet.UnsafeCreateSubrowAt< T >( _index, _subrowIndex ); return true; } From 2ad083e92845ba1345b8abb039891280bfd47399 Mon Sep 17 00:00:00 2001 From: Soreepeong Date: Sat, 17 Aug 2024 21:13:17 +0900 Subject: [PATCH 24/53] Extra format --- src/Lumina/Excel/ExcelModule.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Lumina/Excel/ExcelModule.cs b/src/Lumina/Excel/ExcelModule.cs index 9b466835..212698f4 100644 --- a/src/Lumina/Excel/ExcelModule.cs +++ b/src/Lumina/Excel/ExcelModule.cs @@ -169,8 +169,8 @@ private sealed class InvalidSheet : BaseExcelSheet public Exception Exception { get; private set; } // never actually called - private InvalidSheet( ExcelModule module, ExcelHeaderFile headerFile, Language requestedLanguage, string sheetName ) : base( module, headerFile, - requestedLanguage, sheetName ) + private InvalidSheet( ExcelModule module, ExcelHeaderFile headerFile, Language requestedLanguage, string sheetName ) + : base( module, headerFile, requestedLanguage, sheetName ) { Exception = null!; } From 2a3cee23e1b316192e416913168ac444c6236cac Mon Sep 17 00:00:00 2001 From: Soreepeong Date: Sun, 18 Aug 2024 02:32:58 +0900 Subject: [PATCH 25/53] Remove unnecessary code --- src/Lumina/Excel/BaseExcelSheet.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Lumina/Excel/BaseExcelSheet.cs b/src/Lumina/Excel/BaseExcelSheet.cs index 92c4e5c4..1272dca2 100644 --- a/src/Lumina/Excel/BaseExcelSheet.cs +++ b/src/Lumina/Excel/BaseExcelSheet.cs @@ -163,7 +163,7 @@ private protected BaseExcelSheet( ExcelModule module, ExcelHeaderFile headerFile _rowIndexLookupDict = FrozenDictionary< int, int >.Empty; _rowIndexLookupArray = []; _rowIndexLookupArrayOffset = 0; - _rowOffsetLookupTable = [default]; // so that _rowOffsetLookupTable.UnsafeAt(0) is always valid. + _rowOffsetLookupTable = []; Count = 0; } } From a82a61636d370489ee817e3d1a2ad4b88a785fa6 Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Sat, 17 Aug 2024 12:26:40 -0700 Subject: [PATCH 26/53] Split row and subrow interfaces --- src/Lumina.Tests/SeStringBuilderTests.cs | 5 -- src/Lumina/Excel/BaseExcelSheet.cs | 53 ++++++++++++++++---- src/Lumina/Excel/ExcelModule.cs | 63 ++++++++++++++---------- src/Lumina/Excel/IExcelRow.cs | 29 ++++++----- src/Lumina/Excel/SubrowCollection.cs | 2 +- src/Lumina/Excel/SubrowExcelSheet.cs | 2 +- src/Lumina/GameData.cs | 18 ++----- 7 files changed, 104 insertions(+), 68 deletions(-) diff --git a/src/Lumina.Tests/SeStringBuilderTests.cs b/src/Lumina.Tests/SeStringBuilderTests.cs index 6ce67166..7139d22f 100644 --- a/src/Lumina.Tests/SeStringBuilderTests.cs +++ b/src/Lumina.Tests/SeStringBuilderTests.cs @@ -296,15 +296,10 @@ public readonly struct Addon( ExcelPage page, uint offset, uint row ) : IExcelRo { public uint RowId => row; - ushort IExcelRow< Addon >.SubrowId => throw new NotSupportedException(); - public ReadOnlySeString Text => page.ReadString( offset, offset ); static Addon IExcelRow.Create( ExcelPage page, uint offset, uint row ) => new( page, offset, row ); - - static Addon IExcelRow.Create( ExcelPage page, uint offset, uint row, ushort subrow ) => - throw new NotSupportedException(); } [RequiresGameInstallationFact] diff --git a/src/Lumina/Excel/BaseExcelSheet.cs b/src/Lumina/Excel/BaseExcelSheet.cs index 1272dca2..de57fbc1 100644 --- a/src/Lumina/Excel/BaseExcelSheet.cs +++ b/src/Lumina/Excel/BaseExcelSheet.cs @@ -182,6 +182,10 @@ private protected BaseExcelSheet( ExcelModule module, ExcelHeaderFile headerFile public static BaseExcelSheet From< T >( ExcelModule module ) where T : struct, IExcelRow< T > => From< T >( module, module.Language ); + /// + public static BaseSubrowExcelSheet FromSubrow< T >( ExcelModule module ) where T : struct, IExcelSubrow< T > => + FromSubrow< T >( module, module.Language ); + /// Creates a new instance of , deducing sheet names and column hashes from . /// Type of each row. /// The to access sheet data from. @@ -201,6 +205,15 @@ public static BaseExcelSheet From< T >( ExcelModule module, Language language ) return From< T >( module, language, attribute.Name, attribute.ColumnHash ); } + /// + public static BaseSubrowExcelSheet FromSubrow< T >( ExcelModule module, Language language ) where T : struct, IExcelSubrow< T > + { + var attribute = typeof( T ).GetCustomAttribute() ?? + throw new InvalidOperationException( $"{nameof( T )} has no {nameof( SheetAttribute )}. Use the overload of {nameof( From )} with 4 parameters." ); + + return FromSubrow< T >( module, language, attribute.Name, attribute.ColumnHash ); + } + /// Creates a new instance of . /// Type of each row. /// The to access sheet data from. @@ -211,12 +224,37 @@ public static BaseExcelSheet From< T >( ExcelModule module, Language language ) /// was invalid (hash mismatch). /// Sheet had an unsupported language. /// Header file had a value that is not supported. - /// A new instance of that should be cast to or + /// A new instance of that should be cast to /// before further use. public static BaseExcelSheet From< T >( ExcelModule module, Language language, string sheetName, uint? columnHash = null ) where T : struct, IExcelRow< T > { - var headerFile = module.GameData.GetFile< ExcelHeaderFile >( $"exd/{sheetName}.exh" ) ?? + var headerFile = VerifySheet( module, language, sheetName, columnHash ); + + if( headerFile.Header.Variant != ExcelVariant.Default ) + throw new NotSupportedException( $"Specified sheet variant {headerFile.Header.Variant} is not supported for IExcelRow types." ); + + return new ExcelSheet< T >( module, headerFile, language, sheetName ); + } + + /// Creates a new instance of . + /// A new instance of that should be cast to + /// before further use. + /// + public static BaseSubrowExcelSheet FromSubrow< T >( ExcelModule module, Language language, string sheetName, uint? columnHash = null ) + where T : struct, IExcelSubrow< T > + { + var headerFile = VerifySheet( module, language, sheetName, columnHash ); + + if( headerFile.Header.Variant != ExcelVariant.Subrows ) + throw new NotSupportedException( $"Specified sheet variant {headerFile.Header.Variant} is not supported for IExcelSubrow types." ); + + return new SubrowExcelSheet< T >( module, headerFile, language, sheetName ); + } + + private static ExcelHeaderFile VerifySheet( ExcelModule module, Language language, string sheetName, uint? columnHash = null ) + { + var headerFile = module.GameData.GetFile( $"exd/{sheetName}.exh" ) ?? throw new ArgumentException( "Invalid sheet name", nameof( sheetName ) ); if( module.VerifySheetChecksums && columnHash is { } hash && headerFile.GetColumnsHash() != hash ) @@ -225,12 +263,7 @@ public static BaseExcelSheet From< T >( ExcelModule module, Language language, s if( !headerFile.Languages.Contains( language ) ) throw new UnsupportedLanguageException( nameof( language ), language, null ); - return headerFile.Header.Variant switch - { - ExcelVariant.Default => new ExcelSheet< T >( module, headerFile, language, sheetName ), - ExcelVariant.Subrows => new SubrowExcelSheet< T >( module, headerFile, language, sheetName ), - _ => throw new NotSupportedException( $"Specified sheet variant {headerFile.Header.Variant} is not supported." ), - }; + return headerFile; } /// The number of rows in this sheet. @@ -296,7 +329,7 @@ internal T UnsafeCreateRowAt< T >( int rowIndex ) where T : struct, IExcelRow< T /// Index of the desired row. /// Index of the desired subrow. /// A new instance of . - internal T UnsafeCreateSubrowAt< T >( int rowIndex, ushort subrowId ) where T : struct, IExcelRow< T > => + internal T UnsafeCreateSubrowAt< T >( int rowIndex, ushort subrowId ) where T : struct, IExcelSubrow< T > => UnsafeCreateSubrow< T >( in UnsafeGetRowLookupAt( rowIndex ), subrowId ); /// Creates a row using the given lookup data, without checking for bounds or preconditions. @@ -312,7 +345,7 @@ internal T UnsafeCreateRow< T >( scoped ref readonly RowOffsetLookup lookup ) wh /// Lookup data for the desired row. /// Index of the desired subrow. /// A new instance of . - internal T UnsafeCreateSubrow< T >( scoped ref readonly RowOffsetLookup lookup, ushort subrowId ) where T : struct, IExcelRow< T > => + internal T UnsafeCreateSubrow< T >( scoped ref readonly RowOffsetLookup lookup, ushort subrowId ) where T : struct, IExcelSubrow< T > => T.Create( _pages.UnsafeAt( lookup.PageIndex ), lookup.Offset + 2 + subrowId * ( _subrowDataOffset + 2u ), diff --git a/src/Lumina/Excel/ExcelModule.cs b/src/Lumina/Excel/ExcelModule.cs index 212698f4..f5cf4b07 100644 --- a/src/Lumina/Excel/ExcelModule.cs +++ b/src/Lumina/Excel/ExcelModule.cs @@ -60,30 +60,23 @@ public ExcelModule( GameData gameData ) } /// Loads an . + /// /// Sheet is not of the variant . - /// + /// public ExcelSheet< T > GetSheet< T >( Language? language = null ) where T : struct, IExcelRow< T > => (ExcelSheet< T >) GetBaseSheet< T >( language ); /// Loads an . - /// - /// If the requested language doesn't exist for the file where is not , the language-neutral - /// sheet using will be loaded instead. If the language-neutral sheet does not exist, then the function will fail with - /// . - /// /// Sheet is not of the variant . - /// - public SubrowExcelSheet< T > GetSubrowSheet< T >( Language? language = null ) where T : struct, IExcelRow< T > => - (SubrowExcelSheet< T >) GetBaseSheet< T >( language ); + /// + public SubrowExcelSheet< T > GetSubrowSheet< T >( Language? language = null ) where T : struct, IExcelSubrow< T > => + (SubrowExcelSheet< T >) GetBaseSubrowSheet< T >( language ); /// An excel sheet corresponding to and that may be created anew or /// reused from a previous invocation of this method. /// - /// If the requested language doesn't exist for the file where is not , the language-neutral - /// sheet using will be loaded instead. If the language-neutral sheet does not exist, then the function will fail with - /// . - /// The returned instance of should be cast to or - /// before accessing its rows. + /// The returned instance of should be cast to + /// before accessing its rows. /// /// does not have a valid . /// @@ -91,6 +84,15 @@ public SubrowExcelSheet< T > GetSubrowSheet< T >( Language? language = null ) wh public BaseExcelSheet GetBaseSheet< T >( Language? language = null ) where T : struct, IExcelRow< T > => GetBaseSheet( typeof( T ), language ); + /// + /// The returned instance of should be cast to + /// before accessing its rows. + /// + /// + [EditorBrowsable( EditorBrowsableState.Advanced )] + public BaseSubrowExcelSheet GetBaseSubrowSheet< T >( Language? language = null ) where T : struct, IExcelSubrow< T > => + (BaseSubrowExcelSheet)GetBaseSheet( typeof( T ), language ); + /// Loads an . /// Type of the rows in the sheet. /// The requested sheet language. Leave or empty to use the default language. @@ -98,9 +100,6 @@ public BaseExcelSheet GetBaseSheet< T >( Language? language = null ) where T : s /// reused from a previous invocation of this method. /// /// Only use this method if you need to create a sheet while using reflection. - /// If the requested language doesn't exist for the file where is not , the language-neutral - /// sheet using will be loaded instead. If the language-neutral sheet does not exist, then the function will fail with - /// . /// The returned instance of should be cast to or /// before accessing its rows. /// @@ -119,20 +118,34 @@ public BaseExcelSheet GetBaseSheet( Type rowType, Language? language = null ) MethodInfo m; try { - // As BaseExcelSheet.From has a constraint that T : IExcelRow, it is implicitly required that T is also a struct. + var isSubrowType = key.Type.IsAssignableTo( typeof( IExcelSubrow<> ).MakeGenericType( key.Type ) ); + + // As BaseExcelSheet.From(Subrow) has a constraint that T : IExcel(Row/Subrow), it is implicitly required that T is also a struct. // MakeGenericMethod will check for constraints, and throw ArgumentException if constraints aren't met. - m = typeof( BaseExcelSheet ) - .GetMethod( - nameof( BaseExcelSheet.From ), - BindingFlags.Static | BindingFlags.Public, - [typeof( ExcelModule ), typeof( Language )] )! - .MakeGenericMethod( key.Type ); + if( isSubrowType ) + { + m = typeof( BaseExcelSheet ) + .GetMethod( + nameof( BaseExcelSheet.FromSubrow ), + BindingFlags.Static | BindingFlags.Public, + [typeof( ExcelModule ), typeof( Language )] )! + .MakeGenericMethod( key.Type ); + } + else + { + m = typeof( BaseExcelSheet ) + .GetMethod( + nameof( BaseExcelSheet.From ), + BindingFlags.Static | BindingFlags.Public, + [typeof( ExcelModule ), typeof( Language )] )! + .MakeGenericMethod( key.Type ); + } } catch( ArgumentException e ) { // Exception thrown here will propagate outside ConcurrentDictionary<>.GetOrAdd without touching the data stored inside dictionary. throw new ArgumentException( - $"{key.Type.Name} must implement {typeof( IExcelRow<> ).Name.Split( '`', 2 )[ 0 ]}<{key.Type.Name}>.", + $"{key.Type.Name} must implement either {typeof( IExcelRow<> ).Name.Split( '`', 2 )[ 0 ]}<{key.Type.Name}> or {typeof( IExcelSubrow<> ).Name.Split( '`', 2 )[0]}<{key.Type.Name}>.", nameof( rowType ), e ); } diff --git a/src/Lumina/Excel/IExcelRow.cs b/src/Lumina/Excel/IExcelRow.cs index f006fcb9..de10a2c3 100644 --- a/src/Lumina/Excel/IExcelRow.cs +++ b/src/Lumina/Excel/IExcelRow.cs @@ -1,5 +1,3 @@ -using System; - namespace Lumina.Excel; /// @@ -8,33 +6,38 @@ namespace Lumina.Excel; /// The type that implements the interface. public interface IExcelRow< out T > where T : struct { + /// Gets the row ID. + uint RowId { get; } + /// /// Creates an instance of the current type. Designed only for use within . /// - /// Only used for sheets that are not using subrows, and will throw otherwise. /// /// /// /// A newly created row object. - /// Thrown when the referenced sheet is using subrows. abstract static T Create( ExcelPage page, uint offset, uint row ); +} + +/// +/// Defines a subrow type/schema for an excel sheet. +/// +/// The type that implements the interface. +public interface IExcelSubrow< out T > where T : struct +{ + /// Gets the row ID. + uint RowId { get; } + + /// Gets the subrow ID. + ushort SubrowId { get; } /// /// Creates an instance of the current type. Designed only for use within . /// - /// Only used for sheets that are using subrows, and will throw otherwise. /// /// /// /// /// A newly created subrow object. - /// Thrown when the referenced sheet is not using subrows. abstract static T Create( ExcelPage page, uint offset, uint row, ushort subrow ); - - /// Gets the row ID. - uint RowId { get; } - - /// Gets the subrow ID. - /// Thrown when the referenced sheet is not using subrows. - ushort SubrowId { get; } } \ No newline at end of file diff --git a/src/Lumina/Excel/SubrowCollection.cs b/src/Lumina/Excel/SubrowCollection.cs index 551d4e72..fd09b35b 100644 --- a/src/Lumina/Excel/SubrowCollection.cs +++ b/src/Lumina/Excel/SubrowCollection.cs @@ -8,7 +8,7 @@ namespace Lumina.Excel; /// Collection of subrows under one row. /// Type of the row. public readonly struct SubrowCollection< T > : IList< T >, IReadOnlyList< T > - where T : struct, IExcelRow< T > + where T : struct, IExcelSubrow< T > { private readonly BaseExcelSheet.RowOffsetLookup _lookup; diff --git a/src/Lumina/Excel/SubrowExcelSheet.cs b/src/Lumina/Excel/SubrowExcelSheet.cs index ebfabb8e..ca8d0405 100644 --- a/src/Lumina/Excel/SubrowExcelSheet.cs +++ b/src/Lumina/Excel/SubrowExcelSheet.cs @@ -11,7 +11,7 @@ namespace Lumina.Excel; /// public sealed class SubrowExcelSheet< T > : BaseSubrowExcelSheet, ICollection< SubrowCollection< T > >, IReadOnlyCollection< SubrowCollection< T > > - where T : struct, IExcelRow< T > + where T : struct, IExcelSubrow< T > { internal SubrowExcelSheet( ExcelModule module, ExcelHeaderFile headerFile, Language requestedLanguage, string sheetName ) : base( module, headerFile, requestedLanguage, sheetName ) diff --git a/src/Lumina/GameData.cs b/src/Lumina/GameData.cs index 283c6f5d..c923afb6 100644 --- a/src/Lumina/GameData.cs +++ b/src/Lumina/GameData.cs @@ -291,9 +291,9 @@ public static UInt64 GetFileHash( string path ) /// An excel sheet corresponding to and that may be created anew or /// reused from a previous invocation of this method. /// - /// If the requested language doesn't exist for the file where is not , the + /// If the requested language doesn't exist for the file where is not , the /// language-neutral sheet using will be loaded instead. If the language-neutral sheet does not exist, then the function - /// will return . + /// will return . /// /// Sheet is not of the variant . /// does not have a valid . @@ -313,18 +313,10 @@ public static UInt64 GetFileHash( string path ) } } - /// Loads an . Returns if the sheet does not exist, has an invalid column hash or unsupported variant, or was requested with an unsupported language. - /// The requested sheet language. Leave or empty to use the default language. - /// An excel sheet corresponding to and that may be created anew or - /// reused from a previous invocation of this method. - /// - /// If the requested language doesn't exist for the file where is not , the - /// language-neutral sheet using will be loaded instead. If the language-neutral sheet does not exist, then the function - /// will return . - /// + /// Loads a . Returns if the sheet does not exist, has an invalid column hash or unsupported variant, or was requested with an unsupported language. /// Sheet is not of the variant . - /// does not have a valid . - public SubrowExcelSheet< T >? GetSubrowExcelSheet< T >( Language? language = null ) where T : struct, IExcelRow< T > + /// + public SubrowExcelSheet< T >? GetSubrowExcelSheet< T >( Language? language = null ) where T : struct, IExcelSubrow< T > { try { From 54b7fcd83a24f2d10c6cdf7c700b509701b82c5a Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Sat, 17 Aug 2024 12:46:50 -0700 Subject: [PATCH 27/53] Add subrow support to RowRef --- src/Lumina/Excel/RowRef.cs | 82 +++++++++++++++++++++++++++++++++----- 1 file changed, 71 insertions(+), 11 deletions(-) diff --git a/src/Lumina/Excel/RowRef.cs b/src/Lumina/Excel/RowRef.cs index 0df16726..28a97a71 100644 --- a/src/Lumina/Excel/RowRef.cs +++ b/src/Lumina/Excel/RowRef.cs @@ -1,5 +1,4 @@ using System; -using System.Runtime.CompilerServices; namespace Lumina.Excel; @@ -32,6 +31,10 @@ public readonly struct RowRef( ExcelModule? module, uint rowId, Type? rowType ) public bool Is< T >() where T : struct, IExcelRow< T > => typeof( T ) == rowType; + /// + public bool IsSubrow() where T : struct, IExcelSubrow => + typeof( T ) == rowType; + /// /// Tries to get the referenced row as a specific row type. /// @@ -45,6 +48,15 @@ public bool Is< T >() where T : struct, IExcelRow< T > => return new RowRef< T >( module, rowId ).ValueNullable; } + /// + public T? GetValueOrDefaultSubrow() where T : struct, IExcelSubrow + { + if( !IsSubrow() || module is null ) + return null; + + return new SubrowRef( module, rowId ).ValueNullable; + } + /// /// Tries to get the referenced row as a specific row type. /// @@ -63,6 +75,19 @@ public bool TryGetValue< T >( out T row ) where T : struct, IExcelRow< T > return false; } + /// + public bool TryGetValueSubrow( out T row ) where T : struct, IExcelSubrow + { + if( new SubrowRef( module, rowId ).ValueNullable is { } v ) + { + row = v; + return true; + } + + row = default; + return false; + } + /// /// Attempts to create a to a row id of a list of row types, checking with each type in order. /// @@ -93,6 +118,9 @@ public static RowRef GetFirstValidRowOrUntyped( ExcelModule module, uint rowId, /// A to a row in a . public static RowRef Create< T >( ExcelModule? module, uint rowId ) where T : struct, IExcelRow< T > => new( module, rowId, typeof( T ) ); + /// + public static RowRef CreateSubrow( ExcelModule? module, uint rowId ) where T : struct, IExcelSubrow => new( module, rowId, typeof( T ) ); + /// /// Creates an untyped . /// @@ -109,7 +137,7 @@ public static RowRef GetFirstValidRowOrUntyped( ExcelModule module, uint rowId, /// The referenced row id. public readonly struct RowRef< T >( ExcelModule? module, uint rowId ) where T : struct, IExcelRow< T > { - private readonly BaseExcelSheet? _sheet = module?.GetBaseSheet< T >(); + private readonly ExcelSheet? _sheet = module?.GetSheet< T >(); /// /// The row id of the referenced row. @@ -130,15 +158,7 @@ public readonly struct RowRef< T >( ExcelModule? module, uint rowId ) where T : /// /// Attempts to get the referenced row value. Is null if does not exist in the sheet. /// - public T? ValueNullable { - get { - if( _sheet is null ) - return null; - - ref readonly var lookup = ref _sheet.GetRowLookupOrNullRef( rowId ); - return Unsafe.IsNullRef( in lookup ) ? null : _sheet.UnsafeCreateRow< T >( lookup ); - } - } + public T? ValueNullable => _sheet?.GetRowOrDefault( rowId ); private RowRef ToGeneric() => RowRef.Create< T >( module, rowId ); @@ -147,4 +167,44 @@ public T? ValueNullable { /// /// The to convert. public static explicit operator RowRef( RowRef< T > row ) => row.ToGeneric(); +} + +/// +/// A helper type to concretely reference the first subrow of a row in a specific excel sheet. +/// +/// The row type referenced by the . +/// The to read sheet data from. +/// The referenced row id. +public readonly struct SubrowRef( ExcelModule? module, uint rowId ) where T : struct, IExcelSubrow +{ + private readonly SubrowExcelSheet? _sheet = module?.GetSubrowSheet(); + + /// + /// The row id of the referenced subrow. + /// + public uint RowId => rowId; + + /// + /// Whether the subrow exists in the sheet. + /// + public bool IsValid => _sheet?.HasSubrow( RowId, 0 ) ?? false; + + /// + /// The referenced subrow value itself. + /// + /// Thrown if is false. + public T Value => ValueNullable ?? throw new InvalidOperationException(); + + /// + /// Attempts to get the referenced subrow value. Is null if it does not exist in the sheet. + /// + public T? ValueNullable => _sheet?.GetSubrowOrDefault( rowId, 0 ); + + private RowRef ToGeneric() => RowRef.CreateSubrow( module, rowId ); + + /// + /// Converts a concrete to a generic and dynamically typed . + /// + /// The to convert. + public static explicit operator RowRef( SubrowRef row ) => row.ToGeneric(); } \ No newline at end of file From 45fad7178bf078a8745eea07775fd74c0175731c Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Sat, 17 Aug 2024 13:38:22 -0700 Subject: [PATCH 28/53] Split RowRef.cs, make SubrowRef return a collection --- src/Lumina/Excel/RowRef.cs | 171 +----------------------------- src/Lumina/Excel/SubrowRef.cs | 43 ++++++++ src/Lumina/Excel/UntypedRowRef.cs | 130 +++++++++++++++++++++++ 3 files changed, 175 insertions(+), 169 deletions(-) create mode 100644 src/Lumina/Excel/SubrowRef.cs create mode 100644 src/Lumina/Excel/UntypedRowRef.cs diff --git a/src/Lumina/Excel/RowRef.cs b/src/Lumina/Excel/RowRef.cs index 28a97a71..d0042625 100644 --- a/src/Lumina/Excel/RowRef.cs +++ b/src/Lumina/Excel/RowRef.cs @@ -2,133 +2,6 @@ namespace Lumina.Excel; -/// -/// A helper type to dynamically reference a row in a different excel sheet. -/// -/// The to read sheet data from. -/// The referenced row id. -/// The referenced row's actual . -public readonly struct RowRef( ExcelModule? module, uint rowId, Type? rowType ) -{ - /// - /// The row id of the referenced row. - /// - public uint RowId => rowId; - - /// - /// Whether the is untyped. - /// - /// - /// An untyped is one that doesn't know which sheet it links to. - /// - public bool IsUntyped => rowType == null; - - /// - /// Whether the reference is of a specific row type. - /// - /// The row type/schema to check against. - /// Whether this points to a . - public bool Is< T >() where T : struct, IExcelRow< T > => - typeof( T ) == rowType; - - /// - public bool IsSubrow() where T : struct, IExcelSubrow => - typeof( T ) == rowType; - - /// - /// Tries to get the referenced row as a specific row type. - /// - /// The row type/schema to check against. - /// The referenced row type. Returns null if this does not point to a or if the does not exist in its sheet. - public T? GetValueOrDefault< T >() where T : struct, IExcelRow< T > - { - if( !Is< T >() || module is null ) - return null; - - return new RowRef< T >( module, rowId ).ValueNullable; - } - - /// - public T? GetValueOrDefaultSubrow() where T : struct, IExcelSubrow - { - if( !IsSubrow() || module is null ) - return null; - - return new SubrowRef( module, rowId ).ValueNullable; - } - - /// - /// Tries to get the referenced row as a specific row type. - /// - /// The row type/schema to check against. - /// The output row object. - /// if the type is valid, the row exists, and is written to, and otherwise. - public bool TryGetValue< T >( out T row ) where T : struct, IExcelRow< T > - { - if( new RowRef< T >( module, rowId ).ValueNullable is { } v ) - { - row = v; - return true; - } - - row = default; - return false; - } - - /// - public bool TryGetValueSubrow( out T row ) where T : struct, IExcelSubrow - { - if( new SubrowRef( module, rowId ).ValueNullable is { } v ) - { - row = v; - return true; - } - - row = default; - return false; - } - - /// - /// Attempts to create a to a row id of a list of row types, checking with each type in order. - /// - /// The to read sheet data from. - /// The referenced row id. - /// A list of row types to check against the , in order. - /// A to one of the . If the row id does not exist in any of the sheets, an untyped is returned instead. - public static RowRef GetFirstValidRowOrUntyped( ExcelModule module, uint rowId, params Type[] sheetTypes ) - { - foreach( var sheetType in sheetTypes ) - { - if( module.GetBaseSheet( sheetType ) is { } sheet ) - { - if( sheet.HasRow( rowId ) ) - return new( module, rowId, sheetType ); - } - } - - return CreateUntyped( rowId ); - } - - /// - /// Creates a to a specific row type. - /// - /// The row type referenced by the . - /// The to read sheet data from. - /// The referenced row id. - /// A to a row in a . - public static RowRef Create< T >( ExcelModule? module, uint rowId ) where T : struct, IExcelRow< T > => new( module, rowId, typeof( T ) ); - - /// - public static RowRef CreateSubrow( ExcelModule? module, uint rowId ) where T : struct, IExcelSubrow => new( module, rowId, typeof( T ) ); - - /// - /// Creates an untyped . - /// - /// The referenced row id. - /// An untyped . - public static RowRef CreateUntyped( uint rowId ) => new( null, rowId, null ); -} - /// /// A helper type to concretely reference a row in a specific excel sheet. /// @@ -137,7 +10,7 @@ public static RowRef GetFirstValidRowOrUntyped( ExcelModule module, uint rowId, /// The referenced row id. public readonly struct RowRef< T >( ExcelModule? module, uint rowId ) where T : struct, IExcelRow< T > { - private readonly ExcelSheet? _sheet = module?.GetSheet< T >(); + private readonly ExcelSheet< T >? _sheet = module?.GetSheet< T >(); /// /// The row id of the referenced row. @@ -156,7 +29,7 @@ public readonly struct RowRef< T >( ExcelModule? module, uint rowId ) where T : public T Value => ValueNullable ?? throw new InvalidOperationException(); /// - /// Attempts to get the referenced row value. Is null if does not exist in the sheet. + /// Attempts to get the referenced row value. Is if does not exist in the sheet. /// public T? ValueNullable => _sheet?.GetRowOrDefault( rowId ); @@ -168,43 +41,3 @@ public readonly struct RowRef< T >( ExcelModule? module, uint rowId ) where T : /// The to convert. public static explicit operator RowRef( RowRef< T > row ) => row.ToGeneric(); } - -/// -/// A helper type to concretely reference the first subrow of a row in a specific excel sheet. -/// -/// The row type referenced by the . -/// The to read sheet data from. -/// The referenced row id. -public readonly struct SubrowRef( ExcelModule? module, uint rowId ) where T : struct, IExcelSubrow -{ - private readonly SubrowExcelSheet? _sheet = module?.GetSubrowSheet(); - - /// - /// The row id of the referenced subrow. - /// - public uint RowId => rowId; - - /// - /// Whether the subrow exists in the sheet. - /// - public bool IsValid => _sheet?.HasSubrow( RowId, 0 ) ?? false; - - /// - /// The referenced subrow value itself. - /// - /// Thrown if is false. - public T Value => ValueNullable ?? throw new InvalidOperationException(); - - /// - /// Attempts to get the referenced subrow value. Is null if it does not exist in the sheet. - /// - public T? ValueNullable => _sheet?.GetSubrowOrDefault( rowId, 0 ); - - private RowRef ToGeneric() => RowRef.CreateSubrow( module, rowId ); - - /// - /// Converts a concrete to a generic and dynamically typed . - /// - /// The to convert. - public static explicit operator RowRef( SubrowRef row ) => row.ToGeneric(); -} \ No newline at end of file diff --git a/src/Lumina/Excel/SubrowRef.cs b/src/Lumina/Excel/SubrowRef.cs new file mode 100644 index 00000000..5120322b --- /dev/null +++ b/src/Lumina/Excel/SubrowRef.cs @@ -0,0 +1,43 @@ +using System; + +namespace Lumina.Excel; + +/// +/// A helper type to concretely reference a collection of subrows in a specific excel sheet. +/// +/// The subrow type referenced by the subrows of . +/// The to read sheet data from. +/// The referenced row id. +public readonly struct SubrowRef< T >( ExcelModule? module, uint rowId ) where T : struct, IExcelSubrow< T > +{ + private readonly SubrowExcelSheet< T >? _sheet = module?.GetSubrowSheet(); + + /// + /// The row id of the referenced row. + /// + public uint RowId => rowId; + + /// + /// Whether the exists in the sheet. + /// + public bool IsValid => _sheet?.HasRow( RowId ) ?? false; + + /// + /// The referenced row value itself. + /// + /// Thrown if is false. + public SubrowCollection< T > Value => ValueNullable ?? throw new InvalidOperationException(); + + /// + /// Attempts to get the referenced row value. Is if it does not exist in the sheet. + /// + public SubrowCollection< T >? ValueNullable => _sheet?.GetRowOrDefault( rowId ); + + private RowRef ToGeneric() => RowRef.CreateSubrow< T >( module, rowId ); + + /// + /// Converts a concrete to a generic and dynamically typed . + /// + /// The to convert. + public static explicit operator RowRef( SubrowRef< T > row ) => row.ToGeneric(); +} \ No newline at end of file diff --git a/src/Lumina/Excel/UntypedRowRef.cs b/src/Lumina/Excel/UntypedRowRef.cs new file mode 100644 index 00000000..f37f927e --- /dev/null +++ b/src/Lumina/Excel/UntypedRowRef.cs @@ -0,0 +1,130 @@ +using System; + +namespace Lumina.Excel; + +/// +/// A helper type to dynamically reference a row in a specific excel sheet. +/// +/// The to read sheet data from. +/// The referenced row id. +/// The referenced row's actual . +public readonly struct RowRef( ExcelModule? module, uint rowId, Type? rowType ) +{ + /// + /// The row id of the referenced row. + /// + public uint RowId => rowId; + + /// + /// Whether the is untyped. + /// + /// + /// An untyped is one that doesn't know which sheet it links to. + /// + public bool IsUntyped => rowType == null; + + /// + /// Whether the reference is of a specific row type. + /// + /// The row type/schema to check against. + /// Whether this points to a . + public bool Is< T >() where T : struct, IExcelRow< T > => + typeof( T ) == rowType; + + /// + public bool IsSubrow< T >() where T : struct, IExcelSubrow< T > => + typeof( T ) == rowType; + + /// + /// Tries to get the referenced row as a specific row type. + /// + /// The row type/schema to check against. + /// The referenced row type. Returns null if this does not point to a or if the does not exist in its sheet. + public T? GetValueOrDefault< T >() where T : struct, IExcelRow< T > + { + if( !Is< T >() || module is null ) + return null; + + return new RowRef< T >( module, rowId ).ValueNullable; + } + + /// + public SubrowCollection< T >? GetValueOrDefaultSubrow< T >() where T : struct, IExcelSubrow< T > + { + if( !IsSubrow< T >() || module is null ) + return null; + + return new SubrowRef< T >( module, rowId ).ValueNullable; + } + + /// + /// Tries to get the referenced row as a specific row type. + /// + /// The row type/schema to check against. + /// The output row object. + /// if the type is valid, the row exists, and is written to, and otherwise. + public bool TryGetValue< T >( out T row ) where T : struct, IExcelRow< T > + { + if( new RowRef< T >( module, rowId ).ValueNullable is { } v ) + { + row = v; + return true; + } + + row = default; + return false; + } + + /// + public bool TryGetValueSubrow< T >( out SubrowCollection< T > row ) where T : struct, IExcelSubrow + { + if( new SubrowRef< T >( module, rowId ).ValueNullable is { } v ) + { + row = v; + return true; + } + + row = default; + return false; + } + + /// + /// Attempts to create a to a row id of a list of row types, checking with each type in order. + /// + /// The to read sheet data from. + /// The referenced row id. + /// A list of row types to check against the , in order. + /// A to one of the . If the row id does not exist in any of the sheets, an untyped is returned instead. + public static RowRef GetFirstValidRowOrUntyped( ExcelModule module, uint rowId, params Type[] sheetTypes ) + { + foreach( var sheetType in sheetTypes ) + { + if( module.GetBaseSheet( sheetType ) is { } sheet ) + { + if( sheet.HasRow( rowId ) ) + return new( module, rowId, sheetType ); + } + } + + return CreateUntyped( rowId ); + } + + /// + /// Creates a to a specific row type. + /// + /// The row type referenced by the . + /// The to read sheet data from. + /// The referenced row id. + /// A to a row in a . + public static RowRef Create< T >( ExcelModule? module, uint rowId ) where T : struct, IExcelRow< T > => new( module, rowId, typeof( T ) ); + + /// + public static RowRef CreateSubrow< T >( ExcelModule? module, uint rowId ) where T : struct, IExcelSubrow< T > => new( module, rowId, typeof( T ) ); + + /// + /// Creates an untyped . + /// + /// The referenced row id. + /// An untyped . + public static RowRef CreateUntyped( uint rowId ) => new( null, rowId, null ); +} From 287bc57cdebe498915e1852ba2811283fc4bd3ea Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Sat, 17 Aug 2024 13:38:30 -0700 Subject: [PATCH 29/53] Split IExcelRow.cs --- src/Lumina/Excel/IExcelRow.cs | 23 ----------------------- src/Lumina/Excel/IExcelSubrow.cs | 24 ++++++++++++++++++++++++ 2 files changed, 24 insertions(+), 23 deletions(-) create mode 100644 src/Lumina/Excel/IExcelSubrow.cs diff --git a/src/Lumina/Excel/IExcelRow.cs b/src/Lumina/Excel/IExcelRow.cs index de10a2c3..274e2571 100644 --- a/src/Lumina/Excel/IExcelRow.cs +++ b/src/Lumina/Excel/IExcelRow.cs @@ -18,26 +18,3 @@ public interface IExcelRow< out T > where T : struct /// A newly created row object. abstract static T Create( ExcelPage page, uint offset, uint row ); } - -/// -/// Defines a subrow type/schema for an excel sheet. -/// -/// The type that implements the interface. -public interface IExcelSubrow< out T > where T : struct -{ - /// Gets the row ID. - uint RowId { get; } - - /// Gets the subrow ID. - ushort SubrowId { get; } - - /// - /// Creates an instance of the current type. Designed only for use within . - /// - /// - /// - /// - /// - /// A newly created subrow object. - abstract static T Create( ExcelPage page, uint offset, uint row, ushort subrow ); -} \ No newline at end of file diff --git a/src/Lumina/Excel/IExcelSubrow.cs b/src/Lumina/Excel/IExcelSubrow.cs new file mode 100644 index 00000000..84ac699e --- /dev/null +++ b/src/Lumina/Excel/IExcelSubrow.cs @@ -0,0 +1,24 @@ +namespace Lumina.Excel; + +/// +/// Defines a subrow type/schema for an excel sheet. +/// +/// The type that implements the interface. +public interface IExcelSubrow< out T > where T : struct +{ + /// Gets the row ID. + uint RowId { get; } + + /// Gets the subrow ID. + ushort SubrowId { get; } + + /// + /// Creates an instance of the current type. Designed only for use within . + /// + /// + /// + /// + /// + /// A newly created subrow object. + abstract static T Create( ExcelPage page, uint offset, uint row, ushort subrow ); +} \ No newline at end of file From 349d770cc474ad38ce28643b736b9d442ab6bd4c Mon Sep 17 00:00:00 2001 From: Soreepeong Date: Sun, 18 Aug 2024 05:47:46 +0900 Subject: [PATCH 30/53] Reformat code --- src/Lumina/Excel/BaseExcelSheet.cs | 4 ++-- src/Lumina/Excel/ExcelModule.cs | 4 ++-- src/Lumina/Excel/IExcelRow.cs | 2 +- src/Lumina/Excel/RowRef.cs | 2 +- src/Lumina/Excel/SubrowRef.cs | 2 +- src/Lumina/Excel/UntypedRowRef.cs | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Lumina/Excel/BaseExcelSheet.cs b/src/Lumina/Excel/BaseExcelSheet.cs index de57fbc1..68f1cba9 100644 --- a/src/Lumina/Excel/BaseExcelSheet.cs +++ b/src/Lumina/Excel/BaseExcelSheet.cs @@ -208,7 +208,7 @@ public static BaseExcelSheet From< T >( ExcelModule module, Language language ) /// public static BaseSubrowExcelSheet FromSubrow< T >( ExcelModule module, Language language ) where T : struct, IExcelSubrow< T > { - var attribute = typeof( T ).GetCustomAttribute() ?? + var attribute = typeof( T ).GetCustomAttribute< SheetAttribute >() ?? throw new InvalidOperationException( $"{nameof( T )} has no {nameof( SheetAttribute )}. Use the overload of {nameof( From )} with 4 parameters." ); return FromSubrow< T >( module, language, attribute.Name, attribute.ColumnHash ); @@ -254,7 +254,7 @@ public static BaseSubrowExcelSheet FromSubrow< T >( ExcelModule module, Language private static ExcelHeaderFile VerifySheet( ExcelModule module, Language language, string sheetName, uint? columnHash = null ) { - var headerFile = module.GameData.GetFile( $"exd/{sheetName}.exh" ) ?? + var headerFile = module.GameData.GetFile< ExcelHeaderFile >( $"exd/{sheetName}.exh" ) ?? throw new ArgumentException( "Invalid sheet name", nameof( sheetName ) ); if( module.VerifySheetChecksums && columnHash is { } hash && headerFile.GetColumnsHash() != hash ) diff --git a/src/Lumina/Excel/ExcelModule.cs b/src/Lumina/Excel/ExcelModule.cs index f5cf4b07..8bb489a3 100644 --- a/src/Lumina/Excel/ExcelModule.cs +++ b/src/Lumina/Excel/ExcelModule.cs @@ -91,7 +91,7 @@ public BaseExcelSheet GetBaseSheet< T >( Language? language = null ) where T : s /// [EditorBrowsable( EditorBrowsableState.Advanced )] public BaseSubrowExcelSheet GetBaseSubrowSheet< T >( Language? language = null ) where T : struct, IExcelSubrow< T > => - (BaseSubrowExcelSheet)GetBaseSheet( typeof( T ), language ); + (BaseSubrowExcelSheet) GetBaseSheet( typeof( T ), language ); /// Loads an . /// Type of the rows in the sheet. @@ -145,7 +145,7 @@ public BaseExcelSheet GetBaseSheet( Type rowType, Language? language = null ) { // Exception thrown here will propagate outside ConcurrentDictionary<>.GetOrAdd without touching the data stored inside dictionary. throw new ArgumentException( - $"{key.Type.Name} must implement either {typeof( IExcelRow<> ).Name.Split( '`', 2 )[ 0 ]}<{key.Type.Name}> or {typeof( IExcelSubrow<> ).Name.Split( '`', 2 )[0]}<{key.Type.Name}>.", + $"{key.Type.Name} must implement either {typeof( IExcelRow<> ).Name.Split( '`', 2 )[ 0 ]}<{key.Type.Name}> or {typeof( IExcelSubrow<> ).Name.Split( '`', 2 )[ 0 ]}<{key.Type.Name}>.", nameof( rowType ), e ); } diff --git a/src/Lumina/Excel/IExcelRow.cs b/src/Lumina/Excel/IExcelRow.cs index 274e2571..6d6735d2 100644 --- a/src/Lumina/Excel/IExcelRow.cs +++ b/src/Lumina/Excel/IExcelRow.cs @@ -17,4 +17,4 @@ public interface IExcelRow< out T > where T : struct /// /// A newly created row object. abstract static T Create( ExcelPage page, uint offset, uint row ); -} +} \ No newline at end of file diff --git a/src/Lumina/Excel/RowRef.cs b/src/Lumina/Excel/RowRef.cs index d0042625..6271f1a1 100644 --- a/src/Lumina/Excel/RowRef.cs +++ b/src/Lumina/Excel/RowRef.cs @@ -40,4 +40,4 @@ public readonly struct RowRef< T >( ExcelModule? module, uint rowId ) where T : /// /// The to convert. public static explicit operator RowRef( RowRef< T > row ) => row.ToGeneric(); -} +} \ No newline at end of file diff --git a/src/Lumina/Excel/SubrowRef.cs b/src/Lumina/Excel/SubrowRef.cs index 5120322b..a3566b65 100644 --- a/src/Lumina/Excel/SubrowRef.cs +++ b/src/Lumina/Excel/SubrowRef.cs @@ -10,7 +10,7 @@ namespace Lumina.Excel; /// The referenced row id. public readonly struct SubrowRef< T >( ExcelModule? module, uint rowId ) where T : struct, IExcelSubrow< T > { - private readonly SubrowExcelSheet< T >? _sheet = module?.GetSubrowSheet(); + private readonly SubrowExcelSheet< T >? _sheet = module?.GetSubrowSheet< T >(); /// /// The row id of the referenced row. diff --git a/src/Lumina/Excel/UntypedRowRef.cs b/src/Lumina/Excel/UntypedRowRef.cs index f37f927e..4a103bb8 100644 --- a/src/Lumina/Excel/UntypedRowRef.cs +++ b/src/Lumina/Excel/UntypedRowRef.cs @@ -76,7 +76,7 @@ public bool TryGetValue< T >( out T row ) where T : struct, IExcelRow< T > } /// - public bool TryGetValueSubrow< T >( out SubrowCollection< T > row ) where T : struct, IExcelSubrow + public bool TryGetValueSubrow< T >( out SubrowCollection< T > row ) where T : struct, IExcelSubrow< T > { if( new SubrowRef< T >( module, rowId ).ValueNullable is { } v ) { @@ -127,4 +127,4 @@ public static RowRef GetFirstValidRowOrUntyped( ExcelModule module, uint rowId, /// The referenced row id. /// An untyped . public static RowRef CreateUntyped( uint rowId ) => new( null, rowId, null ); -} +} \ No newline at end of file From a86c80e4a02f721b7cc015e63ce74b782cc5f413 Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Mon, 19 Aug 2024 01:28:04 -0700 Subject: [PATCH 31/53] Add better custom sheet path support --- src/Lumina/Excel/BaseExcelSheet.cs | 50 +++++++++--------- src/Lumina/Excel/ExcelModule.cs | 81 +++++++++++------------------- src/Lumina/GameData.cs | 11 ++-- 3 files changed, 60 insertions(+), 82 deletions(-) diff --git a/src/Lumina/Excel/BaseExcelSheet.cs b/src/Lumina/Excel/BaseExcelSheet.cs index 68f1cba9..04c55410 100644 --- a/src/Lumina/Excel/BaseExcelSheet.cs +++ b/src/Lumina/Excel/BaseExcelSheet.cs @@ -168,8 +168,7 @@ private protected BaseExcelSheet( ExcelModule module, ExcelHeaderFile headerFile } } - /// Creates a new instance of with the 's default language, deducing sheet names and column - /// hashes from . + /// Creates a new instance of with the 's default language, deducing sheet names and column hashes from . /// Type of each row. /// The to access sheet data from. /// does not have a valid . @@ -177,41 +176,42 @@ private protected BaseExcelSheet( ExcelModule module, ExcelHeaderFile headerFile /// was invalid (hash mismatch). /// Sheet had an unsupported language. /// Header file had a value that is not supported. - /// A new instance of that should be cast to or - /// before further use. - public static BaseExcelSheet From< T >( ExcelModule module ) where T : struct, IExcelRow< T > => - From< T >( module, module.Language ); + /// A new instance of . + public static ExcelSheet< T > Create< T >( ExcelModule module ) where T : struct, IExcelRow< T > => + Create< T >( module, module.Language ); - /// - public static BaseSubrowExcelSheet FromSubrow< T >( ExcelModule module ) where T : struct, IExcelSubrow< T > => - FromSubrow< T >( module, module.Language ); + /// A new instance of . + /// + public static SubrowExcelSheet< T > CreateSubrow< T >( ExcelModule module ) where T : struct, IExcelSubrow< T > => + CreateSubrow< T >( module, module.Language ); - /// Creates a new instance of , deducing sheet names and column hashes from . + /// Creates a new instance of , deducing sheet names (unless overridden with ) and column hashes from . /// Type of each row. /// The to access sheet data from. /// The language to use for this sheet. + /// The explicit sheet name, if needed. Leave to use the type's sheet name. Explicit names are necessary for quest/dungeon/cutscene sheets. /// does not have a valid . /// was invalid (invalid sheet name). /// was invalid (hash mismatch). /// Sheet had an unsupported language. /// Header file had a value that is not supported. - /// A new instance of that should be cast to or - /// before further use. - public static BaseExcelSheet From< T >( ExcelModule module, Language language ) where T : struct, IExcelRow< T > + /// A new instance of . + public static ExcelSheet< T > Create< T >( ExcelModule module, Language language, string? sheetName = null ) where T : struct, IExcelRow< T > { var attribute = typeof( T ).GetCustomAttribute< SheetAttribute >() ?? - throw new InvalidOperationException( $"{nameof( T )} has no {nameof( SheetAttribute )}. Use the overload of {nameof( From )} with 4 parameters." ); + throw new InvalidOperationException( $"{nameof( T )} has no {nameof( SheetAttribute )}. Use the overload of {nameof( Create )} with 4 parameters." ); - return From< T >( module, language, attribute.Name, attribute.ColumnHash ); + return Create< T >( module, language, sheetName ?? attribute.Name, attribute.ColumnHash ); } - /// - public static BaseSubrowExcelSheet FromSubrow< T >( ExcelModule module, Language language ) where T : struct, IExcelSubrow< T > + /// A new instance of . + /// + public static SubrowExcelSheet< T > CreateSubrow< T >( ExcelModule module, Language language, string? sheetName = null ) where T : struct, IExcelSubrow< T > { var attribute = typeof( T ).GetCustomAttribute< SheetAttribute >() ?? - throw new InvalidOperationException( $"{nameof( T )} has no {nameof( SheetAttribute )}. Use the overload of {nameof( From )} with 4 parameters." ); + throw new InvalidOperationException( $"{nameof( T )} has no {nameof( SheetAttribute )}. Use the overload of {nameof( Create )} with 4 parameters." ); - return FromSubrow< T >( module, language, attribute.Name, attribute.ColumnHash ); + return CreateSubrow< T >( module, language, sheetName ?? attribute.Name, attribute.ColumnHash ); } /// Creates a new instance of . @@ -224,9 +224,8 @@ public static BaseSubrowExcelSheet FromSubrow< T >( ExcelModule module, Language /// was invalid (hash mismatch). /// Sheet had an unsupported language. /// Header file had a value that is not supported. - /// A new instance of that should be cast to - /// before further use. - public static BaseExcelSheet From< T >( ExcelModule module, Language language, string sheetName, uint? columnHash = null ) + /// A new instance of . + public static ExcelSheet< T > Create< T >( ExcelModule module, Language language, string sheetName, uint? columnHash = null ) where T : struct, IExcelRow< T > { var headerFile = VerifySheet( module, language, sheetName, columnHash ); @@ -238,10 +237,9 @@ public static BaseExcelSheet From< T >( ExcelModule module, Language language, s } /// Creates a new instance of . - /// A new instance of that should be cast to - /// before further use. - /// - public static BaseSubrowExcelSheet FromSubrow< T >( ExcelModule module, Language language, string sheetName, uint? columnHash = null ) + /// A new instance of . + /// + public static SubrowExcelSheet< T > CreateSubrow< T >( ExcelModule module, Language language, string sheetName, uint? columnHash = null ) where T : struct, IExcelSubrow< T > { var headerFile = VerifySheet( module, language, sheetName, columnHash ); diff --git a/src/Lumina/Excel/ExcelModule.cs b/src/Lumina/Excel/ExcelModule.cs index 8bb489a3..e4e58297 100644 --- a/src/Lumina/Excel/ExcelModule.cs +++ b/src/Lumina/Excel/ExcelModule.cs @@ -26,7 +26,7 @@ public class ExcelModule internal ResolveRsvDelegate? RsvResolver => GameData.Options.RsvResolver; - private ConcurrentDictionary< (Type Type, Language Language), BaseExcelSheet > SheetCache { get; } = []; + private ConcurrentDictionary< (Type Type, Language Language, string? Name), BaseExcelSheet > SheetCache { get; } = []; /// /// A delegate provided by the user to resolve RSV strings. @@ -60,44 +60,33 @@ public ExcelModule( GameData gameData ) } /// Loads an . + /// The requested sheet language. Leave or empty to use the default language. + /// The requested explicit sheet name. Leave to use 's sheet name. Explicit names are necessary for quest/dungeon/cutscene sheets. + /// An excel sheet corresponding to , , and + /// that may be created anew or reused from a previous invocation of this method. /// /// Sheet is not of the variant . - /// - public ExcelSheet< T > GetSheet< T >( Language? language = null ) where T : struct, IExcelRow< T > => - (ExcelSheet< T >) GetBaseSheet< T >( language ); + /// + public ExcelSheet< T > GetSheet< T >( Language? language = null, string? sheetName = null ) where T : struct, IExcelRow< T > => + (ExcelSheet< T >)GetBaseSheet( typeof( T ), language, sheetName ); /// Loads an . + /// The requested sheet language. Leave or empty to use the default language. + /// The requested explicit sheet name. Leave to use 's sheet name. Explicit names are necessary for quest/dungeon/cutscene sheets. + /// An excel sheet corresponding to , , and + /// that may be created anew or reused from a previous invocation of this method. + /// /// Sheet is not of the variant . - /// - public SubrowExcelSheet< T > GetSubrowSheet< T >( Language? language = null ) where T : struct, IExcelSubrow< T > => - (SubrowExcelSheet< T >) GetBaseSubrowSheet< T >( language ); - - /// An excel sheet corresponding to and that may be created anew or - /// reused from a previous invocation of this method. - /// - /// The returned instance of should be cast to - /// before accessing its rows. - /// - /// does not have a valid . - /// - [EditorBrowsable( EditorBrowsableState.Advanced )] - public BaseExcelSheet GetBaseSheet< T >( Language? language = null ) where T : struct, IExcelRow< T > => - GetBaseSheet( typeof( T ), language ); - - /// - /// The returned instance of should be cast to - /// before accessing its rows. - /// - /// - [EditorBrowsable( EditorBrowsableState.Advanced )] - public BaseSubrowExcelSheet GetBaseSubrowSheet< T >( Language? language = null ) where T : struct, IExcelSubrow< T > => - (BaseSubrowExcelSheet) GetBaseSheet( typeof( T ), language ); + /// + public SubrowExcelSheet< T > GetSubrowSheet< T >( Language? language = null, string? sheetName = null ) where T : struct, IExcelSubrow< T > => + (SubrowExcelSheet< T >) GetBaseSheet( typeof( T ), language, sheetName ); /// Loads an . /// Type of the rows in the sheet. /// The requested sheet language. Leave or empty to use the default language. - /// An excel sheet corresponding to and that may be created anew or - /// reused from a previous invocation of this method. + /// The requested explicit sheet name. Leave to use 's sheet name. Explicit names are necessary for quest/dungeon/cutscene sheets. + /// An excel sheet corresponding to , , and + /// that may be created anew or reused from a previous invocation of this method. /// /// Only use this method if you need to create a sheet while using reflection. /// The returned instance of should be cast to or @@ -110,10 +99,10 @@ public BaseSubrowExcelSheet GetBaseSubrowSheet< T >( Language? language = null ) /// Sheet had an unsupported . [RequiresDynamicCode( "Creating a generic sheet from a type requires reflection and dynamic code." )] [EditorBrowsable( EditorBrowsableState.Advanced )] - public BaseExcelSheet GetBaseSheet( Type rowType, Language? language = null ) + public BaseExcelSheet GetBaseSheet( Type rowType, Language? language = null, string? sheetName = null ) { var sheet = SheetCache.GetOrAdd( - ( rowType, language ?? Language ), + ( rowType, language ?? Language, sheetName ), static ( key, module ) => { MethodInfo m; try @@ -122,24 +111,14 @@ public BaseExcelSheet GetBaseSheet( Type rowType, Language? language = null ) // As BaseExcelSheet.From(Subrow) has a constraint that T : IExcel(Row/Subrow), it is implicitly required that T is also a struct. // MakeGenericMethod will check for constraints, and throw ArgumentException if constraints aren't met. - if( isSubrowType ) - { - m = typeof( BaseExcelSheet ) - .GetMethod( - nameof( BaseExcelSheet.FromSubrow ), - BindingFlags.Static | BindingFlags.Public, - [typeof( ExcelModule ), typeof( Language )] )! - .MakeGenericMethod( key.Type ); - } - else - { - m = typeof( BaseExcelSheet ) - .GetMethod( - nameof( BaseExcelSheet.From ), - BindingFlags.Static | BindingFlags.Public, - [typeof( ExcelModule ), typeof( Language )] )! - .MakeGenericMethod( key.Type ); - } + m = typeof( BaseExcelSheet ) + .GetMethod( + isSubrowType ? + nameof( BaseExcelSheet.CreateSubrow ) : + nameof( BaseExcelSheet.Create ), + BindingFlags.Static | BindingFlags.Public, + [typeof( ExcelModule ), typeof( Language ), typeof( string )] )! + .MakeGenericMethod( key.Type ); } catch( ArgumentException e ) { @@ -152,7 +131,7 @@ public BaseExcelSheet GetBaseSheet( Type rowType, Language? language = null ) try { - return m.Invoke( null, [module, key.Language] ) as BaseExcelSheet ?? throw new InvalidOperationException( "Something went wrong" ); + return m.Invoke( null, [module, key.Language, key.Name] ) as BaseExcelSheet ?? throw new InvalidOperationException( "Something went wrong" ); } catch( TargetInvocationException e ) { diff --git a/src/Lumina/GameData.cs b/src/Lumina/GameData.cs index c923afb6..2c4fa7ea 100644 --- a/src/Lumina/GameData.cs +++ b/src/Lumina/GameData.cs @@ -288,6 +288,7 @@ public static UInt64 GetFileHash( string path ) /// Loads an . Returns if the sheet does not exist, has an invalid column hash or unsupported variant, or was requested with an unsupported language. /// The requested sheet language. Leave or empty to use the default language. + /// The requested explicit sheet name. Leave to use 's sheet name. Explicit names are necessary for quest/dungeon/cutscene sheets. /// An excel sheet corresponding to and that may be created anew or /// reused from a previous invocation of this method. /// @@ -297,11 +298,11 @@ public static UInt64 GetFileHash( string path ) /// /// Sheet is not of the variant . /// does not have a valid . - public ExcelSheet< T >? GetExcelSheet< T >( Language? language = null ) where T : struct, IExcelRow< T > + public ExcelSheet< T >? GetExcelSheet< T >( Language? language = null, string? name = null ) where T : struct, IExcelRow< T > { try { - return Excel.GetSheet< T >( language ); + return Excel.GetSheet< T >( language, name ); } catch( ArgumentException ) { @@ -315,12 +316,12 @@ public static UInt64 GetFileHash( string path ) /// Loads a . Returns if the sheet does not exist, has an invalid column hash or unsupported variant, or was requested with an unsupported language. /// Sheet is not of the variant . - /// - public SubrowExcelSheet< T >? GetSubrowExcelSheet< T >( Language? language = null ) where T : struct, IExcelSubrow< T > + /// + public SubrowExcelSheet< T >? GetSubrowExcelSheet< T >( Language? language = null, string? name = null ) where T : struct, IExcelSubrow< T > { try { - return Excel.GetSubrowSheet< T >( language ); + return Excel.GetSubrowSheet< T >( language, name ); } catch( ArgumentException ) { From 71cf233ffc9a996fcec629d4f797fe4d1b1af0e3 Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Mon, 19 Aug 2024 01:53:28 -0700 Subject: [PATCH 32/53] sheetName -> name (avoid redundancy) --- src/Lumina/Excel/BaseExcelSheet.cs | 34 +++++++++++++++--------------- src/Lumina/Excel/ExcelModule.cs | 25 +++++++++++----------- 2 files changed, 29 insertions(+), 30 deletions(-) diff --git a/src/Lumina/Excel/BaseExcelSheet.cs b/src/Lumina/Excel/BaseExcelSheet.cs index 04c55410..56621b8a 100644 --- a/src/Lumina/Excel/BaseExcelSheet.cs +++ b/src/Lumina/Excel/BaseExcelSheet.cs @@ -185,75 +185,75 @@ public static ExcelSheet< T > Create< T >( ExcelModule module ) where T : struct public static SubrowExcelSheet< T > CreateSubrow< T >( ExcelModule module ) where T : struct, IExcelSubrow< T > => CreateSubrow< T >( module, module.Language ); - /// Creates a new instance of , deducing sheet names (unless overridden with ) and column hashes from . + /// Creates a new instance of , deducing sheet names (unless overridden with ) and column hashes from . /// Type of each row. /// The to access sheet data from. /// The language to use for this sheet. - /// The explicit sheet name, if needed. Leave to use the type's sheet name. Explicit names are necessary for quest/dungeon/cutscene sheets. + /// The explicit sheet name, if needed. Leave to use the type's sheet name. Explicit names are necessary for quest/dungeon/cutscene sheets. /// does not have a valid . /// was invalid (invalid sheet name). /// was invalid (hash mismatch). /// Sheet had an unsupported language. /// Header file had a value that is not supported. /// A new instance of . - public static ExcelSheet< T > Create< T >( ExcelModule module, Language language, string? sheetName = null ) where T : struct, IExcelRow< T > + public static ExcelSheet< T > Create< T >( ExcelModule module, Language language, string? name = null ) where T : struct, IExcelRow< T > { var attribute = typeof( T ).GetCustomAttribute< SheetAttribute >() ?? throw new InvalidOperationException( $"{nameof( T )} has no {nameof( SheetAttribute )}. Use the overload of {nameof( Create )} with 4 parameters." ); - return Create< T >( module, language, sheetName ?? attribute.Name, attribute.ColumnHash ); + return Create< T >( module, language, name ?? attribute.Name, attribute.ColumnHash ); } /// A new instance of . /// - public static SubrowExcelSheet< T > CreateSubrow< T >( ExcelModule module, Language language, string? sheetName = null ) where T : struct, IExcelSubrow< T > + public static SubrowExcelSheet< T > CreateSubrow< T >( ExcelModule module, Language language, string? name = null ) where T : struct, IExcelSubrow< T > { var attribute = typeof( T ).GetCustomAttribute< SheetAttribute >() ?? throw new InvalidOperationException( $"{nameof( T )} has no {nameof( SheetAttribute )}. Use the overload of {nameof( Create )} with 4 parameters." ); - return CreateSubrow< T >( module, language, sheetName ?? attribute.Name, attribute.ColumnHash ); + return CreateSubrow< T >( module, language, name ?? attribute.Name, attribute.ColumnHash ); } /// Creates a new instance of . /// Type of each row. /// The to access sheet data from. /// The language to use for this sheet. - /// The name of the sheet to read from. + /// The name of the sheet to read from. /// The hash of the columns in the sheet. If , it will not check the hash. - /// was invalid (invalid sheet name). + /// was invalid (invalid sheet name). /// was invalid (hash mismatch). /// Sheet had an unsupported language. /// Header file had a value that is not supported. /// A new instance of . - public static ExcelSheet< T > Create< T >( ExcelModule module, Language language, string sheetName, uint? columnHash = null ) + public static ExcelSheet< T > Create< T >( ExcelModule module, Language language, string name, uint? columnHash = null ) where T : struct, IExcelRow< T > { - var headerFile = VerifySheet( module, language, sheetName, columnHash ); + var headerFile = VerifySheet( module, language, name, columnHash ); if( headerFile.Header.Variant != ExcelVariant.Default ) throw new NotSupportedException( $"Specified sheet variant {headerFile.Header.Variant} is not supported for IExcelRow types." ); - return new ExcelSheet< T >( module, headerFile, language, sheetName ); + return new ExcelSheet< T >( module, headerFile, language, name ); } /// Creates a new instance of . /// A new instance of . /// - public static SubrowExcelSheet< T > CreateSubrow< T >( ExcelModule module, Language language, string sheetName, uint? columnHash = null ) + public static SubrowExcelSheet< T > CreateSubrow< T >( ExcelModule module, Language language, string name, uint? columnHash = null ) where T : struct, IExcelSubrow< T > { - var headerFile = VerifySheet( module, language, sheetName, columnHash ); + var headerFile = VerifySheet( module, language, name, columnHash ); if( headerFile.Header.Variant != ExcelVariant.Subrows ) throw new NotSupportedException( $"Specified sheet variant {headerFile.Header.Variant} is not supported for IExcelSubrow types." ); - return new SubrowExcelSheet< T >( module, headerFile, language, sheetName ); + return new SubrowExcelSheet< T >( module, headerFile, language, name ); } - private static ExcelHeaderFile VerifySheet( ExcelModule module, Language language, string sheetName, uint? columnHash = null ) + private static ExcelHeaderFile VerifySheet( ExcelModule module, Language language, string name, uint? columnHash = null ) { - var headerFile = module.GameData.GetFile< ExcelHeaderFile >( $"exd/{sheetName}.exh" ) ?? - throw new ArgumentException( "Invalid sheet name", nameof( sheetName ) ); + var headerFile = module.GameData.GetFile< ExcelHeaderFile >( $"exd/{name}.exh" ) ?? + throw new ArgumentException( "Invalid sheet name", nameof( name ) ); if( module.VerifySheetChecksums && columnHash is { } hash && headerFile.GetColumnsHash() != hash ) throw new MismatchedColumnHashException( hash, headerFile.GetColumnsHash(), nameof( columnHash ) ); diff --git a/src/Lumina/Excel/ExcelModule.cs b/src/Lumina/Excel/ExcelModule.cs index e4e58297..36b95c21 100644 --- a/src/Lumina/Excel/ExcelModule.cs +++ b/src/Lumina/Excel/ExcelModule.cs @@ -41,7 +41,6 @@ public class ExcelModule /// public IReadOnlyCollection< string > SheetNames { get; } - /// /// Create a new ExcelModule. This will do all the initial discovery of sheets from the EXL but not load any sheets. /// @@ -61,31 +60,31 @@ public ExcelModule( GameData gameData ) /// Loads an . /// The requested sheet language. Leave or empty to use the default language. - /// The requested explicit sheet name. Leave to use 's sheet name. Explicit names are necessary for quest/dungeon/cutscene sheets. - /// An excel sheet corresponding to , , and + /// The requested explicit sheet name. Leave to use 's sheet name. Explicit names are necessary for quest/dungeon/cutscene sheets. + /// An excel sheet corresponding to , , and /// that may be created anew or reused from a previous invocation of this method. /// /// Sheet is not of the variant . /// - public ExcelSheet< T > GetSheet< T >( Language? language = null, string? sheetName = null ) where T : struct, IExcelRow< T > => - (ExcelSheet< T >)GetBaseSheet( typeof( T ), language, sheetName ); + public ExcelSheet< T > GetSheet< T >( Language? language = null, string? name = null ) where T : struct, IExcelRow< T > => + (ExcelSheet< T >)GetBaseSheet( typeof( T ), language, name ); /// Loads an . /// The requested sheet language. Leave or empty to use the default language. - /// The requested explicit sheet name. Leave to use 's sheet name. Explicit names are necessary for quest/dungeon/cutscene sheets. - /// An excel sheet corresponding to , , and + /// The requested explicit sheet name. Leave to use 's sheet name. Explicit names are necessary for quest/dungeon/cutscene sheets. + /// An excel sheet corresponding to , , and /// that may be created anew or reused from a previous invocation of this method. /// /// Sheet is not of the variant . /// - public SubrowExcelSheet< T > GetSubrowSheet< T >( Language? language = null, string? sheetName = null ) where T : struct, IExcelSubrow< T > => - (SubrowExcelSheet< T >) GetBaseSheet( typeof( T ), language, sheetName ); + public SubrowExcelSheet< T > GetSubrowSheet< T >( Language? language = null, string? name = null ) where T : struct, IExcelSubrow< T > => + (SubrowExcelSheet< T >) GetBaseSheet( typeof( T ), language, name ); /// Loads an . /// Type of the rows in the sheet. /// The requested sheet language. Leave or empty to use the default language. - /// The requested explicit sheet name. Leave to use 's sheet name. Explicit names are necessary for quest/dungeon/cutscene sheets. - /// An excel sheet corresponding to , , and + /// The requested explicit sheet name. Leave to use 's sheet name. Explicit names are necessary for quest/dungeon/cutscene sheets. + /// An excel sheet corresponding to , , and /// that may be created anew or reused from a previous invocation of this method. /// /// Only use this method if you need to create a sheet while using reflection. @@ -99,10 +98,10 @@ public SubrowExcelSheet< T > GetSubrowSheet< T >( Language? language = null, str /// Sheet had an unsupported . [RequiresDynamicCode( "Creating a generic sheet from a type requires reflection and dynamic code." )] [EditorBrowsable( EditorBrowsableState.Advanced )] - public BaseExcelSheet GetBaseSheet( Type rowType, Language? language = null, string? sheetName = null ) + public BaseExcelSheet GetBaseSheet( Type rowType, Language? language = null, string? name = null ) { var sheet = SheetCache.GetOrAdd( - ( rowType, language ?? Language, sheetName ), + ( rowType, language ?? Language, name ), static ( key, module ) => { MethodInfo m; try From 0b3082f6b1c94dbfa64742e6dae5d1e0f4b61df4 Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Mon, 19 Aug 2024 02:06:19 -0700 Subject: [PATCH 33/53] Make sheet name optional --- src/Lumina/Excel/BaseExcelSheet.cs | 4 +-- src/Lumina/Excel/SheetAttribute.cs | 44 ++++++++++++++++++++++++++---- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/src/Lumina/Excel/BaseExcelSheet.cs b/src/Lumina/Excel/BaseExcelSheet.cs index 56621b8a..f45312a0 100644 --- a/src/Lumina/Excel/BaseExcelSheet.cs +++ b/src/Lumina/Excel/BaseExcelSheet.cs @@ -201,7 +201,7 @@ public static ExcelSheet< T > Create< T >( ExcelModule module, Language language var attribute = typeof( T ).GetCustomAttribute< SheetAttribute >() ?? throw new InvalidOperationException( $"{nameof( T )} has no {nameof( SheetAttribute )}. Use the overload of {nameof( Create )} with 4 parameters." ); - return Create< T >( module, language, name ?? attribute.Name, attribute.ColumnHash ); + return Create< T >( module, language, name ?? attribute.Name ?? throw new ArgumentNullException( nameof( name ) ), attribute.ColumnHash ); } /// A new instance of . @@ -211,7 +211,7 @@ public static SubrowExcelSheet< T > CreateSubrow< T >( ExcelModule module, Langu var attribute = typeof( T ).GetCustomAttribute< SheetAttribute >() ?? throw new InvalidOperationException( $"{nameof( T )} has no {nameof( SheetAttribute )}. Use the overload of {nameof( Create )} with 4 parameters." ); - return CreateSubrow< T >( module, language, name ?? attribute.Name, attribute.ColumnHash ); + return CreateSubrow< T >( module, language, name ?? attribute.Name ?? throw new ArgumentNullException( nameof( name ) ), attribute.ColumnHash ); } /// Creates a new instance of . diff --git a/src/Lumina/Excel/SheetAttribute.cs b/src/Lumina/Excel/SheetAttribute.cs index c045dd6c..01c764c1 100644 --- a/src/Lumina/Excel/SheetAttribute.cs +++ b/src/Lumina/Excel/SheetAttribute.cs @@ -5,27 +5,59 @@ namespace Lumina.Excel; /// /// An attribute attached to a schema/struct that represents a sheet in an excel file. /// -/// The name of the sheet. -/// The column hash of the sheet; optionally used to check for schema and sheet changes. [AttributeUsage( AttributeTargets.Struct )] -public class SheetAttribute( string name, uint columnHash ) : Attribute +public class SheetAttribute : Attribute { /// /// The name of the sheet. /// - public readonly string Name = name; + /// + /// Can be if the schema is not associated with a specific sheet (i.e. quest/dungeon/cutscene sheets). + /// + public string? Name { get; } /// /// Gets the column hash of the sheet; optionally used to check for schema and sheet changes. /// - public readonly uint? ColumnHash = columnHash; + public uint? ColumnHash { get; } + + /// + /// Creates a new instance of the class. + /// + public SheetAttribute( ) + { + Name = null; + ColumnHash = null; + } /// /// Creates a new instance of the class. /// /// The name of the sheet. - public SheetAttribute( string name ) : this( name, uint.MaxValue ) + public SheetAttribute( string name ) { + Name = name; ColumnHash = null; } + + /// + /// Creates a new instance of the class. + /// + /// The column hash of the sheet; optionally used to check for schema and sheet changes. + public SheetAttribute( uint columnHash ) + { + Name = null; + ColumnHash = columnHash; + } + + /// + /// Creates a new instance of the class. + /// + /// The name of the sheet. + /// The column hash of the sheet; optionally used to check for schema and sheet changes. + public SheetAttribute( string name, uint columnHash ) + { + Name = name; + ColumnHash = columnHash; + } } \ No newline at end of file From 73a0c6fb31296742eed14a22487644ee8799a6b4 Mon Sep 17 00:00:00 2001 From: Soreepeong Date: Mon, 19 Aug 2024 19:44:33 +0900 Subject: [PATCH 34/53] Use constructors directly on Subrow/ExcelSheet, specialize exception types --- src/Lumina/Excel/BaseExcelSheet.cs | 120 +++--------------- src/Lumina/Excel/BaseSubrowExcelSheet.cs | 4 +- src/Lumina/Excel/ExcelModule.cs | 92 +++++++++----- src/Lumina/Excel/ExcelSheet.cs | 38 +++++- .../MismatchedColumnHashException.cs | 0 .../SheetAttributeMissingException.cs | 30 +++++ .../Exceptions/SheetNameEmptyException.cs | 31 +++++ .../Exceptions/SheetNotFoundException.cs | 30 +++++ .../UnsupportedLanguageException.cs | 0 src/Lumina/Excel/SheetAttribute.cs | 2 +- src/Lumina/Excel/SubrowExcelSheet.cs | 39 +++++- 11 files changed, 247 insertions(+), 139 deletions(-) rename src/Lumina/Excel/{ => Exceptions}/MismatchedColumnHashException.cs (100%) create mode 100644 src/Lumina/Excel/Exceptions/SheetAttributeMissingException.cs create mode 100644 src/Lumina/Excel/Exceptions/SheetNameEmptyException.cs create mode 100644 src/Lumina/Excel/Exceptions/SheetNotFoundException.cs rename src/Lumina/Excel/{ => Exceptions}/UnsupportedLanguageException.cs (100%) diff --git a/src/Lumina/Excel/BaseExcelSheet.cs b/src/Lumina/Excel/BaseExcelSheet.cs index f45312a0..6669c407 100644 --- a/src/Lumina/Excel/BaseExcelSheet.cs +++ b/src/Lumina/Excel/BaseExcelSheet.cs @@ -6,7 +6,6 @@ using System.Collections.Frozen; using System.Collections.Generic; using System.Linq; -using System.Reflection; using System.Runtime.CompilerServices; namespace Lumina.Excel; @@ -76,12 +75,27 @@ public abstract class BaseExcelSheet /// Contains information on the columns in this sheet. public IReadOnlyList< ExcelColumnDefinition > Columns { get; } - private protected BaseExcelSheet( ExcelModule module, ExcelHeaderFile headerFile, Language requestedLanguage, string sheetName ) + private protected BaseExcelSheet( ExcelModule module, Language language, string name, uint? columnHash, ExcelVariant expectedVariant ) { + ArgumentNullException.ThrowIfNull( module ); + ArgumentNullException.ThrowIfNull( name ); + + var headerFile = module.GameData.GetFile< ExcelHeaderFile >( $"exd/{name}.exh" ) ?? + throw new SheetNotFoundException( "Invalid sheet name", nameof( name ) ); + + if( module.VerifySheetChecksums && columnHash is { } hash && headerFile.GetColumnsHash() != hash ) + throw new MismatchedColumnHashException( hash, headerFile.GetColumnsHash(), nameof( columnHash ) ); + + if( !headerFile.Languages.Contains( language ) ) + throw new UnsupportedLanguageException( nameof( language ), language, null ); + + if( headerFile.Header.Variant != expectedVariant ) + throw new NotSupportedException( $"Specified sheet variant {headerFile.Header.Variant} is not supported; was expecting {expectedVariant}." ); + var hasSubrows = headerFile.Header.Variant == ExcelVariant.Subrows; Module = module; - Language = headerFile.Languages.Contains( requestedLanguage ) ? requestedLanguage : Language.None; + Language = headerFile.Languages.Contains( language ) ? language : Language.None; Columns = headerFile.ColumnDefinitions; _subrowDataOffset = hasSubrows ? headerFile.Header.DataOffset : (ushort) 0; _pages = new ExcelPage[headerFile.DataPages.Length]; @@ -92,8 +106,8 @@ private protected BaseExcelSheet( ExcelModule module, ExcelHeaderFile headerFile { var pageDef = headerFile.DataPages[ pageIdx ]; var filePath = Language == Language.None - ? $"exd/{sheetName}_{pageDef.StartId}.exd" - : $"exd/{sheetName}_{pageDef.StartId}_{LanguageUtil.GetLanguageStr( Language )}.exd"; + ? $"exd/{name}_{pageDef.StartId}.exd" + : $"exd/{name}_{pageDef.StartId}_{LanguageUtil.GetLanguageStr( Language )}.exd"; var fileData = module.GameData.GetFile< ExcelDataFile >( filePath ); if( fileData == null ) continue; @@ -168,102 +182,6 @@ private protected BaseExcelSheet( ExcelModule module, ExcelHeaderFile headerFile } } - /// Creates a new instance of with the 's default language, deducing sheet names and column hashes from . - /// Type of each row. - /// The to access sheet data from. - /// does not have a valid . - /// was invalid (invalid sheet name). - /// was invalid (hash mismatch). - /// Sheet had an unsupported language. - /// Header file had a value that is not supported. - /// A new instance of . - public static ExcelSheet< T > Create< T >( ExcelModule module ) where T : struct, IExcelRow< T > => - Create< T >( module, module.Language ); - - /// A new instance of . - /// - public static SubrowExcelSheet< T > CreateSubrow< T >( ExcelModule module ) where T : struct, IExcelSubrow< T > => - CreateSubrow< T >( module, module.Language ); - - /// Creates a new instance of , deducing sheet names (unless overridden with ) and column hashes from . - /// Type of each row. - /// The to access sheet data from. - /// The language to use for this sheet. - /// The explicit sheet name, if needed. Leave to use the type's sheet name. Explicit names are necessary for quest/dungeon/cutscene sheets. - /// does not have a valid . - /// was invalid (invalid sheet name). - /// was invalid (hash mismatch). - /// Sheet had an unsupported language. - /// Header file had a value that is not supported. - /// A new instance of . - public static ExcelSheet< T > Create< T >( ExcelModule module, Language language, string? name = null ) where T : struct, IExcelRow< T > - { - var attribute = typeof( T ).GetCustomAttribute< SheetAttribute >() ?? - throw new InvalidOperationException( $"{nameof( T )} has no {nameof( SheetAttribute )}. Use the overload of {nameof( Create )} with 4 parameters." ); - - return Create< T >( module, language, name ?? attribute.Name ?? throw new ArgumentNullException( nameof( name ) ), attribute.ColumnHash ); - } - - /// A new instance of . - /// - public static SubrowExcelSheet< T > CreateSubrow< T >( ExcelModule module, Language language, string? name = null ) where T : struct, IExcelSubrow< T > - { - var attribute = typeof( T ).GetCustomAttribute< SheetAttribute >() ?? - throw new InvalidOperationException( $"{nameof( T )} has no {nameof( SheetAttribute )}. Use the overload of {nameof( Create )} with 4 parameters." ); - - return CreateSubrow< T >( module, language, name ?? attribute.Name ?? throw new ArgumentNullException( nameof( name ) ), attribute.ColumnHash ); - } - - /// Creates a new instance of . - /// Type of each row. - /// The to access sheet data from. - /// The language to use for this sheet. - /// The name of the sheet to read from. - /// The hash of the columns in the sheet. If , it will not check the hash. - /// was invalid (invalid sheet name). - /// was invalid (hash mismatch). - /// Sheet had an unsupported language. - /// Header file had a value that is not supported. - /// A new instance of . - public static ExcelSheet< T > Create< T >( ExcelModule module, Language language, string name, uint? columnHash = null ) - where T : struct, IExcelRow< T > - { - var headerFile = VerifySheet( module, language, name, columnHash ); - - if( headerFile.Header.Variant != ExcelVariant.Default ) - throw new NotSupportedException( $"Specified sheet variant {headerFile.Header.Variant} is not supported for IExcelRow types." ); - - return new ExcelSheet< T >( module, headerFile, language, name ); - } - - /// Creates a new instance of . - /// A new instance of . - /// - public static SubrowExcelSheet< T > CreateSubrow< T >( ExcelModule module, Language language, string name, uint? columnHash = null ) - where T : struct, IExcelSubrow< T > - { - var headerFile = VerifySheet( module, language, name, columnHash ); - - if( headerFile.Header.Variant != ExcelVariant.Subrows ) - throw new NotSupportedException( $"Specified sheet variant {headerFile.Header.Variant} is not supported for IExcelSubrow types." ); - - return new SubrowExcelSheet< T >( module, headerFile, language, name ); - } - - private static ExcelHeaderFile VerifySheet( ExcelModule module, Language language, string name, uint? columnHash = null ) - { - var headerFile = module.GameData.GetFile< ExcelHeaderFile >( $"exd/{name}.exh" ) ?? - throw new ArgumentException( "Invalid sheet name", nameof( name ) ); - - if( module.VerifySheetChecksums && columnHash is { } hash && headerFile.GetColumnsHash() != hash ) - throw new MismatchedColumnHashException( hash, headerFile.GetColumnsHash(), nameof( columnHash ) ); - - if( !headerFile.Languages.Contains( language ) ) - throw new UnsupportedLanguageException( nameof( language ), language, null ); - - return headerFile; - } - /// The number of rows in this sheet. /// /// If this sheet has gaps in row ids, it returns the number of rows that exist, not the highest row id. diff --git a/src/Lumina/Excel/BaseSubrowExcelSheet.cs b/src/Lumina/Excel/BaseSubrowExcelSheet.cs index dcc6fc81..5663a374 100644 --- a/src/Lumina/Excel/BaseSubrowExcelSheet.cs +++ b/src/Lumina/Excel/BaseSubrowExcelSheet.cs @@ -9,8 +9,8 @@ namespace Lumina.Excel; /// An excel sheet of variant. public abstract class BaseSubrowExcelSheet : BaseExcelSheet { - private protected BaseSubrowExcelSheet( ExcelModule module, ExcelHeaderFile headerFile, Language requestedLanguage, string sheetName ) - : base( module, headerFile, requestedLanguage, sheetName ) + private protected BaseSubrowExcelSheet( ExcelModule module, Language requestedLanguage, string sheetName, uint? columnHash ) + : base( module, requestedLanguage, sheetName, columnHash, ExcelVariant.Subrows ) { foreach( var f in OffsetLookupTable ) TotalSubrowCount += f.SubrowCount; diff --git a/src/Lumina/Excel/ExcelModule.cs b/src/Lumina/Excel/ExcelModule.cs index 36b95c21..4bc28a93 100644 --- a/src/Lumina/Excel/ExcelModule.cs +++ b/src/Lumina/Excel/ExcelModule.cs @@ -26,7 +26,8 @@ public class ExcelModule internal ResolveRsvDelegate? RsvResolver => GameData.Options.RsvResolver; - private ConcurrentDictionary< (Type Type, Language Language, string? Name), BaseExcelSheet > SheetCache { get; } = []; + private ConcurrentDictionary< (Type Type, Language Language, string Name), BaseExcelSheet > SheetCache { get; } = []; + private ConcurrentDictionary< Type, SheetAttribute? > SheetAttributeCache { get; } = []; /// /// A delegate provided by the user to resolve RSV strings. @@ -67,7 +68,7 @@ public ExcelModule( GameData gameData ) /// Sheet is not of the variant . /// public ExcelSheet< T > GetSheet< T >( Language? language = null, string? name = null ) where T : struct, IExcelRow< T > => - (ExcelSheet< T >)GetBaseSheet( typeof( T ), language, name ); + (ExcelSheet< T >) GetBaseSheet( typeof( T ), language, name ); /// Loads an . /// The requested sheet language. Leave or empty to use the default language. @@ -91,8 +92,9 @@ public SubrowExcelSheet< T > GetSubrowSheet< T >( Language? language = null, str /// The returned instance of should be cast to or /// before accessing its rows. /// - /// does not have a valid . - /// Sheet does not exist. + /// Sheet name was not specified neither via nor . + /// does not have a valid . + /// Sheet does not exist. /// Sheet had a mismatched column hash. /// Sheet does not support nor . /// Sheet had an unsupported . @@ -100,37 +102,41 @@ public SubrowExcelSheet< T > GetSubrowSheet< T >( Language? language = null, str [EditorBrowsable( EditorBrowsableState.Advanced )] public BaseExcelSheet GetBaseSheet( Type rowType, Language? language = null, string? name = null ) { + var attr = GetSheetAttributes( rowType ) ?? throw new SheetAttributeMissingException( null, nameof( rowType ) ); + name ??= attr.Name ?? throw new SheetNameEmptyException( null, nameof( name ) ); var sheet = SheetCache.GetOrAdd( ( rowType, language ?? Language, name ), - static ( key, module ) => { - MethodInfo m; + static ( key, context ) => { + Type t; try { - var isSubrowType = key.Type.IsAssignableTo( typeof( IExcelSubrow<> ).MakeGenericType( key.Type ) ); - - // As BaseExcelSheet.From(Subrow) has a constraint that T : IExcel(Row/Subrow), it is implicitly required that T is also a struct. - // MakeGenericMethod will check for constraints, and throw ArgumentException if constraints aren't met. - m = typeof( BaseExcelSheet ) - .GetMethod( - isSubrowType ? - nameof( BaseExcelSheet.CreateSubrow ) : - nameof( BaseExcelSheet.Create ), - BindingFlags.Static | BindingFlags.Public, - [typeof( ExcelModule ), typeof( Language ), typeof( string )] )! - .MakeGenericMethod( key.Type ); + t = typeof( ExcelSheet<> ).MakeGenericType( key.Type ); } - catch( ArgumentException e ) + catch( ArgumentException e1 ) { - // Exception thrown here will propagate outside ConcurrentDictionary<>.GetOrAdd without touching the data stored inside dictionary. - throw new ArgumentException( - $"{key.Type.Name} must implement either {typeof( IExcelRow<> ).Name.Split( '`', 2 )[ 0 ]}<{key.Type.Name}> or {typeof( IExcelSubrow<> ).Name.Split( '`', 2 )[ 0 ]}<{key.Type.Name}>.", - nameof( rowType ), - e ); + try + { + t = typeof( SubrowExcelSheet<> ).MakeGenericType( key.Type ); + } + catch( ArgumentException e2 ) + { + // Exception thrown here will propagate outside ConcurrentDictionary<>.GetOrAdd without touching the data stored inside dictionary. + throw new ArgumentException( + $"{key.Type.Name} must implement either {typeof( IExcelRow<> ).Name.Split( '`', 2 )[ 0 ]}<{key.Type.Name}> or {typeof( IExcelSubrow<> ).Name.Split( '`', 2 )[ 0 ]}<{key.Type.Name}>.", + nameof( rowType ), + new AggregateException( e1, e2 ) ); + } } try { - return m.Invoke( null, [module, key.Language, key.Name] ) as BaseExcelSheet ?? throw new InvalidOperationException( "Something went wrong" ); + return Activator.CreateInstance( + t, + BindingFlags.Instance | BindingFlags.Public, + null, + [context.Module, key.Language, key.Name, context.Attribute.ColumnHash], + null ) as BaseExcelSheet ?? + throw new InvalidOperationException( "Something went wrong" ); } catch( TargetInvocationException e ) { @@ -141,7 +147,7 @@ public BaseExcelSheet GetBaseSheet( Type rowType, Language? language = null, str return InvalidSheet.Create( e ); } }, - this ); + ( Module: this, Attribute: attr ) ); if( sheet is not InvalidSheet { Exception: var e } ) return sheet; @@ -155,16 +161,44 @@ public BaseExcelSheet GetBaseSheet( Type rowType, Language? language = null, str throw e; } + /// Unloads cached sheets that reference an assembly. + /// Assembly to look for in the cached sheets. + public void UnloadCachedSheetsOfAssembly( Assembly assembly ) + { + foreach( var c in SheetCache.Keys ) + { + if( c.Type.Assembly == assembly ) + _ = SheetCache.TryRemove( c, out _ ); + } + + foreach( var c in SheetAttributeCache.Keys ) + { + if( c.Assembly == assembly ) + _ = SheetAttributeCache.TryRemove( c, out _ ); + } + } + + /// Gets the sheet attributes for . + /// Type of the row. + /// Sheet attributes, if any. + internal SheetAttribute? GetSheetAttributes< T >() => GetSheetAttributes( typeof( T ) ); + + /// Gets the sheet attributes for . + /// Type of the row. + /// Sheet attributes, if any. + internal SheetAttribute? GetSheetAttributes( Type rowType ) => + SheetAttributeCache.GetOrAdd( + rowType, + static type => type.GetCustomAttribute< SheetAttribute >( false ) ); + private sealed class InvalidSheet : BaseExcelSheet { public Exception Exception { get; private set; } // never actually called private InvalidSheet( ExcelModule module, ExcelHeaderFile headerFile, Language requestedLanguage, string sheetName ) - : base( module, headerFile, requestedLanguage, sheetName ) - { + : base( default!, default, default!, default, default ) => Exception = null!; - } public static InvalidSheet Create( Exception exception ) { diff --git a/src/Lumina/Excel/ExcelSheet.cs b/src/Lumina/Excel/ExcelSheet.cs index a28a43c8..a62c661f 100644 --- a/src/Lumina/Excel/ExcelSheet.cs +++ b/src/Lumina/Excel/ExcelSheet.cs @@ -3,7 +3,6 @@ using System.Collections.Generic; using System.Runtime.CompilerServices; using Lumina.Data; -using Lumina.Data.Files.Excel; using Lumina.Data.Structs.Excel; namespace Lumina.Excel; @@ -12,8 +11,41 @@ namespace Lumina.Excel; /// Type of the rows contained within. public sealed class ExcelSheet< T > : BaseExcelSheet, ICollection< T >, IReadOnlyCollection< T > where T : struct, IExcelRow< T > { - internal ExcelSheet( ExcelModule module, ExcelHeaderFile headerFile, Language requestedLanguage, string sheetName ) - : base( module, headerFile, requestedLanguage, sheetName ) + /// Creates a new instance of , deducing sheet names (unless overridden with ) and column hashes + /// from . + /// Type of each row. + /// The to access sheet data from. + /// The language to use for this sheet. Use to use . + /// The explicit sheet name, if needed. Leave to use the type's sheet name. Explicit names are necessary for + /// quest/dungeon/cutscene sheets. + /// had no and was empty. + /// + /// Sheet does not exist. + /// was invalid (hash mismatch). + /// Sheet had an unsupported language. + /// Header file had a value that is not supported. + /// A new instance of . + public ExcelSheet( ExcelModule module, Language? language = null, string? name = null ) + : this( module, language ?? module.Language, name, module.GetSheetAttributes< T >() ) + { } + + /// Creates a new instance of . + /// Type of each row. + /// The to access sheet data from. + /// The language to use for this sheet. + /// The name of the sheet to read from. + /// The hash of the columns in the sheet. If , it will not check the hash. + /// Sheet does not exist. + /// was invalid (hash mismatch). + /// Sheet had an unsupported language. + /// Header file had a value that is not supported. + /// A new instance of . + public ExcelSheet( ExcelModule module, Language language, string name, uint? columnHash ) + : base( module, language, name, columnHash, ExcelVariant.Default ) + { } + + private ExcelSheet( ExcelModule module, Language language, string? name, SheetAttribute? attribute ) + : this( module, language, name ?? attribute?.Name ?? throw new SheetNameEmptyException( null, nameof( name ) ), attribute?.ColumnHash ) { } bool ICollection< T >.IsReadOnly => true; diff --git a/src/Lumina/Excel/MismatchedColumnHashException.cs b/src/Lumina/Excel/Exceptions/MismatchedColumnHashException.cs similarity index 100% rename from src/Lumina/Excel/MismatchedColumnHashException.cs rename to src/Lumina/Excel/Exceptions/MismatchedColumnHashException.cs diff --git a/src/Lumina/Excel/Exceptions/SheetAttributeMissingException.cs b/src/Lumina/Excel/Exceptions/SheetAttributeMissingException.cs new file mode 100644 index 00000000..8a2a0a7e --- /dev/null +++ b/src/Lumina/Excel/Exceptions/SheetAttributeMissingException.cs @@ -0,0 +1,30 @@ +using System; + +namespace Lumina.Excel; + +/// Exception indicating that sheet attribute was empty. +public sealed class SheetAttributeMissingException : ArgumentException +{ + private const string DefaultMessage = $"Row type has no {nameof( SheetAttribute )}."; + + /// + public SheetAttributeMissingException() : base( DefaultMessage ) + { } + + /// + public SheetAttributeMissingException( string? message ) : base( message ?? DefaultMessage ) + { } + + /// + public SheetAttributeMissingException( string? message, Exception? innerException ) : base( message ?? DefaultMessage, innerException ) + { } + + /// + public SheetAttributeMissingException( string? message, string? paramName ) : base( message ?? DefaultMessage, paramName ) + { } + + /// + public SheetAttributeMissingException( string? message, string? paramName, Exception? innerException ) + : base( message ?? DefaultMessage, paramName, innerException ) + { } +} \ No newline at end of file diff --git a/src/Lumina/Excel/Exceptions/SheetNameEmptyException.cs b/src/Lumina/Excel/Exceptions/SheetNameEmptyException.cs new file mode 100644 index 00000000..c4c7332e --- /dev/null +++ b/src/Lumina/Excel/Exceptions/SheetNameEmptyException.cs @@ -0,0 +1,31 @@ +using System; + +namespace Lumina.Excel; + +/// Exception indicating that sheet name could not be resolved. +public sealed class SheetNameEmptyException : ArgumentException +{ + private const string DefaultMessage = + $"Row type has no {nameof( SheetAttribute )} or its {nameof( SheetAttribute.Name )} is null, and no valid sheet name is specified."; + + /// + public SheetNameEmptyException() : base( DefaultMessage ) + { } + + /// + public SheetNameEmptyException( string? message ) : base( message ?? DefaultMessage ) + { } + + /// + public SheetNameEmptyException( string? message, Exception? innerException ) : base( message ?? DefaultMessage, innerException ) + { } + + /// + public SheetNameEmptyException( string? message, string? paramName ) : base( message ?? DefaultMessage, paramName ) + { } + + /// + public SheetNameEmptyException( string? message, string? paramName, Exception? innerException ) + : base( message ?? DefaultMessage, paramName, innerException ) + { } +} \ No newline at end of file diff --git a/src/Lumina/Excel/Exceptions/SheetNotFoundException.cs b/src/Lumina/Excel/Exceptions/SheetNotFoundException.cs new file mode 100644 index 00000000..0489c2f9 --- /dev/null +++ b/src/Lumina/Excel/Exceptions/SheetNotFoundException.cs @@ -0,0 +1,30 @@ +using System; + +namespace Lumina.Excel; + +/// Exception indicating that no such sheet could be found. +public sealed class SheetNotFoundException : ArgumentException +{ + private const string DefaultMessage = "Sheet could not be found."; + + /// + public SheetNotFoundException() : base( DefaultMessage ) + { } + + /// + public SheetNotFoundException( string? message ) : base( message ?? DefaultMessage ) + { } + + /// + public SheetNotFoundException( string? message, Exception? innerException ) : base( message ?? DefaultMessage, innerException ) + { } + + /// + public SheetNotFoundException( string? message, string? paramName ) : base( message ?? DefaultMessage, paramName ) + { } + + /// + public SheetNotFoundException( string? message, string? paramName, Exception? innerException ) + : base( message ?? DefaultMessage, paramName, innerException ) + { } +} \ No newline at end of file diff --git a/src/Lumina/Excel/UnsupportedLanguageException.cs b/src/Lumina/Excel/Exceptions/UnsupportedLanguageException.cs similarity index 100% rename from src/Lumina/Excel/UnsupportedLanguageException.cs rename to src/Lumina/Excel/Exceptions/UnsupportedLanguageException.cs diff --git a/src/Lumina/Excel/SheetAttribute.cs b/src/Lumina/Excel/SheetAttribute.cs index 01c764c1..cfe80ba3 100644 --- a/src/Lumina/Excel/SheetAttribute.cs +++ b/src/Lumina/Excel/SheetAttribute.cs @@ -24,7 +24,7 @@ public class SheetAttribute : Attribute /// /// Creates a new instance of the class. /// - public SheetAttribute( ) + public SheetAttribute() { Name = null; ColumnHash = null; diff --git a/src/Lumina/Excel/SubrowExcelSheet.cs b/src/Lumina/Excel/SubrowExcelSheet.cs index ca8d0405..1a4a8f98 100644 --- a/src/Lumina/Excel/SubrowExcelSheet.cs +++ b/src/Lumina/Excel/SubrowExcelSheet.cs @@ -1,9 +1,9 @@ -using Lumina.Data.Files.Excel; using Lumina.Data; using System; using System.Collections; using System.Collections.Generic; using System.Runtime.CompilerServices; +using Lumina.Data.Structs.Excel; namespace Lumina.Excel; @@ -13,8 +13,41 @@ public sealed class SubrowExcelSheet< T > : BaseSubrowExcelSheet, ICollection< SubrowCollection< T > >, IReadOnlyCollection< SubrowCollection< T > > where T : struct, IExcelSubrow< T > { - internal SubrowExcelSheet( ExcelModule module, ExcelHeaderFile headerFile, Language requestedLanguage, string sheetName ) - : base( module, headerFile, requestedLanguage, sheetName ) + /// Creates a new instance of , deducing sheet names (unless overridden with ) and column + /// hashes from . + /// Type of each row. + /// The to access sheet data from. + /// The language to use for this sheet. Use to use . + /// The explicit sheet name, if needed. Leave to use the type's sheet name. Explicit names are necessary for + /// quest/dungeon/cutscene sheets. + /// had no and was empty. + /// + /// Sheet does not exist. + /// was invalid (hash mismatch). + /// Sheet had an unsupported language. + /// Header file had a value that is not supported. + /// A new instance of . + public SubrowExcelSheet( ExcelModule module, Language? language = null, string? name = null ) + : this( module, language ?? module.Language, name, module.GetSheetAttributes< T >() ) + { } + + /// Creates a new instance of . + /// Type of each row. + /// The to access sheet data from. + /// The language to use for this sheet. + /// The name of the sheet to read from. + /// The hash of the columns in the sheet. If , it will not check the hash. + /// Sheet does not exist. + /// was invalid (hash mismatch). + /// Sheet had an unsupported language. + /// Header file had a value that is not supported. + /// A new instance of . + public SubrowExcelSheet( ExcelModule module, Language language, string name, uint? columnHash ) + : base( module, language, name, columnHash ) + { } + + private SubrowExcelSheet( ExcelModule module, Language language, string? name, SheetAttribute? attribute ) + : this( module, language, name ?? attribute?.Name ?? throw new SheetNameEmptyException( null, nameof( name ) ), attribute?.ColumnHash ) { } /// From b07aa3284b6484128f014fbf25217d10d901f8bb Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Mon, 19 Aug 2024 12:38:00 -0700 Subject: [PATCH 35/53] Formatting changes --- src/Lumina/Excel/BaseSubrowExcelSheet.cs | 1 - src/Lumina/Excel/Collection.cs | 5 ---- src/Lumina/Excel/ExcelModule.cs | 7 +++-- src/Lumina/Excel/ExcelPage.cs | 2 +- src/Lumina/Excel/ExcelSheet.cs | 7 +---- .../MismatchedColumnHashException.cs | 1 - src/Lumina/Excel/SubrowExcelSheet.cs | 27 +++---------------- 7 files changed, 9 insertions(+), 41 deletions(-) diff --git a/src/Lumina/Excel/BaseSubrowExcelSheet.cs b/src/Lumina/Excel/BaseSubrowExcelSheet.cs index 5663a374..508e6d7b 100644 --- a/src/Lumina/Excel/BaseSubrowExcelSheet.cs +++ b/src/Lumina/Excel/BaseSubrowExcelSheet.cs @@ -1,7 +1,6 @@ using System; using System.Runtime.CompilerServices; using Lumina.Data; -using Lumina.Data.Files.Excel; using Lumina.Data.Structs.Excel; namespace Lumina.Excel; diff --git a/src/Lumina/Excel/Collection.cs b/src/Lumina/Excel/Collection.cs index 3be64629..43227e1d 100644 --- a/src/Lumina/Excel/Collection.cs +++ b/src/Lumina/Excel/Collection.cs @@ -10,11 +10,6 @@ namespace Lumina.Excel; /// /// Mostly an implementation detail for reading excel rows. This type does not store or hold any row data, and is therefore lightweight and trivially constructable. /// A type that wraps a group of fields inside a row. -/// -/// -/// -/// -/// public readonly struct Collection< T >( ExcelPage page, uint parentOffset, uint offset, Func< ExcelPage, uint, uint, uint, T > ctor, int size ) : IList< T >, IReadOnlyList< T > where T : struct { diff --git a/src/Lumina/Excel/ExcelModule.cs b/src/Lumina/Excel/ExcelModule.cs index 4bc28a93..aa7293c7 100644 --- a/src/Lumina/Excel/ExcelModule.cs +++ b/src/Lumina/Excel/ExcelModule.cs @@ -66,7 +66,7 @@ public ExcelModule( GameData gameData ) /// that may be created anew or reused from a previous invocation of this method. /// /// Sheet is not of the variant . - /// + /// public ExcelSheet< T > GetSheet< T >( Language? language = null, string? name = null ) where T : struct, IExcelRow< T > => (ExcelSheet< T >) GetBaseSheet( typeof( T ), language, name ); @@ -77,7 +77,7 @@ public ExcelSheet< T > GetSheet< T >( Language? language = null, string? name = /// that may be created anew or reused from a previous invocation of this method. /// /// Sheet is not of the variant . - /// + /// public SubrowExcelSheet< T > GetSubrowSheet< T >( Language? language = null, string? name = null ) where T : struct, IExcelSubrow< T > => (SubrowExcelSheet< T >) GetBaseSheet( typeof( T ), language, name ); @@ -196,8 +196,7 @@ private sealed class InvalidSheet : BaseExcelSheet public Exception Exception { get; private set; } // never actually called - private InvalidSheet( ExcelModule module, ExcelHeaderFile headerFile, Language requestedLanguage, string sheetName ) - : base( default!, default, default!, default, default ) => + private InvalidSheet() : base( default!, default, default!, default, default ) => Exception = null!; public static InvalidSheet Create( Exception exception ) diff --git a/src/Lumina/Excel/ExcelPage.cs b/src/Lumina/Excel/ExcelPage.cs index 956e1e58..83fccd4d 100644 --- a/src/Lumina/Excel/ExcelPage.cs +++ b/src/Lumina/Excel/ExcelPage.cs @@ -164,7 +164,7 @@ public ulong ReadUInt64( nuint offset ) => /// /// Byte offset of the field inside the page. /// Bit offset of the field inside the byte. (0 - 7) - /// The . + /// The . [MethodImpl( MethodImplOptions.AggressiveInlining )] public bool ReadPackedBool( nuint offset, byte bit ) => ( Read< byte >( offset ) & ( 1 << bit ) ) != 0; diff --git a/src/Lumina/Excel/ExcelSheet.cs b/src/Lumina/Excel/ExcelSheet.cs index a62c661f..71e27985 100644 --- a/src/Lumina/Excel/ExcelSheet.cs +++ b/src/Lumina/Excel/ExcelSheet.cs @@ -13,24 +13,19 @@ public sealed class ExcelSheet< T > : BaseExcelSheet, ICollection< T >, IReadOnl { /// Creates a new instance of , deducing sheet names (unless overridden with ) and column hashes /// from . - /// Type of each row. /// The to access sheet data from. /// The language to use for this sheet. Use to use . /// The explicit sheet name, if needed. Leave to use the type's sheet name. Explicit names are necessary for /// quest/dungeon/cutscene sheets. /// had no and was empty. /// - /// Sheet does not exist. /// was invalid (hash mismatch). - /// Sheet had an unsupported language. - /// Header file had a value that is not supported. - /// A new instance of . + /// public ExcelSheet( ExcelModule module, Language? language = null, string? name = null ) : this( module, language ?? module.Language, name, module.GetSheetAttributes< T >() ) { } /// Creates a new instance of . - /// Type of each row. /// The to access sheet data from. /// The language to use for this sheet. /// The name of the sheet to read from. diff --git a/src/Lumina/Excel/Exceptions/MismatchedColumnHashException.cs b/src/Lumina/Excel/Exceptions/MismatchedColumnHashException.cs index 155385e2..1545cffd 100644 --- a/src/Lumina/Excel/Exceptions/MismatchedColumnHashException.cs +++ b/src/Lumina/Excel/Exceptions/MismatchedColumnHashException.cs @@ -3,7 +3,6 @@ namespace Lumina.Excel; /// Exception indicating that the requested row type's column hash is different from game data. -/// public sealed class MismatchedColumnHashException( uint typeHash, uint gameHash, string? paramName ) : ArgumentException( $"The requested row type has a column hash that is different from game data. (Type: {typeHash:X08}, Game: {gameHash:X08})", diff --git a/src/Lumina/Excel/SubrowExcelSheet.cs b/src/Lumina/Excel/SubrowExcelSheet.cs index 1a4a8f98..55bb36ce 100644 --- a/src/Lumina/Excel/SubrowExcelSheet.cs +++ b/src/Lumina/Excel/SubrowExcelSheet.cs @@ -3,7 +3,6 @@ using System.Collections; using System.Collections.Generic; using System.Runtime.CompilerServices; -using Lumina.Data.Structs.Excel; namespace Lumina.Excel; @@ -15,33 +14,15 @@ public sealed class SubrowExcelSheet< T > { /// Creates a new instance of , deducing sheet names (unless overridden with ) and column /// hashes from . - /// Type of each row. - /// The to access sheet data from. - /// The language to use for this sheet. Use to use . - /// The explicit sheet name, if needed. Leave to use the type's sheet name. Explicit names are necessary for - /// quest/dungeon/cutscene sheets. - /// had no and was empty. - /// - /// Sheet does not exist. - /// was invalid (hash mismatch). - /// Sheet had an unsupported language. - /// Header file had a value that is not supported. - /// A new instance of . + /// A new instance of . + /// public SubrowExcelSheet( ExcelModule module, Language? language = null, string? name = null ) : this( module, language ?? module.Language, name, module.GetSheetAttributes< T >() ) { } /// Creates a new instance of . - /// Type of each row. - /// The to access sheet data from. - /// The language to use for this sheet. - /// The name of the sheet to read from. - /// The hash of the columns in the sheet. If , it will not check the hash. - /// Sheet does not exist. - /// was invalid (hash mismatch). - /// Sheet had an unsupported language. - /// Header file had a value that is not supported. - /// A new instance of . + /// A new instance of . + /// public SubrowExcelSheet( ExcelModule module, Language language, string name, uint? columnHash ) : base( module, language, name, columnHash ) { } From 030bfd331229851e6826264ef85a5966312e22cf Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Mon, 19 Aug 2024 12:38:43 -0700 Subject: [PATCH 36/53] Create exceptions namespace --- src/Lumina/Excel/Exceptions/MismatchedColumnHashException.cs | 2 +- src/Lumina/Excel/Exceptions/SheetAttributeMissingException.cs | 2 +- src/Lumina/Excel/Exceptions/SheetNameEmptyException.cs | 2 +- src/Lumina/Excel/Exceptions/SheetNotFoundException.cs | 2 +- src/Lumina/Excel/Exceptions/UnsupportedLanguageException.cs | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Lumina/Excel/Exceptions/MismatchedColumnHashException.cs b/src/Lumina/Excel/Exceptions/MismatchedColumnHashException.cs index 1545cffd..6af72cea 100644 --- a/src/Lumina/Excel/Exceptions/MismatchedColumnHashException.cs +++ b/src/Lumina/Excel/Exceptions/MismatchedColumnHashException.cs @@ -1,6 +1,6 @@ using System; -namespace Lumina.Excel; +namespace Lumina.Excel.Exceptions; /// Exception indicating that the requested row type's column hash is different from game data. public sealed class MismatchedColumnHashException( uint typeHash, uint gameHash, string? paramName ) : diff --git a/src/Lumina/Excel/Exceptions/SheetAttributeMissingException.cs b/src/Lumina/Excel/Exceptions/SheetAttributeMissingException.cs index 8a2a0a7e..12f9bc96 100644 --- a/src/Lumina/Excel/Exceptions/SheetAttributeMissingException.cs +++ b/src/Lumina/Excel/Exceptions/SheetAttributeMissingException.cs @@ -1,6 +1,6 @@ using System; -namespace Lumina.Excel; +namespace Lumina.Excel.Exceptions; /// Exception indicating that sheet attribute was empty. public sealed class SheetAttributeMissingException : ArgumentException diff --git a/src/Lumina/Excel/Exceptions/SheetNameEmptyException.cs b/src/Lumina/Excel/Exceptions/SheetNameEmptyException.cs index c4c7332e..5d8fb152 100644 --- a/src/Lumina/Excel/Exceptions/SheetNameEmptyException.cs +++ b/src/Lumina/Excel/Exceptions/SheetNameEmptyException.cs @@ -1,6 +1,6 @@ using System; -namespace Lumina.Excel; +namespace Lumina.Excel.Exceptions; /// Exception indicating that sheet name could not be resolved. public sealed class SheetNameEmptyException : ArgumentException diff --git a/src/Lumina/Excel/Exceptions/SheetNotFoundException.cs b/src/Lumina/Excel/Exceptions/SheetNotFoundException.cs index 0489c2f9..63785d94 100644 --- a/src/Lumina/Excel/Exceptions/SheetNotFoundException.cs +++ b/src/Lumina/Excel/Exceptions/SheetNotFoundException.cs @@ -1,6 +1,6 @@ using System; -namespace Lumina.Excel; +namespace Lumina.Excel.Exceptions; /// Exception indicating that no such sheet could be found. public sealed class SheetNotFoundException : ArgumentException diff --git a/src/Lumina/Excel/Exceptions/UnsupportedLanguageException.cs b/src/Lumina/Excel/Exceptions/UnsupportedLanguageException.cs index b68958eb..18848ff8 100644 --- a/src/Lumina/Excel/Exceptions/UnsupportedLanguageException.cs +++ b/src/Lumina/Excel/Exceptions/UnsupportedLanguageException.cs @@ -1,6 +1,6 @@ using System; -namespace Lumina.Excel; +namespace Lumina.Excel.Exceptions; /// Exception indicating that the requested language is not supported by the requested sheet. public sealed class UnsupportedLanguageException : ArgumentOutOfRangeException From e3a39d80677e0f8e7689c00c23a65d8870345ced Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Mon, 19 Aug 2024 12:51:14 -0700 Subject: [PATCH 37/53] Fix compiler error & exception handling --- src/Lumina/Excel/BaseExcelSheet.cs | 1 + src/Lumina/Excel/ExcelModule.cs | 1 + src/Lumina/Excel/ExcelSheet.cs | 1 + src/Lumina/Excel/SubrowExcelSheet.cs | 1 + src/Lumina/GameData.cs | 17 +++++------------ 5 files changed, 9 insertions(+), 12 deletions(-) diff --git a/src/Lumina/Excel/BaseExcelSheet.cs b/src/Lumina/Excel/BaseExcelSheet.cs index 6669c407..46ea62f1 100644 --- a/src/Lumina/Excel/BaseExcelSheet.cs +++ b/src/Lumina/Excel/BaseExcelSheet.cs @@ -1,6 +1,7 @@ using Lumina.Data; using Lumina.Data.Files.Excel; using Lumina.Data.Structs.Excel; +using Lumina.Excel.Exceptions; using Lumina.Extensions; using System; using System.Collections.Frozen; diff --git a/src/Lumina/Excel/ExcelModule.cs b/src/Lumina/Excel/ExcelModule.cs index aa7293c7..cfd2aa54 100644 --- a/src/Lumina/Excel/ExcelModule.cs +++ b/src/Lumina/Excel/ExcelModule.cs @@ -10,6 +10,7 @@ using System.Reflection; using System.Runtime.CompilerServices; using Lumina.Data.Structs.Excel; +using Lumina.Excel.Exceptions; namespace Lumina.Excel; diff --git a/src/Lumina/Excel/ExcelSheet.cs b/src/Lumina/Excel/ExcelSheet.cs index 71e27985..f8a391e1 100644 --- a/src/Lumina/Excel/ExcelSheet.cs +++ b/src/Lumina/Excel/ExcelSheet.cs @@ -4,6 +4,7 @@ using System.Runtime.CompilerServices; using Lumina.Data; using Lumina.Data.Structs.Excel; +using Lumina.Excel.Exceptions; namespace Lumina.Excel; diff --git a/src/Lumina/Excel/SubrowExcelSheet.cs b/src/Lumina/Excel/SubrowExcelSheet.cs index 55bb36ce..ce7dec89 100644 --- a/src/Lumina/Excel/SubrowExcelSheet.cs +++ b/src/Lumina/Excel/SubrowExcelSheet.cs @@ -1,4 +1,5 @@ using Lumina.Data; +using Lumina.Excel.Exceptions; using System; using System.Collections; using System.Collections.Generic; diff --git a/src/Lumina/GameData.cs b/src/Lumina/GameData.cs index 2c4fa7ea..505a3885 100644 --- a/src/Lumina/GameData.cs +++ b/src/Lumina/GameData.cs @@ -8,6 +8,7 @@ using Lumina.Data.Structs; using Lumina.Data.Structs.Excel; using Lumina.Excel; +using Lumina.Excel.Exceptions; using Lumina.Misc; // ReSharper disable MemberCanBePrivate.Global @@ -296,19 +297,15 @@ public static UInt64 GetFileHash( string path ) /// language-neutral sheet using will be loaded instead. If the language-neutral sheet does not exist, then the function /// will return . /// - /// Sheet is not of the variant . - /// does not have a valid . + /// Sheet name was not specified neither via 's nor . + /// does not have a valid . public ExcelSheet< T >? GetExcelSheet< T >( Language? language = null, string? name = null ) where T : struct, IExcelRow< T > { try { return Excel.GetSheet< T >( language, name ); } - catch( ArgumentException ) - { - return null; - } - catch( NotSupportedException ) + catch( Exception e ) when ( e is SheetNotFoundException or MismatchedColumnHashException or NotSupportedException or UnsupportedLanguageException ) { return null; } @@ -323,11 +320,7 @@ public static UInt64 GetFileHash( string path ) { return Excel.GetSubrowSheet< T >( language, name ); } - catch( ArgumentException ) - { - return null; - } - catch ( NotSupportedException ) + catch( Exception e ) when ( e is SheetNotFoundException or MismatchedColumnHashException or NotSupportedException or UnsupportedLanguageException ) { return null; } From b70f89e8b4e2955a7ab3ec4f412cf1b844b6fcf9 Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Mon, 19 Aug 2024 12:54:30 -0700 Subject: [PATCH 38/53] Extra exception docs formatting --- src/Lumina/Excel/ExcelModule.cs | 4 ++-- src/Lumina/GameData.cs | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Lumina/Excel/ExcelModule.cs b/src/Lumina/Excel/ExcelModule.cs index cfd2aa54..391ec574 100644 --- a/src/Lumina/Excel/ExcelModule.cs +++ b/src/Lumina/Excel/ExcelModule.cs @@ -66,7 +66,7 @@ public ExcelModule( GameData gameData ) /// An excel sheet corresponding to , , and /// that may be created anew or reused from a previous invocation of this method. /// - /// Sheet is not of the variant . + /// Sheet was not a . /// public ExcelSheet< T > GetSheet< T >( Language? language = null, string? name = null ) where T : struct, IExcelRow< T > => (ExcelSheet< T >) GetBaseSheet( typeof( T ), language, name ); @@ -77,7 +77,7 @@ public ExcelSheet< T > GetSheet< T >( Language? language = null, string? name = /// An excel sheet corresponding to , , and /// that may be created anew or reused from a previous invocation of this method. /// - /// Sheet is not of the variant . + /// Sheet was not a . /// public SubrowExcelSheet< T > GetSubrowSheet< T >( Language? language = null, string? name = null ) where T : struct, IExcelSubrow< T > => (SubrowExcelSheet< T >) GetBaseSheet( typeof( T ), language, name ); diff --git a/src/Lumina/GameData.cs b/src/Lumina/GameData.cs index 505a3885..1fa740d5 100644 --- a/src/Lumina/GameData.cs +++ b/src/Lumina/GameData.cs @@ -312,7 +312,6 @@ public static UInt64 GetFileHash( string path ) } /// Loads a . Returns if the sheet does not exist, has an invalid column hash or unsupported variant, or was requested with an unsupported language. - /// Sheet is not of the variant . /// public SubrowExcelSheet< T >? GetSubrowExcelSheet< T >( Language? language = null, string? name = null ) where T : struct, IExcelSubrow< T > { From 0a137402bf6ef369d2c7f446b3a17aee0a6b290e Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Fri, 23 Aug 2024 01:12:04 -0700 Subject: [PATCH 39/53] Additional sheet changes (ty kizer) --- src/Lumina.Benchmark/Bench.cs | 172 ++++++++++++ src/Lumina.Benchmark/Lumina.Benchmark.csproj | 23 ++ src/Lumina.Benchmark/Program.cs | 9 + src/Lumina.sln | 10 +- src/Lumina/Excel/BaseSubrowExcelSheet.cs | 65 ----- src/Lumina/Excel/ExcelModule.cs | 250 ++++++++++++------ src/Lumina/Excel/ExcelSheet.cs | 80 +++--- .../Exceptions/SheetNameEmptyException.cs | 13 +- src/Lumina/Excel/IExcelSheet.cs | 39 +++ src/Lumina/Excel/ISubrowExcelSheet.cs | 36 +++ .../{BaseExcelSheet.cs => RawExcelSheet.cs} | 54 ++-- src/Lumina/Excel/RawSubrowExcelSheet.cs | 49 ++++ src/Lumina/Excel/RowRef.cs | 119 +++++++-- src/Lumina/Excel/RowRef{T}.cs | 43 +++ src/Lumina/Excel/SheetAttribute.cs | 2 +- src/Lumina/Excel/SubrowCollection.cs | 12 +- src/Lumina/Excel/SubrowExcelSheet.cs | 95 ++++--- .../Excel/{SubrowRef.cs => SubrowRef{T}.cs} | 0 src/Lumina/Excel/UntypedRowRef.cs | 130 --------- 19 files changed, 771 insertions(+), 430 deletions(-) create mode 100644 src/Lumina.Benchmark/Bench.cs create mode 100644 src/Lumina.Benchmark/Lumina.Benchmark.csproj create mode 100644 src/Lumina.Benchmark/Program.cs delete mode 100644 src/Lumina/Excel/BaseSubrowExcelSheet.cs create mode 100644 src/Lumina/Excel/IExcelSheet.cs create mode 100644 src/Lumina/Excel/ISubrowExcelSheet.cs rename src/Lumina/Excel/{BaseExcelSheet.cs => RawExcelSheet.cs} (83%) create mode 100644 src/Lumina/Excel/RawSubrowExcelSheet.cs create mode 100644 src/Lumina/Excel/RowRef{T}.cs rename src/Lumina/Excel/{SubrowRef.cs => SubrowRef{T}.cs} (100%) delete mode 100644 src/Lumina/Excel/UntypedRowRef.cs diff --git a/src/Lumina.Benchmark/Bench.cs b/src/Lumina.Benchmark/Bench.cs new file mode 100644 index 00000000..d9560dd1 --- /dev/null +++ b/src/Lumina.Benchmark/Bench.cs @@ -0,0 +1,172 @@ +using BenchmarkDotNet.Attributes; +using Lumina.Data.Files.Excel; +using Lumina.Data.Structs.Excel; +using Lumina.Data; +using System.Runtime.CompilerServices; +using Lumina.Excel; + +namespace Lumina.Benchmark; + +[DisassemblyDiagnoser( maxDepth: 500, exportGithubMarkdown: false, exportHtml: true )] +public class Bench +{ + private GameData gameData; + private ExcelSheet addonSheet; + private uint[] addonRows; + private ExcelSheet itemSheet; + private uint[] itemRows; + private SubrowExcelSheet subrowSheet; + private uint[] subrowRows; + + [GlobalSetup] + public void Setup() + { + gameData = new GameData( @"J:\Programs\Steam\steamapps\common\FINAL FANTASY XIV Online\game\sqpack", new() + { + PanicOnSheetChecksumMismatch = false, + } ); + + addonSheet = gameData.GetExcelSheet()!; + addonRows = addonSheet.Select( x => x.RowId ).ToArray(); + + itemSheet = gameData.GetExcelSheet()!; + itemRows = itemSheet.Select( x => x.RowId ).ToArray(); + + subrowSheet = gameData.GetSubrowExcelSheet()!; + subrowRows = subrowSheet.Select( x => x.RowId ).ToArray(); + } + + [Benchmark] + [MethodImpl( MethodImplOptions.NoInlining )] + public void TestAllSheets() + { + var gaps = new List(); + foreach( var sheetName in gameData.Excel.SheetNames ) + { + if( gameData.GetFile( $"exd/{sheetName}.exh" ) is not { } headerFile ) + continue; + var lang = headerFile.Languages.Contains( Language.English ) ? Language.English : Language.None; + switch( headerFile.Header.Variant ) + { + case ExcelVariant.Default: + { + var sheet = gameData.Excel.GetSheet( lang, sheetName ); + gaps.Add( + sheet.Count == 0 ? 0 : sheet.GetRowAt( sheet.Count - 1 ).RowId - sheet.GetRowAt( 0 ).RowId + 1 - (uint)sheet.Count ); + break; + } + case ExcelVariant.Subrows: + { + var sheet = gameData.Excel.GetSubrowSheet( lang, sheetName ); + gaps.Add( sheet.Count == 0 ? 0 : sheet.GetRowAt( sheet.Count - 1 ).RowId - sheet.GetRowAt( 0 ).RowId + 1 - (uint)sheet.Count ); + break; + } + } + } + + gaps.Sort(); + var countAcc = 0; + var wasteAcc = 0; + var test = string.Join( + "\n", + gaps + .GroupBy( x => x / 1024, x => x ) + .Select( x => + $"{( x.Key + 1 ) * 1024,8}, {( countAcc += x.Count() ) * 100f / gaps.Count,6:00.00}%, {( wasteAcc += x.Sum( y => (int)y ) * 4 ) / 1024,5}KB" ) ); + } + + [Benchmark] + [MethodImpl( MethodImplOptions.NoInlining )] + public ulong AddonRowAccessor() + { + ulong ret = 0; + foreach( var x in addonRows ) + ret += addonSheet[x].Data; + return ret; + } + + [Benchmark] + [MethodImpl( MethodImplOptions.NoInlining )] + public ulong AddonRowEnumerator() + { + ulong ret = 0; + foreach( var x in addonSheet ) + ret += x.Data; + return ret; + } + + [Benchmark] + [MethodImpl( MethodImplOptions.NoInlining )] + public ulong ItemRowAccessor() + { + ulong ret = 0; + foreach( var x in itemRows ) + ret += itemSheet[x].Data; + return ret; + } + + [Benchmark] + [MethodImpl( MethodImplOptions.NoInlining )] + public ulong ItemRowEnumerator() + { + ulong ret = 0; + foreach( var x in itemSheet ) + ret += x.Data; + return ret; + } + + [Benchmark] + [MethodImpl( MethodImplOptions.NoInlining )] + public ulong SubrowAccessor() + { + ulong ret = 0; + foreach( var x in subrowRows ) + { + var sc = subrowSheet.GetSubrowCount( x ); + for( ushort j = 0; j < sc; j++ ) + ret += subrowSheet[x, j].Data; + } + return ret; + } + + [Benchmark] + [MethodImpl( MethodImplOptions.NoInlining )] + public ulong SubrowEnumerator() + { + ulong ret = 0; + foreach( var x in subrowSheet.Flatten() ) + ret += x.RowId; + return ret; + } + + [Sheet( "Addon" )] + public readonly struct Addon( ExcelPage page, uint offset, uint row ) : IExcelRow + { + public uint RowId => row; + + public uint Data => row & offset; + + static Addon IExcelRow.Create( ExcelPage page, uint offset, uint row ) => new( page, offset, row ); + } + + [Sheet( "Item" )] + public readonly struct Item( ExcelPage page, uint offset, uint row ) : IExcelRow + { + public uint RowId => row; + + public uint Data => row & offset; + + static Item IExcelRow.Create( ExcelPage page, uint offset, uint row ) => new( page, offset, row ); + } + + [Sheet( "QuestLinkMarker" )] + public readonly struct QuestLinkMarker( ExcelPage page, uint offset, uint row, ushort subrow ) : IExcelSubrow + { + public uint RowId => row; + public ushort SubrowId => subrow; + + public uint Data => row & offset; + + static QuestLinkMarker IExcelSubrow.Create( ExcelPage page, uint offset, uint row, ushort subrow ) => new( page, offset, row, subrow ); + } +} diff --git a/src/Lumina.Benchmark/Lumina.Benchmark.csproj b/src/Lumina.Benchmark/Lumina.Benchmark.csproj new file mode 100644 index 00000000..008b036f --- /dev/null +++ b/src/Lumina.Benchmark/Lumina.Benchmark.csproj @@ -0,0 +1,23 @@ + + + + net8.0 + Exe + enable + enable + + + + + + + + + + + + + + + + diff --git a/src/Lumina.Benchmark/Program.cs b/src/Lumina.Benchmark/Program.cs new file mode 100644 index 00000000..b11205cb --- /dev/null +++ b/src/Lumina.Benchmark/Program.cs @@ -0,0 +1,9 @@ +namespace Lumina.Benchmark; + +internal static class Program +{ + static void Main( string[] args ) + { + BenchmarkDotNet.Running.BenchmarkSwitcher.FromAssembly( typeof( Program ).Assembly ).Run( args ); + } +} diff --git a/src/Lumina.sln b/src/Lumina.sln index 4934cb21..47fafa3b 100644 --- a/src/Lumina.sln +++ b/src/Lumina.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.30011.22 +# Visual Studio Version 17 +VisualStudioVersion = 17.10.35013.160 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Lumina", "Lumina\Lumina.csproj", "{4B3957D0-3904-4492-8CFA-86791E1DDD6D}" EndProject @@ -13,6 +13,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Lumina.Cmd", "Lumina.Cmd\Lu EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tools", "Tools", "{C3DB5F15-1172-4938-B946-281EC006FEB1}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Lumina.Benchmark", "Lumina.Benchmark\Lumina.Benchmark.csproj", "{221FB03B-B726-44D1-8A68-2D92D080137D}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -35,6 +37,10 @@ Global {3637C5FA-3A4F-4275-9C61-4C8CAC955B49}.Debug|Any CPU.Build.0 = Debug|Any CPU {3637C5FA-3A4F-4275-9C61-4C8CAC955B49}.Release|Any CPU.ActiveCfg = Release|Any CPU {3637C5FA-3A4F-4275-9C61-4C8CAC955B49}.Release|Any CPU.Build.0 = Release|Any CPU + {221FB03B-B726-44D1-8A68-2D92D080137D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {221FB03B-B726-44D1-8A68-2D92D080137D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {221FB03B-B726-44D1-8A68-2D92D080137D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {221FB03B-B726-44D1-8A68-2D92D080137D}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/Lumina/Excel/BaseSubrowExcelSheet.cs b/src/Lumina/Excel/BaseSubrowExcelSheet.cs deleted file mode 100644 index 508e6d7b..00000000 --- a/src/Lumina/Excel/BaseSubrowExcelSheet.cs +++ /dev/null @@ -1,65 +0,0 @@ -using System; -using System.Runtime.CompilerServices; -using Lumina.Data; -using Lumina.Data.Structs.Excel; - -namespace Lumina.Excel; - -/// An excel sheet of variant. -public abstract class BaseSubrowExcelSheet : BaseExcelSheet -{ - private protected BaseSubrowExcelSheet( ExcelModule module, Language requestedLanguage, string sheetName, uint? columnHash ) - : base( module, requestedLanguage, sheetName, columnHash, ExcelVariant.Subrows ) - { - foreach( var f in OffsetLookupTable ) - TotalSubrowCount += f.SubrowCount; - } - - /// - /// The total number of subrows in this sheet across all rows. - /// - public int TotalSubrowCount { get; } - - /// - /// Whether this sheet has a subrow with the given and . - /// - /// The row id to check. - /// The subrow id to check. - /// Whether the subrow exists. - public bool HasSubrow( uint rowId, ushort subrowId ) - { - ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); - return !Unsafe.IsNullRef( in lookup ) && subrowId < lookup.SubrowCount; - } - - /// - /// Tries to get the number of subrows in the th row in this sheet. - /// - /// The row id to get. - /// The number of subrows in this row. - /// if the row exists and is written to and otherwise. - public bool TryGetSubrowCount( uint rowId, out ushort subrowCount ) - { - ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); - if( Unsafe.IsNullRef( in lookup ) ) - { - subrowCount = 0; - return false; - } - - subrowCount = lookup.SubrowCount; - return true; - } - - /// - /// Gets the number of subrows in the th row in this sheet. Throws if the row does not exist. - /// - /// The row id to get. - /// The number of subrows in this row. Returns null if the row does not exist. - /// Thrown if the sheet does not have a row at that . - public ushort GetSubrowCount( uint rowId ) - { - ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); - return Unsafe.IsNullRef( in lookup ) ? throw new ArgumentOutOfRangeException( nameof( rowId ), rowId, null ) : lookup.SubrowCount; - } -} \ No newline at end of file diff --git a/src/Lumina/Excel/ExcelModule.cs b/src/Lumina/Excel/ExcelModule.cs index 391ec574..34785ef8 100644 --- a/src/Lumina/Excel/ExcelModule.cs +++ b/src/Lumina/Excel/ExcelModule.cs @@ -8,9 +8,12 @@ using System.Diagnostics.CodeAnalysis; using System.IO; using System.Reflection; -using System.Runtime.CompilerServices; using Lumina.Data.Structs.Excel; using Lumina.Excel.Exceptions; +using System.Collections.Frozen; +using System.Linq; +using System.Runtime.CompilerServices; +using Lumina.Data.Parsing.Layer; namespace Lumina.Excel; @@ -26,8 +29,9 @@ public class ExcelModule internal bool VerifySheetChecksums => GameData.Options.PanicOnSheetChecksumMismatch; internal ResolveRsvDelegate? RsvResolver => GameData.Options.RsvResolver; - - private ConcurrentDictionary< (Type Type, Language Language, string Name), BaseExcelSheet > SheetCache { get; } = []; + + private FrozenDictionary< string, SheetData > DefinedSheetCache { get; } + private ConcurrentDictionary< string, SheetData > AdhocSheetCache { get; } private ConcurrentDictionary< Type, SheetAttribute? > SheetAttributeCache { get; } = []; /// @@ -58,6 +62,16 @@ public ExcelModule( GameData gameData ) GameData.Logger?.Information( "got {ExltEntryCount} exlt entries", files.ExdMap.Count ); SheetNames = [.. files.ExdMap.Keys]; + + DefinedSheetCache = SheetNames + .Select( name => ( name, GameData.GetFile< ExcelHeaderFile >( $"exd/{name}.exh") ) ) + .Where( sheet => sheet.Item2 is not null ) + .ToFrozenDictionary( + sheet => sheet.name, + sheet => new SheetData( this, sheet.Item2!, sheet.name ), + StringComparer.OrdinalIgnoreCase + ); + AdhocSheetCache = new( StringComparer.OrdinalIgnoreCase ); } /// Loads an . @@ -68,8 +82,21 @@ public ExcelModule( GameData gameData ) /// /// Sheet was not a . /// - public ExcelSheet< T > GetSheet< T >( Language? language = null, string? name = null ) where T : struct, IExcelRow< T > => - (ExcelSheet< T >) GetBaseSheet( typeof( T ), language, name ); + public ExcelSheet< T > GetSheet< T >( Language? language = null, string? name = null ) where T : struct, IExcelRow< T > + { + var attr = GetSheetAttributes( typeof( T ) ) ?? throw new SheetAttributeMissingException( null, nameof( T ) ); + name ??= attr.Name ?? throw new SheetNameEmptyException( nameof( name ) ); + + var rawSheet = GetRawSheetCore( name, language, out var variant ); + + if( VerifySheetChecksums && attr?.ColumnHash is { } hash && hash != rawSheet.ColumnHash ) + throw new MismatchedColumnHashException( hash, rawSheet.ColumnHash, nameof( T ) ); + + if( variant != ExcelVariant.Default ) + throw new NotSupportedException( $"Specified sheet variant {variant} is not supported; was expecting {ExcelVariant.Default}." ); + + return new ExcelSheet< T >( rawSheet ); + } /// Loads an . /// The requested sheet language. Leave or empty to use the default language. @@ -79,10 +106,23 @@ public ExcelSheet< T > GetSheet< T >( Language? language = null, string? name = /// /// Sheet was not a . /// - public SubrowExcelSheet< T > GetSubrowSheet< T >( Language? language = null, string? name = null ) where T : struct, IExcelSubrow< T > => - (SubrowExcelSheet< T >) GetBaseSheet( typeof( T ), language, name ); + public SubrowExcelSheet< T > GetSubrowSheet< T >( Language? language = null, string? name = null ) where T : struct, IExcelSubrow< T > + { + var attr = GetSheetAttributes( typeof( T ) ) ?? throw new SheetAttributeMissingException( null, nameof( T ) ); + name ??= attr.Name ?? throw new SheetNameEmptyException( nameof( name ) ); + + var rawSheet = GetRawSheetCore( name, language, out var variant ); + + if( VerifySheetChecksums && attr?.ColumnHash is { } hash && hash != rawSheet.ColumnHash ) + throw new MismatchedColumnHashException( hash, rawSheet.ColumnHash, nameof( T ) ); + + if ( variant != ExcelVariant.Subrows ) + throw new NotSupportedException( $"Specified sheet variant {variant} is not supported; was expecting {ExcelVariant.Subrows}." ); - /// Loads an . + return new SubrowExcelSheet< T >( (RawSubrowExcelSheet)rawSheet ); + } + + /// Loads a typed . /// Type of the rows in the sheet. /// The requested sheet language. Leave or empty to use the default language. /// The requested explicit sheet name. Leave to use 's sheet name. Explicit names are necessary for quest/dungeon/cutscene sheets. @@ -90,88 +130,102 @@ public SubrowExcelSheet< T > GetSubrowSheet< T >( Language? language = null, str /// that may be created anew or reused from a previous invocation of this method. /// /// Only use this method if you need to create a sheet while using reflection. - /// The returned instance of should be cast to or + /// The returned instance of should be cast to or /// before accessing its rows. /// - /// Sheet name was not specified neither via nor . /// does not have a valid . + /// Sheet name was not specified neither via nor . /// Sheet does not exist. + /// Sheet supports neither nor . + /// Sheet has an unsupported . /// Sheet had a mismatched column hash. - /// Sheet does not support nor . - /// Sheet had an unsupported . [RequiresDynamicCode( "Creating a generic sheet from a type requires reflection and dynamic code." )] [EditorBrowsable( EditorBrowsableState.Advanced )] - public BaseExcelSheet GetBaseSheet( Type rowType, Language? language = null, string? name = null ) + public IExcelSheet GetBaseSheet( Type rowType, Language? language = null, string? name = null ) { var attr = GetSheetAttributes( rowType ) ?? throw new SheetAttributeMissingException( null, nameof( rowType ) ); - name ??= attr.Name ?? throw new SheetNameEmptyException( null, nameof( name ) ); - var sheet = SheetCache.GetOrAdd( - ( rowType, language ?? Language, name ), - static ( key, context ) => { - Type t; - try - { - t = typeof( ExcelSheet<> ).MakeGenericType( key.Type ); - } - catch( ArgumentException e1 ) - { - try - { - t = typeof( SubrowExcelSheet<> ).MakeGenericType( key.Type ); - } - catch( ArgumentException e2 ) - { - // Exception thrown here will propagate outside ConcurrentDictionary<>.GetOrAdd without touching the data stored inside dictionary. - throw new ArgumentException( - $"{key.Type.Name} must implement either {typeof( IExcelRow<> ).Name.Split( '`', 2 )[ 0 ]}<{key.Type.Name}> or {typeof( IExcelSubrow<> ).Name.Split( '`', 2 )[ 0 ]}<{key.Type.Name}>.", - nameof( rowType ), - new AggregateException( e1, e2 ) ); - } - } + name ??= attr.Name ?? throw new SheetNameEmptyException( nameof( name ) ); - try - { - return Activator.CreateInstance( - t, - BindingFlags.Instance | BindingFlags.Public, - null, - [context.Module, key.Language, key.Name, context.Attribute.ColumnHash], - null ) as BaseExcelSheet ?? - throw new InvalidOperationException( "Something went wrong" ); - } - catch( TargetInvocationException e ) - { - return InvalidSheet.Create( e.InnerException ?? e ); - } - catch( Exception e ) - { - return InvalidSheet.Create( e ); - } - }, - ( Module: this, Attribute: attr ) ); + var rawSheet = GetRawSheetCore( name, language, out var variant ); - if( sheet is not InvalidSheet { Exception: var e } ) - return sheet; - if( e is UnsupportedLanguageException ) + if (VerifySheetChecksums && attr?.ColumnHash is { } hash && hash != rawSheet.ColumnHash ) + throw new MismatchedColumnHashException( hash, rawSheet.ColumnHash, nameof( rowType ) ); + + ExcelVariant expectedVariant; + Type returnType; + if( typeof( IExcelRow<> ).MakeGenericType( rowType ).IsAssignableFrom( rowType ) ) + { + expectedVariant = ExcelVariant.Default; + returnType = typeof( ExcelSheet<> ); + } + else if( typeof( IExcelSubrow<> ).MakeGenericType( rowType ).IsAssignableFrom( rowType ) ) { - if( language == Language.None ) - throw new UnsupportedLanguageException( nameof( language ), language, null ); - return GetBaseSheet( rowType, Language.None ); + expectedVariant = ExcelVariant.Subrows; + returnType = typeof( SubrowExcelSheet<> ); } + else + throw new NotSupportedException( $"Type \"{rowType}\" is not a valid row type." ); + + if ( variant != expectedVariant ) + throw new NotSupportedException( $"Sheet \"{name}\" is not of {variant} variant." ); - throw e; + return Activator.CreateInstance( + returnType.MakeGenericType( rowType ), + BindingFlags.Instance | BindingFlags.Public, + null, + [rawSheet], + null ) as ISubrowExcelSheet ?? + throw new InvalidOperationException( "Something went wrong" ); } - /// Unloads cached sheets that reference an assembly. - /// Assembly to look for in the cached sheets. - public void UnloadCachedSheetsOfAssembly( Assembly assembly ) + /// Loads a . + /// The requested sheet name. + /// The requested sheet language. Leave or empty to use the default language. + /// A raw excel sheet corresponding to , and + /// that may be created anew or reused from a previous invocation of this method. + /// + /// The returned instance of can be cast to if the underlying sheet is an variant. + /// + /// is null. + /// Sheet does not exist. + /// Sheet supports neither nor . + /// Sheet has an unsupported . + [EditorBrowsable( EditorBrowsableState.Advanced )] + public RawExcelSheet GetRawSheet( string name, Language? language = null ) => + GetRawSheetCore( name, language, out _ ); + + private RawExcelSheet GetRawSheetCore( string name, Language? language, out ExcelVariant variant ) { - foreach( var c in SheetCache.Keys ) + ArgumentNullException.ThrowIfNull( name ); + language ??= Language; + + ref readonly var definedData = ref DefinedSheetCache.GetValueRefOrNullRef( name ); + SheetData data; + if( !Unsafe.IsNullRef( in definedData ) ) + data = definedData; + else { - if( c.Type.Assembly == assembly ) - _ = SheetCache.TryRemove( c, out _ ); + data = AdhocSheetCache.GetOrAdd( + name, + static ( key, self ) => { + var headerFile = self.GameData.GetFile< ExcelHeaderFile >( $"exd/{key}.exh" ) ?? + throw new SheetNotFoundException( null, nameof( key ) ); + + return new SheetData( self, headerFile, key ); + }, + this + ); } + variant = data.Variant; + + return data.GetRawSheet( language.Value ); + } + + /// Unloads cached values that reference an assembly. + /// Assembly to look for in the cache. + public void UnloadTypedCache( Assembly assembly ) + { foreach( var c in SheetAttributeCache.Keys ) { if( c.Assembly == assembly ) @@ -182,29 +236,63 @@ public void UnloadCachedSheetsOfAssembly( Assembly assembly ) /// Gets the sheet attributes for . /// Type of the row. /// Sheet attributes, if any. - internal SheetAttribute? GetSheetAttributes< T >() => GetSheetAttributes( typeof( T ) ); + private SheetAttribute? GetSheetAttributes< T >() => GetSheetAttributes( typeof( T ) ); /// Gets the sheet attributes for . /// Type of the row. /// Sheet attributes, if any. - internal SheetAttribute? GetSheetAttributes( Type rowType ) => + private SheetAttribute? GetSheetAttributes( Type rowType ) => SheetAttributeCache.GetOrAdd( rowType, static type => type.GetCustomAttribute< SheetAttribute >( false ) ); - private sealed class InvalidSheet : BaseExcelSheet + private sealed class SheetData { - public Exception Exception { get; private set; } + public ExcelHeaderFile HeaderFile { get; } + public Lazy< RawExcelSheet >?[] LanguageCache { get; } + + public ExcelVariant Variant => HeaderFile.Header.Variant; + + public SheetData( ExcelModule module, ExcelHeaderFile headerFile, string name ) + { + HeaderFile = headerFile; + var langs = headerFile.Languages.Prepend( Language.None ); + var maxLang = langs.Cast< ushort >().Max(); + LanguageCache = new Lazy< RawExcelSheet >?[maxLang + 1]; + foreach ( var lang in langs ) + LanguageCache[(ushort)lang] = new( () => CreateFor( module, lang, name ) ); + } - // never actually called - private InvalidSheet() : base( default!, default, default!, default, default ) => - Exception = null!; + private RawExcelSheet CreateFor( ExcelModule module, Language lang, string name ) => + Variant switch + { + ExcelVariant.Default => new RawExcelSheet( module, HeaderFile, lang, name ), + ExcelVariant.Subrows => new RawSubrowExcelSheet( module, HeaderFile, lang, name ), + var v => throw new NotSupportedException( $"Sheet variant {v} is not supported." ), + }; - public static InvalidSheet Create( Exception exception ) + private Lazy< RawExcelSheet >? GetRawSheetCore( Language language ) { - var ret = (InvalidSheet) RuntimeHelpers.GetUninitializedObject( typeof( InvalidSheet ) ); - ret.Exception = exception; - return ret; + if( LanguageCache.Length <= (ushort)language ) + return null; + return LanguageCache[(ushort)language]; + } + + public RawExcelSheet GetRawSheet( Language language ) + { + var entry = GetRawSheetCore( language ); + if( entry == null ) + { + if( language == Language.None ) + throw new UnsupportedLanguageException( nameof( language ), language, null ); + else + { + entry = GetRawSheetCore( Language.None ) ?? + throw new UnsupportedLanguageException( nameof( language ), language, null ); + } + } + + return entry.Value; } } } \ No newline at end of file diff --git a/src/Lumina/Excel/ExcelSheet.cs b/src/Lumina/Excel/ExcelSheet.cs index f8a391e1..f937a82e 100644 --- a/src/Lumina/Excel/ExcelSheet.cs +++ b/src/Lumina/Excel/ExcelSheet.cs @@ -4,45 +4,31 @@ using System.Runtime.CompilerServices; using Lumina.Data; using Lumina.Data.Structs.Excel; -using Lumina.Excel.Exceptions; namespace Lumina.Excel; /// An excel sheet of variant. /// Type of the rows contained within. -public sealed class ExcelSheet< T > : BaseExcelSheet, ICollection< T >, IReadOnlyCollection< T > where T : struct, IExcelRow< T > +/// Creates a new instance of . +/// The to access sheet data from. +/// A new instance of . +/// This constructor does not perform any type checks. +public sealed class ExcelSheet< T >( RawExcelSheet sheet ) : IExcelSheet, ICollection< T >, IReadOnlyCollection< T > where T : struct, IExcelRow< T > { - /// Creates a new instance of , deducing sheet names (unless overridden with ) and column hashes - /// from . - /// The to access sheet data from. - /// The language to use for this sheet. Use to use . - /// The explicit sheet name, if needed. Leave to use the type's sheet name. Explicit names are necessary for - /// quest/dungeon/cutscene sheets. - /// had no and was empty. - /// - /// was invalid (hash mismatch). - /// - public ExcelSheet( ExcelModule module, Language? language = null, string? name = null ) - : this( module, language ?? module.Language, name, module.GetSheetAttributes< T >() ) - { } - - /// Creates a new instance of . - /// The to access sheet data from. - /// The language to use for this sheet. - /// The name of the sheet to read from. - /// The hash of the columns in the sheet. If , it will not check the hash. - /// Sheet does not exist. - /// was invalid (hash mismatch). - /// Sheet had an unsupported language. - /// Header file had a value that is not supported. - /// A new instance of . - public ExcelSheet( ExcelModule module, Language language, string name, uint? columnHash ) - : base( module, language, name, columnHash, ExcelVariant.Default ) - { } - - private ExcelSheet( ExcelModule module, Language language, string? name, SheetAttribute? attribute ) - : this( module, language, name ?? attribute?.Name ?? throw new SheetNameEmptyException( null, nameof( name ) ), attribute?.ColumnHash ) - { } + /// Gets the raw sheet this typed sheet is based on. + public RawExcelSheet RawSheet { get; } = sheet; + + /// + public ExcelModule Module => RawSheet.Module; + + /// + public Language Language => RawSheet.Language; + + /// + public IReadOnlyList< ExcelColumnDefinition > Columns => RawSheet.Columns; + + /// + public int Count => RawSheet.Count; bool ICollection< T >.IsReadOnly => true; @@ -56,8 +42,8 @@ private ExcelSheet( ExcelModule module, Language language, string? name, SheetAt /// A nullable row object. Returns if the row does not exist. public T? GetRowOrDefault( uint rowId ) { - ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); - return Unsafe.IsNullRef( in lookup ) ? null : UnsafeCreateRow< T >( in lookup ); + ref readonly var lookup = ref RawSheet.GetRowLookupOrNullRef( rowId ); + return Unsafe.IsNullRef( in lookup ) ? null : RawSheet.UnsafeCreateRow< T >( in lookup ); } /// @@ -68,14 +54,14 @@ private ExcelSheet( ExcelModule module, Language language, string? name, SheetAt /// if the row exists and is written to and otherwise. public bool TryGetRow( uint rowId, out T row ) { - ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); + ref readonly var lookup = ref RawSheet.GetRowLookupOrNullRef( rowId ); if( Unsafe.IsNullRef( in lookup ) ) { row = default; return false; } - row = UnsafeCreateRow< T >( in lookup ); + row = RawSheet.UnsafeCreateRow< T >( in lookup ); return true; } @@ -87,8 +73,8 @@ public bool TryGetRow( uint rowId, out T row ) /// Throws when the row id does not have a row attached to it. public T GetRow( uint rowId ) { - ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); - return Unsafe.IsNullRef( in lookup ) ? throw new ArgumentOutOfRangeException( nameof( rowId ), rowId, null ) : UnsafeCreateRow< T >( in lookup ); + ref readonly var lookup = ref RawSheet.GetRowLookupOrNullRef( rowId ); + return Unsafe.IsNullRef( in lookup ) ? throw new ArgumentOutOfRangeException( nameof( rowId ), rowId, null ) : RawSheet.UnsafeCreateRow< T >( in lookup ); } /// @@ -100,11 +86,17 @@ public T GetRow( uint rowId ) public T GetRowAt( int rowIndex ) { ArgumentOutOfRangeException.ThrowIfNegative( rowIndex ); - ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual( rowIndex, OffsetLookupTable.Length ); + ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual( rowIndex, RawSheet.OffsetLookupTable.Length ); - return UnsafeCreateRowAt< T >( rowIndex ); + return RawSheet.UnsafeCreateRowAt< T >( rowIndex ); } + /// + public ushort GetColumnOffset( int columnIdx ) => RawSheet.GetColumnOffset( columnIdx ); + + /// + public bool HasRow( uint rowId ) => RawSheet.HasRow( rowId ); + /// public bool Contains( T item ) => TryGetRow( item.RowId, out var row ) && EqualityComparer< T >.Default.Equals( item, row ); @@ -115,8 +107,8 @@ public void CopyTo( T[] array, int arrayIndex ) ArgumentOutOfRangeException.ThrowIfNegative( arrayIndex ); if( Count > array.Length - arrayIndex ) throw new ArgumentException( "The number of elements in the source list is greater than the available space." ); - foreach( var lookup in OffsetLookupTable ) - array[ arrayIndex++ ] = UnsafeCreateRow< T >( in lookup ); + foreach( var lookup in RawSheet.OffsetLookupTable ) + array[ arrayIndex++ ] = RawSheet.UnsafeCreateRow< T >( in lookup ); } void ICollection< T >.Add( T item ) => throw new NotSupportedException(); @@ -151,7 +143,7 @@ public bool MoveNext() // UnsafeCreateRowAt must be called only when the preconditions are validated. // If it is to be called on-demand from get_Current, then it may end up being called with invalid parameters, // so we create the instance in advance here. - Current = sheet.UnsafeCreateRowAt< T >( _index ); + Current = sheet.RawSheet.UnsafeCreateRowAt< T >( _index ); return true; } diff --git a/src/Lumina/Excel/Exceptions/SheetNameEmptyException.cs b/src/Lumina/Excel/Exceptions/SheetNameEmptyException.cs index 5d8fb152..97e4f61b 100644 --- a/src/Lumina/Excel/Exceptions/SheetNameEmptyException.cs +++ b/src/Lumina/Excel/Exceptions/SheetNameEmptyException.cs @@ -3,17 +3,17 @@ namespace Lumina.Excel.Exceptions; /// Exception indicating that sheet name could not be resolved. -public sealed class SheetNameEmptyException : ArgumentException +public sealed class SheetNameEmptyException : ArgumentNullException { private const string DefaultMessage = - $"Row type has no {nameof( SheetAttribute )} or its {nameof( SheetAttribute.Name )} is null, and no valid sheet name is specified."; + $"Sheet name must be specified via parameter or sheet attributes."; /// public SheetNameEmptyException() : base( DefaultMessage ) { } /// - public SheetNameEmptyException( string? message ) : base( message ?? DefaultMessage ) + public SheetNameEmptyException( string? paramName ) : base( paramName, DefaultMessage ) { } /// @@ -21,11 +21,6 @@ public SheetNameEmptyException( string? message, Exception? innerException ) : b { } /// - public SheetNameEmptyException( string? message, string? paramName ) : base( message ?? DefaultMessage, paramName ) - { } - - /// - public SheetNameEmptyException( string? message, string? paramName, Exception? innerException ) - : base( message ?? DefaultMessage, paramName, innerException ) + public SheetNameEmptyException( string? paramName, string? message ) : base( paramName, message ?? DefaultMessage ) { } } \ No newline at end of file diff --git a/src/Lumina/Excel/IExcelSheet.cs b/src/Lumina/Excel/IExcelSheet.cs new file mode 100644 index 00000000..05136378 --- /dev/null +++ b/src/Lumina/Excel/IExcelSheet.cs @@ -0,0 +1,39 @@ +using Lumina.Data; +using Lumina.Data.Structs.Excel; +using System; +using System.Collections.Generic; + +namespace Lumina.Excel; + +/// A wrapper around an excel sheet. +public interface IExcelSheet +{ + /// The module that this sheet belongs to. + ExcelModule Module { get; } + + /// The language of the rows in this sheet. + /// This can be different from the requested language if it wasn't supported. + Language Language { get; } + + /// Contains information on the columns in this sheet. + IReadOnlyList< ExcelColumnDefinition > Columns { get; } + + /// The number of rows in this sheet. + /// + /// If this sheet has gaps in row ids, it returns the number of rows that exist, not the highest row id. + /// If this sheet has subrows, this will still return the number of rows and not the total number of subrows. + /// + int Count { get; } + + /// Gets the offset of the column at in the row data. + /// The index of the column. + /// The offset of the column. + /// Thrown when the column index is invalid. It must be less than .Count. + ushort GetColumnOffset( int columnIdx ); + + /// Whether this sheet has a row with the given . + /// If this sheet has subrows, this will check if the row id has any subrows. + /// The row id to check. + /// Whether the row exists. + bool HasRow( uint rowId ); +} \ No newline at end of file diff --git a/src/Lumina/Excel/ISubrowExcelSheet.cs b/src/Lumina/Excel/ISubrowExcelSheet.cs new file mode 100644 index 00000000..ea738f13 --- /dev/null +++ b/src/Lumina/Excel/ISubrowExcelSheet.cs @@ -0,0 +1,36 @@ +using System; + +namespace Lumina.Excel; + +/// A wrapper around an excel subrow sheet. +public interface ISubrowExcelSheet : IExcelSheet +{ + /// + /// The total number of subrows in this sheet across all rows. + /// + int TotalSubrowCount { get; } + + /// + /// Whether this sheet has a subrow with the given and . + /// + /// The row id to check. + /// The subrow id to check. + /// Whether the subrow exists. + bool HasSubrow( uint rowId, ushort subrowId ); + + /// + /// Tries to get the number of subrows in the th row in this sheet. + /// + /// The row id to get. + /// The number of subrows in this row. + /// if the row exists and is written to and otherwise. + bool TryGetSubrowCount( uint rowId, out ushort subrowCount ); + + /// + /// Gets the number of subrows in the th row in this sheet. Throws if the row does not exist. + /// + /// The row id to get. + /// The number of subrows in this row. Returns null if the row does not exist. + /// Thrown if the sheet does not have a row at that . + ushort GetSubrowCount( uint rowId ); +} \ No newline at end of file diff --git a/src/Lumina/Excel/BaseExcelSheet.cs b/src/Lumina/Excel/RawExcelSheet.cs similarity index 83% rename from src/Lumina/Excel/BaseExcelSheet.cs rename to src/Lumina/Excel/RawExcelSheet.cs index 46ea62f1..e2dcadc7 100644 --- a/src/Lumina/Excel/BaseExcelSheet.cs +++ b/src/Lumina/Excel/RawExcelSheet.cs @@ -11,8 +11,8 @@ namespace Lumina.Excel; -/// A wrapper around an excel sheet. -public abstract class BaseExcelSheet +/// An excel sheet of variant. +public class RawExcelSheet : IExcelSheet { /// Number of items in that may resolve to no entry. // 7.05h: across 7292 sheets that exist and are referenced from exlt file, following ratio can be represented solely using lookup array of certain sizes. @@ -53,7 +53,7 @@ public abstract class BaseExcelSheet // 9832448, 99.99%, 79643KB // 10146816, 100.00%, 119276KB // We're allowing up to 65536 lookup items in _rowOffsetLookupTable, at cost of up to 3293KB of lookup items that resolve to nonexistence per language. - private const int MaxUnusedLookupItemCount = 65536; + private const int MaxUnusedLookupItemCount = 0x10000; private readonly ExcelPage[] _pages; private readonly RowOffsetLookup[] _rowOffsetLookupTable; @@ -66,38 +66,35 @@ public abstract class BaseExcelSheet private readonly int[] _rowIndexLookupArray; private readonly uint _rowIndexLookupArrayOffset; - /// The module that this sheet belongs to. + internal uint ColumnHash { get; } + + /// public ExcelModule Module { get; } - /// The language of the rows in this sheet. - /// This can be different from the requested language if it wasn't supported. + /// public Language Language { get; } - /// Contains information on the columns in this sheet. + /// public IReadOnlyList< ExcelColumnDefinition > Columns { get; } - private protected BaseExcelSheet( ExcelModule module, Language language, string name, uint? columnHash, ExcelVariant expectedVariant ) + /// + public int Count { get; } + + internal RawExcelSheet( ExcelModule module, ExcelHeaderFile headerFile, Language language, string name ) { ArgumentNullException.ThrowIfNull( module ); + ArgumentNullException.ThrowIfNull( headerFile ); ArgumentNullException.ThrowIfNull( name ); - var headerFile = module.GameData.GetFile< ExcelHeaderFile >( $"exd/{name}.exh" ) ?? - throw new SheetNotFoundException( "Invalid sheet name", nameof( name ) ); - - if( module.VerifySheetChecksums && columnHash is { } hash && headerFile.GetColumnsHash() != hash ) - throw new MismatchedColumnHashException( hash, headerFile.GetColumnsHash(), nameof( columnHash ) ); - if( !headerFile.Languages.Contains( language ) ) throw new UnsupportedLanguageException( nameof( language ), language, null ); - if( headerFile.Header.Variant != expectedVariant ) - throw new NotSupportedException( $"Specified sheet variant {headerFile.Header.Variant} is not supported; was expecting {expectedVariant}." ); - var hasSubrows = headerFile.Header.Variant == ExcelVariant.Subrows; Module = module; - Language = headerFile.Languages.Contains( language ) ? language : Language.None; + Language = language; Columns = headerFile.ColumnDefinitions; + ColumnHash = headerFile.GetColumnsHash(); _subrowDataOffset = hasSubrows ? headerFile.Header.DataOffset : (ushort) 0; _pages = new ExcelPage[headerFile.DataPages.Length]; _rowOffsetLookupTable = new RowOffsetLookup[headerFile.Header.RowCount]; @@ -183,26 +180,13 @@ private protected BaseExcelSheet( ExcelModule module, Language language, string } } - /// The number of rows in this sheet. - /// - /// If this sheet has gaps in row ids, it returns the number of rows that exist, not the highest row id. - /// If this sheet has subrows, this will still return the number of rows and not the total number of subrows. - /// - public int Count { get; } - /// Gets the offset lookup table. - private protected ReadOnlySpan< RowOffsetLookup > OffsetLookupTable => _rowOffsetLookupTable; + internal ReadOnlySpan< RowOffsetLookup > OffsetLookupTable => _rowOffsetLookupTable; - /// Gets the offset of the column at in the row data. - /// The index of the column. - /// The offset of the column. - /// Thrown when the column index is invalid. It must be less than .Count. + /// public ushort GetColumnOffset( int columnIdx ) => Columns[ columnIdx ].Offset; - /// Whether this sheet has a row with the given . - /// If this sheet has subrows, this will check if the row id has any subrows. - /// The row id to check. - /// Whether the row exists. + /// public bool HasRow( uint rowId ) { ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); @@ -275,4 +259,4 @@ internal T UnsafeCreateSubrow< T >( scoped ref readonly RowOffsetLookup lookup, /// Index of the page that contains the data for this row. /// Number of subrows in the row, or 1 if the sheet does not support subrows. internal readonly record struct RowOffsetLookup( uint RowId, uint Offset, ushort PageIndex, ushort SubrowCount ); -} \ No newline at end of file +} diff --git a/src/Lumina/Excel/RawSubrowExcelSheet.cs b/src/Lumina/Excel/RawSubrowExcelSheet.cs new file mode 100644 index 00000000..e6a0ee72 --- /dev/null +++ b/src/Lumina/Excel/RawSubrowExcelSheet.cs @@ -0,0 +1,49 @@ +using System; +using System.Runtime.CompilerServices; +using Lumina.Data; +using Lumina.Data.Files.Excel; +using Lumina.Data.Structs.Excel; + +namespace Lumina.Excel; + +/// An excel sheet of variant. +public sealed class RawSubrowExcelSheet : RawExcelSheet, ISubrowExcelSheet +{ + internal RawSubrowExcelSheet( ExcelModule module, ExcelHeaderFile headerFile, Language language, string name ) + : base( module, headerFile, language, name ) + { + foreach( var f in OffsetLookupTable ) + TotalSubrowCount += f.SubrowCount; + } + + /// + public int TotalSubrowCount { get; } + + /// + public bool HasSubrow( uint rowId, ushort subrowId ) + { + ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); + return !Unsafe.IsNullRef( in lookup ) && subrowId < lookup.SubrowCount; + } + + /// + public bool TryGetSubrowCount( uint rowId, out ushort subrowCount ) + { + ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); + if( Unsafe.IsNullRef( in lookup ) ) + { + subrowCount = 0; + return false; + } + + subrowCount = lookup.SubrowCount; + return true; + } + + /// + public ushort GetSubrowCount( uint rowId ) + { + ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); + return Unsafe.IsNullRef( in lookup ) ? throw new ArgumentOutOfRangeException( nameof( rowId ), rowId, null ) : lookup.SubrowCount; + } +} \ No newline at end of file diff --git a/src/Lumina/Excel/RowRef.cs b/src/Lumina/Excel/RowRef.cs index 6271f1a1..3bc8e4cb 100644 --- a/src/Lumina/Excel/RowRef.cs +++ b/src/Lumina/Excel/RowRef.cs @@ -3,41 +3,128 @@ namespace Lumina.Excel; /// -/// A helper type to concretely reference a row in a specific excel sheet. +/// A helper type to dynamically reference a row in a specific excel sheet. /// -/// The row type referenced by the . /// The to read sheet data from. /// The referenced row id. -public readonly struct RowRef< T >( ExcelModule? module, uint rowId ) where T : struct, IExcelRow< T > +/// The referenced row's actual . +public readonly struct RowRef( ExcelModule? module, uint rowId, Type? rowType ) { - private readonly ExcelSheet< T >? _sheet = module?.GetSheet< T >(); - /// /// The row id of the referenced row. /// public uint RowId => rowId; /// - /// Whether the exists in the sheet. + /// Whether the is untyped. + /// + /// + /// An untyped is one that doesn't know which sheet it links to. + /// + public bool IsUntyped => rowType == null; + + /// + /// Whether the reference is of a specific row type. + /// + /// The row type/schema to check against. + /// Whether this points to a . + public bool Is< T >() where T : struct, IExcelRow< T > => + typeof( T ) == rowType; + + /// + public bool IsSubrow< T >() where T : struct, IExcelSubrow< T > => + typeof( T ) == rowType; + + /// + /// Tries to get the referenced row as a specific row type. + /// + /// The row type/schema to check against. + /// The referenced row type. Returns null if this does not point to a or if the does not exist in its sheet. + public T? GetValueOrDefault< T >() where T : struct, IExcelRow< T > + { + if( !Is< T >() || module is null ) + return null; + + return new RowRef< T >( module, rowId ).ValueNullable; + } + + /// + public SubrowCollection< T >? GetValueOrDefaultSubrow< T >() where T : struct, IExcelSubrow< T > + { + if( !IsSubrow< T >() || module is null ) + return null; + + return new SubrowRef< T >( module, rowId ).ValueNullable; + } + + /// + /// Tries to get the referenced row as a specific row type. /// - public bool IsValid => _sheet?.HasRow( RowId ) ?? false; + /// The row type/schema to check against. + /// The output row object. + /// if the type is valid, the row exists, and is written to, and otherwise. + public bool TryGetValue< T >( out T row ) where T : struct, IExcelRow< T > + { + if( new RowRef< T >( module, rowId ).ValueNullable is { } v ) + { + row = v; + return true; + } + + row = default; + return false; + } + + /// + public bool TryGetValueSubrow< T >( out SubrowCollection< T > row ) where T : struct, IExcelSubrow< T > + { + if( new SubrowRef< T >( module, rowId ).ValueNullable is { } v ) + { + row = v; + return true; + } + + row = default; + return false; + } /// - /// The referenced row value itself. + /// Attempts to create a to a row id of a list of row types, checking with each type in order. /// - /// Thrown if is false. - public T Value => ValueNullable ?? throw new InvalidOperationException(); + /// The to read sheet data from. + /// The referenced row id. + /// A list of row types to check against the , in order. + /// A to one of the . If the row id does not exist in any of the sheets, an untyped is returned instead. + public static RowRef GetFirstValidRowOrUntyped( ExcelModule module, uint rowId, params Type[] sheetTypes ) + { + foreach( var sheetType in sheetTypes ) + { + if( module.GetBaseSheet( sheetType ) is { } sheet ) + { + if( sheet.HasRow( rowId ) ) + return new( module, rowId, sheetType ); + } + } + + return CreateUntyped( rowId ); + } /// - /// Attempts to get the referenced row value. Is if does not exist in the sheet. + /// Creates a to a specific row type. /// - public T? ValueNullable => _sheet?.GetRowOrDefault( rowId ); + /// The row type referenced by the . + /// The to read sheet data from. + /// The referenced row id. + /// A to a row in a . + public static RowRef Create< T >( ExcelModule? module, uint rowId ) where T : struct, IExcelRow< T > => new( module, rowId, typeof( T ) ); - private RowRef ToGeneric() => RowRef.Create< T >( module, rowId ); + /// + public static RowRef CreateSubrow< T >( ExcelModule? module, uint rowId ) where T : struct, IExcelSubrow< T > => new( module, rowId, typeof( T ) ); /// - /// Converts a concrete to a generic and dynamically typed . + /// Creates an untyped . /// - /// The to convert. - public static explicit operator RowRef( RowRef< T > row ) => row.ToGeneric(); + /// The referenced row id. + /// An untyped . + public static RowRef CreateUntyped( uint rowId ) => new( null, rowId, null ); } \ No newline at end of file diff --git a/src/Lumina/Excel/RowRef{T}.cs b/src/Lumina/Excel/RowRef{T}.cs new file mode 100644 index 00000000..6271f1a1 --- /dev/null +++ b/src/Lumina/Excel/RowRef{T}.cs @@ -0,0 +1,43 @@ +using System; + +namespace Lumina.Excel; + +/// +/// A helper type to concretely reference a row in a specific excel sheet. +/// +/// The row type referenced by the . +/// The to read sheet data from. +/// The referenced row id. +public readonly struct RowRef< T >( ExcelModule? module, uint rowId ) where T : struct, IExcelRow< T > +{ + private readonly ExcelSheet< T >? _sheet = module?.GetSheet< T >(); + + /// + /// The row id of the referenced row. + /// + public uint RowId => rowId; + + /// + /// Whether the exists in the sheet. + /// + public bool IsValid => _sheet?.HasRow( RowId ) ?? false; + + /// + /// The referenced row value itself. + /// + /// Thrown if is false. + public T Value => ValueNullable ?? throw new InvalidOperationException(); + + /// + /// Attempts to get the referenced row value. Is if does not exist in the sheet. + /// + public T? ValueNullable => _sheet?.GetRowOrDefault( rowId ); + + private RowRef ToGeneric() => RowRef.Create< T >( module, rowId ); + + /// + /// Converts a concrete to a generic and dynamically typed . + /// + /// The to convert. + public static explicit operator RowRef( RowRef< T > row ) => row.ToGeneric(); +} \ No newline at end of file diff --git a/src/Lumina/Excel/SheetAttribute.cs b/src/Lumina/Excel/SheetAttribute.cs index cfe80ba3..65759e13 100644 --- a/src/Lumina/Excel/SheetAttribute.cs +++ b/src/Lumina/Excel/SheetAttribute.cs @@ -6,7 +6,7 @@ namespace Lumina.Excel; /// An attribute attached to a schema/struct that represents a sheet in an excel file. /// [AttributeUsage( AttributeTargets.Struct )] -public class SheetAttribute : Attribute +public sealed class SheetAttribute : Attribute { /// /// The name of the sheet. diff --git a/src/Lumina/Excel/SubrowCollection.cs b/src/Lumina/Excel/SubrowCollection.cs index fd09b35b..ce032a0d 100644 --- a/src/Lumina/Excel/SubrowCollection.cs +++ b/src/Lumina/Excel/SubrowCollection.cs @@ -10,9 +10,9 @@ namespace Lumina.Excel; public readonly struct SubrowCollection< T > : IList< T >, IReadOnlyList< T > where T : struct, IExcelSubrow< T > { - private readonly BaseExcelSheet.RowOffsetLookup _lookup; + private readonly RawExcelSheet.RowOffsetLookup _lookup; - internal SubrowCollection( SubrowExcelSheet< T > sheet, scoped ref readonly BaseExcelSheet.RowOffsetLookup lookup ) + internal SubrowCollection( SubrowExcelSheet< T > sheet, scoped ref readonly RawExcelSheet.RowOffsetLookup lookup ) { Sheet = sheet; _lookup = lookup; @@ -35,7 +35,7 @@ public T this[ int index ] { get { ArgumentOutOfRangeException.ThrowIfNegative( index ); ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual( index, Count ); - return Sheet.UnsafeCreateSubrow< T >( in _lookup, unchecked( (ushort) index ) ); + return Sheet.RawSheet.UnsafeCreateSubrow< T >( in _lookup, unchecked( (ushort) index ) ); } } @@ -61,7 +61,7 @@ public int IndexOf( T item ) if( item.RowId != RowId || item.SubrowId >= Count ) return -1; - var row = Sheet.UnsafeCreateSubrow< T >( in _lookup, item.SubrowId ); + var row = Sheet.RawSheet.UnsafeCreateSubrow< T >( in _lookup, item.SubrowId ); return EqualityComparer< T >.Default.Equals( item, row ) ? item.SubrowId : -1; } @@ -76,7 +76,7 @@ public void CopyTo( T[] array, int arrayIndex ) if( Count > array.Length - arrayIndex ) throw new ArgumentException( "The number of elements in the source list is greater than the available space." ); for( var i = 0; i < Count; i++ ) - array[ arrayIndex++ ] = Sheet.UnsafeCreateSubrow< T >( in _lookup, unchecked( (ushort) i ) ); + array[ arrayIndex++ ] = Sheet.RawSheet.UnsafeCreateSubrow< T >( in _lookup, unchecked( (ushort) i ) ); } /// @@ -105,7 +105,7 @@ public bool MoveNext() // UnsafeCreateSubrow must be called only when the preconditions are validated. // If it is to be called on-demand from get_Current, then it may end up being called with invalid parameters, // so we create the instance in advance here. - Current = subrowCollection.Sheet.UnsafeCreateSubrow< T >( in subrowCollection._lookup, unchecked( (ushort) _index ) ); + Current = subrowCollection.Sheet.RawSheet.UnsafeCreateSubrow< T >( in subrowCollection._lookup, unchecked( (ushort) _index ) ); return true; } diff --git a/src/Lumina/Excel/SubrowExcelSheet.cs b/src/Lumina/Excel/SubrowExcelSheet.cs index ce7dec89..42cc49d5 100644 --- a/src/Lumina/Excel/SubrowExcelSheet.cs +++ b/src/Lumina/Excel/SubrowExcelSheet.cs @@ -1,4 +1,5 @@ using Lumina.Data; +using Lumina.Data.Structs.Excel; using Lumina.Excel.Exceptions; using System; using System.Collections; @@ -8,29 +9,26 @@ namespace Lumina.Excel; /// Type of the rows contained within. -/// -public sealed class SubrowExcelSheet< T > - : BaseSubrowExcelSheet, ICollection< SubrowCollection< T > >, IReadOnlyCollection< SubrowCollection< T > > - where T : struct, IExcelSubrow< T > +/// +public sealed class SubrowExcelSheet< T >( RawSubrowExcelSheet sheet ) : ISubrowExcelSheet, ICollection< SubrowCollection< T > >, IReadOnlyCollection< SubrowCollection< T > > where T : struct, IExcelSubrow< T > { - /// Creates a new instance of , deducing sheet names (unless overridden with ) and column - /// hashes from . - /// A new instance of . - /// - public SubrowExcelSheet( ExcelModule module, Language? language = null, string? name = null ) - : this( module, language ?? module.Language, name, module.GetSheetAttributes< T >() ) - { } - - /// Creates a new instance of . - /// A new instance of . - /// - public SubrowExcelSheet( ExcelModule module, Language language, string name, uint? columnHash ) - : base( module, language, name, columnHash ) - { } - - private SubrowExcelSheet( ExcelModule module, Language language, string? name, SheetAttribute? attribute ) - : this( module, language, name ?? attribute?.Name ?? throw new SheetNameEmptyException( null, nameof( name ) ), attribute?.ColumnHash ) - { } + /// Gets the raw sheet this typed sheet is based on. + public RawSubrowExcelSheet RawSheet { get; } = sheet; + + /// + public ExcelModule Module => RawSheet.Module; + + /// + public Language Language => RawSheet.Language; + + /// + public IReadOnlyList< ExcelColumnDefinition > Columns => RawSheet.Columns; + + /// + public int Count => RawSheet.Count; + + /// + public int TotalSubrowCount => RawSheet.TotalSubrowCount; /// bool ICollection< SubrowCollection< T > >.IsReadOnly => true; @@ -48,7 +46,7 @@ private SubrowExcelSheet( ExcelModule module, Language language, string? name, S /// A nullable subrow collection object. Returns if the row does not exist. public SubrowCollection< T >? GetRowOrDefault( uint rowId ) { - ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); + ref readonly var lookup = ref RawSheet.GetRowLookupOrNullRef( rowId ); return Unsafe.IsNullRef( in lookup ) ? null : new( this, in lookup ); } @@ -60,7 +58,7 @@ private SubrowExcelSheet( ExcelModule module, Language language, string? name, S /// if the row exists and is written to and otherwise. public bool TryGetRow( uint rowId, out SubrowCollection< T > row ) { - ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); + ref readonly var lookup = ref RawSheet.GetRowLookupOrNullRef( rowId ); if( Unsafe.IsNullRef( in lookup ) ) { row = default; @@ -79,7 +77,7 @@ public bool TryGetRow( uint rowId, out SubrowCollection< T > row ) /// Thrown if the sheet does not have a row at that . public SubrowCollection< T > GetRow( uint rowId ) { - ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); + ref readonly var lookup = ref RawSheet.GetRowLookupOrNullRef( rowId ); return Unsafe.IsNullRef( in lookup ) ? throw new ArgumentOutOfRangeException( nameof( rowId ), rowId, null ) : new( this, in lookup ); } @@ -92,9 +90,9 @@ public SubrowCollection< T > GetRow( uint rowId ) public SubrowCollection< T > GetRowAt( int rowIndex ) { ArgumentOutOfRangeException.ThrowIfNegative( rowIndex ); - ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual( rowIndex, OffsetLookupTable.Length ); + ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual( rowIndex, RawSheet.OffsetLookupTable.Length ); - return new( this, in UnsafeGetRowLookupAt( rowIndex ) ); + return new( this, in RawSheet.UnsafeGetRowLookupAt( rowIndex ) ); } /// @@ -105,8 +103,8 @@ public SubrowCollection< T > GetRowAt( int rowIndex ) /// A nullable row object. Returns null if the subrow does not exist. public T? GetSubrowOrDefault( uint rowId, ushort subrowId ) { - ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); - return Unsafe.IsNullRef( in lookup ) || subrowId >= lookup.SubrowCount ? null : UnsafeCreateSubrow< T >( in lookup, subrowId ); + ref readonly var lookup = ref RawSheet.GetRowLookupOrNullRef( rowId ); + return Unsafe.IsNullRef( in lookup ) || subrowId >= lookup.SubrowCount ? null : RawSheet.UnsafeCreateSubrow< T >( in lookup, subrowId ); } /// @@ -118,14 +116,14 @@ public SubrowCollection< T > GetRowAt( int rowIndex ) /// if the subrow exists and is written to and otherwise. public bool TryGetSubrow( uint rowId, ushort subrowId, out T subrow ) { - ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); + ref readonly var lookup = ref RawSheet.GetRowLookupOrNullRef( rowId ); if( Unsafe.IsNullRef( in lookup ) || subrowId >= lookup.SubrowCount ) { subrow = default; return false; } - subrow = UnsafeCreateSubrow< T >( in lookup, subrowId ); + subrow = RawSheet.UnsafeCreateSubrow< T >( in lookup, subrowId ); return true; } @@ -138,13 +136,13 @@ public bool TryGetSubrow( uint rowId, ushort subrowId, out T subrow ) /// Thrown if the sheet does not have a row at that . public T GetSubrow( uint rowId, ushort subrowId ) { - ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); + ref readonly var lookup = ref RawSheet.GetRowLookupOrNullRef( rowId ); if( Unsafe.IsNullRef( in lookup ) ) throw new ArgumentOutOfRangeException( nameof( rowId ), rowId, null ); ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual( subrowId, lookup.SubrowCount ); - return UnsafeCreateSubrow< T >( in lookup, subrowId ); + return RawSheet.UnsafeCreateSubrow< T >( in lookup, subrowId ); } /// @@ -156,18 +154,33 @@ public T GetSubrow( uint rowId, ushort subrowId ) /// A row object. public T GetSubrowAt( int rowIndex, ushort subrowId ) { - var offsetLookupTable = OffsetLookupTable; + var offsetLookupTable = RawSheet.OffsetLookupTable; ArgumentOutOfRangeException.ThrowIfNegative( rowIndex ); ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual( rowIndex, offsetLookupTable.Length ); - ref readonly var lookup = ref UnsafeGetRowLookupAt( rowIndex ); + ref readonly var lookup = ref RawSheet.UnsafeGetRowLookupAt( rowIndex ); ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual( subrowId, lookup.SubrowCount ); - return UnsafeCreateSubrow< T >( in lookup, subrowId ); + return RawSheet.UnsafeCreateSubrow< T >( in lookup, subrowId ); } /// - public bool Contains( SubrowCollection< T > item ) => ReferenceEquals( item.Sheet, this ) && HasRow( item.RowId ); + public bool HasSubrow( uint rowId, ushort subrowId ) => RawSheet.HasSubrow( rowId, subrowId ); + + /// + public bool TryGetSubrowCount( uint rowId, out ushort subrowCount ) => RawSheet.TryGetSubrowCount( rowId, out subrowCount ); + + /// + public ushort GetSubrowCount( uint rowId ) => RawSheet.GetSubrowCount( rowId ); + + /// + public ushort GetColumnOffset( int columnIdx ) => RawSheet.GetColumnOffset( columnIdx ); + + /// + public bool HasRow( uint rowId ) => RawSheet.HasRow( rowId ); + + /// + public bool Contains( SubrowCollection< T > item ) => ReferenceEquals( item.Sheet, this ) && RawSheet.HasRow( item.RowId ); /// public void CopyTo( SubrowCollection< T >[] array, int arrayIndex ) @@ -176,7 +189,7 @@ public void CopyTo( SubrowCollection< T >[] array, int arrayIndex ) ArgumentOutOfRangeException.ThrowIfNegative( arrayIndex ); if( Count > array.Length - arrayIndex ) throw new ArgumentException( "The number of elements in the source list is greater than the available space." ); - foreach( var lookup in OffsetLookupTable ) + foreach( var lookup in RawSheet.OffsetLookupTable ) array[ arrayIndex++ ] = new( this, in lookup ); } @@ -216,7 +229,7 @@ public bool MoveNext() // UnsafeGetRowLookupAt must be called only when the preconditions are validated. // If it is to be called on-demand from get_Current, then it may end up being called with invalid parameters, // so we create the instance in advance here. - Current = new( sheet, in sheet.UnsafeGetRowLookupAt( _index ) ); + Current = new( sheet, in sheet.RawSheet.UnsafeGetRowLookupAt( _index ) ); return true; } @@ -260,7 +273,7 @@ public bool MoveNext() return false; } - _subrowCount = sheet.UnsafeGetRowLookupAt( _index ).SubrowCount; + _subrowCount = sheet.RawSheet.UnsafeGetRowLookupAt( _index ).SubrowCount; if( _subrowCount == 0 ) continue; @@ -272,7 +285,7 @@ public bool MoveNext() // UnsafeCreateSubrowAt must be called only when the preconditions are validated. // If it is to be called on-demand from get_Current, then it may end up being called with invalid parameters, // so we create the instance in advance here. - Current = sheet.UnsafeCreateSubrowAt< T >( _index, _subrowIndex ); + Current = sheet.RawSheet.UnsafeCreateSubrowAt< T >( _index, _subrowIndex ); return true; } diff --git a/src/Lumina/Excel/SubrowRef.cs b/src/Lumina/Excel/SubrowRef{T}.cs similarity index 100% rename from src/Lumina/Excel/SubrowRef.cs rename to src/Lumina/Excel/SubrowRef{T}.cs diff --git a/src/Lumina/Excel/UntypedRowRef.cs b/src/Lumina/Excel/UntypedRowRef.cs deleted file mode 100644 index 4a103bb8..00000000 --- a/src/Lumina/Excel/UntypedRowRef.cs +++ /dev/null @@ -1,130 +0,0 @@ -using System; - -namespace Lumina.Excel; - -/// -/// A helper type to dynamically reference a row in a specific excel sheet. -/// -/// The to read sheet data from. -/// The referenced row id. -/// The referenced row's actual . -public readonly struct RowRef( ExcelModule? module, uint rowId, Type? rowType ) -{ - /// - /// The row id of the referenced row. - /// - public uint RowId => rowId; - - /// - /// Whether the is untyped. - /// - /// - /// An untyped is one that doesn't know which sheet it links to. - /// - public bool IsUntyped => rowType == null; - - /// - /// Whether the reference is of a specific row type. - /// - /// The row type/schema to check against. - /// Whether this points to a . - public bool Is< T >() where T : struct, IExcelRow< T > => - typeof( T ) == rowType; - - /// - public bool IsSubrow< T >() where T : struct, IExcelSubrow< T > => - typeof( T ) == rowType; - - /// - /// Tries to get the referenced row as a specific row type. - /// - /// The row type/schema to check against. - /// The referenced row type. Returns null if this does not point to a or if the does not exist in its sheet. - public T? GetValueOrDefault< T >() where T : struct, IExcelRow< T > - { - if( !Is< T >() || module is null ) - return null; - - return new RowRef< T >( module, rowId ).ValueNullable; - } - - /// - public SubrowCollection< T >? GetValueOrDefaultSubrow< T >() where T : struct, IExcelSubrow< T > - { - if( !IsSubrow< T >() || module is null ) - return null; - - return new SubrowRef< T >( module, rowId ).ValueNullable; - } - - /// - /// Tries to get the referenced row as a specific row type. - /// - /// The row type/schema to check against. - /// The output row object. - /// if the type is valid, the row exists, and is written to, and otherwise. - public bool TryGetValue< T >( out T row ) where T : struct, IExcelRow< T > - { - if( new RowRef< T >( module, rowId ).ValueNullable is { } v ) - { - row = v; - return true; - } - - row = default; - return false; - } - - /// - public bool TryGetValueSubrow< T >( out SubrowCollection< T > row ) where T : struct, IExcelSubrow< T > - { - if( new SubrowRef< T >( module, rowId ).ValueNullable is { } v ) - { - row = v; - return true; - } - - row = default; - return false; - } - - /// - /// Attempts to create a to a row id of a list of row types, checking with each type in order. - /// - /// The to read sheet data from. - /// The referenced row id. - /// A list of row types to check against the , in order. - /// A to one of the . If the row id does not exist in any of the sheets, an untyped is returned instead. - public static RowRef GetFirstValidRowOrUntyped( ExcelModule module, uint rowId, params Type[] sheetTypes ) - { - foreach( var sheetType in sheetTypes ) - { - if( module.GetBaseSheet( sheetType ) is { } sheet ) - { - if( sheet.HasRow( rowId ) ) - return new( module, rowId, sheetType ); - } - } - - return CreateUntyped( rowId ); - } - - /// - /// Creates a to a specific row type. - /// - /// The row type referenced by the . - /// The to read sheet data from. - /// The referenced row id. - /// A to a row in a . - public static RowRef Create< T >( ExcelModule? module, uint rowId ) where T : struct, IExcelRow< T > => new( module, rowId, typeof( T ) ); - - /// - public static RowRef CreateSubrow< T >( ExcelModule? module, uint rowId ) where T : struct, IExcelSubrow< T > => new( module, rowId, typeof( T ) ); - - /// - /// Creates an untyped . - /// - /// The referenced row id. - /// An untyped . - public static RowRef CreateUntyped( uint rowId ) => new( null, rowId, null ); -} \ No newline at end of file From 130f03643a07f61fe4f9e563779fe557d89f6cb5 Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Tue, 27 Aug 2024 14:50:17 -0700 Subject: [PATCH 40/53] Refactor SheetNames property --- src/Lumina/Excel/ExcelModule.cs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/Lumina/Excel/ExcelModule.cs b/src/Lumina/Excel/ExcelModule.cs index 34785ef8..eb5ed497 100644 --- a/src/Lumina/Excel/ExcelModule.cs +++ b/src/Lumina/Excel/ExcelModule.cs @@ -13,7 +13,6 @@ using System.Collections.Frozen; using System.Linq; using System.Runtime.CompilerServices; -using Lumina.Data.Parsing.Layer; namespace Lumina.Excel; @@ -45,7 +44,7 @@ public class ExcelModule /// /// Get the names of all available sheets, parsed from root.exl. /// - public IReadOnlyCollection< string > SheetNames { get; } + public IReadOnlyList< string > SheetNames => DefinedSheetCache.Keys; /// /// Create a new ExcelModule. This will do all the initial discovery of sheets from the EXL but not load any sheets. @@ -61,14 +60,12 @@ public ExcelModule( GameData gameData ) GameData.Logger?.Information( "got {ExltEntryCount} exlt entries", files.ExdMap.Count ); - SheetNames = [.. files.ExdMap.Keys]; - - DefinedSheetCache = SheetNames - .Select( name => ( name, GameData.GetFile< ExcelHeaderFile >( $"exd/{name}.exh") ) ) - .Where( sheet => sheet.Item2 is not null ) + DefinedSheetCache = files.ExdMap.Keys + .Select( name => ( Name: name, Header: GameData.GetFile< ExcelHeaderFile >( $"exd/{name}.exh") ) ) + .Where( sheet => sheet.Header is not null ) .ToFrozenDictionary( - sheet => sheet.name, - sheet => new SheetData( this, sheet.Item2!, sheet.name ), + sheet => sheet.Name, + sheet => new SheetData( this, sheet.Header!, sheet.Name ), StringComparer.OrdinalIgnoreCase ); AdhocSheetCache = new( StringComparer.OrdinalIgnoreCase ); From 78a00f01c8ff36cfd63bc55cc8c069a99d6f00f7 Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Tue, 27 Aug 2024 15:53:16 -0700 Subject: [PATCH 41/53] Fix invalid cast --- src/Lumina/Excel/ExcelModule.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Lumina/Excel/ExcelModule.cs b/src/Lumina/Excel/ExcelModule.cs index eb5ed497..8a1d5d58 100644 --- a/src/Lumina/Excel/ExcelModule.cs +++ b/src/Lumina/Excel/ExcelModule.cs @@ -171,7 +171,7 @@ public IExcelSheet GetBaseSheet( Type rowType, Language? language = null, string BindingFlags.Instance | BindingFlags.Public, null, [rawSheet], - null ) as ISubrowExcelSheet ?? + null ) as IExcelSheet ?? throw new InvalidOperationException( "Something went wrong" ); } From ddce871e9dcad5649f2bebe00125b5b26ad9466e Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Tue, 27 Aug 2024 15:53:28 -0700 Subject: [PATCH 42/53] Add rowref benchmarks --- src/Lumina.Benchmark/Bench.cs | 172 ------------------------------ src/Lumina.Benchmark/ExcelRows.cs | 67 ++++++++++++ src/Lumina.Benchmark/Row.cs | 92 ++++++++++++++++ src/Lumina.Benchmark/RowRef.cs | 56 ++++++++++ src/Lumina.Benchmark/Sheets.cs | 49 +++++++++ 5 files changed, 264 insertions(+), 172 deletions(-) delete mode 100644 src/Lumina.Benchmark/Bench.cs create mode 100644 src/Lumina.Benchmark/ExcelRows.cs create mode 100644 src/Lumina.Benchmark/Row.cs create mode 100644 src/Lumina.Benchmark/RowRef.cs create mode 100644 src/Lumina.Benchmark/Sheets.cs diff --git a/src/Lumina.Benchmark/Bench.cs b/src/Lumina.Benchmark/Bench.cs deleted file mode 100644 index d9560dd1..00000000 --- a/src/Lumina.Benchmark/Bench.cs +++ /dev/null @@ -1,172 +0,0 @@ -using BenchmarkDotNet.Attributes; -using Lumina.Data.Files.Excel; -using Lumina.Data.Structs.Excel; -using Lumina.Data; -using System.Runtime.CompilerServices; -using Lumina.Excel; - -namespace Lumina.Benchmark; - -[DisassemblyDiagnoser( maxDepth: 500, exportGithubMarkdown: false, exportHtml: true )] -public class Bench -{ - private GameData gameData; - private ExcelSheet addonSheet; - private uint[] addonRows; - private ExcelSheet itemSheet; - private uint[] itemRows; - private SubrowExcelSheet subrowSheet; - private uint[] subrowRows; - - [GlobalSetup] - public void Setup() - { - gameData = new GameData( @"J:\Programs\Steam\steamapps\common\FINAL FANTASY XIV Online\game\sqpack", new() - { - PanicOnSheetChecksumMismatch = false, - } ); - - addonSheet = gameData.GetExcelSheet()!; - addonRows = addonSheet.Select( x => x.RowId ).ToArray(); - - itemSheet = gameData.GetExcelSheet()!; - itemRows = itemSheet.Select( x => x.RowId ).ToArray(); - - subrowSheet = gameData.GetSubrowExcelSheet()!; - subrowRows = subrowSheet.Select( x => x.RowId ).ToArray(); - } - - [Benchmark] - [MethodImpl( MethodImplOptions.NoInlining )] - public void TestAllSheets() - { - var gaps = new List(); - foreach( var sheetName in gameData.Excel.SheetNames ) - { - if( gameData.GetFile( $"exd/{sheetName}.exh" ) is not { } headerFile ) - continue; - var lang = headerFile.Languages.Contains( Language.English ) ? Language.English : Language.None; - switch( headerFile.Header.Variant ) - { - case ExcelVariant.Default: - { - var sheet = gameData.Excel.GetSheet( lang, sheetName ); - gaps.Add( - sheet.Count == 0 ? 0 : sheet.GetRowAt( sheet.Count - 1 ).RowId - sheet.GetRowAt( 0 ).RowId + 1 - (uint)sheet.Count ); - break; - } - case ExcelVariant.Subrows: - { - var sheet = gameData.Excel.GetSubrowSheet( lang, sheetName ); - gaps.Add( sheet.Count == 0 ? 0 : sheet.GetRowAt( sheet.Count - 1 ).RowId - sheet.GetRowAt( 0 ).RowId + 1 - (uint)sheet.Count ); - break; - } - } - } - - gaps.Sort(); - var countAcc = 0; - var wasteAcc = 0; - var test = string.Join( - "\n", - gaps - .GroupBy( x => x / 1024, x => x ) - .Select( x => - $"{( x.Key + 1 ) * 1024,8}, {( countAcc += x.Count() ) * 100f / gaps.Count,6:00.00}%, {( wasteAcc += x.Sum( y => (int)y ) * 4 ) / 1024,5}KB" ) ); - } - - [Benchmark] - [MethodImpl( MethodImplOptions.NoInlining )] - public ulong AddonRowAccessor() - { - ulong ret = 0; - foreach( var x in addonRows ) - ret += addonSheet[x].Data; - return ret; - } - - [Benchmark] - [MethodImpl( MethodImplOptions.NoInlining )] - public ulong AddonRowEnumerator() - { - ulong ret = 0; - foreach( var x in addonSheet ) - ret += x.Data; - return ret; - } - - [Benchmark] - [MethodImpl( MethodImplOptions.NoInlining )] - public ulong ItemRowAccessor() - { - ulong ret = 0; - foreach( var x in itemRows ) - ret += itemSheet[x].Data; - return ret; - } - - [Benchmark] - [MethodImpl( MethodImplOptions.NoInlining )] - public ulong ItemRowEnumerator() - { - ulong ret = 0; - foreach( var x in itemSheet ) - ret += x.Data; - return ret; - } - - [Benchmark] - [MethodImpl( MethodImplOptions.NoInlining )] - public ulong SubrowAccessor() - { - ulong ret = 0; - foreach( var x in subrowRows ) - { - var sc = subrowSheet.GetSubrowCount( x ); - for( ushort j = 0; j < sc; j++ ) - ret += subrowSheet[x, j].Data; - } - return ret; - } - - [Benchmark] - [MethodImpl( MethodImplOptions.NoInlining )] - public ulong SubrowEnumerator() - { - ulong ret = 0; - foreach( var x in subrowSheet.Flatten() ) - ret += x.RowId; - return ret; - } - - [Sheet( "Addon" )] - public readonly struct Addon( ExcelPage page, uint offset, uint row ) : IExcelRow - { - public uint RowId => row; - - public uint Data => row & offset; - - static Addon IExcelRow.Create( ExcelPage page, uint offset, uint row ) => new( page, offset, row ); - } - - [Sheet( "Item" )] - public readonly struct Item( ExcelPage page, uint offset, uint row ) : IExcelRow - { - public uint RowId => row; - - public uint Data => row & offset; - - static Item IExcelRow.Create( ExcelPage page, uint offset, uint row ) => new( page, offset, row ); - } - - [Sheet( "QuestLinkMarker" )] - public readonly struct QuestLinkMarker( ExcelPage page, uint offset, uint row, ushort subrow ) : IExcelSubrow - { - public uint RowId => row; - public ushort SubrowId => subrow; - - public uint Data => row & offset; - - static QuestLinkMarker IExcelSubrow.Create( ExcelPage page, uint offset, uint row, ushort subrow ) => new( page, offset, row, subrow ); - } -} diff --git a/src/Lumina.Benchmark/ExcelRows.cs b/src/Lumina.Benchmark/ExcelRows.cs new file mode 100644 index 00000000..98eac797 --- /dev/null +++ b/src/Lumina.Benchmark/ExcelRows.cs @@ -0,0 +1,67 @@ +using Lumina.Excel; + +namespace Lumina.Benchmark; + +[Sheet( "Addon" )] +public readonly struct Addon( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + public uint Data => row & offset; + + static Addon IExcelRow.Create( ExcelPage page, uint offset, uint row ) => new( page, offset, row ); +} + +[Sheet( "Item" )] +public readonly struct Item( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + public uint Data => row & offset; + + static Item IExcelRow.Create( ExcelPage page, uint offset, uint row ) => new( page, offset, row ); +} + +[Sheet( "QuestLinkMarker" )] +public readonly struct QuestLinkMarker( ExcelPage page, uint offset, uint row, ushort subrow ) : IExcelSubrow +{ + public uint RowId => row; + public ushort SubrowId => subrow; + public uint Data => row & offset; + + static QuestLinkMarker IExcelSubrow.Create( ExcelPage page, uint offset, uint row, ushort subrow ) => new( page, offset, row, subrow ); +} + + +[Sheet( "GatheringItem" )] +public readonly struct GatheringItem( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + public uint Data => row & offset; + + public readonly RowRef Item => RowRef.GetFirstValidRowOrUntyped( page.Module, (uint)page.ReadInt32( offset + 8 ), [typeof( Item ), typeof( EventItem )] ); + public readonly RowRef GatheringItemLevel => new( page.Module, (uint)page.ReadUInt16( offset + 12 ) ); + + static GatheringItem IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "EventItem" )] +public readonly struct EventItem( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + public uint Data => row & offset; + + static EventItem IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "GatheringItemLevelConvertTable" )] +public readonly struct GatheringItemLevelConvertTable( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + public uint Data => row & offset; + + public readonly byte GatheringItemLevel => page.ReadUInt8( offset ); + + static GatheringItemLevelConvertTable IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} diff --git a/src/Lumina.Benchmark/Row.cs b/src/Lumina.Benchmark/Row.cs new file mode 100644 index 00000000..fb5bbc21 --- /dev/null +++ b/src/Lumina.Benchmark/Row.cs @@ -0,0 +1,92 @@ +using BenchmarkDotNet.Attributes; +using Lumina.Excel; + +namespace Lumina.Benchmark; + +[DisassemblyDiagnoser( maxDepth: 500, exportGithubMarkdown: false, exportHtml: true )] +public class RowBench +{ + private GameData gameData; + private ExcelSheet addonSheet; + private uint[] addonRows; + private ExcelSheet itemSheet; + private uint[] itemRows; + private SubrowExcelSheet subrowSheet; + private uint[] subrowRows; + + [GlobalSetup] + public void Setup() + { + gameData = new GameData( @"J:\Programs\Steam\steamapps\common\FINAL FANTASY XIV Online\game\sqpack", new() + { + PanicOnSheetChecksumMismatch = false, + } ); + + addonSheet = gameData.GetExcelSheet()!; + addonRows = addonSheet.Select( x => x.RowId ).ToArray(); + + itemSheet = gameData.GetExcelSheet()!; + itemRows = itemSheet.Select( x => x.RowId ).ToArray(); + + subrowSheet = gameData.GetSubrowExcelSheet()!; + subrowRows = subrowSheet.Select( x => x.RowId ).ToArray(); + } + + [Benchmark] + public ulong AddonRowAccessor() + { + ulong ret = 0; + foreach( var x in addonRows ) + ret += addonSheet[x].Data; + return ret; + } + + [Benchmark] + public ulong AddonRowEnumerator() + { + ulong ret = 0; + foreach( var x in addonSheet ) + ret += x.Data; + return ret; + } + + [Benchmark] + public ulong ItemRowAccessor() + { + ulong ret = 0; + foreach( var x in itemRows ) + ret += itemSheet[x].Data; + return ret; + } + + [Benchmark] + public ulong ItemRowEnumerator() + { + ulong ret = 0; + foreach( var x in itemSheet ) + ret += x.Data; + return ret; + } + + [Benchmark] + public ulong SubrowAccessor() + { + ulong ret = 0; + foreach( var x in subrowRows ) + { + var sc = subrowSheet.GetSubrowCount( x ); + for( ushort j = 0; j < sc; j++ ) + ret += subrowSheet[x, j].Data; + } + return ret; + } + + [Benchmark] + public ulong SubrowEnumerator() + { + ulong ret = 0; + foreach( var x in subrowSheet.Flatten() ) + ret += x.RowId; + return ret; + } +} diff --git a/src/Lumina.Benchmark/RowRef.cs b/src/Lumina.Benchmark/RowRef.cs new file mode 100644 index 00000000..ccd6804b --- /dev/null +++ b/src/Lumina.Benchmark/RowRef.cs @@ -0,0 +1,56 @@ +using BenchmarkDotNet.Attributes; +using Lumina.Excel; + +namespace Lumina.Benchmark; + +[DisassemblyDiagnoser( maxDepth: 500, exportGithubMarkdown: false, exportHtml: true )] +public class RowRefBench +{ + private GameData gameData; + private ExcelSheet rowRefSheet; + private RowRef rowRef; + + [GlobalSetup] + public void Setup() + { + gameData = new GameData( @"J:\Programs\Steam\steamapps\common\FINAL FANTASY XIV Online\game\sqpack", new() + { + PanicOnSheetChecksumMismatch = false, + } ); + + rowRefSheet = gameData.GetExcelSheet()!; + rowRef = rowRefSheet.First().GatheringItemLevel; + } + + [Benchmark] + public ulong RowRefEnumerator() + { + ulong ret = 0; + foreach( var x in rowRefSheet ) + { + if( x.RowId == 0 || x.Item.RowId == 0 ) + continue; + + ret += x.GatheringItemLevel.RowId; + } + return ret; + } + + [Benchmark] + public byte RowRefOne() => + rowRef.Value.GatheringItemLevel; + + [Benchmark] + public ulong RowRefSheet() + { + ulong ret = 0; + foreach( var x in rowRefSheet ) + { + if( x.RowId == 0 || x.Item.RowId == 0 ) + continue; + + ret += x.GatheringItemLevel.Value.GatheringItemLevel; + } + return ret; + } +} diff --git a/src/Lumina.Benchmark/Sheets.cs b/src/Lumina.Benchmark/Sheets.cs new file mode 100644 index 00000000..e1178050 --- /dev/null +++ b/src/Lumina.Benchmark/Sheets.cs @@ -0,0 +1,49 @@ +using BenchmarkDotNet.Attributes; +using Lumina.Data.Files.Excel; +using Lumina.Data.Structs.Excel; +using Lumina.Data; + +namespace Lumina.Benchmark; + +[DisassemblyDiagnoser( maxDepth: 500, exportGithubMarkdown: false, exportHtml: true )] +public class SheetsBench +{ + private GameData gameData; + + [GlobalSetup] + public void Setup() + { + gameData = new GameData( @"J:\Programs\Steam\steamapps\common\FINAL FANTASY XIV Online\game\sqpack", new() + { + PanicOnSheetChecksumMismatch = false, + } ); + } + + [Benchmark] + public ulong TestAllSheets() + { + ulong ret = 0; + foreach( var sheetName in gameData.Excel.SheetNames ) + { + if( gameData.GetFile( $"exd/{sheetName}.exh" ) is not { } headerFile ) + continue; + var lang = headerFile.Languages.Contains( Language.English ) ? Language.English : Language.None; + switch( headerFile.Header.Variant ) + { + case ExcelVariant.Default: + { + var sheet = gameData.Excel.GetSheet( lang, sheetName ); + ret += sheet.Count == 0 ? 0 : sheet.GetRowAt( sheet.Count - 1 ).RowId - sheet.GetRowAt( 0 ).RowId + 1 - (uint)sheet.Count; + break; + } + case ExcelVariant.Subrows: + { + var sheet = gameData.Excel.GetSubrowSheet( lang, sheetName ); + ret += sheet.Count == 0 ? 0 : sheet.GetRowAt( sheet.Count - 1 ).RowId - sheet.GetRowAt( 0 ).RowId + 1 - (uint)sheet.Count ; + break; + } + } + } + return ret; + } +} From 6a28cd286842132edf16db1c9d4142e6aebce1d8 Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Tue, 27 Aug 2024 17:07:41 -0700 Subject: [PATCH 43/53] More benchmarks --- src/Lumina.Benchmark/ExcelRows.cs | 4 +++ src/Lumina.Benchmark/RowRef.cs | 45 ++++++++++++++++++++++++++----- 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/src/Lumina.Benchmark/ExcelRows.cs b/src/Lumina.Benchmark/ExcelRows.cs index 98eac797..81fc52a1 100644 --- a/src/Lumina.Benchmark/ExcelRows.cs +++ b/src/Lumina.Benchmark/ExcelRows.cs @@ -17,6 +17,8 @@ public readonly struct Item( ExcelPage page, uint offset, uint row ) : IExcelRow public uint RowId => row; public uint Data => row & offset; + public readonly ushort Icon => page.ReadUInt16( offset + 136 ); + static Item IExcelRow.Create( ExcelPage page, uint offset, uint row ) => new( page, offset, row ); } @@ -50,6 +52,8 @@ public readonly struct EventItem( ExcelPage page, uint offset, uint row ) : IExc public uint RowId => row; public uint Data => row & offset; + public readonly ushort Icon => page.ReadUInt16( offset + 24 ); + static EventItem IExcelRow.Create( ExcelPage page, uint offset, uint row ) => new( page, offset, row ); } diff --git a/src/Lumina.Benchmark/RowRef.cs b/src/Lumina.Benchmark/RowRef.cs index ccd6804b..7f941ac6 100644 --- a/src/Lumina.Benchmark/RowRef.cs +++ b/src/Lumina.Benchmark/RowRef.cs @@ -28,7 +28,7 @@ public ulong RowRefEnumerator() ulong ret = 0; foreach( var x in rowRefSheet ) { - if( x.RowId == 0 || x.Item.RowId == 0 ) + if( x.RowId == 0 || x.GatheringItemLevel.RowId == 0 ) continue; ret += x.GatheringItemLevel.RowId; @@ -37,20 +37,53 @@ public ulong RowRefEnumerator() } [Benchmark] - public byte RowRefOne() => - rowRef.Value.GatheringItemLevel; + public ulong RowRefResolver() + { + ulong ret = 0; + foreach( var x in rowRefSheet ) + { + if( x.RowId == 0 || x.GatheringItemLevel.RowId == 0 ) + continue; + + ret += x.GatheringItemLevel.Value.GatheringItemLevel; + } + return ret; + } [Benchmark] - public ulong RowRefSheet() + public ulong RowMultiRefEnumerator() { ulong ret = 0; foreach( var x in rowRefSheet ) { - if( x.RowId == 0 || x.Item.RowId == 0 ) + var item = x.Item; + if( x.RowId == 0 || item.RowId == 0 ) continue; - ret += x.GatheringItemLevel.Value.GatheringItemLevel; + ret += item.RowId; + } + return ret; + } + + [Benchmark] + public ulong RowMultiRefResolver() + { + ulong ret = 0; + foreach( var x in rowRefSheet ) + { + var item = x.Item; + if( x.RowId == 0 || item.RowId == 0 ) + continue; + + if( item.GetValueOrDefault() is { } itemVal ) + ret += itemVal.Icon; + else if( item.GetValueOrDefault() is { } itemVal2 ) + ret += itemVal2.Icon; } return ret; } + + [Benchmark] + public byte RowRefOne() => + rowRef.Value.GatheringItemLevel; } From 46dbd7b99dd5e3539819cb5fa610a1013128c1bf Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Tue, 27 Aug 2024 17:40:47 -0700 Subject: [PATCH 44/53] Benchmark whitespace --- src/Lumina.Benchmark/ExcelRows.cs | 26 +++++++++++++------------- src/Lumina.Benchmark/Row.cs | 12 ++++++------ src/Lumina.Benchmark/RowRef.cs | 6 +++--- src/Lumina.Benchmark/Sheets.cs | 6 +++--- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/src/Lumina.Benchmark/ExcelRows.cs b/src/Lumina.Benchmark/ExcelRows.cs index 81fc52a1..69433191 100644 --- a/src/Lumina.Benchmark/ExcelRows.cs +++ b/src/Lumina.Benchmark/ExcelRows.cs @@ -3,69 +3,69 @@ namespace Lumina.Benchmark; [Sheet( "Addon" )] -public readonly struct Addon( ExcelPage page, uint offset, uint row ) : IExcelRow +public readonly struct Addon( ExcelPage page, uint offset, uint row ) : IExcelRow< Addon > { public uint RowId => row; public uint Data => row & offset; - static Addon IExcelRow.Create( ExcelPage page, uint offset, uint row ) => new( page, offset, row ); + static Addon IExcelRow< Addon >.Create( ExcelPage page, uint offset, uint row ) => new( page, offset, row ); } [Sheet( "Item" )] -public readonly struct Item( ExcelPage page, uint offset, uint row ) : IExcelRow +public readonly struct Item( ExcelPage page, uint offset, uint row ) : IExcelRow< Item > { public uint RowId => row; public uint Data => row & offset; public readonly ushort Icon => page.ReadUInt16( offset + 136 ); - static Item IExcelRow.Create( ExcelPage page, uint offset, uint row ) => new( page, offset, row ); + static Item IExcelRow< Item >.Create( ExcelPage page, uint offset, uint row ) => new( page, offset, row ); } [Sheet( "QuestLinkMarker" )] -public readonly struct QuestLinkMarker( ExcelPage page, uint offset, uint row, ushort subrow ) : IExcelSubrow +public readonly struct QuestLinkMarker( ExcelPage page, uint offset, uint row, ushort subrow ) : IExcelSubrow< QuestLinkMarker > { public uint RowId => row; public ushort SubrowId => subrow; public uint Data => row & offset; - static QuestLinkMarker IExcelSubrow.Create( ExcelPage page, uint offset, uint row, ushort subrow ) => new( page, offset, row, subrow ); + static QuestLinkMarker IExcelSubrow< QuestLinkMarker >.Create( ExcelPage page, uint offset, uint row, ushort subrow ) => new( page, offset, row, subrow ); } [Sheet( "GatheringItem" )] -public readonly struct GatheringItem( ExcelPage page, uint offset, uint row ) : IExcelRow +public readonly struct GatheringItem( ExcelPage page, uint offset, uint row ) : IExcelRow< GatheringItem > { public uint RowId => row; public uint Data => row & offset; public readonly RowRef Item => RowRef.GetFirstValidRowOrUntyped( page.Module, (uint)page.ReadInt32( offset + 8 ), [typeof( Item ), typeof( EventItem )] ); - public readonly RowRef GatheringItemLevel => new( page.Module, (uint)page.ReadUInt16( offset + 12 ) ); + public readonly RowRef< GatheringItemLevelConvertTable > GatheringItemLevel => new( page.Module, page.ReadUInt16( offset + 12 ) ); - static GatheringItem IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + static GatheringItem IExcelRow< GatheringItem >.Create( ExcelPage page, uint offset, uint row ) => new( page, offset, row ); } [Sheet( "EventItem" )] -public readonly struct EventItem( ExcelPage page, uint offset, uint row ) : IExcelRow +public readonly struct EventItem( ExcelPage page, uint offset, uint row ) : IExcelRow< EventItem > { public uint RowId => row; public uint Data => row & offset; public readonly ushort Icon => page.ReadUInt16( offset + 24 ); - static EventItem IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + static EventItem IExcelRow< EventItem >.Create( ExcelPage page, uint offset, uint row ) => new( page, offset, row ); } [Sheet( "GatheringItemLevelConvertTable" )] -public readonly struct GatheringItemLevelConvertTable( ExcelPage page, uint offset, uint row ) : IExcelRow +public readonly struct GatheringItemLevelConvertTable( ExcelPage page, uint offset, uint row ) : IExcelRow< GatheringItemLevelConvertTable > { public uint RowId => row; public uint Data => row & offset; public readonly byte GatheringItemLevel => page.ReadUInt8( offset ); - static GatheringItemLevelConvertTable IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + static GatheringItemLevelConvertTable IExcelRow< GatheringItemLevelConvertTable >.Create( ExcelPage page, uint offset, uint row ) => new( page, offset, row ); } diff --git a/src/Lumina.Benchmark/Row.cs b/src/Lumina.Benchmark/Row.cs index fb5bbc21..9ce222d1 100644 --- a/src/Lumina.Benchmark/Row.cs +++ b/src/Lumina.Benchmark/Row.cs @@ -7,11 +7,11 @@ namespace Lumina.Benchmark; public class RowBench { private GameData gameData; - private ExcelSheet addonSheet; + private ExcelSheet< Addon > addonSheet; private uint[] addonRows; - private ExcelSheet itemSheet; + private ExcelSheet< Item > itemSheet; private uint[] itemRows; - private SubrowExcelSheet subrowSheet; + private SubrowExcelSheet< QuestLinkMarker > subrowSheet; private uint[] subrowRows; [GlobalSetup] @@ -22,13 +22,13 @@ public void Setup() PanicOnSheetChecksumMismatch = false, } ); - addonSheet = gameData.GetExcelSheet()!; + addonSheet = gameData.GetExcelSheet< Addon >()!; addonRows = addonSheet.Select( x => x.RowId ).ToArray(); - itemSheet = gameData.GetExcelSheet()!; + itemSheet = gameData.GetExcelSheet< Item >()!; itemRows = itemSheet.Select( x => x.RowId ).ToArray(); - subrowSheet = gameData.GetSubrowExcelSheet()!; + subrowSheet = gameData.GetSubrowExcelSheet< QuestLinkMarker >()!; subrowRows = subrowSheet.Select( x => x.RowId ).ToArray(); } diff --git a/src/Lumina.Benchmark/RowRef.cs b/src/Lumina.Benchmark/RowRef.cs index 7f941ac6..0634b385 100644 --- a/src/Lumina.Benchmark/RowRef.cs +++ b/src/Lumina.Benchmark/RowRef.cs @@ -7,8 +7,8 @@ namespace Lumina.Benchmark; public class RowRefBench { private GameData gameData; - private ExcelSheet rowRefSheet; - private RowRef rowRef; + private ExcelSheet< GatheringItem > rowRefSheet; + private RowRef< GatheringItemLevelConvertTable > rowRef; [GlobalSetup] public void Setup() @@ -18,7 +18,7 @@ public void Setup() PanicOnSheetChecksumMismatch = false, } ); - rowRefSheet = gameData.GetExcelSheet()!; + rowRefSheet = gameData.GetExcelSheet< GatheringItem >()!; rowRef = rowRefSheet.First().GatheringItemLevel; } diff --git a/src/Lumina.Benchmark/Sheets.cs b/src/Lumina.Benchmark/Sheets.cs index e1178050..dbc57446 100644 --- a/src/Lumina.Benchmark/Sheets.cs +++ b/src/Lumina.Benchmark/Sheets.cs @@ -25,20 +25,20 @@ public ulong TestAllSheets() ulong ret = 0; foreach( var sheetName in gameData.Excel.SheetNames ) { - if( gameData.GetFile( $"exd/{sheetName}.exh" ) is not { } headerFile ) + if( gameData.GetFile< ExcelHeaderFile >( $"exd/{sheetName}.exh" ) is not { } headerFile ) continue; var lang = headerFile.Languages.Contains( Language.English ) ? Language.English : Language.None; switch( headerFile.Header.Variant ) { case ExcelVariant.Default: { - var sheet = gameData.Excel.GetSheet( lang, sheetName ); + var sheet = gameData.Excel.GetSheet< Addon >( lang, sheetName ); ret += sheet.Count == 0 ? 0 : sheet.GetRowAt( sheet.Count - 1 ).RowId - sheet.GetRowAt( 0 ).RowId + 1 - (uint)sheet.Count; break; } case ExcelVariant.Subrows: { - var sheet = gameData.Excel.GetSubrowSheet( lang, sheetName ); + var sheet = gameData.Excel.GetSubrowSheet< QuestLinkMarker >( lang, sheetName ); ret += sheet.Count == 0 ? 0 : sheet.GetRowAt( sheet.Count - 1 ).RowId - sheet.GetRowAt( 0 ).RowId + 1 - (uint)sheet.Count ; break; } From a562e312850239ab974a0b873f25f2c0d57787a8 Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Tue, 27 Aug 2024 17:45:16 -0700 Subject: [PATCH 45/53] RowRef performance updates --- src/Lumina/Excel/ExcelModule.cs | 19 +++++++++++++++++++ src/Lumina/Excel/RawExcelSheet.cs | 11 +++++++++-- src/Lumina/Excel/RowRef{T}.cs | 19 +++++++++++++------ src/Lumina/Excel/SubrowExcelSheet.cs | 1 - src/Lumina/Excel/SubrowRef{T}.cs | 19 +++++++++++++------ 5 files changed, 54 insertions(+), 15 deletions(-) diff --git a/src/Lumina/Excel/ExcelModule.cs b/src/Lumina/Excel/ExcelModule.cs index 8a1d5d58..0da9d381 100644 --- a/src/Lumina/Excel/ExcelModule.cs +++ b/src/Lumina/Excel/ExcelModule.cs @@ -32,6 +32,7 @@ public class ExcelModule private FrozenDictionary< string, SheetData > DefinedSheetCache { get; } private ConcurrentDictionary< string, SheetData > AdhocSheetCache { get; } private ConcurrentDictionary< Type, SheetAttribute? > SheetAttributeCache { get; } = []; + private ConcurrentDictionary< Type, IExcelSheet > DirectBaseSheetCache { get; } = []; /// /// A delegate provided by the user to resolve RSV strings. @@ -119,6 +120,24 @@ public SubrowExcelSheet< T > GetSubrowSheet< T >( Language? language = null, str return new SubrowExcelSheet< T >( (RawSubrowExcelSheet)rawSheet ); } + /// Loads a typed using the default language and name. + /// An excel sheet corresponding to , , and + /// that may be created anew or reused from a previous invocation of this method. + /// + /// This particular overload caches sheets based on . + /// Only use this method if you need to create a sheet while using reflection. + /// The returned instance of should be cast to or + /// before accessing its rows. + /// + /// does not have a valid . + /// Sheet name was not specified via . + /// Sheet supports neither nor . + /// + [RequiresDynamicCode( "Creating a generic sheet from a type requires reflection and dynamic code." )] + [EditorBrowsable( EditorBrowsableState.Advanced )] + public IExcelSheet GetBaseSheet( Type rowType ) => + DirectBaseSheetCache.GetOrAdd( rowType, static ( t, self ) => self.GetBaseSheet( t, null, null ), this ); + /// Loads a typed . /// Type of the rows in the sheet. /// The requested sheet language. Leave or empty to use the default language. diff --git a/src/Lumina/Excel/RawExcelSheet.cs b/src/Lumina/Excel/RawExcelSheet.cs index e2dcadc7..54aa1cb4 100644 --- a/src/Lumina/Excel/RawExcelSheet.cs +++ b/src/Lumina/Excel/RawExcelSheet.cs @@ -189,8 +189,15 @@ internal RawExcelSheet( ExcelModule module, ExcelHeaderFile headerFile, Language /// public bool HasRow( uint rowId ) { - ref readonly var lookup = ref GetRowLookupOrNullRef( rowId ); - return !Unsafe.IsNullRef( in lookup ) && lookup.SubrowCount > 0; + var lookupArrayIndex = unchecked( rowId - _rowIndexLookupArrayOffset ); + if( lookupArrayIndex < _rowIndexLookupArray.Length ) + { + var rowIndex = _rowIndexLookupArray.UnsafeAt( (int) lookupArrayIndex ); + return rowIndex != -1; + } + + ref readonly var rowIndexRef = ref _rowIndexLookupDict.GetValueRefOrNullRef( (int) rowId ); + return !Unsafe.IsNullRef( in rowIndexRef ); } /// Gets a row lookup at the given index, if possible. diff --git a/src/Lumina/Excel/RowRef{T}.cs b/src/Lumina/Excel/RowRef{T}.cs index 6271f1a1..e8d815fa 100644 --- a/src/Lumina/Excel/RowRef{T}.cs +++ b/src/Lumina/Excel/RowRef{T}.cs @@ -8,19 +8,26 @@ namespace Lumina.Excel; /// The row type referenced by the . /// The to read sheet data from. /// The referenced row id. -public readonly struct RowRef< T >( ExcelModule? module, uint rowId ) where T : struct, IExcelRow< T > +public struct RowRef< T >( ExcelModule? module, uint rowId ) where T : struct, IExcelRow< T > { - private readonly ExcelSheet< T >? _sheet = module?.GetSheet< T >(); + private ExcelSheet< T >? _sheet = null; + private ExcelSheet< T >? Sheet { + get { + if( module == null ) + return null; + return _sheet ??= module.GetSheet< T >(); + } + } /// /// The row id of the referenced row. /// - public uint RowId => rowId; + public readonly uint RowId => rowId; /// /// Whether the exists in the sheet. /// - public bool IsValid => _sheet?.HasRow( RowId ) ?? false; + public bool IsValid => Sheet?.HasRow( RowId ) ?? false; /// /// The referenced row value itself. @@ -31,9 +38,9 @@ public readonly struct RowRef< T >( ExcelModule? module, uint rowId ) where T : /// /// Attempts to get the referenced row value. Is if does not exist in the sheet. /// - public T? ValueNullable => _sheet?.GetRowOrDefault( rowId ); + public T? ValueNullable => Sheet?.GetRowOrDefault( rowId ); - private RowRef ToGeneric() => RowRef.Create< T >( module, rowId ); + private readonly RowRef ToGeneric() => RowRef.Create< T >( module, rowId ); /// /// Converts a concrete to a generic and dynamically typed . diff --git a/src/Lumina/Excel/SubrowExcelSheet.cs b/src/Lumina/Excel/SubrowExcelSheet.cs index 42cc49d5..2629ac1a 100644 --- a/src/Lumina/Excel/SubrowExcelSheet.cs +++ b/src/Lumina/Excel/SubrowExcelSheet.cs @@ -1,6 +1,5 @@ using Lumina.Data; using Lumina.Data.Structs.Excel; -using Lumina.Excel.Exceptions; using System; using System.Collections; using System.Collections.Generic; diff --git a/src/Lumina/Excel/SubrowRef{T}.cs b/src/Lumina/Excel/SubrowRef{T}.cs index a3566b65..4b727cf2 100644 --- a/src/Lumina/Excel/SubrowRef{T}.cs +++ b/src/Lumina/Excel/SubrowRef{T}.cs @@ -8,19 +8,26 @@ namespace Lumina.Excel; /// The subrow type referenced by the subrows of . /// The to read sheet data from. /// The referenced row id. -public readonly struct SubrowRef< T >( ExcelModule? module, uint rowId ) where T : struct, IExcelSubrow< T > +public struct SubrowRef< T >( ExcelModule? module, uint rowId ) where T : struct, IExcelSubrow< T > { - private readonly SubrowExcelSheet< T >? _sheet = module?.GetSubrowSheet< T >(); + private SubrowExcelSheet< T >? _sheet = null; + private SubrowExcelSheet< T >? Sheet { + get { + if( module == null ) + return null; + return _sheet ??= module.GetSubrowSheet< T >(); + } + } /// /// The row id of the referenced row. /// - public uint RowId => rowId; + public readonly uint RowId => rowId; /// /// Whether the exists in the sheet. /// - public bool IsValid => _sheet?.HasRow( RowId ) ?? false; + public bool IsValid => Sheet?.HasRow( RowId ) ?? false; /// /// The referenced row value itself. @@ -31,9 +38,9 @@ public readonly struct SubrowRef< T >( ExcelModule? module, uint rowId ) where T /// /// Attempts to get the referenced row value. Is if it does not exist in the sheet. /// - public SubrowCollection< T >? ValueNullable => _sheet?.GetRowOrDefault( rowId ); + public SubrowCollection< T >? ValueNullable => Sheet?.GetRowOrDefault( rowId ); - private RowRef ToGeneric() => RowRef.CreateSubrow< T >( module, rowId ); + private readonly RowRef ToGeneric() => RowRef.CreateSubrow< T >( module, rowId ); /// /// Converts a concrete to a generic and dynamically typed . From 4c000732f3ecea17d1340192db321504d7aa0a66 Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Thu, 29 Aug 2024 18:42:03 -0700 Subject: [PATCH 46/53] Fix RowRef.GetValueOrDefault behavior --- src/Lumina/Excel/RowRef.cs | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/src/Lumina/Excel/RowRef.cs b/src/Lumina/Excel/RowRef.cs index 3bc8e4cb..ab2ecf40 100644 --- a/src/Lumina/Excel/RowRef.cs +++ b/src/Lumina/Excel/RowRef.cs @@ -65,27 +65,27 @@ public bool IsSubrow< T >() where T : struct, IExcelSubrow< T > => /// if the type is valid, the row exists, and is written to, and otherwise. public bool TryGetValue< T >( out T row ) where T : struct, IExcelRow< T > { - if( new RowRef< T >( module, rowId ).ValueNullable is { } v ) + if( !Is< T >() || module is null ) { - row = v; - return true; + row = default; + return false; } - row = default; - return false; + row = new RowRef< T >( module, rowId ).Value; + return true; } /// public bool TryGetValueSubrow< T >( out SubrowCollection< T > row ) where T : struct, IExcelSubrow< T > { - if( new SubrowRef< T >( module, rowId ).ValueNullable is { } v ) + if( !IsSubrow< T >() || module is null ) { - row = v; - return true; + row = default; + return false; } - row = default; - return false; + row = new SubrowRef< T >( module, rowId ).Value; + return true; } /// @@ -93,18 +93,13 @@ public bool TryGetValueSubrow< T >( out SubrowCollection< T > row ) where T : st /// /// The to read sheet data from. /// The referenced row id. + /// A hash of ; must be unique in every permutation. /// A list of row types to check against the , in order. /// A to one of the . If the row id does not exist in any of the sheets, an untyped is returned instead. - public static RowRef GetFirstValidRowOrUntyped( ExcelModule module, uint rowId, params Type[] sheetTypes ) + public static RowRef GetFirstValidRowOrUntyped( ExcelModule module, uint rowId, int typeHash, params ReadOnlySpan sheetTypes ) { - foreach( var sheetType in sheetTypes ) - { - if( module.GetBaseSheet( sheetType ) is { } sheet ) - { - if( sheet.HasRow( rowId ) ) - return new( module, rowId, sheetType ); - } - } + if( module.FindRowInterval( rowId, sheetTypes, typeHash ) is { } type ) + return new( module, rowId, type ); return CreateUntyped( rowId ); } From 1de6d4071bcdbbe8094b86b401d66185c4022624 Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Fri, 30 Aug 2024 00:33:17 -0700 Subject: [PATCH 47/53] Increase generic RowRef resolution speed through interval trees --- src/Lumina.Benchmark/ExcelRows.cs | 208 ++++++++++++++++++- src/Lumina.Benchmark/Lumina.Benchmark.csproj | 1 + src/Lumina.Benchmark/Npc.cs | 97 +++++++++ src/Lumina.Benchmark/Program.cs | 8 + src/Lumina.Benchmark/Row.cs | 5 +- src/Lumina.Benchmark/RowRef.cs | 5 +- src/Lumina.Benchmark/Sheets.cs | 5 +- src/Lumina/Excel/Collection.cs | 2 +- src/Lumina/Excel/ExcelModule.cs | 36 ++-- src/Lumina/Excel/RowRef.cs | 45 +++- src/Lumina/Excel/RowRefIntervalTree.cs | 114 ++++++++++ 11 files changed, 488 insertions(+), 38 deletions(-) create mode 100644 src/Lumina.Benchmark/Npc.cs create mode 100644 src/Lumina/Excel/RowRefIntervalTree.cs diff --git a/src/Lumina.Benchmark/ExcelRows.cs b/src/Lumina.Benchmark/ExcelRows.cs index 69433191..97ee3f6a 100644 --- a/src/Lumina.Benchmark/ExcelRows.cs +++ b/src/Lumina.Benchmark/ExcelRows.cs @@ -39,7 +39,7 @@ public readonly struct GatheringItem( ExcelPage page, uint offset, uint row ) : public uint RowId => row; public uint Data => row & offset; - public readonly RowRef Item => RowRef.GetFirstValidRowOrUntyped( page.Module, (uint)page.ReadInt32( offset + 8 ), [typeof( Item ), typeof( EventItem )] ); + public readonly RowRef Item => RowRef.GetFirstValidRowOrUntyped( page.Module, (uint)page.ReadInt32( offset + 8 ), [typeof( Item ), typeof( EventItem )], 0 ); public readonly RowRef< GatheringItemLevelConvertTable > GatheringItemLevel => new( page.Module, page.ReadUInt16( offset + 12 ) ); static GatheringItem IExcelRow< GatheringItem >.Create( ExcelPage page, uint offset, uint row ) => @@ -69,3 +69,209 @@ public readonly struct GatheringItemLevelConvertTable( ExcelPage page, uint offs static GatheringItemLevelConvertTable IExcelRow< GatheringItemLevelConvertTable >.Create( ExcelPage page, uint offset, uint row ) => new( page, offset, row ); } + +[Sheet( "ENpcBase" )] +public readonly unsafe struct ENpcBase( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + private static RowRef ENpcDataCtor( ExcelPage page, uint parentOffset, uint offset, uint i ) => RowRef.GetFirstValidRowOrUntyped( page.Module, page.ReadUInt32( offset + i * 4 ), [typeof( ChocoboTaxiStand ), typeof( CraftLeve ), typeof( CustomTalk ), typeof( DefaultTalk ), typeof( FccShop ), typeof( GCShop ), typeof( GilShop ), typeof( GuildleveAssignment ), typeof( GuildOrderGuide ), typeof( GuildOrderOfficer ), typeof( Quest ), typeof( SpecialShop ), typeof( Story ), typeof( SwitchTalk ), typeof( TopicSelect ), typeof( TripleTriad ), typeof( Warp )], 1 ); + public readonly Collection ENpcData => new( page, offset, offset, &ENpcDataCtor, 32 ); + + static ENpcBase IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "ChocoboTaxiStand" )] +public readonly struct ChocoboTaxiStand( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static ChocoboTaxiStand IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "CraftLeve" )] +public readonly struct CraftLeve( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static CraftLeve IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "CustomTalk" )] +public readonly struct CustomTalk( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static CustomTalk IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "DefaultTalk" )] +public readonly struct DefaultTalk( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static DefaultTalk IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "FccShop" )] +public readonly struct FccShop( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static FccShop IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "GCShop" )] +public readonly struct GCShop( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static GCShop IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "GilShop" )] +public readonly struct GilShop( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static GilShop IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "GuildleveAssignment" )] +public readonly struct GuildleveAssignment( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static GuildleveAssignment IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "GuildOrderGuide" )] +public readonly struct GuildOrderGuide( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static GuildOrderGuide IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "GuildOrderOfficer" )] +public readonly struct GuildOrderOfficer( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static GuildOrderOfficer IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "Quest" )] +public readonly struct Quest( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static Quest IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "SpecialShop" )] +public readonly struct SpecialShop( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static SpecialShop IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "Story" )] +public readonly struct Story( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static Story IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "SwitchTalk" )] +public readonly struct SwitchTalk( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static SwitchTalk IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "TopicSelect" )] +public readonly unsafe struct TopicSelect( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + private static RowRef ShopCtor( ExcelPage page, uint parentOffset, uint offset, uint i ) => RowRef.GetFirstValidRowOrUntyped( page.Module, page.ReadUInt32( offset + 4 + i * 4 ), [typeof( SpecialShop ), typeof( GilShop ), typeof( PreHandler )], 2 ); + public readonly Collection Shop => new( page, offset, offset, &ShopCtor, 10 ); + + static TopicSelect IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "TripleTriad" )] +public readonly struct TripleTriad( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static TripleTriad IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "Warp" )] +public readonly struct Warp( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static Warp IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "PreHandler" )] +public readonly struct PreHandler( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + public readonly RowRef Target => RowRef.GetFirstValidRowOrUntyped( page.Module, page.ReadUInt32( offset + 8 ), [typeof( CollectablesShop ), typeof( InclusionShop ), typeof( GilShop ), typeof( SpecialShop ), typeof( Description )], 3 ); + + static PreHandler IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "CollectablesShop" )] +public readonly struct CollectablesShop( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static CollectablesShop IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "InclusionShop" )] +public readonly struct InclusionShop( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static InclusionShop IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "Description" )] +public readonly struct Description( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static Description IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} \ No newline at end of file diff --git a/src/Lumina.Benchmark/Lumina.Benchmark.csproj b/src/Lumina.Benchmark/Lumina.Benchmark.csproj index 008b036f..2deb3251 100644 --- a/src/Lumina.Benchmark/Lumina.Benchmark.csproj +++ b/src/Lumina.Benchmark/Lumina.Benchmark.csproj @@ -5,6 +5,7 @@ Exe enable enable + true diff --git a/src/Lumina.Benchmark/Npc.cs b/src/Lumina.Benchmark/Npc.cs new file mode 100644 index 00000000..8218786d --- /dev/null +++ b/src/Lumina.Benchmark/Npc.cs @@ -0,0 +1,97 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Diagnostics.dotTrace; +using Lumina.Excel; + +namespace Lumina.Benchmark; + +[DisassemblyDiagnoser( maxDepth: 500, exportGithubMarkdown: false, exportHtml: true )] +[DotTraceDiagnoser] +public class NpcBench +{ + private GameData gameData; + private ExcelSheet npcSheet; + + [GlobalSetup] + public void Setup() + { + gameData = Program.CreateGameData(); + + npcSheet = gameData.GetExcelSheet()!; + } + + [Benchmark] + public Lookups BuildDataMap() + { + var ret = new Lookups(); + + foreach( var npc in npcSheet ) + { + foreach( var variable in npc.ENpcData ) + { + ret.EvaluateRowRef( npc, variable ); + } + } + + return ret; + } + + public readonly struct Lookups() + { + public readonly Dictionary> npcIdToSpecialShopIdLookup = []; + public readonly Dictionary> npcIdToFccShopIdLookup = []; + public readonly Dictionary> npcIdToGilShopIdLookup = []; + public readonly Dictionary> npcIdToGcShopIdLookup = []; + + public readonly Dictionary> specialShopIdToNpcIdLookup = []; + public readonly Dictionary> fccShopIdToNpcIdLookup = []; + public readonly Dictionary> gilShopIdToNpcIdLookup = []; + public readonly Dictionary> gcShopIdToNpcIdLookup = []; + + public void EvaluateRowRef( ENpcBase npcBase, RowRef rowRef ) + { + if( rowRef.Is() ) + { + fccShopIdToNpcIdLookup.TryAdd( rowRef.RowId, [] ); + fccShopIdToNpcIdLookup[rowRef.RowId].Add( npcBase.RowId ); + + npcIdToFccShopIdLookup.TryAdd( npcBase.RowId, [] ); + npcIdToFccShopIdLookup[npcBase.RowId].Add( rowRef.RowId ); + } + else if( rowRef.Is() ) + { + gcShopIdToNpcIdLookup.TryAdd( rowRef.RowId, [] ); + gcShopIdToNpcIdLookup[rowRef.RowId].Add( npcBase.RowId ); + + npcIdToGcShopIdLookup.TryAdd( npcBase.RowId, [] ); + npcIdToGcShopIdLookup[npcBase.RowId].Add( rowRef.RowId ); + } + else if( rowRef.Is() ) + { + gilShopIdToNpcIdLookup.TryAdd( rowRef.RowId, [] ); + gilShopIdToNpcIdLookup[rowRef.RowId].Add( npcBase.RowId ); + + npcIdToGilShopIdLookup.TryAdd( npcBase.RowId, [] ); + npcIdToGilShopIdLookup[npcBase.RowId].Add( rowRef.RowId ); + } + else if( rowRef.Is() ) + { + specialShopIdToNpcIdLookup.TryAdd( rowRef.RowId, [] ); + specialShopIdToNpcIdLookup[rowRef.RowId].Add( npcBase.RowId ); + + npcIdToSpecialShopIdLookup.TryAdd( npcBase.RowId, [] ); + npcIdToSpecialShopIdLookup[npcBase.RowId].Add( rowRef.RowId ); + } + else if( rowRef.TryGetValue(out var topicSelect) ) + { + foreach( var topicShop in topicSelect.Shop ) + { + EvaluateRowRef( npcBase, topicShop ); + } + } + else if( rowRef.TryGetValue( out var preHandler ) ) + { + EvaluateRowRef( npcBase, preHandler.Target ); + } + } + } +} diff --git a/src/Lumina.Benchmark/Program.cs b/src/Lumina.Benchmark/Program.cs index b11205cb..5c0b46b1 100644 --- a/src/Lumina.Benchmark/Program.cs +++ b/src/Lumina.Benchmark/Program.cs @@ -6,4 +6,12 @@ static void Main( string[] args ) { BenchmarkDotNet.Running.BenchmarkSwitcher.FromAssembly( typeof( Program ).Assembly ).Run( args ); } + + public static GameData CreateGameData() + { + return new( @"J:\Programs\Steam\steamapps\common\FINAL FANTASY XIV Online\game\sqpack", new() + { + PanicOnSheetChecksumMismatch = false, + } ); + } } diff --git a/src/Lumina.Benchmark/Row.cs b/src/Lumina.Benchmark/Row.cs index 9ce222d1..b9b879b6 100644 --- a/src/Lumina.Benchmark/Row.cs +++ b/src/Lumina.Benchmark/Row.cs @@ -17,10 +17,7 @@ public class RowBench [GlobalSetup] public void Setup() { - gameData = new GameData( @"J:\Programs\Steam\steamapps\common\FINAL FANTASY XIV Online\game\sqpack", new() - { - PanicOnSheetChecksumMismatch = false, - } ); + gameData = Program.CreateGameData(); addonSheet = gameData.GetExcelSheet< Addon >()!; addonRows = addonSheet.Select( x => x.RowId ).ToArray(); diff --git a/src/Lumina.Benchmark/RowRef.cs b/src/Lumina.Benchmark/RowRef.cs index 0634b385..5b98c11f 100644 --- a/src/Lumina.Benchmark/RowRef.cs +++ b/src/Lumina.Benchmark/RowRef.cs @@ -13,10 +13,7 @@ public class RowRefBench [GlobalSetup] public void Setup() { - gameData = new GameData( @"J:\Programs\Steam\steamapps\common\FINAL FANTASY XIV Online\game\sqpack", new() - { - PanicOnSheetChecksumMismatch = false, - } ); + gameData = Program.CreateGameData(); rowRefSheet = gameData.GetExcelSheet< GatheringItem >()!; rowRef = rowRefSheet.First().GatheringItemLevel; diff --git a/src/Lumina.Benchmark/Sheets.cs b/src/Lumina.Benchmark/Sheets.cs index dbc57446..4e2ab4ca 100644 --- a/src/Lumina.Benchmark/Sheets.cs +++ b/src/Lumina.Benchmark/Sheets.cs @@ -13,10 +13,7 @@ public class SheetsBench [GlobalSetup] public void Setup() { - gameData = new GameData( @"J:\Programs\Steam\steamapps\common\FINAL FANTASY XIV Online\game\sqpack", new() - { - PanicOnSheetChecksumMismatch = false, - } ); + gameData = Program.CreateGameData(); } [Benchmark] diff --git a/src/Lumina/Excel/Collection.cs b/src/Lumina/Excel/Collection.cs index 43227e1d..53e56982 100644 --- a/src/Lumina/Excel/Collection.cs +++ b/src/Lumina/Excel/Collection.cs @@ -10,7 +10,7 @@ namespace Lumina.Excel; /// /// Mostly an implementation detail for reading excel rows. This type does not store or hold any row data, and is therefore lightweight and trivially constructable. /// A type that wraps a group of fields inside a row. -public readonly struct Collection< T >( ExcelPage page, uint parentOffset, uint offset, Func< ExcelPage, uint, uint, uint, T > ctor, int size ) +public readonly unsafe struct Collection< T >( ExcelPage page, uint parentOffset, uint offset, delegate* managed ctor, int size ) : IList< T >, IReadOnlyList< T > where T : struct { /// diff --git a/src/Lumina/Excel/ExcelModule.cs b/src/Lumina/Excel/ExcelModule.cs index 0da9d381..e22957eb 100644 --- a/src/Lumina/Excel/ExcelModule.cs +++ b/src/Lumina/Excel/ExcelModule.cs @@ -32,7 +32,7 @@ public class ExcelModule private FrozenDictionary< string, SheetData > DefinedSheetCache { get; } private ConcurrentDictionary< string, SheetData > AdhocSheetCache { get; } private ConcurrentDictionary< Type, SheetAttribute? > SheetAttributeCache { get; } = []; - private ConcurrentDictionary< Type, IExcelSheet > DirectBaseSheetCache { get; } = []; + private ConcurrentDictionary< int, RowRefIntervalTree > RowRefIntervalCache { get; } = []; /// /// A delegate provided by the user to resolve RSV strings. @@ -120,24 +120,6 @@ public SubrowExcelSheet< T > GetSubrowSheet< T >( Language? language = null, str return new SubrowExcelSheet< T >( (RawSubrowExcelSheet)rawSheet ); } - /// Loads a typed using the default language and name. - /// An excel sheet corresponding to , , and - /// that may be created anew or reused from a previous invocation of this method. - /// - /// This particular overload caches sheets based on . - /// Only use this method if you need to create a sheet while using reflection. - /// The returned instance of should be cast to or - /// before accessing its rows. - /// - /// does not have a valid . - /// Sheet name was not specified via . - /// Sheet supports neither nor . - /// - [RequiresDynamicCode( "Creating a generic sheet from a type requires reflection and dynamic code." )] - [EditorBrowsable( EditorBrowsableState.Advanced )] - public IExcelSheet GetBaseSheet( Type rowType ) => - DirectBaseSheetCache.GetOrAdd( rowType, static ( t, self ) => self.GetBaseSheet( t, null, null ), this ); - /// Loads a typed . /// Type of the rows in the sheet. /// The requested sheet language. Leave or empty to use the default language. @@ -249,6 +231,22 @@ public void UnloadTypedCache( Assembly assembly ) } } + internal unsafe Type? FindRowInterval( uint rowId, ReadOnlySpan types, [ConstantExpected] int typeHash ) + { + // We do not need atomicity here. If the cache reallocates/changes (i.e. ConcurrentDictionary._tables is reallocated/rehashed), + // TryAdd can simply fail and the GC will take care of the additional interval tree. + if( !RowRefIntervalCache.TryGetValue( typeHash, out var ret ) ) + RowRefIntervalCache.TryAdd( typeHash, ret = new( this, types ) ); + return ret.Get( rowId ); + } + + internal RawExcelSheet GetSheetByType( Type type ) + { + var attr = GetSheetAttributes( type ) ?? throw new SheetAttributeMissingException( null, nameof( type ) ); + var name = attr.Name ?? throw new SheetNameEmptyException( nameof( type ) ); + return GetRawSheet( name ); + } + /// Gets the sheet attributes for . /// Type of the row. /// Sheet attributes, if any. diff --git a/src/Lumina/Excel/RowRef.cs b/src/Lumina/Excel/RowRef.cs index ab2ecf40..82ffc6c8 100644 --- a/src/Lumina/Excel/RowRef.cs +++ b/src/Lumina/Excel/RowRef.cs @@ -1,4 +1,6 @@ using System; +using System.ComponentModel; +using System.Diagnostics.CodeAnalysis; namespace Lumina.Excel; @@ -93,17 +95,50 @@ public bool TryGetValueSubrow< T >( out SubrowCollection< T > row ) where T : st /// /// The to read sheet data from. /// The referenced row id. - /// A hash of ; must be unique in every permutation. - /// A list of row types to check against the , in order. - /// A to one of the . If the row id does not exist in any of the sheets, an untyped is returned instead. - public static RowRef GetFirstValidRowOrUntyped( ExcelModule module, uint rowId, int typeHash, params ReadOnlySpan sheetTypes ) + /// A list of / types to check against , in order. + /// The order-sensitive hash of . + /// A to one of the . If the row id does not exist in any of the sheets, an untyped is returned instead. + /// Use to generate a . It's recommended to make this a compile-time constant if possible to improve performance. + public static RowRef GetFirstValidRowOrUntyped( ExcelModule module, uint rowId, ReadOnlySpan< Type > types, [ConstantExpected] int typeHash ) { - if( module.FindRowInterval( rowId, sheetTypes, typeHash ) is { } type ) + if( module.FindRowInterval( rowId, types, typeHash ) is { } type ) return new( module, rowId, type ); return CreateUntyped( rowId ); } + /// + /// + [Obsolete( "It's recommended to use the other overload and to manually generate a typeHash. Only this overload if you are explicitly disregarding performance." )] + public static RowRef GetFirstValidRowOrUntyped( ExcelModule module, uint rowId, ReadOnlySpan< Type > types ) + { + var hash = new HashCode(); + foreach( var hashType in types ) + hash.Add( hashType.TypeHandle.Value ); + +#pragma warning disable CA1857 // ConstantExpectedAttribute is explicitly ignored; we are re-emitting a warning with ObsoleteAttribute. + if( module.FindRowInterval( rowId, types, hash.ToHashCode() ) is { } type ) +#pragma warning restore CA1857 + return new( module, rowId, type ); + + return CreateUntyped( rowId ); + } + + /// + /// Creates an order-sensitive hash of . + /// + /// A list of ordered / types. + /// A typeHash for use in + /// It is not recommended to call this at runtime because of the performance hit. Use if you do not have a typeHash at compile-time. + [EditorBrowsable( EditorBrowsableState.Advanced )] + public static int CreateTypeHash( ReadOnlySpan< Type > types ) + { + var ret = new HashCode(); + foreach( var type in types ) + ret.Add( type.AssemblyQualifiedName ); + return ret.ToHashCode(); + } + /// /// Creates a to a specific row type. /// diff --git a/src/Lumina/Excel/RowRefIntervalTree.cs b/src/Lumina/Excel/RowRefIntervalTree.cs new file mode 100644 index 00000000..0cf1f07e --- /dev/null +++ b/src/Lumina/Excel/RowRefIntervalTree.cs @@ -0,0 +1,114 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics; +using System.Runtime.CompilerServices; + +namespace Lumina.Excel; + +internal sealed class RowRefIntervalTree +{ + private readonly record struct Interval( uint From, uint To, Type Type ); + + private readonly struct Point( uint rowId ) : IComparable< Interval > + { + [MethodImpl( MethodImplOptions.AggressiveInlining )] + public int CompareTo( Interval other ) + { + if( rowId < other.From ) + return -1; + else if( rowId >= other.To ) + return 1; + return 0; + } + } + + private readonly Interval[] Intervals; + + public RowRefIntervalTree( ExcelModule module, ReadOnlySpan< Type > types ) + { + List< Interval > retIntervals = []; + + List< Interval > currentIntervals = []; + foreach( var type in types ) + { + var sheet = module.GetSheetByType( type ); + + currentIntervals.Clear(); + uint? from = null; + uint to = 0; + foreach( var row in sheet.OffsetLookupTable ) + { + var id = row.RowId; + if( !from.HasValue ) + from = to = id; + else if( row.RowId == to + 1 ) + to = id; + else + { + currentIntervals.Add( new( from.Value, to, type ) ); + from = to = id; + } + } + if( from.HasValue ) + currentIntervals.Add( new( from.Value, to, type ) ); + + var lstI = 0; + var curI = 0; + while( lstI < retIntervals.Count && curI < currentIntervals.Count ) + { + var lst = retIntervals[lstI]; + var cur = currentIntervals[curI]; + // list item is fully before current item + if( lst.To <= cur.From ) + lstI++; + // current item is fully before list item + else if( cur.To <= lst.From ) + curI++; + // list item is before or begins at current item + else if( lst.From <= cur.From ) + { + // list item fully contains current item + if( lst.To >= cur.To ) + curI++; + // current item ends ahead of list item + else + { + cur = cur with { From = lst.To }; + retIntervals.Insert( lstI + 1, cur ); + curI++; + lstI += 2; + } + } + // current item is before list item + else if( cur.From < lst.From ) + { + // current item fully contains list item + if( cur.To >= lst.To ) + lstI++; + // list item ends ahead of current item + else + { + cur = cur with { To = lst.From }; + retIntervals.Insert( lstI, cur ); + curI++; + lstI++; + } + } + else + throw new UnreachableException(); + } + while( curI < currentIntervals.Count ) + retIntervals.Add( currentIntervals[curI++] ); + } + + Intervals = [.. retIntervals]; + } + + public Type? Get( uint rowId ) + { + ReadOnlySpan< Interval > intervals = Intervals.AsSpan(); + var idx = intervals.BinarySearch( new Point( rowId ) ); + return idx >= 0 ? intervals[idx].Type : null; + } +} From 651599d794ead3805de357cc994f422f86fe0b10 Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Fri, 30 Aug 2024 01:09:03 -0700 Subject: [PATCH 48/53] Add RowRefIntervalTree tests --- src/Lumina.Tests/ExcelRows.cs | 212 ++++++++++++++++++ src/Lumina.Tests/ExcelTests.cs | 93 ++++++++ src/Lumina.Tests/Lumina.Tests.csproj | 3 +- .../RequiresGameInstallationFact.cs | 8 + 4 files changed, 315 insertions(+), 1 deletion(-) create mode 100644 src/Lumina.Tests/ExcelRows.cs create mode 100644 src/Lumina.Tests/ExcelTests.cs diff --git a/src/Lumina.Tests/ExcelRows.cs b/src/Lumina.Tests/ExcelRows.cs new file mode 100644 index 00000000..4f57693a --- /dev/null +++ b/src/Lumina.Tests/ExcelRows.cs @@ -0,0 +1,212 @@ +using Lumina.Excel; + +namespace Lumina.Tests; + +[Sheet( "ENpcBase" )] +public readonly unsafe struct ENpcBase( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + private static uint ENpcDataCtor( ExcelPage page, uint parentOffset, uint offset, uint i ) => page.ReadUInt32( offset + i * 4 ); + public readonly Collection ENpcData => new( page, offset, offset, &ENpcDataCtor, 32 ); + + private static RowRef ENpcDataCtor2( ExcelPage page, uint parentOffset, uint offset, uint i ) => RowRef.GetFirstValidRowOrUntyped( page.Module, page.ReadUInt32( offset + i * 4 ), [typeof( ChocoboTaxiStand ), typeof( CraftLeve ), typeof( CustomTalk ), typeof( DefaultTalk ), typeof( FccShop ), typeof( GCShop ), typeof( GilShop ), typeof( GuildleveAssignment ), typeof( GuildOrderGuide ), typeof( GuildOrderOfficer ), typeof( Quest ), typeof( SpecialShop ), typeof( Story ), typeof( SwitchTalk ), typeof( TopicSelect ), typeof( TripleTriad ), typeof( Warp )], 1 ); + public readonly Collection ENpcData2 => new( page, offset, offset, &ENpcDataCtor2, 32 ); + + static ENpcBase IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "ChocoboTaxiStand" )] +public readonly struct ChocoboTaxiStand( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static ChocoboTaxiStand IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "CraftLeve" )] +public readonly struct CraftLeve( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static CraftLeve IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "CustomTalk" )] +public readonly struct CustomTalk( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static CustomTalk IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "DefaultTalk" )] +public readonly struct DefaultTalk( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static DefaultTalk IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "FccShop" )] +public readonly struct FccShop( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static FccShop IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "GCShop" )] +public readonly struct GCShop( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static GCShop IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "GilShop" )] +public readonly struct GilShop( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static GilShop IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "GuildleveAssignment" )] +public readonly struct GuildleveAssignment( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static GuildleveAssignment IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "GuildOrderGuide" )] +public readonly struct GuildOrderGuide( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static GuildOrderGuide IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "GuildOrderOfficer" )] +public readonly struct GuildOrderOfficer( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static GuildOrderOfficer IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "Quest" )] +public readonly struct Quest( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static Quest IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "SpecialShop" )] +public readonly struct SpecialShop( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static SpecialShop IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "Story" )] +public readonly struct Story( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static Story IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "SwitchTalk" )] +public readonly struct SwitchTalk( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static SwitchTalk IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "TopicSelect" )] +public readonly unsafe struct TopicSelect( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + private static RowRef ShopCtor( ExcelPage page, uint parentOffset, uint offset, uint i ) => RowRef.GetFirstValidRowOrUntyped( page.Module, page.ReadUInt32( offset + 4 + i * 4 ), [typeof( SpecialShop ), typeof( GilShop ), typeof( PreHandler )], 2 ); + public readonly Collection Shop => new( page, offset, offset, &ShopCtor, 10 ); + + static TopicSelect IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "TripleTriad" )] +public readonly struct TripleTriad( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static TripleTriad IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "Warp" )] +public readonly struct Warp( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static Warp IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "PreHandler" )] +public readonly struct PreHandler( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + public readonly RowRef Target => RowRef.GetFirstValidRowOrUntyped( page.Module, page.ReadUInt32( offset + 8 ), [typeof( CollectablesShop ), typeof( InclusionShop ), typeof( GilShop ), typeof( SpecialShop ), typeof( Description )], 3 ); + + static PreHandler IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "CollectablesShop" )] +public readonly struct CollectablesShop( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static CollectablesShop IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "InclusionShop" )] +public readonly struct InclusionShop( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static InclusionShop IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} + +[Sheet( "Description" )] +public readonly struct Description( ExcelPage page, uint offset, uint row ) : IExcelRow +{ + public uint RowId => row; + + static Description IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); +} \ No newline at end of file diff --git a/src/Lumina.Tests/ExcelTests.cs b/src/Lumina.Tests/ExcelTests.cs new file mode 100644 index 00000000..ba036b98 --- /dev/null +++ b/src/Lumina.Tests/ExcelTests.cs @@ -0,0 +1,93 @@ +using Lumina.Excel; +using System; +using System.Collections.Generic; +using System.Text; +using Xunit; + +namespace Lumina.Tests; + +public class ExcelTests +{ + [RequiresGameInstallationFact] + public void RowRefIntervalTree() + { + var gameData = RequiresGameInstallationFact.CreateGameData(); + + var types = new Type[] { typeof( ChocoboTaxiStand ), typeof( CraftLeve ), typeof( CustomTalk ), typeof( DefaultTalk ), typeof( FccShop ), typeof( GCShop ), typeof( GilShop ), typeof( GuildleveAssignment ), typeof( GuildOrderGuide ), typeof( GuildOrderOfficer ), typeof( Quest ), typeof( SpecialShop ), typeof( Story ), typeof( SwitchTalk ), typeof( TopicSelect ), typeof( TripleTriad ), typeof( Warp ) }; + var sheets = new List<(string, RawExcelSheet)>(); + foreach( var t in types ) + { + var n = t.Name; + var sheet = gameData.Excel.GetRawSheet( n ); + sheets.Add( (n, sheet) ); + } + + var s = new StringBuilder(); + foreach( var npc in gameData.GetExcelSheet()! ) + { + foreach( var variable in npc.ENpcData ) + { + var found = false; + foreach( var (name, sheet) in sheets ) + { + if( sheet.HasRow( variable ) ) + { + found = true; + s.AppendLine( name ); + break; + } + } + if( !found ) + s.AppendLine( "unk" ); + } + } + var brute = s.ToString(); + + s.Clear(); + foreach( var npc in gameData.GetExcelSheet()! ) + { + foreach( var variable in npc.ENpcData2 ) + { + if( variable.Is() ) + s.AppendLine( "ChocoboTaxiStand" ); + else if( variable.Is() ) + s.AppendLine( "CraftLeve" ); + else if( variable.Is() ) + s.AppendLine( "CustomTalk" ); + else if( variable.Is() ) + s.AppendLine( "DefaultTalk" ); + else if( variable.Is() ) + s.AppendLine( "FccShop" ); + else if( variable.Is() ) + s.AppendLine( "GCShop" ); + else if( variable.Is() ) + s.AppendLine( "GilShop" ); + else if( variable.Is() ) + s.AppendLine( "GuildleveAssignment" ); + else if( variable.Is() ) + s.AppendLine( "GuildOrderGuide" ); + else if( variable.Is() ) + s.AppendLine( "GuildOrderOfficer" ); + else if( variable.Is() ) + s.AppendLine( "Quest" ); + else if( variable.Is() ) + s.AppendLine( "SpecialShop" ); + else if( variable.Is() ) + s.AppendLine( "Story" ); + else if( variable.Is() ) + s.AppendLine( "SwitchTalk" ); + else if( variable.Is() ) + s.AppendLine( "TopicSelect" ); + else if( variable.Is() ) + s.AppendLine( "TripleTriad" ); + else if( variable.Is() ) + s.AppendLine( "Warp" ); + else + s.AppendLine( "unk" ); + } + } + var interval = s.ToString(); + + Assert.Equal( brute, interval ); + } +} diff --git a/src/Lumina.Tests/Lumina.Tests.csproj b/src/Lumina.Tests/Lumina.Tests.csproj index 1abee6f6..ee227a98 100644 --- a/src/Lumina.Tests/Lumina.Tests.csproj +++ b/src/Lumina.Tests/Lumina.Tests.csproj @@ -1,9 +1,10 @@ - + net8.0 false + true diff --git a/src/Lumina.Tests/RequiresGameInstallationFact.cs b/src/Lumina.Tests/RequiresGameInstallationFact.cs index 3ff252af..39a966af 100644 --- a/src/Lumina.Tests/RequiresGameInstallationFact.cs +++ b/src/Lumina.Tests/RequiresGameInstallationFact.cs @@ -12,4 +12,12 @@ public RequiresGameInstallationFact() if( !Directory.Exists( path ) ) Skip = "Game installation is not found at the default path."; } + + public static GameData CreateGameData() + { + return new( path, new() + { + PanicOnSheetChecksumMismatch = false, + } ); + } } \ No newline at end of file From 6b55c217711a0990fc7129cf90f973eae5a9d898 Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Fri, 30 Aug 2024 01:09:26 -0700 Subject: [PATCH 49/53] Fix IntervalTree construction --- src/Lumina/Excel/RowRefIntervalTree.cs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/Lumina/Excel/RowRefIntervalTree.cs b/src/Lumina/Excel/RowRefIntervalTree.cs index 0cf1f07e..b3b19435 100644 --- a/src/Lumina/Excel/RowRefIntervalTree.cs +++ b/src/Lumina/Excel/RowRefIntervalTree.cs @@ -41,13 +41,17 @@ public RowRefIntervalTree( ExcelModule module, ReadOnlySpan< Type > types ) { var id = row.RowId; if( !from.HasValue ) - from = to = id; - else if( row.RowId == to + 1 ) - to = id; + { + from = id; + to = id + 1; + } + else if( row.RowId == to ) + to = id + 1; else { currentIntervals.Add( new( from.Value, to, type ) ); - from = to = id; + from = id; + to = id + 1; } } if( from.HasValue ) @@ -64,7 +68,11 @@ public RowRefIntervalTree( ExcelModule module, ReadOnlySpan< Type > types ) lstI++; // current item is fully before list item else if( cur.To <= lst.From ) + { + retIntervals.Insert( lstI, cur ); curI++; + lstI++; + } // list item is before or begins at current item else if( lst.From <= cur.From ) { From 2c844d93518cbbab8b392dad5cb7890c7b77c62b Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Fri, 30 Aug 2024 03:25:58 -0700 Subject: [PATCH 50/53] Change hashing to be consistent with Lumina.Excel --- src/Lumina/Excel/RowRef.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Lumina/Excel/RowRef.cs b/src/Lumina/Excel/RowRef.cs index 82ffc6c8..ec2f34b9 100644 --- a/src/Lumina/Excel/RowRef.cs +++ b/src/Lumina/Excel/RowRef.cs @@ -135,7 +135,7 @@ public static int CreateTypeHash( ReadOnlySpan< Type > types ) { var ret = new HashCode(); foreach( var type in types ) - ret.Add( type.AssemblyQualifiedName ); + ret.Add( $"{type.Assembly.FullName};{type.FullName}" ); return ret.ToHashCode(); } From 9b0aab33904ef2182976838db1ef554c692b1c95 Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Sun, 15 Sep 2024 15:20:53 -0700 Subject: [PATCH 51/53] Add custom language support to RowRefs --- src/Lumina.Benchmark/ExcelRows.cs | 10 +++---- src/Lumina/Excel/ExcelPage.cs | 9 +++++- src/Lumina/Excel/RawExcelSheet.cs | 2 +- src/Lumina/Excel/RowRef.cs | 47 ++++++++++++++++++++----------- src/Lumina/Excel/RowRef{T}.cs | 18 +++++++++--- src/Lumina/Excel/SubrowRef{T}.cs | 14 +++++++-- 6 files changed, 70 insertions(+), 30 deletions(-) diff --git a/src/Lumina.Benchmark/ExcelRows.cs b/src/Lumina.Benchmark/ExcelRows.cs index 97ee3f6a..e6cdd08c 100644 --- a/src/Lumina.Benchmark/ExcelRows.cs +++ b/src/Lumina.Benchmark/ExcelRows.cs @@ -39,8 +39,8 @@ public readonly struct GatheringItem( ExcelPage page, uint offset, uint row ) : public uint RowId => row; public uint Data => row & offset; - public readonly RowRef Item => RowRef.GetFirstValidRowOrUntyped( page.Module, (uint)page.ReadInt32( offset + 8 ), [typeof( Item ), typeof( EventItem )], 0 ); - public readonly RowRef< GatheringItemLevelConvertTable > GatheringItemLevel => new( page.Module, page.ReadUInt16( offset + 12 ) ); + public readonly RowRef Item => RowRef.GetFirstValidRowOrUntyped( page.Module, (uint)page.ReadInt32( offset + 8 ), [typeof( Item ), typeof( EventItem )], 0, page.Language ); + public readonly RowRef< GatheringItemLevelConvertTable > GatheringItemLevel => new( page.Module, page.ReadUInt16( offset + 12 ), page.Language ); static GatheringItem IExcelRow< GatheringItem >.Create( ExcelPage page, uint offset, uint row ) => new( page, offset, row ); @@ -75,7 +75,7 @@ public readonly unsafe struct ENpcBase( ExcelPage page, uint offset, uint row ) { public uint RowId => row; - private static RowRef ENpcDataCtor( ExcelPage page, uint parentOffset, uint offset, uint i ) => RowRef.GetFirstValidRowOrUntyped( page.Module, page.ReadUInt32( offset + i * 4 ), [typeof( ChocoboTaxiStand ), typeof( CraftLeve ), typeof( CustomTalk ), typeof( DefaultTalk ), typeof( FccShop ), typeof( GCShop ), typeof( GilShop ), typeof( GuildleveAssignment ), typeof( GuildOrderGuide ), typeof( GuildOrderOfficer ), typeof( Quest ), typeof( SpecialShop ), typeof( Story ), typeof( SwitchTalk ), typeof( TopicSelect ), typeof( TripleTriad ), typeof( Warp )], 1 ); + private static RowRef ENpcDataCtor( ExcelPage page, uint parentOffset, uint offset, uint i ) => RowRef.GetFirstValidRowOrUntyped( page.Module, page.ReadUInt32( offset + i * 4 ), [typeof( ChocoboTaxiStand ), typeof( CraftLeve ), typeof( CustomTalk ), typeof( DefaultTalk ), typeof( FccShop ), typeof( GCShop ), typeof( GilShop ), typeof( GuildleveAssignment ), typeof( GuildOrderGuide ), typeof( GuildOrderOfficer ), typeof( Quest ), typeof( SpecialShop ), typeof( Story ), typeof( SwitchTalk ), typeof( TopicSelect ), typeof( TripleTriad ), typeof( Warp )], 1, page.Language ); public readonly Collection ENpcData => new( page, offset, offset, &ENpcDataCtor, 32 ); static ENpcBase IExcelRow.Create( ExcelPage page, uint offset, uint row ) => @@ -213,7 +213,7 @@ public readonly unsafe struct TopicSelect( ExcelPage page, uint offset, uint row { public uint RowId => row; - private static RowRef ShopCtor( ExcelPage page, uint parentOffset, uint offset, uint i ) => RowRef.GetFirstValidRowOrUntyped( page.Module, page.ReadUInt32( offset + 4 + i * 4 ), [typeof( SpecialShop ), typeof( GilShop ), typeof( PreHandler )], 2 ); + private static RowRef ShopCtor( ExcelPage page, uint parentOffset, uint offset, uint i ) => RowRef.GetFirstValidRowOrUntyped( page.Module, page.ReadUInt32( offset + 4 + i * 4 ), [typeof( SpecialShop ), typeof( GilShop ), typeof( PreHandler )], 2, page.Language ); public readonly Collection Shop => new( page, offset, offset, &ShopCtor, 10 ); static TopicSelect IExcelRow.Create( ExcelPage page, uint offset, uint row ) => @@ -243,7 +243,7 @@ public readonly struct PreHandler( ExcelPage page, uint offset, uint row ) : IEx { public uint RowId => row; - public readonly RowRef Target => RowRef.GetFirstValidRowOrUntyped( page.Module, page.ReadUInt32( offset + 8 ), [typeof( CollectablesShop ), typeof( InclusionShop ), typeof( GilShop ), typeof( SpecialShop ), typeof( Description )], 3 ); + public readonly RowRef Target => RowRef.GetFirstValidRowOrUntyped( page.Module, page.ReadUInt32( offset + 8 ), [typeof( CollectablesShop ), typeof( InclusionShop ), typeof( GilShop ), typeof( SpecialShop ), typeof( Description )], 3, page.Language ); static PreHandler IExcelRow.Create( ExcelPage page, uint offset, uint row ) => new( page, offset, row ); diff --git a/src/Lumina/Excel/ExcelPage.cs b/src/Lumina/Excel/ExcelPage.cs index 83fccd4d..e31e977e 100644 --- a/src/Lumina/Excel/ExcelPage.cs +++ b/src/Lumina/Excel/ExcelPage.cs @@ -1,3 +1,4 @@ +using Lumina.Data; using Lumina.Extensions; using Lumina.Text.ReadOnly; using System; @@ -22,14 +23,20 @@ public sealed class ExcelPage /// public ExcelModule Module { get; } + /// + /// The associated language of the page. + /// + public Language Language { get; } + private readonly byte[] data; private ReadOnlyMemory< byte > Data => data; private readonly ushort dataOffset; - internal ExcelPage( ExcelModule module, byte[] pageData, ushort headerDataOffset ) + internal ExcelPage( ExcelModule module, Language language, byte[] pageData, ushort headerDataOffset ) { Module = module; + Language = language; data = pageData; dataOffset = headerDataOffset; } diff --git a/src/Lumina/Excel/RawExcelSheet.cs b/src/Lumina/Excel/RawExcelSheet.cs index 54aa1cb4..301f48ca 100644 --- a/src/Lumina/Excel/RawExcelSheet.cs +++ b/src/Lumina/Excel/RawExcelSheet.cs @@ -110,7 +110,7 @@ internal RawExcelSheet( ExcelModule module, ExcelHeaderFile headerFile, Language if( fileData == null ) continue; - var newPage = _pages[ pageIdx ] = new( Module, fileData.Data, headerFile.Header.DataOffset ); + var newPage = _pages[ pageIdx ] = new( Module, Language, fileData.Data, headerFile.Header.DataOffset ); // If row count information from exh file is incorrect, cope with it. if( i + fileData.RowData.Count > _rowOffsetLookupTable.Length ) diff --git a/src/Lumina/Excel/RowRef.cs b/src/Lumina/Excel/RowRef.cs index ec2f34b9..437d758f 100644 --- a/src/Lumina/Excel/RowRef.cs +++ b/src/Lumina/Excel/RowRef.cs @@ -1,3 +1,4 @@ +using Lumina.Data; using System; using System.ComponentModel; using System.Diagnostics.CodeAnalysis; @@ -10,7 +11,8 @@ namespace Lumina.Excel; /// The to read sheet data from. /// The referenced row id. /// The referenced row's actual . -public readonly struct RowRef( ExcelModule? module, uint rowId, Type? rowType ) +/// The associated language of the referenced row. Leave to use 's default language. +public readonly struct RowRef( ExcelModule? module, uint rowId, Type? rowType, Language? language = null ) { /// /// The row id of the referenced row. @@ -25,6 +27,14 @@ public readonly struct RowRef( ExcelModule? module, uint rowId, Type? rowType ) /// public bool IsUntyped => rowType == null; + /// + /// The associated language of this row. + /// + /// + /// Can be if this has no associated . + /// + public Language? Language => language ?? module?.Language; + /// /// Whether the reference is of a specific row type. /// @@ -47,7 +57,7 @@ public bool IsSubrow< T >() where T : struct, IExcelSubrow< T > => if( !Is< T >() || module is null ) return null; - return new RowRef< T >( module, rowId ).ValueNullable; + return new RowRef< T >( module, rowId, language ).ValueNullable; } /// @@ -56,7 +66,7 @@ public bool IsSubrow< T >() where T : struct, IExcelSubrow< T > => if( !IsSubrow< T >() || module is null ) return null; - return new SubrowRef< T >( module, rowId ).ValueNullable; + return new SubrowRef< T >( module, rowId, language ).ValueNullable; } /// @@ -73,7 +83,7 @@ public bool TryGetValue< T >( out T row ) where T : struct, IExcelRow< T > return false; } - row = new RowRef< T >( module, rowId ).Value; + row = new RowRef< T >( module, rowId, language ).Value; return true; } @@ -86,7 +96,7 @@ public bool TryGetValueSubrow< T >( out SubrowCollection< T > row ) where T : st return false; } - row = new SubrowRef< T >( module, rowId ).Value; + row = new SubrowRef< T >( module, rowId, language ).Value; return true; } @@ -97,20 +107,21 @@ public bool TryGetValueSubrow< T >( out SubrowCollection< T > row ) where T : st /// The referenced row id. /// A list of / types to check against , in order. /// The order-sensitive hash of . + /// The associated language of the row. Leave to use 's default language instead. /// A to one of the . If the row id does not exist in any of the sheets, an untyped is returned instead. /// Use to generate a . It's recommended to make this a compile-time constant if possible to improve performance. - public static RowRef GetFirstValidRowOrUntyped( ExcelModule module, uint rowId, ReadOnlySpan< Type > types, [ConstantExpected] int typeHash ) + public static RowRef GetFirstValidRowOrUntyped( ExcelModule module, uint rowId, ReadOnlySpan< Type > types, [ConstantExpected] int typeHash, Language? language = null ) { if( module.FindRowInterval( rowId, types, typeHash ) is { } type ) - return new( module, rowId, type ); + return new( module, rowId, type, language ); - return CreateUntyped( rowId ); + return CreateUntyped( rowId, language ?? module.Language ); } /// - /// + /// [Obsolete( "It's recommended to use the other overload and to manually generate a typeHash. Only this overload if you are explicitly disregarding performance." )] - public static RowRef GetFirstValidRowOrUntyped( ExcelModule module, uint rowId, ReadOnlySpan< Type > types ) + public static RowRef GetFirstValidRowOrUntyped( ExcelModule module, uint rowId, ReadOnlySpan< Type > types, Language? language = null ) { var hash = new HashCode(); foreach( var hashType in types ) @@ -119,9 +130,9 @@ public static RowRef GetFirstValidRowOrUntyped( ExcelModule module, uint rowId, #pragma warning disable CA1857 // ConstantExpectedAttribute is explicitly ignored; we are re-emitting a warning with ObsoleteAttribute. if( module.FindRowInterval( rowId, types, hash.ToHashCode() ) is { } type ) #pragma warning restore CA1857 - return new( module, rowId, type ); + return new( module, rowId, type, language ); - return CreateUntyped( rowId ); + return CreateUntyped( rowId, language ?? module.Language ); } /// @@ -129,7 +140,7 @@ public static RowRef GetFirstValidRowOrUntyped( ExcelModule module, uint rowId, /// /// A list of ordered / types. /// A typeHash for use in - /// It is not recommended to call this at runtime because of the performance hit. Use if you do not have a typeHash at compile-time. + /// It is not recommended to call this at runtime because of the performance hit. Use if you do not have a typeHash at compile-time. [EditorBrowsable( EditorBrowsableState.Advanced )] public static int CreateTypeHash( ReadOnlySpan< Type > types ) { @@ -145,16 +156,18 @@ public static int CreateTypeHash( ReadOnlySpan< Type > types ) /// The row type referenced by the . /// The to read sheet data from. /// The referenced row id. + /// The associated language of the row. Leave to use 's default language instead. /// A to a row in a . - public static RowRef Create< T >( ExcelModule? module, uint rowId ) where T : struct, IExcelRow< T > => new( module, rowId, typeof( T ) ); + public static RowRef Create< T >( ExcelModule? module, uint rowId, Language? language = null ) where T : struct, IExcelRow< T > => new( module, rowId, typeof( T ), language ); - /// - public static RowRef CreateSubrow< T >( ExcelModule? module, uint rowId ) where T : struct, IExcelSubrow< T > => new( module, rowId, typeof( T ) ); + /// + public static RowRef CreateSubrow< T >( ExcelModule? module, uint rowId, Language? language = null ) where T : struct, IExcelSubrow< T > => new( module, rowId, typeof( T ), language ); /// /// Creates an untyped . /// /// The referenced row id. + /// The associated language of the row, if there is any. /// An untyped . - public static RowRef CreateUntyped( uint rowId ) => new( null, rowId, null ); + public static RowRef CreateUntyped( uint rowId, Language? language = null ) => new( null, rowId, null, language ); } \ No newline at end of file diff --git a/src/Lumina/Excel/RowRef{T}.cs b/src/Lumina/Excel/RowRef{T}.cs index e8d815fa..1f83e87e 100644 --- a/src/Lumina/Excel/RowRef{T}.cs +++ b/src/Lumina/Excel/RowRef{T}.cs @@ -1,3 +1,4 @@ +using Lumina.Data; using System; namespace Lumina.Excel; @@ -8,14 +9,15 @@ namespace Lumina.Excel; /// The row type referenced by the . /// The to read sheet data from. /// The referenced row id. -public struct RowRef< T >( ExcelModule? module, uint rowId ) where T : struct, IExcelRow< T > +/// The associated language of the referenced row. Leave to use 's default language. +public struct RowRef< T >( ExcelModule? module, uint rowId, Language? language = null ) where T : struct, IExcelRow< T > { - private ExcelSheet< T >? _sheet = null; + private ExcelSheet< T >? _sheet = null; private ExcelSheet< T >? Sheet { get { if( module == null ) return null; - return _sheet ??= module.GetSheet< T >(); + return _sheet ??= module.GetSheet< T >(language); } } @@ -24,6 +26,14 @@ private ExcelSheet< T >? Sheet { /// public readonly uint RowId => rowId; + /// + /// The associated language of this row. + /// + /// + /// Can be if this has no associated . + /// + public readonly Language? Language => language ?? module?.Language; + /// /// Whether the exists in the sheet. /// @@ -40,7 +50,7 @@ private ExcelSheet< T >? Sheet { /// public T? ValueNullable => Sheet?.GetRowOrDefault( rowId ); - private readonly RowRef ToGeneric() => RowRef.Create< T >( module, rowId ); + private readonly RowRef ToGeneric() => RowRef.Create< T >( module, rowId, language ); /// /// Converts a concrete to a generic and dynamically typed . diff --git a/src/Lumina/Excel/SubrowRef{T}.cs b/src/Lumina/Excel/SubrowRef{T}.cs index 4b727cf2..52c749a7 100644 --- a/src/Lumina/Excel/SubrowRef{T}.cs +++ b/src/Lumina/Excel/SubrowRef{T}.cs @@ -1,3 +1,4 @@ +using Lumina.Data; using System; namespace Lumina.Excel; @@ -8,14 +9,15 @@ namespace Lumina.Excel; /// The subrow type referenced by the subrows of . /// The to read sheet data from. /// The referenced row id. -public struct SubrowRef< T >( ExcelModule? module, uint rowId ) where T : struct, IExcelSubrow< T > +/// The associated language of the referenced row. Leave to use 's default language. +public struct SubrowRef< T >( ExcelModule? module, uint rowId, Language? language = null ) where T : struct, IExcelSubrow< T > { private SubrowExcelSheet< T >? _sheet = null; private SubrowExcelSheet< T >? Sheet { get { if( module == null ) return null; - return _sheet ??= module.GetSubrowSheet< T >(); + return _sheet ??= module.GetSubrowSheet< T >(language); } } @@ -24,6 +26,14 @@ private SubrowExcelSheet< T >? Sheet { /// public readonly uint RowId => rowId; + /// + /// The associated language of this row. + /// + /// + /// Can be if this has no associated . + /// + public readonly Language? Language => language ?? module?.Language; + /// /// Whether the exists in the sheet. /// From 843f9765a2a1dbfdffc084e2f7fded573059d39b Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Sun, 15 Sep 2024 16:43:12 -0700 Subject: [PATCH 52/53] Formatting changes --- src/Lumina/Excel/RowRef{T}.cs | 2 +- src/Lumina/Excel/SubrowRef{T}.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Lumina/Excel/RowRef{T}.cs b/src/Lumina/Excel/RowRef{T}.cs index 1f83e87e..523e3217 100644 --- a/src/Lumina/Excel/RowRef{T}.cs +++ b/src/Lumina/Excel/RowRef{T}.cs @@ -17,7 +17,7 @@ private ExcelSheet< T >? Sheet { get { if( module == null ) return null; - return _sheet ??= module.GetSheet< T >(language); + return _sheet ??= module.GetSheet< T >( language ); } } diff --git a/src/Lumina/Excel/SubrowRef{T}.cs b/src/Lumina/Excel/SubrowRef{T}.cs index 52c749a7..856641c5 100644 --- a/src/Lumina/Excel/SubrowRef{T}.cs +++ b/src/Lumina/Excel/SubrowRef{T}.cs @@ -17,7 +17,7 @@ private SubrowExcelSheet< T >? Sheet { get { if( module == null ) return null; - return _sheet ??= module.GetSubrowSheet< T >(language); + return _sheet ??= module.GetSubrowSheet< T >( language ); } } From 2ce372ad3aa883b0fd991edb3ac4bef54f8530d2 Mon Sep 17 00:00:00 2001 From: Asriel Camora Date: Sun, 29 Sep 2024 16:13:54 -0700 Subject: [PATCH 53/53] Fix compiler errors --- .../RequiresGameInstallationFact.cs | 6 +-- src/Lumina.Tests/SeStringBuilderTests.cs | 40 ++++++++++++------- src/Lumina/Excel/ExcelModule.cs | 2 +- 3 files changed, 29 insertions(+), 19 deletions(-) diff --git a/src/Lumina.Tests/RequiresGameInstallationFact.cs b/src/Lumina.Tests/RequiresGameInstallationFact.cs index 39a966af..4c539211 100644 --- a/src/Lumina.Tests/RequiresGameInstallationFact.cs +++ b/src/Lumina.Tests/RequiresGameInstallationFact.cs @@ -5,17 +5,17 @@ namespace Lumina.Tests; public sealed class RequiresGameInstallationFact : FactAttribute { - private const string path = @"C:\Program Files (x86)\SquareEnix\FINAL FANTASY XIV - A Realm Reborn\game\sqpack"; + private const string Path = @"C:\Program Files (x86)\SquareEnix\FINAL FANTASY XIV - A Realm Reborn\game\sqpack"; public RequiresGameInstallationFact() { - if( !Directory.Exists( path ) ) + if( !Directory.Exists( Path ) ) Skip = "Game installation is not found at the default path."; } public static GameData CreateGameData() { - return new( path, new() + return new( Path, new() { PanicOnSheetChecksumMismatch = false, } ); diff --git a/src/Lumina.Tests/SeStringBuilderTests.cs b/src/Lumina.Tests/SeStringBuilderTests.cs index cb0ab45a..f008a888 100644 --- a/src/Lumina.Tests/SeStringBuilderTests.cs +++ b/src/Lumina.Tests/SeStringBuilderTests.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Reflection; using System.Text; using Lumina.Data; @@ -311,7 +312,7 @@ static Addon IExcelRow.Create( ExcelPage page, uint offset, uint row ) => [RequiresGameInstallationFact] public void AddonIsParsedCorrectly() { - var gameData = new GameData( @"C:\Program Files (x86)\SquareEnix\FINAL FANTASY XIV - A Realm Reborn\game\sqpack" ); + var gameData = RequiresGameInstallationFact.CreateGameData(); var addon = gameData.Excel.GetSheet< Addon >(); var ssb = new SeStringBuilder(); var expected = new Dictionary< uint, ReadOnlySeString > @@ -438,7 +439,7 @@ public void AddonIsParsedCorrectly() { _outputHelper.WriteLine( $"{row.RowId}\t{row.Text.ExtractText()}\t{row.Text}" ); if( expected.TryGetValue( row.RowId, out var expectedSeString ) ) - Assert.StrictEqual( expectedSeString, r ); + Assert.StrictEqual( expectedSeString, row.Text ); } } @@ -589,28 +590,25 @@ public void FriendlyErrorMessage() [RequiresGameInstallationFact] public void AllSheetsTextColumnCodec() { - var gameData = new GameData( @"C:\Program Files (x86)\SquareEnix\FINAL FANTASY XIV - A Realm Reborn\game\sqpack" ); + var gameData = RequiresGameInstallationFact.CreateGameData(); var ssb = new SeStringBuilder(); - foreach( var sheetName in gameData.Excel.GetSheetNames() ) + foreach( var sheetName in gameData.Excel.SheetNames ) { - var languages = gameData.GetFile< ExcelHeaderFile >( ExcelModule.BuildExcelHeaderPath( sheetName ) )?.Languages ?? [Language.None]; + var header = gameData.GetFile( $"exd/{sheetName}.exh" ); + if( header?.Header.Variant == ExcelVariant.Subrows ) + continue; + var languages = header?.Languages ?? [Language.None]; foreach( var language in languages ) { - if( gameData.Excel.GetSheetRaw( sheetName, language ) is not { } sheet ) - continue; - - // CustomTalkDefineClient: it currently fails at reading string columns in sheets of subrow variant. - if( sheet.Variant != ExcelVariant.Default ) + if( gameData.Excel.GetSheet( language, sheetName ) is not { } sheet ) continue; + var stringColumns = sheet.Columns.Where( c => c.Type == ExcelColumnDataType.String ).Select( c => c.Offset ).ToArray(); foreach( var row in sheet ) { - for( var i = 0; i < sheet.Columns.Length; i++ ) + foreach( var columnOffset in stringColumns ) { - if( sheet.Columns[ i ].Type != ExcelColumnDataType.String ) - continue; - - var test1 = row.ReadColumn< SeString >( i ).AsReadOnly(); + var test1 = row.ReadString(columnOffset); if( test1.Data.Span.IndexOf( "payload:"u8 ) != -1 ) throw new( $"Unsupported payload at {sheetName}#{row.RowId}; {test1}" ); @@ -630,4 +628,16 @@ public void AllSheetsTextColumnCodec() } } } + + [Sheet] + public readonly struct RawRow( ExcelPage page, uint offset, uint row ) : IExcelRow + { + public uint RowId => row; + + public ReadOnlySeString ReadString( ushort off ) => + page.ReadString( off + offset, offset ); + + static RawRow IExcelRow.Create( ExcelPage page, uint offset, uint row ) => + new( page, offset, row ); + } } \ No newline at end of file diff --git a/src/Lumina/Excel/ExcelModule.cs b/src/Lumina/Excel/ExcelModule.cs index e22957eb..706f3820 100644 --- a/src/Lumina/Excel/ExcelModule.cs +++ b/src/Lumina/Excel/ExcelModule.cs @@ -62,7 +62,7 @@ public ExcelModule( GameData gameData ) GameData.Logger?.Information( "got {ExltEntryCount} exlt entries", files.ExdMap.Count ); DefinedSheetCache = files.ExdMap.Keys - .Select( name => ( Name: name, Header: GameData.GetFile< ExcelHeaderFile >( $"exd/{name}.exh") ) ) + .Select( name => ( Name: name, Header: GameData.GetFile< ExcelHeaderFile >( $"exd/{name}.exh" ) ) ) .Where( sheet => sheet.Header is not null ) .ToFrozenDictionary( sheet => sheet.Name,