Skip to content

Commit

Permalink
fix: Fixes serializing strings and uses memory mapped files for loadi…
Browse files Browse the repository at this point in the history 
…ng (#1133)
  • Loading branch information
@kamronbatman
kamronbatman authored Aug 16, 2022
1 parent bc746db commit 2ae7629
Show file tree
Hide file tree
Showing 5 changed files with 324 additions and 285 deletions.
101 changes: 74 additions & 27 deletions Projects/Server/Serialization/AdhocPersistence.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,44 +15,91 @@

using System;
using System.IO;
using System.IO.MemoryMappedFiles;
using System.Runtime.CompilerServices;
using System.Threading.Tasks;

namespace Server
namespace Server;

public static class AdhocPersistence
{
public static class AdhocPersistence
/**
* Serializes to memory synchronously. Optional buffer can be provided.
* Note: The buffer may not be the same after returning from the function if more data is written
* than the initial buffer can handle.
*/
public static BufferWriter Serialize(Action<IGenericWriter> serializer)
{
var saveBuffer = new BufferWriter(true);
serializer(saveBuffer);
return saveBuffer;
}

/**
* Writes a buffer to disk. This function should be called asynchronously.
*/
public static void WriteSnapshot(string filePath, Span<byte> buffer)
{
public static void Serialize(string filePath, Action<IGenericWriter> serializer)
var fullPath = PathUtility.GetFullPath(filePath, Core.BaseDirectory);
var file = new FileInfo(fullPath);
PathUtility.EnsureDirectory(file.DirectoryName);

using var fs = new FileStream(fullPath, FileMode.Create, FileAccess.Write);
fs.Write(buffer);
}

public static void SerializeAndSnapshot(string filePath, Action<IGenericWriter> serializer)
{
var saveBuffer = Serialize(serializer);
Task.Run(() => { WriteSnapshot(filePath, saveBuffer.Buffer.AsSpan(0, (int)saveBuffer.Position)); });
}

public static void Deserialize(string filePath, Action<IGenericReader> deserializer)
{
var fullPath = PathUtility.GetFullPath(filePath, Core.BaseDirectory);
var file = new FileInfo(fullPath);

if (!file.Exists)
{
return;
}

var fileLength = file.Length;
if (fileLength == 0)
{
var fullPath = Path.Combine(Core.BaseDirectory, filePath);
var file = new FileInfo(fullPath);
file.Directory?.Create();
return;
}

string error;

using var bin = new BinaryFileWriter(fullPath, true);
serializer(bin);
try
{
using var mmf = MemoryMappedFile.CreateFromFile(fullPath, FileMode.Open);
using var stream = mmf.CreateViewStream();
using var br = new BinaryFileReader(stream);
deserializer(br);

error = br.Position != fileLength
? $"Serialized {fileLength} bytes, but {br.Position} bytes deserialized"
: null;
}
catch (Exception e)
{
error = e.ToString();
}

public static void Deserialize(string filePath, Action<IGenericReader> deserializer)
if (error != null)
{
var fullPath = Path.Combine(Core.BaseDirectory, filePath);
var file = new FileInfo(fullPath);
file.Directory?.Create();
Console.WriteLine($"***** Bad deserialize of {file.FullName} *****");
Console.WriteLine(error);

if (!file.Exists)
{
return;
}
Console.WriteLine("Skip this file and continue? (y/n)");

try
{
using FileStream fs = new FileStream(fullPath, FileMode.Open, FileAccess.Read, FileShare.Read);
using var br = new BinaryFileReader(fs);
deserializer(br);
}
catch (Exception e)
var pressedKey = Console.ReadKey(true).Key;

if (pressedKey != ConsoleKey.Y)
{
Utility.PushColor(ConsoleColor.Red);
Console.WriteLine($"***** Bad deserialize of {file.FullName} *****");
Console.WriteLine(e.ToString());
Utility.PopColor();
throw new Exception("Deserialization failed.");
}
}
}
Expand Down
122 changes: 73 additions & 49 deletions Projects/Server/Serialization/BinaryFileReader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,82 +16,106 @@
using System;
using System.IO;
using System.Runtime.CompilerServices;
using System.Text;
using Server.Buffers;
using Server.Collections;
using Server.Text;

namespace Server
namespace Server;

public class BinaryFileReader : IGenericReader, IDisposable
{
public class BinaryFileReader : IGenericReader, IDisposable
private BinaryReader _reader;
private Encoding _encoding;

public BinaryFileReader(BinaryReader br, Encoding encoding = null)
{
private readonly BinaryReader _reader;
_reader = br;
_encoding = encoding ?? TextEncoding.UTF8;
}

public BinaryFileReader(BinaryReader br) => _reader = br;
public BinaryFileReader(Stream stream, Encoding encoding = null) : this(new BinaryReader(stream), encoding)
{
}

public BinaryFileReader(Stream stream) => _reader = new BinaryReader(stream);
public long Position => _reader.BaseStream.Position;

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Close() => _reader.Close();
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Close() => _reader.Close();

public DateTime LastSerialized { get; init; }
public DateTime LastSerialized { get; init; }

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public string ReadString(bool intern = false)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public string ReadString(bool intern = false)
{
if (!ReadBool())
{
var str = _reader.ReadString();
return intern ? Utility.Intern(str) : str;
return null;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public long ReadLong() => _reader.ReadInt64();
var length = ((IGenericReader)this).ReadEncodedInt();
if (length <= 0)
{
return intern ? Utility.Intern("") : "";
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public ulong ReadULong() => _reader.ReadUInt64();
byte[] buffer = STArrayPool<byte>.Shared.Rent(length);
var str = TextEncoding.GetString(buffer.AsSpan(0, length), _encoding);
STArrayPool<byte>.Shared.Return(buffer);
return intern ? Utility.Intern(str) : str;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public int ReadInt() => _reader.ReadInt32();
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public long ReadLong() => _reader.ReadInt64();

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public uint ReadUInt() => _reader.ReadUInt32();
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public ulong ReadULong() => _reader.ReadUInt64();

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public short ReadShort() => _reader.ReadInt16();
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public int ReadInt() => _reader.ReadInt32();

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public ushort ReadUShort() => _reader.ReadUInt16();
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public uint ReadUInt() => _reader.ReadUInt32();

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public double ReadDouble() => _reader.ReadDouble();
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public short ReadShort() => _reader.ReadInt16();

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public float ReadFloat() => _reader.ReadSingle();
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public ushort ReadUShort() => _reader.ReadUInt16();

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public byte ReadByte() => _reader.ReadByte();
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public double ReadDouble() => _reader.ReadDouble();

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public sbyte ReadSByte() => _reader.ReadSByte();
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public float ReadFloat() => _reader.ReadSingle();

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool ReadBool() => _reader.ReadBoolean();
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public byte ReadByte() => _reader.ReadByte();

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public Serial ReadSerial() => (Serial)_reader.ReadUInt32();
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public sbyte ReadSByte() => _reader.ReadSByte();

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public int Read(Span<byte> buffer) => _reader.Read(buffer);
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool ReadBool() => _reader.ReadBoolean();

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public BitArray ReadBitArray()
{
var length = ((IGenericReader)this).ReadEncodedInt();
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public Serial ReadSerial() => (Serial)_reader.ReadUInt32();

// BinaryReader doesn't expose a Span slice of the buffer, so we use a custom ctor
return new BitArray(_reader, length);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public int Read(Span<byte> buffer) => _reader.Read(buffer);

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public long Seek(long offset, SeekOrigin origin) => _reader.BaseStream.Seek(offset, origin);
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public BitArray ReadBitArray()
{
var length = ((IGenericReader)this).ReadEncodedInt();

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Dispose() => Close();
// BinaryReader doesn't expose a Span slice of the buffer, so we use a custom ctor
return new BitArray(_reader, length);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public long Seek(long offset, SeekOrigin origin) => _reader.BaseStream.Seek(offset, origin);

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Dispose() => Close();
}
Loading

0 comments on commit 2ae7629

Please sign in to comment.