From ec6e32c6f49ac18130a03b4b68f0da58b916e965 Mon Sep 17 00:00:00 2001 From: andrew <1297077+andrewmd5@users.noreply.github.com> Date: Fri, 9 Aug 2024 13:39:19 +0900 Subject: [PATCH 1/4] fix(cs/serialization): Improve precision in date serialization - Modify `BebopWriter.WriteDate` method to use a more precise algorithm - Introduce constants for ticks between epochs and date mask - Update date conversion to use milliseconds since Unix epoch - Apply bitmask to ensure consistent 64-bit representation This change should resolve hash discrepancies caused by date fields when serializing and deserializing objects. Testing: - Verified consistent hashing of original and round-tripped data - Ensured compatibility with existing implementations --- .env | 4 ++-- .github/workflows/integration-tests.yml | 2 ++ Runtime/C#/Runtime/BebopWriter.cs | 15 ++++++++++----- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.env b/.env index 0f7fd5d2..0832629f 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ -VERSION="3.0.12" +VERSION="3.0.13" MAJOR=3 MINOR=0 -PATCH=12 +PATCH=13 diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 8ceee95a..88a9acb2 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -18,6 +18,8 @@ jobs: uses: actions/setup-dotnet@v3 with: dotnet-version: 9.0.x + dotnet-quality: 'preview' + - name: Setup Rust uses: actions-rs/toolchain@v1 with: diff --git a/Runtime/C#/Runtime/BebopWriter.cs b/Runtime/C#/Runtime/BebopWriter.cs index 3d6430af..1081041e 100644 --- a/Runtime/C#/Runtime/BebopWriter.cs +++ b/Runtime/C#/Runtime/BebopWriter.cs @@ -13,6 +13,9 @@ namespace Bebop.Runtime /// public ref struct BebopWriter { + const long TicksBetweenEpochs = 621355968000000000L; + const long DateMask = 0x3fffffffffffffffL; + // ReSharper disable once InconsistentNaming private static readonly UTF8Encoding UTF8 = new(); @@ -287,7 +290,9 @@ public void WriteFloat64(double value) [MethodImpl(BebopConstants.HotPath)] public void WriteDate(DateTime date) { - WriteInt64(date.ToUniversalTime().ToBinary()); + long ms = (long)(date.ToUniversalTime() - new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc)).TotalMilliseconds; + long ticks = ms * 10000L + TicksBetweenEpochs; + WriteInt64(ticks & DateMask); } /// @@ -321,7 +326,7 @@ public void WriteString(string value) fixed (char* c = value) { var size = UTF8.GetByteCount(c, value.Length); - WriteUInt32(unchecked((uint) size)); + WriteUInt32(unchecked((uint)size)); var index = Length; GrowBy(size); fixed (byte* o = _buffer.Slice(index, size)) @@ -365,7 +370,7 @@ public void FillRecordLength(int position, uint messageLength) [MethodImpl(BebopConstants.HotPath)] public void WriteFloat32S(float[] value) { - WriteUInt32(unchecked((uint) value.Length)); + WriteUInt32(unchecked((uint)value.Length)); var index = Length; var floatBytes = AsBytes(value); if (floatBytes.IsEmpty) @@ -382,7 +387,7 @@ public void WriteFloat32S(float[] value) [MethodImpl(BebopConstants.HotPath)] public void WriteFloat64S(double[] value) { - WriteUInt32(unchecked((uint) value.Length)); + WriteUInt32(unchecked((uint)value.Length)); var index = Length; var doubleBytes = AsBytes(value); if (doubleBytes.IsEmpty) @@ -417,7 +422,7 @@ public void WriteBytes(ImmutableArray value) [MethodImpl(BebopConstants.HotPath)] public void WriteBytes(byte[] value) { - WriteUInt32(unchecked((uint) value.Length)); + WriteUInt32(unchecked((uint)value.Length)); if (value.Length == 0) { return; From 7a2de39adbf2402c7f6630d4e9bdb13831721c58 Mon Sep 17 00:00:00 2001 From: andrew <1297077+andrewmd5@users.noreply.github.com> Date: Fri, 9 Aug 2024 13:48:14 +0900 Subject: [PATCH 2/4] fix(cs/deserialization): Align date deserialization with other implementations This commit modifies the date deserialization process in Bebop to ensure consistency with the other implementations and corrects a type mismatch issue. This change complements the previous fix for date serialization, providing a complete and correct solution for date handling across languages. - Modify `BebopReader.ReadDate()` method to use a more precise algorithm - Introduce constants for ticks between epochs and date mask - Update date conversion to use milliseconds since Unix epoch - Ensure UTC DateTime is always returned - Fix type mismatch between ulong and long in date calculation - Use unchecked conversion to handle potential overflow This change resolves inconsistencies in date representation when deserializing data. --- Runtime/C#/Runtime/BebopReader.cs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Runtime/C#/Runtime/BebopReader.cs b/Runtime/C#/Runtime/BebopReader.cs index e6bff96b..b8630170 100644 --- a/Runtime/C#/Runtime/BebopReader.cs +++ b/Runtime/C#/Runtime/BebopReader.cs @@ -14,6 +14,9 @@ namespace Bebop.Runtime /// public ref struct BebopReader { + private const long TicksBetweenEpochs = 621355968000000000L; + private const ulong DateMask = 0x3fffffffffffffffUL; + // ReSharper disable once InconsistentNaming private static readonly UTF8Encoding UTF8 = new(); @@ -193,9 +196,13 @@ public double ReadFloat64() /// /// A UTC instance. [MethodImpl(BebopConstants.HotPath)] - public DateTime ReadDate() => - // make sure it always reads it as UTC by setting the first bits to `01`. - DateTime.FromBinary((ReadInt64() & 0x7fffffffffffffff) | 0x4000000000000000); + public DateTime ReadDate() + { + ulong rawTicks = ReadUInt64() & DateMask; + long ticks = unchecked((long)rawTicks); + long ms = (ticks - TicksBetweenEpochs) / 10000L; + return new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddMilliseconds(ms); + } /// /// Reads a length prefixed string from the underlying buffer and advances the current position by that many bytes. From 4e6426f67380d15703d3f06b2d9eba3635346d7b Mon Sep 17 00:00:00 2001 From: andrew <1297077+andrewmd5@users.noreply.github.com> Date: Fri, 9 Aug 2024 13:53:12 +0900 Subject: [PATCH 3/4] Revert "fix(cs/deserialization): Align date deserialization with other implementations" This reverts commit 7a2de39adbf2402c7f6630d4e9bdb13831721c58. --- Runtime/C#/Runtime/BebopReader.cs | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/Runtime/C#/Runtime/BebopReader.cs b/Runtime/C#/Runtime/BebopReader.cs index b8630170..e6bff96b 100644 --- a/Runtime/C#/Runtime/BebopReader.cs +++ b/Runtime/C#/Runtime/BebopReader.cs @@ -14,9 +14,6 @@ namespace Bebop.Runtime /// public ref struct BebopReader { - private const long TicksBetweenEpochs = 621355968000000000L; - private const ulong DateMask = 0x3fffffffffffffffUL; - // ReSharper disable once InconsistentNaming private static readonly UTF8Encoding UTF8 = new(); @@ -196,13 +193,9 @@ public double ReadFloat64() /// /// A UTC instance. [MethodImpl(BebopConstants.HotPath)] - public DateTime ReadDate() - { - ulong rawTicks = ReadUInt64() & DateMask; - long ticks = unchecked((long)rawTicks); - long ms = (ticks - TicksBetweenEpochs) / 10000L; - return new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddMilliseconds(ms); - } + public DateTime ReadDate() => + // make sure it always reads it as UTC by setting the first bits to `01`. + DateTime.FromBinary((ReadInt64() & 0x7fffffffffffffff) | 0x4000000000000000); /// /// Reads a length prefixed string from the underlying buffer and advances the current position by that many bytes. From dde054a08df280b1c4f6ced615cbe37b6a565548 Mon Sep 17 00:00:00 2001 From: andrew <1297077+andrewmd5@users.noreply.github.com> Date: Fri, 9 Aug 2024 13:57:01 +0900 Subject: [PATCH 4/4] test(cs): Improve DateTime precision handling in WriteRead test --- Laboratory/C#/Test/UnitTest1.cs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Laboratory/C#/Test/UnitTest1.cs b/Laboratory/C#/Test/UnitTest1.cs index 5805cd92..38a73a7f 100644 --- a/Laboratory/C#/Test/UnitTest1.cs +++ b/Laboratory/C#/Test/UnitTest1.cs @@ -22,8 +22,8 @@ public void Setup() [Test] public void WriteRead() { - var testBytes = new byte[] {0x1, 0x2, 0x3}; - var testFloats = new float[] {float.MinValue, float.MaxValue}; + var testBytes = new byte[] { 0x1, 0x2, 0x3 }; + var testFloats = new float[] { float.MinValue, float.MaxValue }; var testDoubles = new double[] { double.MinValue, double.MaxValue }; var testGuid = Guid.Parse("81c6987b-48b7-495f-ad01-ec20cc5f5be1"); const string testString = @"Hello 明 World!😊"; @@ -65,7 +65,7 @@ public void WriteRead() // test float / double Assert.AreEqual(float.MaxValue, output.ReadFloat32()); Assert.AreEqual(double.MaxValue, output.ReadFloat64()); - // test float array + // test float array var floatArrayLength = output.ReadUInt32(); Assert.AreEqual(testFloats.Length, floatArrayLength); var parsedFloats = new float[floatArrayLength]; @@ -88,7 +88,10 @@ public void WriteRead() // test guid Assert.AreEqual(testGuid, output.ReadGuid()); // test date - Assert.AreEqual(testDate, output.ReadDate()); + var readDate = output.ReadDate(); + Console.WriteLine($"Read date: {readDate:O}"); + Assert.That(readDate, Is.EqualTo(testDate).Within(TimeSpan.FromMilliseconds(1)), + $"Dates should match within 1ms. Difference: {(readDate - testDate).TotalMilliseconds}ms"); // test byte array CollectionAssert.AreEqual(testBytes, output.ReadBytes()); @@ -109,7 +112,7 @@ public void RoundTrip() new Musician {Name = "Miles Davis", Plays = Instrument.Trumpet} } }; - var library = new Library {Songs = new Dictionary {{testGuid, song}}}; + var library = new Library { Songs = new Dictionary { { testGuid, song } } }; var decodedLibrary = Library.Decode(library.EncodeImmutably()); Assert.AreEqual(library, decodedLibrary); }