Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

LibraryImport conversion #13

Draft
wants to merge 45 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
45 commits
Select commit Hold shift + click to select a range
5865e66
Target only net7.0 and drop netstandard2.0/net6.0 support
jzebedee May 15, 2023
66b1d5c
Enable unsafe code
jzebedee May 15, 2023
4bcb92f
Replace DllImport with LibraryImport
jzebedee May 15, 2023
d040962
Add function pointer overload of libdeflate_set_memory_allocator and …
jzebedee May 15, 2023
3583793
Make benchmarks a friend of LibDeflate
jzebedee May 15, 2023
9c327a1
Use NativeMemory for unsafe malloc/free tests
jzebedee May 15, 2023
d402710
Use top-level main in benchmarks project
jzebedee May 15, 2023
766f50c
Add custom allocator benchmarks
jzebedee May 15, 2023
514df5a
Bump test package versions
jzebedee Sep 17, 2023
6f97def
Bump benchmark package versions
jzebedee Sep 17, 2023
72528f0
Bump polysharp version
jzebedee Sep 17, 2023
ebcd829
Add libdeflate_options
jzebedee Sep 18, 2023
fe8bda6
Add CustomMemoryAllocatorTests
jzebedee Sep 18, 2023
3caba62
Add more documentation to libdeflate_options
jzebedee Sep 18, 2023
84744d2
Add libdeflate_alloc_compressor_ex
jzebedee Sep 18, 2023
4414230
Add libdeflate_alloc_decompressor_ex
jzebedee Sep 18, 2023
2091152
Bump package version and native package version to v1.19.0
jzebedee Sep 18, 2023
309a690
Break out libdeflate_options from CustomMemoryAllocator
jzebedee Sep 18, 2023
cc63207
Only enable trimming for TFMs that support it
jzebedee Sep 19, 2023
dbc0167
Fix expected and actual being reversed
jzebedee Sep 19, 2023
ef2a213
Fixup style
jzebedee Sep 19, 2023
896afd3
Fixup style
jzebedee Sep 19, 2023
5c56cde
Use range slices
jzebedee Sep 19, 2023
c5165c8
Add ExactSpelling = true to native imports
jzebedee Sep 19, 2023
db9df5a
Add throw helpers
jzebedee Sep 19, 2023
0dfe191
Move calling convention for native imports into Constants
jzebedee Sep 19, 2023
5f63118
Rebuild ZlibStream without vulnerable deps and target netstandard2.0
jzebedee Sep 20, 2023
2aa3813
Update test projects
jzebedee Sep 20, 2023
905c127
Set langVersion to preview
jzebedee Sep 20, 2023
3724f4c
Target only net7.0 and drop netstandard2.0/net6.0 support
jzebedee May 15, 2023
5e10776
Enable unsafe code
jzebedee May 15, 2023
472cfe3
Replace DllImport with LibraryImport
jzebedee May 15, 2023
f578c88
Add function pointer overload of libdeflate_set_memory_allocator and …
jzebedee May 15, 2023
9a8da23
Make benchmarks a friend of LibDeflate
jzebedee May 15, 2023
f6c43c9
Use NativeMemory for unsafe malloc/free tests
jzebedee May 15, 2023
f863236
Use top-level main in benchmarks project
jzebedee May 15, 2023
aa984d3
Add custom allocator benchmarks
jzebedee May 15, 2023
b58827b
Add broken benchmarks
jzebedee May 20, 2023
727f08e
Use unsafe options
jzebedee Jan 28, 2024
f0019ca
Merge local
jzebedee Jan 28, 2024
38b683c
Use compact collection expression
jzebedee Jan 28, 2024
1881194
Use collection expressions
jzebedee Jan 28, 2024
56cc137
Simplify checksum impls and add readonly modifiers
jzebedee Jan 28, 2024
c357dce
Simplify type names
jzebedee Jan 28, 2024
16558da
Disable runtime marshalling
jzebedee Jan 28, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ jobs:
- uses: actions/setup-dotnet@v3
with:
dotnet-version: |
6.0.x
7.0.x
- name: Install dependencies
run: dotnet restore
Expand Down
1 change: 0 additions & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ jobs:
- uses: actions/setup-dotnet@v3
with:
dotnet-version: |
6.0.x
7.0.x
- name: Pack
run: dotnet pack -c Release -o pkg
Expand Down
57 changes: 57 additions & 0 deletions bench/LibDeflate.Benchmarks/CustomAllocatorBenchmarks.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
using BenchmarkDotNet.Attributes;
using LibDeflate.Imports;
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;

namespace LibDeflate.Benchmarks;

[MemoryDiagnoser]
[SimpleJob]
public class CustomAllocatorBenchmarks
{
[GlobalSetup(Target = nameof(CompressorAllocCustom))]
public void SetCustomAllocator()
{
Console.WriteLine("Custom Allocator: set");
CustomMemoryAllocator.libdeflate_set_memory_allocator(malloc, free);

static nint malloc(nuint len) => Marshal.AllocHGlobal((nint)len);

static void free(nint alloc) => Marshal.FreeHGlobal(alloc);
}

[GlobalSetup(Target = nameof(CompressorAllocCustomUnsafe))]
public unsafe void SetCustomAllocatorUnsafe()
{
Console.WriteLine("Custom Unsafe Allocator: set");
CustomMemoryAllocator.libdeflate_set_memory_allocator_unsafe(&malloc_unsafe, &free_unsafe);

[UnmanagedCallersOnly(CallConvs = new[] { typeof(CallConvCdecl) })]
static unsafe void* malloc_unsafe(nuint len) => NativeMemory.Alloc(len);

[UnmanagedCallersOnly(CallConvs = new[] { typeof(CallConvCdecl) })]
static unsafe void free_unsafe(void* alloc) => NativeMemory.Free(alloc);
}

[Benchmark(Baseline = true)]
public void CompressorAlloc()
{
var compressor = Compression.libdeflate_alloc_compressor(0);
Compression.libdeflate_free_compressor(compressor);
}

[Benchmark]
public void CompressorAllocCustom()
{
var compressor = Compression.libdeflate_alloc_compressor(0);
Compression.libdeflate_free_compressor(compressor);
}

[Benchmark]
public void CompressorAllocCustomUnsafe()
{
var compressor = Compression.libdeflate_alloc_compressor(0);
Compression.libdeflate_free_compressor(compressor);
}
}
84 changes: 63 additions & 21 deletions bench/LibDeflate.Benchmarks/DeflateCompressorBenchmarks.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,56 +4,98 @@
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Runtime.CompilerServices;

namespace LibDeflate.Benchmarks;

[MemoryDiagnoser]
[SimpleJob]
public class DeflateCompressorBenchmarks
{
public static IEnumerable<object[]> Inputs => from key in TestFiles.Keys
from level in Levels
select new object[] { key, level };
private static string AssetBase
{
get
{
var cwd = Directory.GetCurrentDirectory();

private static Dictionary<string, byte[]> TestFiles { get; } = Directory.EnumerateFiles(@"texts/")
.Where(fn => !Path.GetExtension(fn).Equals(".gz", StringComparison.OrdinalIgnoreCase))
.ToDictionary(fn => Path.GetFileNameWithoutExtension(fn), File.ReadAllBytes);
string assetsDir;
while (!Directory.Exists(assetsDir = Path.Join(cwd, "assets")))
{
cwd = Path.GetDirectoryName(cwd);
}

return assetsDir;
}
}

private static IEnumerable<int> Levels
{
get
{
//yield return 0;
yield return 0;
yield return 1;
//yield return 6;
//yield return 9;
yield return -1;
yield return 9;
}
}

[Benchmark(Baseline = true)]
[ArgumentsSource(nameof(Inputs))]
public void DeflateSIO(string testFile, int level)
[GlobalSetup]
public static void PrepareTestAssets()
{
var compressionLevel = level switch
var assetsFolder = Path.Join(AssetBase, "UncompressedTestFiles");
var testFiles = new Dictionary<string, byte[]>();
foreach (var file in Directory.EnumerateFiles(assetsFolder, null, SearchOption.AllDirectories))
{
0 => CompressionLevel.NoCompression,
1 => CompressionLevel.Fastest,
6 => CompressionLevel.Optimal,
9 => CompressionLevel.SmallestSize
};
var key = Path.GetRelativePath(assetsFolder, file);
testFiles.Add(key, File.ReadAllBytes(file));
}

TestFiles = testFiles;
}

public static Dictionary<string, byte[]> TestFiles { get; set; }

public static IEnumerable<object[]> GetInputs() => from key in TestFiles.Keys
from level in Levels
select new object[] { key, level };

#pragma warning disable CS8509 // The switch expression does not handle all possible values of its input type (it is not exhaustive).
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static CompressionLevel ToLevelEnum(int level) => level switch
{
0 => CompressionLevel.NoCompression,
1 => CompressionLevel.Fastest,
-1 => CompressionLevel.Optimal,
9 => CompressionLevel.SmallestSize
};
#pragma warning restore CS8509 // The switch expression does not handle all possible values of its input type (it is not exhaustive).

[Benchmark(Baseline = true)]
[ArgumentsSource(nameof(GetInputs))]
public void DeflateSIO(string testFile, int level)
{
var input = TestFiles[testFile];
using var outputMs = new MemoryStream(input.Length);
using var deflateStream = new DeflateStream(outputMs, compressionLevel);
using var deflateStream = new DeflateStream(outputMs, ToLevelEnum(level));
deflateStream.Write(input);
}

[Benchmark]
[ArgumentsSource(nameof(Inputs))]
public void DeflateLibdeflate(string testFile, int level)
[ArgumentsSource(nameof(GetInputs))]
public void DeflateLibdeflate_MemoryOwner(string testFile, int level)
{
var input = TestFiles[testFile];
using var compressor = new DeflateCompressor(level);
using var owner = compressor.Compress(input);
}

[Benchmark]
[ArgumentsSource(nameof(GetInputs))]
public void DeflateLibdeflate_Buffer(string testFile, int level)
{
var input = TestFiles[testFile];
using var compressor = new DeflateCompressor(level);
var output = new byte[input.Length];
var bytesWritten = compressor.Compress(input, output);
}
}
5 changes: 3 additions & 2 deletions bench/LibDeflate.Benchmarks/LibDeflate.Benchmarks.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,13 @@

<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net6.0</TargetFramework>
<TargetFramework>net7.0</TargetFramework>
<IsPackable>false</IsPackable>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="BenchmarkDotNet" Version="0.13.5" />
<PackageReference Include="BenchmarkDotNet" Version="0.13.8" />
</ItemGroup>

<ItemGroup>
Expand Down
7 changes: 1 addition & 6 deletions bench/LibDeflate.Benchmarks/Program.cs
Original file line number Diff line number Diff line change
@@ -1,8 +1,3 @@
using BenchmarkDotNet.Running;

namespace LibDeflate.Benchmarks;

public class Program
{
public static void Main(string[] args) => BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args);
}
BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args);
21 changes: 5 additions & 16 deletions src/LibDeflate/Checksums/Adler32.cs
Original file line number Diff line number Diff line change
Expand Up @@ -7,25 +7,14 @@ namespace LibDeflate.Checksums;

public struct Adler32
{
private bool _initialized;
//because we have to supply 1 as the initial value
//we must always retrieve this through Hash
private uint _currentAdler;
public uint Hash
{
get
{
if (!_initialized)
{
_currentAdler = 1;
_initialized = true;
}
return _currentAdler;
}
}

public Adler32() => _currentAdler = 1;

public readonly uint Hash => _currentAdler;

public void Append(ReadOnlySpan<byte> input)
=> _currentAdler = AppendCore(Hash, input);
=> _currentAdler = AppendCore(_currentAdler, input);

public uint Compute(ReadOnlySpan<byte> input)
=> _currentAdler = AppendCore(1, input);
Expand Down
2 changes: 1 addition & 1 deletion src/LibDeflate/Checksums/Crc32.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ public struct Crc32
{
private uint _currentCrc;

public uint Hash => _currentCrc;
public readonly uint Hash => _currentCrc;

public void Append(ReadOnlySpan<byte> input)
=> _currentCrc = AppendCore(_currentCrc, input);
Expand Down
2 changes: 1 addition & 1 deletion src/LibDeflate/Compressor.cs
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ protected Compressor(int compressionLevel)
return null;
}

return output.Slice(0, (int)bytesWritten);
return output[..(int)bytesWritten];
}
catch
{
Expand Down
16 changes: 9 additions & 7 deletions src/LibDeflate/Imports/Checksums.cs
Original file line number Diff line number Diff line change
@@ -1,27 +1,29 @@
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;

namespace LibDeflate.Imports;

using size_t = System.UIntPtr;
using size_t = nuint;

internal static class Checksums
internal static partial class Checksums
{
///<summary>
/// libdeflate_adler32() updates a running Adler-32 checksum with 'len' bytes of
/// data and returns the updated checksum. When starting a new checksum, the
/// required initial value for 'adler' is 1. This value is also returned when
/// 'buffer' is specified as NULL.
///</summary>
[DllImport(Constants.DllName, CallingConvention = CallingConvention.Cdecl)]
public static extern UInt32 libdeflate_adler32(UInt32 adler, in byte buffer, size_t len);
[LibraryImport(Constants.DllName)]
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
public static partial uint libdeflate_adler32(uint adler, in byte buffer, size_t len);

///<summary>
/// libdeflate_crc32() updates a running CRC-32 checksum with 'len' bytes of data
/// and returns the updated checksum. When starting a new checksum, the required
/// initial value for 'crc' is 0. This value is also returned when 'buffer' is
/// specified as NULL.
///</summary>
[DllImport(Constants.DllName, CallingConvention = CallingConvention.Cdecl)]
public static extern UInt32 libdeflate_crc32(UInt32 crc, in byte buffer, size_t len);
[LibraryImport(Constants.DllName)]
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
public static partial uint libdeflate_crc32(uint crc, in byte buffer, size_t len);
}
Loading
Loading