Merge pull request #6470 from frenzibyte/ios-vimage

Use Accelerate framework for texture image processing on iOS and fix premultiplication issues

osu.Framework.Tests/Visual/Graphics/TestSceneTexturePremultiplication.cs

@@ -0,0 +1,128 @@
+// Copyright (c) ppy Pty Ltd <[email protected]>. Licensed under the MIT Licence.
+// See the LICENCE file in the repository root for full licence text.
+
+using System.Collections.Generic;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+using NUnit.Framework;
+using osu.Framework.Allocation;
+using osu.Framework.Graphics;
+using osu.Framework.Graphics.Containers;
+using osu.Framework.Graphics.Shapes;
+using osu.Framework.Graphics.Sprites;
+using osu.Framework.Graphics.Textures;
+using osu.Framework.IO.Stores;
+using osu.Framework.Platform;
+using osuTK;
+using osuTK.Graphics;
+using SixLabors.ImageSharp;
+using SixLabors.ImageSharp.PixelFormats;
+
+namespace osu.Framework.Tests.Visual.Graphics
+{
+    public partial class TestSceneTexturePremultiplication : FrameworkTestScene
+    {
+        private TextureStore textures = null!;
+
+        [BackgroundDependencyLoader]
+        private void load(GameHost host)
+        {
+            textures = new TextureStore(host.Renderer, host.CreateTextureLoaderStore(new CustomResourceStore()), false, TextureFilteringMode.Nearest);
+        }
+
+        [Test]
+        public void TestComparison()
+        {
+            AddStep("setup", () =>
+            {
+                Child = new FillFlowContainer
+                {
+                    Anchor = Anchor.Centre,
+                    Origin = Anchor.Centre,
+                    AutoSizeAxes = Axes.Both,
+                    Spacing = new Vector2(0f, 5f),
+                    Direction = FillDirection.Vertical,
+                    Children = new Drawable[]
+                    {
+                        new Container
+                        {
+                            AutoSizeAxes = Axes.Both,
+                            Children = new[]
+                            {
+                                new Box
+                                {
+                                    Size = new Vector2(256, 128),
+                                    Colour = Color4.Blue,
+                                },
+                                new Sprite
+                                {
+                                    Texture = textures.Get("zero-to-red"),
+                                    Size = new Vector2(256, 128),
+                                }
+                            },
+                        },
+                        new SpriteText
+                        {
+                            Text = "Rendering of the sprite above should be identical to the one below",
+                        },
+                        new Sprite
+                        {
+                            Texture = textures.Get("blue-to-red"),
+                            Size = new Vector2(256, 128),
+                        },
+                    }
+                };
+            });
+        }
+
+        private class CustomResourceStore : IResourceStore<byte[]>
+        {
+            public byte[] Get(string name) => throw new System.NotImplementedException();
+            public Task<byte[]> GetAsync(string name, CancellationToken cancellationToken = default) => throw new System.NotImplementedException();
+
+            public Stream GetStream(string name)
+            {
+                switch (name)
+                {
+                    case "zero-to-red":
+                    {
+                        var memoryStream = new MemoryStream();
+
+                        Image<Rgba32> image = new Image<Rgba32>(256, 1);
+
+                        for (int i = 0; i < 256; i++)
+                            image[i, 0] = new Rgba32(255, 0, 0, (byte)i);
+
+                        image.SaveAsPng(memoryStream);
+                        memoryStream.Seek(0, SeekOrigin.Begin);
+                        return memoryStream;
+                    }
+
+                    case "blue-to-red":
+                    {
+                        var memoryStream = new MemoryStream();
+
+                        Image<Rgba32> image = new Image<Rgba32>(256, 1);
+
+                        for (int i = 0; i < 256; i++)
+                            image[i, 0] = new Rgba32((byte)i, 0, (byte)(255 - i), 255);
+
+                        image.SaveAsPng(memoryStream);
+                        memoryStream.Seek(0, SeekOrigin.Begin);
+                        return memoryStream;
+                    }
+
+                    default:
+                        return Stream.Null;
+                }
+            }
+
+            public IEnumerable<string> GetAvailableResources() => new[] { "zero-to-red", "blue-to-red" };
+
+            public void Dispose()
+            {
+            }
+        }
+    }
+}

osu.Framework.iOS/Graphics/Textures/IOSTextureLoaderStore.cs

@@ -2,12 +2,17 @@
 // See the LICENCE file in the repository root for full licence text.
 
 using System;
+using System.Diagnostics;
 using System.IO;
+using System.Runtime.InteropServices;
+using Accelerate;
 using CoreGraphics;
 using Foundation;
+using ObjCRuntime;
 using osu.Framework.Graphics.Textures;
 using osu.Framework.IO.Stores;
 using SixLabors.ImageSharp;
+using SixLabors.ImageSharp.Processing;
 using UIKit;
 
 namespace osu.Framework.iOS.Graphics.Textures
@@ -19,7 +24,7 @@ public IOSTextureLoaderStore(IResourceStore<byte[]> store)
         {
         }
 
-        protected override Image<TPixel> ImageFromStream<TPixel>(Stream stream)
+        protected override unsafe Image<TPixel> ImageFromStream<TPixel>(Stream stream)
         {
             using (var nativeData = NSData.FromStream(stream))
             {
@@ -33,17 +38,64 @@ protected override Image<TPixel> ImageFromStream<TPixel>(Stream stream)
                     int width = (int)uiImage.Size.Width;
                     int height = (int)uiImage.Size.Height;
 
-                    // TODO: Use pool/memory when builds success with Xamarin.
-                    // Probably at .NET Core 3.1 time frame.
-                    byte[] data = new byte[width * height * 4];
-                    using (CGBitmapContext textureContext = new CGBitmapContext(data, width, height, 8, width * 4, CGColorSpace.CreateDeviceRGB(), CGImageAlphaInfo.PremultipliedLast))
-                        textureContext.DrawImage(new CGRect(0, 0, width, height), uiImage.CGImage);
+                    var format = new vImage_CGImageFormat
+                    {
+                        BitsPerComponent = 8,
+                        BitsPerPixel = 32,
+                        ColorSpace = CGColorSpace.CreateDeviceRGB().Handle,
+                        // notably, iOS generally uses premultiplied alpha when rendering image to pixels via CGBitmapContext or otherwise,
+                        // but vImage offers using straight alpha directly without any conversion from our side (by specifying Last instead of PremultipliedLast).
+                        BitmapInfo = (CGBitmapFlags)CGImageAlphaInfo.Last,
+                        Decode = null,
+                        RenderingIntent = CGColorRenderingIntent.Default,
+                    };
 
-                    var image = Image.LoadPixelData<TPixel>(data, width, height);
+                    vImageBuffer accelerateImage = default;
 
+                    // perform initial call to retrieve preferred alignment and bytes-per-row values for the given image dimensions.
+                    long alignment = (long)vImageBuffer_Init(&accelerateImage, (uint)height, (uint)width, 32, vImageFlags.NoAllocate);
+                    Debug.Assert(alignment > 0);
+
+                    // allocate aligned memory region to contain image pixel data.
+                    int bytesCount = accelerateImage.BytesPerRow * accelerateImage.Height;
+                    accelerateImage.Data = (IntPtr)NativeMemory.AlignedAlloc((nuint)(accelerateImage.BytesPerRow * accelerateImage.Height), (nuint)alignment);
+
+                    var result = vImageBuffer_InitWithCGImage(&accelerateImage, &format, null, uiImage.CGImage!.Handle, vImageFlags.NoAllocate);
+                    Debug.Assert(result == vImageError.NoError);
+
+                    var dataSpan = new ReadOnlySpan<byte>(accelerateImage.Data.ToPointer(), bytesCount);
+
+                    int stride = accelerateImage.BytesPerRow / 4;
+                    var image = Image.LoadPixelData<TPixel>(dataSpan, stride, height);
+                    image.Mutate(i => i.Crop(width, height));
+
+                    NativeMemory.AlignedFree(accelerateImage.Data.ToPointer());
                     return image;
                 }
             }
         }
+
+        #region Accelerate API
+
+        [DllImport(Constants.AccelerateLibrary)]
+        private static extern unsafe vImageError vImageBuffer_Init(vImageBuffer* buf, uint height, uint width, uint pixelBits, vImageFlags flags);
+
+        [DllImport(Constants.AccelerateLibrary)]
+        private static extern unsafe vImageError vImageBuffer_InitWithCGImage(vImageBuffer* buf, vImage_CGImageFormat* format, nfloat* backgroundColour, NativeHandle image, vImageFlags flags);
+
+        // ReSharper disable once InconsistentNaming
+        [StructLayout(LayoutKind.Sequential)]
+        public unsafe struct vImage_CGImageFormat
+        {
+            public uint BitsPerComponent;
+            public uint BitsPerPixel;
+            public NativeHandle ColorSpace;
+            public CGBitmapFlags BitmapInfo;
+            public uint Version;
+            public nfloat* Decode;
+            public CGColorRenderingIntent RenderingIntent;
+        }
+
+        #endregion
     }
 }