ONNX Runtime GenAI C# API

.gitignore

@@ -1,6 +1,13 @@
 # Files to tell git to ignore
 
+.vscode
 /src/.vs
+/src/csharp/.vs
+src/csharp/bin/
+src/csharp/obj/
+test/csharp/.vs
+test/csharp/bin/
+test/csharp/obj/
 /ort
 /build
 /test_models

CMakeLists.txt

@@ -10,6 +10,7 @@ include(cmake/options.cmake)
 include(cmake/external/onnxruntime_external_deps.cmake)
 # Checking if CUDA is supported
 include(CheckLanguage)
+add_compile_definitions(BUILDING_ORT_GENAI_C)
 if(USE_CUDA)
   check_language(CUDA)
 endif()
@@ -129,6 +130,8 @@ if(USE_CUDA AND NOT EXISTS "${ORT_LIB_DIR}/${ONNXRUNTIME_PROVIDERS_CUDA_LIB}")
 endif()
 
 file(GLOB onnxruntime_libs "${ORT_LIB_DIR}/${ONNXRUNTIME_FILES}")
+target_link_directories(onnxruntime-genai PRIVATE ${ORT_LIB_DIR})
+target_link_libraries(onnxruntime-genai PRIVATE ${ONNXRUNTIME_LIB})
 
 if(USE_CUDA AND CMAKE_CUDA_COMPILER)
   set_target_properties(onnxruntime-genai PROPERTIES LINKER_LANGUAGE CUDA)

build.py

@@ -102,6 +102,7 @@ def build(
     cudnn_home: str | bytes | os.PathLike | None = None,
     cmake_generator: str | None = None,
     ort_home: str | bytes | os.PathLike | None = None,
+    enable_csharp: bool = False,
 ):
     """Generates the CMake build tree and builds the project.
 
@@ -160,6 +161,19 @@ def build(
     make_command = ["cmake", "--build", ".", "--config", "Release"]
     run_subprocess(make_command, cwd="build", env=env).check_returncode()
 
+    if enable_csharp:
+        if not is_windows():
+            raise RuntimeError("C# API is only supported on Windows.")
+
+        dotnet = resolve_executable_path("dotnet")
+        csharp_build_command = [dotnet, "build", ".", "-c", "Release"]
+        run_subprocess(csharp_build_command, cwd=os.path.join("src", "csharp")).check_returncode()
+        properties = []
+        if ort_home:
+            properties += [f"/p:OrtHome={ort_home}"]
+        run_subprocess(csharp_build_command + properties, cwd=os.path.join("test", "csharp")).check_returncode()
+        run_subprocess([dotnet, "test", "-c", "Release"] + properties, cwd=os.path.join("test", "csharp")).check_returncode()
+
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
@@ -192,6 +206,7 @@ def build(
     )
     parser.add_argument("--skip_wheel", action="store_true", help="Skip building the Python wheel.")
     parser.add_argument("--ort_home", default=None, help="Root directory of onnxruntime.")
+    parser.add_argument("--enable_csharp", action="store_true", help="Build the C# API.")
     args = parser.parse_args()
 
     update_submodules()
@@ -201,4 +216,5 @@ def build(
         cudnn_home=args.cudnn_home,
         cmake_generator=args.cmake_generator,
         ort_home=args.ort_home,
+        enable_csharp=args.enable_csharp,
     )

cmake/options.cmake

@@ -2,7 +2,7 @@ include(CMakeDependentOption)
 
 option(USE_CUDA "Build with CUDA support" ON)
 option(NO_TOKENIZER "Don't include the Tokenizer" OFF)
-option(ENABLE_PYTHON "Enable python buildings" ON)
+option(ENABLE_PYTHON "Build the Python API." ON)
 option(ENABLE_TESTS "Enable tests" ON)
 
 cmake_dependent_option(BUILD_WHEEL "Build the python wheel" ON "ENABLE_PYTHON" OFF)

src/csharp/Generator.cs

@@ -0,0 +1,64 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+using System.Runtime.InteropServices;
+
+namespace Microsoft.ML.OnnxRuntimeGenAI
+{
+    public class Generator : IDisposable
+    {
+        private IntPtr _generatorHandle;
+
+        public Generator(Model model, GeneratorParams generatorParams)
+        {
+            Result.VerifySuccess(NativeMethods.OgaCreateGenerator(model.Handle, generatorParams.Handle, out _generatorHandle));
+        }
+
+        public bool IsDone()
+        {
+            return NativeMethods.OgaGenerator_IsDone(_generatorHandle);
+        }
+
+        public void ComputeLogits()
+        {
+            Result.VerifySuccess(NativeMethods.OgaGenerator_ComputeLogits(_generatorHandle));
+        }
+
+        public void GenerateNextTokenTop()
+        {
+            Result.VerifySuccess(NativeMethods.OgaGenerator_GenerateNextToken_Top(_generatorHandle));
+        }
+
+        public int[] GetSequence(int index)
+        {
+            IntPtr nullPtr = IntPtr.Zero;
+            UIntPtr sequenceLength = NativeMethods.OgaGenerator_GetSequenceLength(_generatorHandle, (IntPtr)index);
+            int[] sequence = new int[sequenceLength.ToUInt64()];
+            IntPtr sequencePtr = NativeMethods.OgaGenerator_GetSequence(_generatorHandle, (IntPtr)index);
+            Marshal.Copy(sequencePtr, sequence, 0, sequence.Length);
+
+            return sequence;
+        }
+
+        ~Generator()
+        {
+            Dispose(false);
+        }
+
+        public void Dispose()
+        {
+            Dispose(true);
+            GC.SuppressFinalize(this);
+        }
+
+        protected virtual void Dispose(bool disposing)
+        {
+            if (_generatorHandle != IntPtr.Zero)
+            {
+                NativeMethods.OgaDestroyGenerator(_generatorHandle);
+                _generatorHandle = IntPtr.Zero;
+            }
+        }
+    }
+}

src/csharp/GeneratorParams.cs

@@ -0,0 +1,53 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.InteropServices;
+
+namespace Microsoft.ML.OnnxRuntimeGenAI
+{
+    public class GeneratorParams : IDisposable
+    {
+        private IntPtr _generatorParamsHandle;
+
+        public GeneratorParams(Model model)
+        {
+            Result.VerifySuccess(NativeMethods.OgaCreateGeneratorParams(model.Handle, out _generatorParamsHandle));
+        }
+
+        internal IntPtr Handle { get { return _generatorParamsHandle; } }
+
+        public void SetMaxLength(int maxLength)
+        {
+            Result.VerifySuccess(NativeMethods.OgaGeneratorParamsSetMaxLength(_generatorParamsHandle, (IntPtr)maxLength));
+        }
+
+        public void SetInputIDs(IReadOnlyCollection<int> inputIDs, ulong sequenceLength, ulong batchSize)
+        {
+            int[] inputIDsArray = inputIDs.ToArray();
+            Result.VerifySuccess(NativeMethods.OgaGeneratorParamsSetInputIDs(_generatorParamsHandle, inputIDsArray, (UIntPtr)inputIDsArray.Length, (UIntPtr)sequenceLength, (UIntPtr)batchSize));
+        }
+
+        ~GeneratorParams()
+        {
+            Dispose(false);
+        }
+
+        public void Dispose()
+        {
+            Dispose(true);
+            GC.SuppressFinalize(this);
+        }
+
+        protected virtual void Dispose(bool disposing)
+        {
+            if (_generatorParamsHandle != IntPtr.Zero)
+            {
+                NativeMethods.OgaDestroyGeneratorParams(_generatorParamsHandle);
+                _generatorParamsHandle = IntPtr.Zero;
+            }
+        }
+    }
+}

src/csharp/Microsoft.ML.OnnxRuntimeGenAI.csproj

@@ -0,0 +1,55 @@
+<Project Sdk="MSBuild.Sdk.Extras/3.0.22">
+  <PropertyGroup>
+    <TargetFrameworks>netstandard2.1</TargetFrameworks>
+    <Platforms>x64</Platforms>
+    <LangVersion>default</LangVersion>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+
+    <!--internal build related properties-->
+    <SrcRoot>$(ProjectDir)..\..\src</SrcRoot>
+    <CsharpSrcRoot>$(SrcRoot)\csharp</CsharpSrcRoot>
+    <TargetArchitecture>x64</TargetArchitecture>
+
+    <RootNamespace>Microsoft.ML.OnnxRuntimeGenAI</RootNamespace>
+    <AssemblyName>Microsoft.ML.OnnxRuntimeGenAI</AssemblyName>
+    <EnableDefaultItems>false</EnableDefaultItems>
+
+    <PackageId>Microsoft.ML.OnnxRuntimeGenAI.Managed</PackageId>
+    <Authors>Microsoft</Authors>
+    <PackageVersion>1.0.0</PackageVersion>
+    <Version>$(PackageVersion)</Version>
+    <Description>This package contains ONNX Runtime Generative AI API for .Net platforms</Description>
+    <PackageTags>ONNX;ONNX Runtime;ONNX Runtime Gen AI;Machine Learning</PackageTags>
+    <PackageProjectUrl>https://github.com/microsoft/onnxruntime-genai</PackageProjectUrl>
+    <Copyright>© Microsoft Corporation. All rights reserved.</Copyright>
+    <PackageLicenseFile>LICENSE.txt</PackageLicenseFile>
+    <!-- sourcelink flags -->
+    <PublishRepositoryUrl>false</PublishRepositoryUrl>
+
+    <GenerateTargetFrameworkAttribute>false</GenerateTargetFrameworkAttribute>
+    <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
+    <Configurations>Debug;Release;RelWithDebInfo</Configurations>
+
+    <IsLinuxBuild Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Linux)))' == 'true'">true</IsLinuxBuild>
+    <IsWindowsBuild Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Windows)))' == 'true'">true</IsWindowsBuild>
+    <IsMacOSBuild Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::OSX)))' == 'true'">true</IsMacOSBuild>
+  </PropertyGroup>
+
+
+  <PropertyGroup Condition="'$(IsWindowsBuild)'=='true'">
+      <OnnxRuntimeGenAIBuildDirectory>$(CsharpSrcRoot)\..\..\build</OnnxRuntimeGenAIBuildDirectory>
+      <NativeBuildOutputDir>$(OnnxRuntimeGenAIBuildDirectory)\$(Configuration)\</NativeBuildOutputDir>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <None Include="$(NativeBuildOutputDir)\onnxruntime-genai.lib" Condition="Exists('$(NativeBuildOutputDir)\onnxruntime-genai.lib')" PackagePath="" Pack="false" CopyToOutputDirectory="Never" Visible="false" />
+    <None Include="$(NativeBuildOutputDir)\onnxruntime-genai.dll" Condition="Exists('$(NativeBuildOutputDir)\onnxruntime-genai.dll')" PackagePath="" Pack="false" CopyToOutputDirectory="PreserveNewest" Visible="false" />
+    <None Include="$(NativeBuildOutputDir)\onnxruntime.dll" Condition="Exists('$(NativeBuildOutputDir)\onnxruntime.dll')" PackagePath="" Pack="false" CopyToOutputDirectory="PreserveNewest" Visible="false" />
+    <None Include="$(CsharpSrcRoot)\..\..\LICENSE" PackagePath="" Pack="true" Visible="false" CopyToOutputDirectory="PreserveNewest" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Compile Include="**\*.cs" Link="%(Filename)%(Extension)" />
+  </ItemGroup>
+
+</Project>

src/csharp/Model.cs

@@ -0,0 +1,72 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+using System.Runtime.InteropServices;
+
+namespace Microsoft.ML.OnnxRuntimeGenAI
+{
+    public enum DeviceType : long
+    {
+        Auto = 0,
+        CPU = 1,
+        CUDA = 2
+    }
+
+    public class Model : IDisposable
+    {
+        private IntPtr _modelHandle;
+
+        public Model(string modelPath, DeviceType deviceType)
+        {
+            Result.VerifySuccess(NativeMethods.OgaCreateModel(Utils.ToUtf8(modelPath), deviceType, out _modelHandle));
+        }
+
+        internal IntPtr Handle { get { return _modelHandle; } }
+
+        public int[][] Generate(GeneratorParams generatorParams)
+        {
+            IntPtr nativeSequences = IntPtr.Zero;
+            Result.VerifySuccess(NativeMethods.OgaGenerate(_modelHandle, generatorParams.Handle, out nativeSequences));
+            try
+            {
+                ulong batchSize = NativeMethods.OgaSequencesCount(nativeSequences).ToUInt64();
+                int[][] sequences = new int[batchSize][];
+
+                for (ulong sequenceIndex = 0; sequenceIndex < batchSize; sequenceIndex++)
+                {
+                    ulong sequenceLength = NativeMethods.OgaSequencesGetSequenceCount(nativeSequences, (UIntPtr)sequenceIndex).ToUInt64();
+                    sequences[sequenceIndex] = new int[sequenceLength];
+                    IntPtr sequencePtr = NativeMethods.OgaSequencesGetSequenceData(nativeSequences, (UIntPtr)sequenceIndex);
+                    Marshal.Copy(sequencePtr, sequences[sequenceIndex], 0, sequences[sequenceIndex].Length);
+                }
+
+                return sequences;
+            }
+            finally
+            {
+                NativeMethods.OgaDestroySequences(nativeSequences);
+            }
+        }
+
+        ~Model()
+        {
+            Dispose(false);
+        }
+
+        public void Dispose()
+        {
+            Dispose(true);
+            GC.SuppressFinalize(this);
+        }
+
+        protected virtual void Dispose(bool disposing)
+        {
+            if (_modelHandle != IntPtr.Zero)
+            {
+                NativeMethods.OgaDestroyModel(_modelHandle);
+                _modelHandle = IntPtr.Zero;
+            }
+        }
+    }
+}