diff --git a/.gitignore b/.gitignore index 60b60827f..9f3d82131 100644 --- a/.gitignore +++ b/.gitignore @@ -22,4 +22,6 @@ examples/python/genai_models examples/python/hf_cache !test/test_models/hf-internal-testing/ -!test/test_models/hf-internal-testing/tiny-random-gpt2*/*.onnx \ No newline at end of file +!test/test_models/hf-internal-testing/tiny-random-gpt2*/*.onnx +/onnxruntime-android-1.17.1 +/onnxruntime-win-x64-1.17.0 diff --git a/examples/android/ORTGenAIDemo/.gitignore b/examples/android/ORTGenAIDemo/.gitignore new file mode 100644 index 000000000..951e883e1 --- /dev/null +++ b/examples/android/ORTGenAIDemo/.gitignore @@ -0,0 +1,16 @@ +*.iml +.gradle +/local.properties +/.idea/caches +/.idea/libraries +/.idea/modules.xml +/.idea/workspace.xml +/.idea/navEditor.xml +/.idea/assetWizardSettings.xml +.DS_Store +/build +/captures +.externalNativeBuild +.cxx +local.properties + diff --git a/examples/android/ORTGenAIDemo/DeviceExplorerScreenshot.png b/examples/android/ORTGenAIDemo/DeviceExplorerScreenshot.png new file mode 100644 index 000000000..7e2a43464 Binary files /dev/null and b/examples/android/ORTGenAIDemo/DeviceExplorerScreenshot.png differ diff --git a/examples/android/ORTGenAIDemo/ReadMe.md b/examples/android/ORTGenAIDemo/ReadMe.md new file mode 100644 index 000000000..1d332f859 --- /dev/null +++ b/examples/android/ORTGenAIDemo/ReadMe.md @@ -0,0 +1,20 @@ +Follow these instructions to get phi-2: https://github.com/microsoft/onnxruntime-genai/blob/main/examples/csharp/README.md + +The folder is too large to add to the app `assets` folder so we have to manually get it onto the device. + +From Android Studio: + - create (if necessary) and run your emulator + - make sure it has at least 8GB of internal storage + - debug/run the app so it's deployed to the device and creates it's `files` directory + - expected to be `/data/data/ai.onnxruntime.genai.demo/files` + - this is the path returned by `getFilesDir()` in MainActivity + - not sure what triggers the creation of this directory. + - if you debug the app and set a breakpoint on the call to create the GenAIWrapper instance in MainActivity.onCreate it should exist when the breakpoint is hit + - Open Device Explorer in Android Studio + - Navigate to `/data/data/ai.onnxruntime.genai.demo/files` + - adjust as needed if the value returned by getFilesDir() differs for your emulator + - copy the whole phi-2 folder (should be called phi2-int4-cpu) to the `files` directory + - it should look like this: + - ![device explorer screenshot](DeviceExplorerScreenshot.png) + +The rest _should_ work OOTB (in theory). diff --git a/examples/android/ORTGenAIDemo/app/.gitignore b/examples/android/ORTGenAIDemo/app/.gitignore new file mode 100644 index 000000000..42afabfd2 --- /dev/null +++ b/examples/android/ORTGenAIDemo/app/.gitignore @@ -0,0 +1 @@ +/build \ No newline at end of file diff --git a/examples/android/ORTGenAIDemo/app/build.gradle.kts b/examples/android/ORTGenAIDemo/app/build.gradle.kts new file mode 100644 index 000000000..3598433ca --- /dev/null +++ b/examples/android/ORTGenAIDemo/app/build.gradle.kts @@ -0,0 +1,63 @@ +plugins { + id("com.android.application") +} + +android { + namespace = "ai.onnxruntime.genai.demo" + compileSdk = 33 + + defaultConfig { + applicationId = "ai.onnxruntime.genai.demo" + minSdk = 27 + targetSdk = 33 + versionCode = 1 + versionName = "1.0" + + testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + externalNativeBuild { + cmake { + cppFlags += "-std=c++17" + } + } + + ndk { + //noinspection ChromeOsAbiSupport + abiFilters += listOf("arm64-v8a", "x86_64") + } + } + + buildTypes { + release { + isMinifyEnabled = false + proguardFiles( + getDefaultProguardFile("proguard-android-optimize.txt"), + "proguard-rules.pro" + ) + } + } + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 + } + + externalNativeBuild { + cmake { + path = file("src/main/cpp/CMakeLists.txt") + version = "3.22.1" + } + } + buildFeatures { + viewBinding = true + } +} + +dependencies { + + implementation("androidx.appcompat:appcompat:1.6.1") + implementation("com.google.android.material:material:1.9.0") + implementation("androidx.constraintlayout:constraintlayout:2.1.4") + testImplementation("junit:junit:4.13.2") + androidTestImplementation("androidx.test.ext:junit:1.1.5") + androidTestImplementation("androidx.test.espresso:espresso-core:3.5.1") +} \ No newline at end of file diff --git a/examples/android/ORTGenAIDemo/app/proguard-rules.pro b/examples/android/ORTGenAIDemo/app/proguard-rules.pro new file mode 100644 index 000000000..481bb4348 --- /dev/null +++ b/examples/android/ORTGenAIDemo/app/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile \ No newline at end of file diff --git a/examples/android/ORTGenAIDemo/app/src/androidTest/java/ai/onnxruntime/genai/demo/ExampleInstrumentedTest.java b/examples/android/ORTGenAIDemo/app/src/androidTest/java/ai/onnxruntime/genai/demo/ExampleInstrumentedTest.java new file mode 100644 index 000000000..d6a9ae130 --- /dev/null +++ b/examples/android/ORTGenAIDemo/app/src/androidTest/java/ai/onnxruntime/genai/demo/ExampleInstrumentedTest.java @@ -0,0 +1,26 @@ +package ai.onnxruntime.genai.demo; + +import android.content.Context; + +import androidx.test.platform.app.InstrumentationRegistry; +import androidx.test.ext.junit.runners.AndroidJUnit4; + +import org.junit.Test; +import org.junit.runner.RunWith; + +import static org.junit.Assert.*; + +/** + * Instrumented test, which will execute on an Android device. + * + * @see Testing documentation + */ +@RunWith(AndroidJUnit4.class) +public class ExampleInstrumentedTest { + @Test + public void useAppContext() { + // Context of the app under test. + Context appContext = InstrumentationRegistry.getInstrumentation().getTargetContext(); + assertEquals("ai.onnxruntime.genai.demo", appContext.getPackageName()); + } +} \ No newline at end of file diff --git a/examples/android/ORTGenAIDemo/app/src/main/AndroidManifest.xml b/examples/android/ORTGenAIDemo/app/src/main/AndroidManifest.xml new file mode 100644 index 000000000..c735c2d83 --- /dev/null +++ b/examples/android/ORTGenAIDemo/app/src/main/AndroidManifest.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/examples/android/ORTGenAIDemo/app/src/main/cpp/CMakeLists.txt b/examples/android/ORTGenAIDemo/app/src/main/cpp/CMakeLists.txt new file mode 100644 index 000000000..2401f11dc --- /dev/null +++ b/examples/android/ORTGenAIDemo/app/src/main/cpp/CMakeLists.txt @@ -0,0 +1,43 @@ +# For more information about using CMake with Android Studio, read the +# documentation: https://d.android.com/studio/projects/add-native-code.html. +# For more examples on how to use CMake, see https://github.com/android/ndk-samples. + +# Sets the minimum CMake version required for this project. +cmake_minimum_required(VERSION 3.22.1) + +# Declares the project name. The project name can be accessed via ${ PROJECT_NAME}, +# Since this is the top level CMakeLists.txt, the project name is also accessible +# with ${CMAKE_PROJECT_NAME} (both CMake variables are in-sync within the top level +# build script scope). +project("genai") + +#set(APP_LIB_DIR ${PROJECT_SOURCE_DIR}/../../../libs) + +# Creates and names a library, sets it as either STATIC +# or SHARED, and provides the relative paths to its source code. +# You can define multiple libraries, and CMake builds them for you. +# Gradle automatically packages shared libraries with your APK. +# +# In this top level CMakeLists.txt, ${CMAKE_PROJECT_NAME} is used to define +# the target library name; in the sub-module's CMakeLists.txt, ${PROJECT_NAME} +# is preferred for the same purpose. +# +# In order to load a library into your app from Java/Kotlin, you must call +# System.loadLibrary() and pass the name of the library defined here; +# for GameActivity/NativeActivity derived applications, the same library name must be +# used in the AndroidManifest.xml file. +add_library(${CMAKE_PROJECT_NAME} SHARED + # List C/C++ source files with relative paths to this CMakeLists.txt. + native-lib.cpp) + +target_link_directories(${CMAKE_PROJECT_NAME} PRIVATE ${PROJECT_SOURCE_DIR}/../jniLibs/${CMAKE_ANDROID_ARCH_ABI}) + +# Specifies libraries CMake should link to your target library. You +# can link libraries from various origins, such as libraries defined in this +# build script, prebuilt third-party libraries, or Android system libraries. +target_link_libraries(${CMAKE_PROJECT_NAME} + # List libraries link to the target library + onnxruntime-genai + onnxruntime + android + log) diff --git a/examples/android/ORTGenAIDemo/app/src/main/cpp/native-lib.cpp b/examples/android/ORTGenAIDemo/app/src/main/cpp/native-lib.cpp new file mode 100644 index 000000000..008a7a56f --- /dev/null +++ b/examples/android/ORTGenAIDemo/app/src/main/cpp/native-lib.cpp @@ -0,0 +1,201 @@ +#include +#include +#include + +#include + +#include "ort_genai_c.h" + +namespace { + void ThrowException(JNIEnv *env, OgaResult *result) { + __android_log_write(ANDROID_LOG_DEBUG, "native", "ThrowException"); + // copy error so we can release the OgaResult + jstring jerr_msg = env->NewStringUTF(OgaResultGetError(result)); + OgaDestroyResult(result); + + static const char *className = "ai/onnxruntime/genai/demo/GenAIException"; + jclass exClazz = env->FindClass(className); + jmethodID exConstructor = env->GetMethodID(exClazz, "", "(Ljava/lang/String;)V"); + jobject javaException = env->NewObject(exClazz, exConstructor, jerr_msg); + env->Throw(static_cast(javaException)); + } + + void ThrowIfError(JNIEnv *env, OgaResult *result) { + if (result != nullptr) { + ThrowException(env, result); + } + } + + // handle conversion/release of jstring to const char* + struct CString { + CString(JNIEnv *env, jstring str) + : env_{env}, str_{str}, cstr{env->GetStringUTFChars(str, /* isCopy */ nullptr)} { + } + + const char *cstr; + + operator const char *() const { return cstr; } + + ~CString() { + env_->ReleaseStringUTFChars(str_, cstr); + } + + private: + JNIEnv *env_; + jstring str_; + }; +} + +extern "C" JNIEXPORT jlong JNICALL +Java_ai_onnxruntime_genai_demo_GenAIWrapper_loadModel(JNIEnv *env, jobject thiz, jstring model_path) { + CString path{env, model_path}; + __android_log_print(ANDROID_LOG_DEBUG, "native", "loadModel %s", path.cstr); + + OgaModel *model = nullptr; + OgaResult *result = OgaCreateModel(path, &model); + __android_log_print(ANDROID_LOG_DEBUG, "native", "model address %p", model); + + ThrowIfError(env, result); + + return (jlong)model; +} + +extern "C" JNIEXPORT void JNICALL +Java_ai_onnxruntime_genai_demo_GenAIWrapper_releaseModel(JNIEnv *env, jobject thiz, jlong native_model) { + auto* model = reinterpret_cast(native_model); + __android_log_print(ANDROID_LOG_DEBUG, "native", "releaseModel: %p", model); + OgaDestroyModel(model); +} + +extern "C" JNIEXPORT jlong JNICALL +Java_ai_onnxruntime_genai_demo_GenAIWrapper_createTokenizer(JNIEnv *env, jobject thiz, jlong native_model) { + const auto* model = reinterpret_cast(native_model); + OgaTokenizer *tokenizer = nullptr; + OgaResult* result = OgaCreateTokenizer(model, &tokenizer); + __android_log_print(ANDROID_LOG_DEBUG, "native", "tokenizer address: %p", tokenizer); + + ThrowIfError(env, result); + + return (jlong)tokenizer; +} + +extern "C" JNIEXPORT void JNICALL +Java_ai_onnxruntime_genai_demo_GenAIWrapper_releaseTokenizer(JNIEnv *env, jobject thiz, jlong native_tokenizer) { + auto* tokenizer = reinterpret_cast(native_tokenizer); + __android_log_print(ANDROID_LOG_DEBUG, "native", "releaseTokenizer: %p", tokenizer); + OgaDestroyTokenizer(tokenizer); +} + +extern "C" +JNIEXPORT jstring JNICALL +Java_ai_onnxruntime_genai_demo_GenAIWrapper_run(JNIEnv *env, jobject thiz, jlong native_model, jlong native_tokenizer, + jstring jprompt, jboolean use_callback) { + using SequencesPtr = std::unique_ptr>; + using GeneratorParamsPtr = std::unique_ptr>; + using TokenizerStreamPtr = std::unique_ptr>; + using GeneratorPtr = std::unique_ptr>; + + auto* model = reinterpret_cast(native_model); + auto* tokenizer = reinterpret_cast(native_tokenizer); + + CString prompt{env, jprompt}; + + const auto check_result = [env](OgaResult* result) { + ThrowIfError(env, result); + }; + + // var sequences = tokenizer.Encode(prompt); + OgaSequences* sequences = nullptr; + check_result(OgaCreateSequences(&sequences)); + SequencesPtr seq_cleanup{sequences, OgaDestroySequences}; + + check_result(OgaTokenizerEncode(tokenizer, prompt, sequences)); + + // using GeneratorParams generatorParams = new GeneratorParams(model); + OgaGeneratorParams* generator_params = nullptr; + check_result(OgaCreateGeneratorParams(model, &generator_params)); + GeneratorParamsPtr gp_cleanup{generator_params, OgaDestroyGeneratorParams}; + + // generatorParams.SetSearchOption("max_length", 200); + check_result(OgaGeneratorParamsSetSearchNumber(generator_params, "max_length", 200)); // TODO: Rename this API. 'search number' is really opaque + // generatorParams.SetInputSequences(sequences); + check_result(OgaGeneratorParamsSetInputSequences(generator_params, sequences)); + + __android_log_print(ANDROID_LOG_DEBUG, "native", "starting token generation"); + + const auto decode_tokens = [&](const int32_t* tokens, size_t num_tokens){ + const char* output_text = nullptr; + check_result(OgaTokenizerDecode(tokenizer, tokens, num_tokens, &output_text)); + jstring text = env->NewStringUTF(output_text); + OgaDestroyString(output_text); + return text; + }; + + jstring output_text; + + if (!use_callback) { + // var outputSequences = model.Generate(generatorParams); + OgaSequences *output_sequences = nullptr; + check_result(OgaGenerate(model, generator_params, &output_sequences)); + SequencesPtr output_seq_cleanup(output_sequences, OgaDestroySequences); + + size_t num_sequences = OgaSequencesCount(output_sequences); + __android_log_print(ANDROID_LOG_DEBUG, "native", "%zu sequences generated", num_sequences); + + // var outputString = tokenizer.Decode(outputSequences[0]); + // TODO: Is there only ever 1 sequence in the output? Handling just one for simplicity for now. + const int32_t* tokens = OgaSequencesGetSequenceData(output_sequences, 0); + size_t num_tokens = OgaSequencesGetSequenceCount(output_sequences, 0); + + output_text = decode_tokens(tokens, num_tokens); + } + else { + // using var tokenizerStream = tokenizer.CreateStream(); + OgaTokenizerStream* tokenizer_stream = nullptr; + check_result(OgaCreateTokenizerStream(tokenizer, &tokenizer_stream)); + TokenizerStreamPtr stream_cleanup(tokenizer_stream, OgaDestroyTokenizerStream); + + // using var generator = new Generator(model, generatorParams); + OgaGenerator *generator = nullptr; + check_result(OgaCreateGenerator(model, generator_params, &generator)); + GeneratorPtr gen_cleanup(generator, OgaDestroyGenerator); + + // setup the callback to GenAIWrapper::gotNextToken + jclass genai_wrapper = env->GetObjectClass(thiz); + jmethodID callback_id = env->GetMethodID(genai_wrapper, "gotNextToken", "(Ljava/lang/String;)V"); + const auto do_callback = [&](const char* token){ + jstring jtoken = env->NewStringUTF(token); + env->CallVoidMethod(thiz, callback_id, jtoken); + env->DeleteLocalRef(jtoken); + }; + + // while (!generator.IsDone()) + while (!OgaGenerator_IsDone(generator)) { + // generator.ComputeLogits(); + // generator.GenerateNextTokenTop(); + check_result(OgaGenerator_ComputeLogits(generator)); + check_result(OgaGenerator_GenerateNextToken_Top(generator)); + + // TODO: Do we need to do something to ensure there's only one sequence being generated? + // TODO: seem to lack a way to get the number of sequences in the generator as there's no equivalent to + // OgaSequencesCount + const int32_t* seq = OgaGenerator_GetSequence(generator, 0); + size_t seq_len = OgaGenerator_GetSequenceLength(generator, 0); // last token + const char* token = nullptr; + check_result(OgaTokenizerStreamDecode(tokenizer_stream, seq[seq_len - 1], &token)); + do_callback(token); + // Destroy is (assumably) not required for OgaTokenizerStreamDecode based on this which seems to indicate + // the tokenizer is re-using memory for each call. + // `'out' is valid until the next call to OgaTokenizerStreamDecode + // or when the OgaTokenizerStream is destroyed` + // OgaDestroyString(token); This causes 'Scudo ERROR: misaligned pointer when deallocating address' + } + + // decode overall + const int32_t* tokens = OgaGenerator_GetSequence(generator, 0); + size_t num_tokens = OgaGenerator_GetSequenceLength(generator, 0); + output_text = decode_tokens(tokens, num_tokens); + } + + return output_text; +} diff --git a/examples/android/ORTGenAIDemo/app/src/main/cpp/ort_genai_c.h b/examples/android/ORTGenAIDemo/app/src/main/cpp/ort_genai_c.h new file mode 100644 index 000000000..e702082fc --- /dev/null +++ b/examples/android/ORTGenAIDemo/app/src/main/cpp/ort_genai_c.h @@ -0,0 +1,240 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef _WIN32 +#ifdef BUILDING_ORT_GENAI_C +#define OGA_EXPORT __declspec(dllexport) +#else +#define OGA_EXPORT __declspec(dllimport) +#endif +#define OGA_API_CALL _stdcall +#else +// To make symbols visible on macOS/iOS +#ifdef __APPLE__ +#define OGA_EXPORT __attribute__((visibility("default"))) +#else +#define OGA_EXPORT +#endif +#define OGA_API_CALL +#endif + +// ONNX Runtime Generative AI C API +// This API is not thread safe. + +typedef struct OgaResult OgaResult; +typedef struct OgaGeneratorParams OgaGeneratorParams; +typedef struct OgaGenerator OgaGenerator; +typedef struct OgaModel OgaModel; +// OgaSequences is an array of token arrays where the number of token arrays can be obtained using +// OgaSequencesCount and the number of tokens in each token array can be obtained using OgaSequencesGetSequenceCount. +typedef struct OgaSequences OgaSequences; +typedef struct OgaTokenizer OgaTokenizer; +typedef struct OgaTokenizerStream OgaTokenizerStream; + +/* + * \param[in] result OgaResult that contains the error message. + * \return Error message contained in the OgaResult. The const char* is owned by the OgaResult + * and can will be freed when the OgaResult is destroyed. + */ +OGA_EXPORT const char* OGA_API_CALL OgaResultGetError(OgaResult* result); + +/* + * \param[in] result OgaResult to be destroyed. + */ +OGA_EXPORT void OGA_API_CALL OgaDestroyResult(OgaResult*); +OGA_EXPORT void OGA_API_CALL OgaDestroyString(const char*); + +OGA_EXPORT OgaResult* OGA_API_CALL OgaCreateSequences(OgaSequences** out); + +/* + * \param[in] sequences OgaSequences to be destroyed. + */ +OGA_EXPORT void OGA_API_CALL OgaDestroySequences(OgaSequences* sequences); + +/* + * \brief Returns the number of sequences in the OgaSequences + * \param[in] sequences + * \return The number of sequences in the OgaSequences + */ +OGA_EXPORT size_t OGA_API_CALL OgaSequencesCount(const OgaSequences* sequences); + +/* + * \brief Returns the number of tokens in the sequence at the given index + * \param[in] sequences + * \return The number of tokens in the sequence at the given index + */ +OGA_EXPORT size_t OGA_API_CALL OgaSequencesGetSequenceCount(const OgaSequences* sequences, size_t sequence_index); + +/* + * \brief Returns a pointer to the sequence data at the given index. The number of tokens in the sequence + * is given by OgaSequencesGetSequenceCount + * \param[in] sequences + * \return The pointer to the sequence data at the given index. The pointer is valid until the OgaSequences is destroyed. + */ +OGA_EXPORT const int32_t* OGA_API_CALL OgaSequencesGetSequenceData(const OgaSequences* sequences, size_t sequence_index); + +/* + * \brief Creates a model from the given configuration directory and device type. + * \param[in] config_path The path to the model configuration directory. The path is expected to be encoded in UTF-8. + * \param[in] device_type The device type to use for the model. + * \param[out] out The created model. + * \return OgaResult containing the error message if the model creation failed. + */ +OGA_EXPORT OgaResult* OGA_API_CALL OgaCreateModel(const char* config_path, OgaModel** out); + +/* + * \brief Destroys the given model. + * \param[in] model The model to be destroyed. + */ +OGA_EXPORT void OGA_API_CALL OgaDestroyModel(OgaModel* model); + +/* + * \brief Generates an array of token arrays from the model execution based on the given generator params. + * \param[in] model The model to use for generation. + * \param[in] generator_params The parameters to use for generation. + * \param[out] out The generated sequences of tokens. The caller is responsible for freeing the sequences using OgaDestroySequences + * after it is done using the sequences. + * \return OgaResult containing the error message if the generation failed. + */ +OGA_EXPORT OgaResult* OGA_API_CALL OgaGenerate(const OgaModel* model, const OgaGeneratorParams* generator_params, OgaSequences** out); + +/* + * \brief Creates a OgaGeneratorParams from the given model. + * \param[in] model The model to use for generation. + * \param[out] out The created generator params. + * \return OgaResult containing the error message if the generator params creation failed. + */ +OGA_EXPORT OgaResult* OGA_API_CALL OgaCreateGeneratorParams(const OgaModel* model, OgaGeneratorParams** out); + +/* + * \brief Destroys the given generator params. + * \param[in] generator_params The generator params to be destroyed. + */ +OGA_EXPORT void OGA_API_CALL OgaDestroyGeneratorParams(OgaGeneratorParams* generator_params); + +OGA_EXPORT OgaResult* OGA_API_CALL OgaGeneratorParamsSetSearchNumber(OgaGeneratorParams* generator_params, const char* name, double value); +OGA_EXPORT OgaResult* OGA_API_CALL OgaGeneratorParamsSetSearchBool(OgaGeneratorParams* generator_params, const char* name, bool value); + +/* + * \brief Sets the input ids for the generator params. The input ids are used to seed the generation. + * \param[in] generator_params The generator params to set the input ids on. + * \param[in] input_ids The input ids array of size input_ids_count = batch_size * sequence_length. + * \param[in] input_ids_count The total number of input ids. + * \param[in] sequence_length The sequence length of the input ids. + * \param[in] batch_size The batch size of the input ids. + * \return OgaResult containing the error message if the setting of the input ids failed. + */ +OGA_EXPORT OgaResult* OGA_API_CALL OgaGeneratorParamsSetInputIDs(OgaGeneratorParams* generator_params, const int32_t* input_ids, + size_t input_ids_count, size_t sequence_length, size_t batch_size); + +/* + * \brief Sets the input id sequences for the generator params. The input id sequences are used to seed the generation. + * \param[in] generator_params The generator params to set the input ids on. + * \param[in] sequences The input id sequences. + * \return OgaResult containing the error message if the setting of the input id sequences failed. + */ +OGA_EXPORT OgaResult* OGA_API_CALL OgaGeneratorParamsSetInputSequences(OgaGeneratorParams* generator_params, const OgaSequences* sequences); + +OGA_EXPORT OgaResult* OGA_API_CALL OgaGeneratorParamsSetWhisperInputFeatures(OgaGeneratorParams*, const int32_t* inputs, size_t count); +OGA_EXPORT OgaResult* OGA_API_CALL OgaGeneratorParamsSetWhisperDecoderInputIDs(OgaGeneratorParams*, const int32_t* input_ids, size_t input_ids_count); + +/* + * \brief Creates a generator from the given model and generator params. + * \param[in] model The model to use for generation. + * \param[in] params The parameters to use for generation. + * \param[out] out The created generator. + * \return OgaResult containing the error message if the generator creation failed. + */ +OGA_EXPORT OgaResult* OGA_API_CALL OgaCreateGenerator(const OgaModel* model, const OgaGeneratorParams* params, OgaGenerator** out); + +/* + * \brief Destroys the given generator. + * \param[in] generator The generator to be destroyed. + */ +OGA_EXPORT void OGA_API_CALL OgaDestroyGenerator(OgaGenerator* generator); + +/* + * \brief Returns true if the generator has finished generating all the sequences. + * \param[in] generator The generator to check if it is done with generating all sequences. + * \return True if the generator has finished generating all the sequences, false otherwise. + */ +OGA_EXPORT bool OGA_API_CALL OgaGenerator_IsDone(const OgaGenerator* generator); + +/* + * \brief Computes the logits from the model based on the input ids and the past state. The computed logits are stored in the generator. + * \param[in] generator The generator to compute the logits for. + * \return OgaResult containing the error message if the computation of the logits failed. + */ +OGA_EXPORT OgaResult* OGA_API_CALL OgaGenerator_ComputeLogits(OgaGenerator* generator); + +/* + * \brief Generates the next token based on the computed logits using the greedy search. + * \param[in] generator The generator to generate the next token for. + * \return OgaResult containing the error message if the generation of the next token failed. + */ +OGA_EXPORT OgaResult* OGA_API_CALL OgaGenerator_GenerateNextToken_Top(OgaGenerator* generator); + +/* Top-K sampling: most probable words from the model's output probability distribution for the next word + */ +OGA_EXPORT OgaResult* OGA_API_CALL OgaGenerator_GenerateNextToken_TopK(OgaGenerator* generator, int k, float t); + +/*Top-P sampling selects words from the smallest set of words whose cumulative probability exceeds a predefined threshold (p) + */ +OGA_EXPORT OgaResult* OGA_API_CALL OgaGenerator_GenerateNextToken_TopP(OgaGenerator* generator, float p, float t); + +OGA_EXPORT OgaResult* OGA_API_CALL OgaGenerator_GenerateNextToken_TopK_TopP(OgaGenerator* generator, int k, float p, float t); +OGA_EXPORT OgaResult* OGA_API_CALL OgaGenerator_GenerateNextToken(OgaGenerator* generator); + +/* + * \brief Returns the number of tokens in the sequence at the given index. + * \param[in] generator The generator to get the count of the tokens for the sequence at the given index. + * \return The number tokens in the sequence at the given index. + */ +OGA_EXPORT size_t OGA_API_CALL OgaGenerator_GetSequenceLength(const OgaGenerator* generator, size_t index); + +/* + * \brief Returns a pointer to the sequence data at the given index. The number of tokens in the sequence + * is given by OgaGenerator_GetSequenceLength + * \param[in] generator The generator to get the sequence data for the sequence at the given index. + * \return The pointer to the sequence data at the given index. The sequence data is owned by the OgaGenerator + * and will be freed when the OgaGenerator is destroyed. The caller must copy the data if it needs to + * be used after the OgaGenerator is destroyed. + */ +OGA_EXPORT const int32_t* OGA_API_CALL OgaGenerator_GetSequence(const OgaGenerator* generator, size_t index); + +OGA_EXPORT OgaResult* OGA_API_CALL OgaCreateTokenizer(const OgaModel* model, OgaTokenizer** out); +OGA_EXPORT void OGA_API_CALL OgaDestroyTokenizer(OgaTokenizer*); + +/* Encodes a single string and adds the encoded sequence of tokens to the OgaSequences. The OgaSequences must be freed with OgaDestroySequences + when it is no longer needed. + */ +OGA_EXPORT OgaResult* OGA_API_CALL OgaTokenizerEncode(const OgaTokenizer*, const char* str, OgaSequences* sequences); + +/* Decode a single token sequence and returns a null terminated utf8 string. out_string must be freed with OgaDestroyString + */ +OGA_EXPORT OgaResult* OGA_API_CALL OgaTokenizerDecode(const OgaTokenizer*, const int32_t* tokens, size_t token_count, const char** out_string); + +/* OgaTokenizerStream is to decoded token strings incrementally, one token at a time. + */ +OGA_EXPORT OgaResult* OGA_API_CALL OgaCreateTokenizerStream(const OgaTokenizer*, OgaTokenizerStream** out); +OGA_EXPORT void OGA_API_CALL OgaDestroyTokenizerStream(OgaTokenizerStream*); + +/* + * Decode a single token in the stream. If this results in a word being generated, it will be returned in 'out'. + * The caller is responsible for concatenating each chunk together to generate the complete result. + * 'out' is valid until the next call to OgaTokenizerStreamDecode or when the OgaTokenizerStream is destroyed + */ +OGA_EXPORT OgaResult* OGA_API_CALL OgaTokenizerStreamDecode(OgaTokenizerStream*, int32_t token, const char** out); + +OGA_EXPORT OgaResult* OGA_API_CALL OgaSetCurrentGpuDeviceId(int device_id); +OGA_EXPORT OgaResult* OGA_API_CALL OgaGetCurrentGpuDeviceId(int* device_id); + +#ifdef __cplusplus +} +#endif diff --git a/examples/android/ORTGenAIDemo/app/src/main/java/ai/onnxruntime/genai/demo/GenAIException.java b/examples/android/ORTGenAIDemo/app/src/main/java/ai/onnxruntime/genai/demo/GenAIException.java new file mode 100644 index 000000000..94ba3c27b --- /dev/null +++ b/examples/android/ORTGenAIDemo/app/src/main/java/ai/onnxruntime/genai/demo/GenAIException.java @@ -0,0 +1,7 @@ +package ai.onnxruntime.genai.demo; + +public class GenAIException extends Exception { + public GenAIException(String message) { + super(message); + } +} \ No newline at end of file diff --git a/examples/android/ORTGenAIDemo/app/src/main/java/ai/onnxruntime/genai/demo/GenAIWrapper.java b/examples/android/ORTGenAIDemo/app/src/main/java/ai/onnxruntime/genai/demo/GenAIWrapper.java new file mode 100644 index 000000000..04a688266 --- /dev/null +++ b/examples/android/ORTGenAIDemo/app/src/main/java/ai/onnxruntime/genai/demo/GenAIWrapper.java @@ -0,0 +1,50 @@ +package ai.onnxruntime.genai.demo; + +import android.util.Log; + +public class GenAIWrapper implements AutoCloseable { + // Load the GenAI library on application startup. + // TODO: Do we need to load the onnxruntime library explicitly? + // TODO: Will this work with them under + static { + System.loadLibrary("genai"); // JNI layer + System.loadLibrary("onnxruntime-genai"); + System.loadLibrary("onnxruntime"); + } + + private final long nativeModel; + private final long nativeTokenizer; + + GenAIWrapper(String modelPath) { + + nativeModel = loadModel(modelPath); + nativeTokenizer = createTokenizer(nativeModel); + } + + String run(String prompt) { + return run(nativeModel, nativeTokenizer, prompt, /* useCallback*/ true); + } + + @Override + public void close() throws Exception { + if (nativeTokenizer != 0) { + releaseTokenizer(nativeTokenizer); + } + + if (nativeModel != 0) { + releaseModel(nativeModel); + } + } + + public void gotNextToken(String token) { + // TODO: Hook this up with the caller providing the callback func to the ctor of this class, + // or alternatively to run() with it being passed into the run method + Log.i("GenAI", "gotNextToken: " + token); + } + private native long loadModel(String modelPath); + private native void releaseModel(long nativeModel); + private native long createTokenizer(long nativeModel); + private native void releaseTokenizer(long nativeTokenizer); + + private native String run(long nativeModel, long nativeTokenizer, String prompt, boolean useCallback); +} diff --git a/examples/android/ORTGenAIDemo/app/src/main/java/ai/onnxruntime/genai/demo/MainActivity.java b/examples/android/ORTGenAIDemo/app/src/main/java/ai/onnxruntime/genai/demo/MainActivity.java new file mode 100644 index 000000000..626e81d9e --- /dev/null +++ b/examples/android/ORTGenAIDemo/app/src/main/java/ai/onnxruntime/genai/demo/MainActivity.java @@ -0,0 +1,35 @@ +package ai.onnxruntime.genai.demo; + +import androidx.appcompat.app.AppCompatActivity; + +import android.os.Bundle; +import android.widget.TextView; + +import java.io.File; + +import ai.onnxruntime.genai.demo.databinding.ActivityMainBinding; + +public class MainActivity extends AppCompatActivity { + private ActivityMainBinding binding; + private GenAIWrapper genAIWrapper; + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + + binding = ActivityMainBinding.inflate(getLayoutInflater()); + setContentView(binding.getRoot()); + + // manually upload the model. easiest from Android Studio. + // Create emulator. Make sure it has at least 8GB of internal storage! + // Debug app to do initial copy + // In Device Explorer navigate to /data/data/ai.onnxruntime.genai.demo/files + // Right-click on the files folder an update the phi-int4-cpu folder. + File fd = getFilesDir(); + genAIWrapper = new GenAIWrapper(fd.getPath() + "/phi2-int4-cpu"); + String output = genAIWrapper.run("What is the square root of pi."); + + TextView tv = binding.sampleText; + tv.setText(output); + } + +} \ No newline at end of file diff --git a/examples/android/ORTGenAIDemo/app/src/main/jniLibs/arm64-v8a/libonnxruntime-genai.so b/examples/android/ORTGenAIDemo/app/src/main/jniLibs/arm64-v8a/libonnxruntime-genai.so new file mode 100644 index 000000000..2095f1ce8 Binary files /dev/null and b/examples/android/ORTGenAIDemo/app/src/main/jniLibs/arm64-v8a/libonnxruntime-genai.so differ diff --git a/examples/android/ORTGenAIDemo/app/src/main/jniLibs/arm64-v8a/libonnxruntime.so b/examples/android/ORTGenAIDemo/app/src/main/jniLibs/arm64-v8a/libonnxruntime.so new file mode 100644 index 000000000..14d4c5faa Binary files /dev/null and b/examples/android/ORTGenAIDemo/app/src/main/jniLibs/arm64-v8a/libonnxruntime.so differ diff --git a/examples/android/ORTGenAIDemo/app/src/main/jniLibs/arm64-v8a/libonnxruntime4j_jni.so b/examples/android/ORTGenAIDemo/app/src/main/jniLibs/arm64-v8a/libonnxruntime4j_jni.so new file mode 100644 index 000000000..ac975152d Binary files /dev/null and b/examples/android/ORTGenAIDemo/app/src/main/jniLibs/arm64-v8a/libonnxruntime4j_jni.so differ diff --git a/examples/android/ORTGenAIDemo/app/src/main/jniLibs/x86_64/libonnxruntime-genai.so b/examples/android/ORTGenAIDemo/app/src/main/jniLibs/x86_64/libonnxruntime-genai.so new file mode 100644 index 000000000..aa1c9de97 Binary files /dev/null and b/examples/android/ORTGenAIDemo/app/src/main/jniLibs/x86_64/libonnxruntime-genai.so differ diff --git a/examples/android/ORTGenAIDemo/app/src/main/jniLibs/x86_64/libonnxruntime.so b/examples/android/ORTGenAIDemo/app/src/main/jniLibs/x86_64/libonnxruntime.so new file mode 100644 index 000000000..4f4b6d070 Binary files /dev/null and b/examples/android/ORTGenAIDemo/app/src/main/jniLibs/x86_64/libonnxruntime.so differ diff --git a/examples/android/ORTGenAIDemo/app/src/main/jniLibs/x86_64/libonnxruntime4j_jni.so b/examples/android/ORTGenAIDemo/app/src/main/jniLibs/x86_64/libonnxruntime4j_jni.so new file mode 100644 index 000000000..a34325a94 Binary files /dev/null and b/examples/android/ORTGenAIDemo/app/src/main/jniLibs/x86_64/libonnxruntime4j_jni.so differ diff --git a/examples/android/ORTGenAIDemo/app/src/main/res/drawable/ic_launcher_background.xml b/examples/android/ORTGenAIDemo/app/src/main/res/drawable/ic_launcher_background.xml new file mode 100644 index 000000000..07d5da9cb --- /dev/null +++ b/examples/android/ORTGenAIDemo/app/src/main/res/drawable/ic_launcher_background.xml @@ -0,0 +1,170 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/android/ORTGenAIDemo/app/src/main/res/drawable/ic_launcher_foreground.xml b/examples/android/ORTGenAIDemo/app/src/main/res/drawable/ic_launcher_foreground.xml new file mode 100644 index 000000000..2b068d114 --- /dev/null +++ b/examples/android/ORTGenAIDemo/app/src/main/res/drawable/ic_launcher_foreground.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/examples/android/ORTGenAIDemo/app/src/main/res/layout/activity_main.xml b/examples/android/ORTGenAIDemo/app/src/main/res/layout/activity_main.xml new file mode 100644 index 000000000..0fdf98529 --- /dev/null +++ b/examples/android/ORTGenAIDemo/app/src/main/res/layout/activity_main.xml @@ -0,0 +1,19 @@ + + + + + + \ No newline at end of file diff --git a/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-anydpi/ic_launcher.xml b/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-anydpi/ic_launcher.xml new file mode 100644 index 000000000..6f3b755bf --- /dev/null +++ b/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-anydpi/ic_launcher.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-anydpi/ic_launcher_round.xml b/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-anydpi/ic_launcher_round.xml new file mode 100644 index 000000000..6f3b755bf --- /dev/null +++ b/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-anydpi/ic_launcher_round.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-hdpi/ic_launcher.webp b/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-hdpi/ic_launcher.webp new file mode 100644 index 000000000..c209e78ec Binary files /dev/null and b/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-hdpi/ic_launcher.webp differ diff --git a/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-hdpi/ic_launcher_round.webp b/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-hdpi/ic_launcher_round.webp new file mode 100644 index 000000000..b2dfe3d1b Binary files /dev/null and b/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-hdpi/ic_launcher_round.webp differ diff --git a/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-mdpi/ic_launcher.webp b/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-mdpi/ic_launcher.webp new file mode 100644 index 000000000..4f0f1d64e Binary files /dev/null and b/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-mdpi/ic_launcher.webp differ diff --git a/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-mdpi/ic_launcher_round.webp b/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-mdpi/ic_launcher_round.webp new file mode 100644 index 000000000..62b611da0 Binary files /dev/null and b/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-mdpi/ic_launcher_round.webp differ diff --git a/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-xhdpi/ic_launcher.webp b/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-xhdpi/ic_launcher.webp new file mode 100644 index 000000000..948a3070f Binary files /dev/null and b/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-xhdpi/ic_launcher.webp differ diff --git a/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-xhdpi/ic_launcher_round.webp b/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-xhdpi/ic_launcher_round.webp new file mode 100644 index 000000000..1b9a6956b Binary files /dev/null and b/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-xhdpi/ic_launcher_round.webp differ diff --git a/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-xxhdpi/ic_launcher.webp b/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-xxhdpi/ic_launcher.webp new file mode 100644 index 000000000..28d4b77f9 Binary files /dev/null and b/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-xxhdpi/ic_launcher.webp differ diff --git a/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.webp b/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.webp new file mode 100644 index 000000000..9287f5083 Binary files /dev/null and b/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.webp differ diff --git a/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-xxxhdpi/ic_launcher.webp b/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-xxxhdpi/ic_launcher.webp new file mode 100644 index 000000000..aa7d6427e Binary files /dev/null and b/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-xxxhdpi/ic_launcher.webp differ diff --git a/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.webp b/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.webp new file mode 100644 index 000000000..9126ae37c Binary files /dev/null and b/examples/android/ORTGenAIDemo/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.webp differ diff --git a/examples/android/ORTGenAIDemo/app/src/main/res/values-night/themes.xml b/examples/android/ORTGenAIDemo/app/src/main/res/values-night/themes.xml new file mode 100644 index 000000000..e4601ce2f --- /dev/null +++ b/examples/android/ORTGenAIDemo/app/src/main/res/values-night/themes.xml @@ -0,0 +1,16 @@ + + + + \ No newline at end of file diff --git a/examples/android/ORTGenAIDemo/app/src/main/res/values/colors.xml b/examples/android/ORTGenAIDemo/app/src/main/res/values/colors.xml new file mode 100644 index 000000000..f8c6127d3 --- /dev/null +++ b/examples/android/ORTGenAIDemo/app/src/main/res/values/colors.xml @@ -0,0 +1,10 @@ + + + #FFBB86FC + #FF6200EE + #FF3700B3 + #FF03DAC5 + #FF018786 + #FF000000 + #FFFFFFFF + \ No newline at end of file diff --git a/examples/android/ORTGenAIDemo/app/src/main/res/values/strings.xml b/examples/android/ORTGenAIDemo/app/src/main/res/values/strings.xml new file mode 100644 index 000000000..64784e49b --- /dev/null +++ b/examples/android/ORTGenAIDemo/app/src/main/res/values/strings.xml @@ -0,0 +1,3 @@ + + ORT GenAI Demo + \ No newline at end of file diff --git a/examples/android/ORTGenAIDemo/app/src/main/res/values/themes.xml b/examples/android/ORTGenAIDemo/app/src/main/res/values/themes.xml new file mode 100644 index 000000000..80cfbbc73 --- /dev/null +++ b/examples/android/ORTGenAIDemo/app/src/main/res/values/themes.xml @@ -0,0 +1,16 @@ + + + + \ No newline at end of file diff --git a/examples/android/ORTGenAIDemo/app/src/main/res/xml/backup_rules.xml b/examples/android/ORTGenAIDemo/app/src/main/res/xml/backup_rules.xml new file mode 100644 index 000000000..fa0f996d2 --- /dev/null +++ b/examples/android/ORTGenAIDemo/app/src/main/res/xml/backup_rules.xml @@ -0,0 +1,13 @@ + + + + \ No newline at end of file diff --git a/examples/android/ORTGenAIDemo/app/src/main/res/xml/data_extraction_rules.xml b/examples/android/ORTGenAIDemo/app/src/main/res/xml/data_extraction_rules.xml new file mode 100644 index 000000000..9ee9997b0 --- /dev/null +++ b/examples/android/ORTGenAIDemo/app/src/main/res/xml/data_extraction_rules.xml @@ -0,0 +1,19 @@ + + + + + + + \ No newline at end of file diff --git a/examples/android/ORTGenAIDemo/app/src/test/java/ai/onnxruntime/genai/demo/ExampleUnitTest.java b/examples/android/ORTGenAIDemo/app/src/test/java/ai/onnxruntime/genai/demo/ExampleUnitTest.java new file mode 100644 index 000000000..c11e6262d --- /dev/null +++ b/examples/android/ORTGenAIDemo/app/src/test/java/ai/onnxruntime/genai/demo/ExampleUnitTest.java @@ -0,0 +1,17 @@ +package ai.onnxruntime.genai.demo; + +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * Example local unit test, which will execute on the development machine (host). + * + * @see Testing documentation + */ +public class ExampleUnitTest { + @Test + public void addition_isCorrect() { + assertEquals(4, 2 + 2); + } +} \ No newline at end of file diff --git a/examples/android/ORTGenAIDemo/build.gradle.kts b/examples/android/ORTGenAIDemo/build.gradle.kts new file mode 100644 index 000000000..1c3d467e4 --- /dev/null +++ b/examples/android/ORTGenAIDemo/build.gradle.kts @@ -0,0 +1,4 @@ +// Top-level build file where you can add configuration options common to all sub-projects/modules. +plugins { + id("com.android.application") version "8.1.0" apply false +} \ No newline at end of file diff --git a/examples/android/ORTGenAIDemo/gradle.properties b/examples/android/ORTGenAIDemo/gradle.properties new file mode 100644 index 000000000..2c53b8e1f --- /dev/null +++ b/examples/android/ORTGenAIDemo/gradle.properties @@ -0,0 +1,21 @@ +# Project-wide Gradle settings. +# IDE (e.g. Android Studio) users: +# Gradle settings configured through the IDE *will override* +# any settings specified in this file. +# For more details on how to configure your build environment visit +# http://www.gradle.org/docs/current/userguide/build_environment.html +# Specifies the JVM arguments used for the daemon process. +# The setting is particularly useful for tweaking memory settings. +org.gradle.jvmargs=-Xmx8192m -Dfile.encoding=UTF-8 +# When configured, Gradle will run in incubating parallel mode. +# This option should only be used with decoupled projects. More details, visit +# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects +# org.gradle.parallel=true +# AndroidX package structure to make it clearer which packages are bundled with the +# Android operating system, and which are packaged with your app's APK +# https://developer.android.com/topic/libraries/support-library/androidx-rn +android.useAndroidX=true +# Enables namespacing of each library's R class so that its R class includes only the +# resources declared in the library itself and none from the library's dependencies, +# thereby reducing the size of the R class for that library +android.nonTransitiveRClass=true \ No newline at end of file diff --git a/examples/android/ORTGenAIDemo/gradle/wrapper/gradle-wrapper.jar b/examples/android/ORTGenAIDemo/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000..e708b1c02 Binary files /dev/null and b/examples/android/ORTGenAIDemo/gradle/wrapper/gradle-wrapper.jar differ diff --git a/examples/android/ORTGenAIDemo/gradle/wrapper/gradle-wrapper.properties b/examples/android/ORTGenAIDemo/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..2cc0653c4 --- /dev/null +++ b/examples/android/ORTGenAIDemo/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Mon Mar 25 10:44:29 AEST 2024 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.0-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/examples/android/ORTGenAIDemo/gradlew b/examples/android/ORTGenAIDemo/gradlew new file mode 100644 index 000000000..4f906e0c8 --- /dev/null +++ b/examples/android/ORTGenAIDemo/gradlew @@ -0,0 +1,185 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/examples/android/ORTGenAIDemo/gradlew.bat b/examples/android/ORTGenAIDemo/gradlew.bat new file mode 100644 index 000000000..107acd32c --- /dev/null +++ b/examples/android/ORTGenAIDemo/gradlew.bat @@ -0,0 +1,89 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/examples/android/ORTGenAIDemo/settings.gradle.kts b/examples/android/ORTGenAIDemo/settings.gradle.kts new file mode 100644 index 000000000..9c0dc707c --- /dev/null +++ b/examples/android/ORTGenAIDemo/settings.gradle.kts @@ -0,0 +1,18 @@ +pluginManagement { + repositories { + google() + mavenCentral() + gradlePluginPortal() + } +} +dependencyResolutionManagement { + repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS) + repositories { + google() + mavenCentral() + } +} + +rootProject.name = "ORT GenAI Demo" +include(":app") + \ No newline at end of file