Remove NO_TOKENIZER build option (#353)

We always build with tokenizer now, and the option was broken anyways as
it wasn't being updated.

CMakeLists.txt

@@ -179,16 +179,11 @@ if(USE_DML)
   add_dependencies(onnxruntime-genai-static RESTORE_PACKAGES)
 endif()
 
-if(NO_TOKENIZEROOT)
-  add_compile_definitions(NO_TOKENIZER=1)
-  message("----------------Tokenizer Disabled------------------")
-else()
-  add_subdirectory("${CMAKE_SOURCE_DIR}/src/tokenizer")
-  target_include_directories(onnxruntime-genai PRIVATE ${TOKENIZER_ROOT})
-  target_include_directories(onnxruntime-genai-static PUBLIC ${TOKENIZER_ROOT})
-  target_link_libraries(onnxruntime-genai PRIVATE tokenizer)
-  target_link_libraries(onnxruntime-genai-static PUBLIC tokenizer)
-endif()
+add_subdirectory("${CMAKE_SOURCE_DIR}/src/tokenizer")
+target_include_directories(onnxruntime-genai PRIVATE ${TOKENIZER_ROOT})
+target_include_directories(onnxruntime-genai-static PUBLIC ${TOKENIZER_ROOT})
+target_link_libraries(onnxruntime-genai PRIVATE tokenizer)
+target_link_libraries(onnxruntime-genai-static PUBLIC tokenizer)
 
 if(ENABLE_TESTS)
   add_subdirectory("${CMAKE_SOURCE_DIR}/test")

cmake/options.cmake

@@ -2,7 +2,6 @@ include(CMakeDependentOption)
 
 option(USE_CUDA "Build with CUDA support" ON)
 option(USE_DML "Build with DML support" OFF)
-option(NO_TOKENIZER "Don't include the Tokenizer" OFF)
 option(ENABLE_PYTHON "Build the Python API." ON)
 option(ENABLE_TESTS "Enable tests" ON)
 option(TEST_PHI2 "Enable tests for Phi2" OFF)

src/models/model.cpp

@@ -94,26 +94,6 @@ std::vector<int32_t> PadInputs(std::span<std::span<const int32_t>> sequences, in
   return result;
 }
 
-#ifdef NO_TOKENIZER
-const std::string& TokenizerStream::Decode(int32_t token) {
-  throw std::runtime_error("Tokenizer not enabled");
-}
-
-std::unique_ptr<TokenizerStream> Tokenizer::CreateStream() const {
-  return std::make_unique<TokenizerStream>();
-}
-
-Tokenizer::Tokenizer(Config& config) {
-}
-
-std::vector<int32_t> Tokenizer::Encode(const char* text) const {
-  throw std::runtime_error("Tokenizer not enabled");
-}
-
-std::string Tokenizer::Decode(std::span<int32_t> tokens) const {
-  throw std::runtime_error("Tokenizer not enabled");
-}
-#else
 void CheckResult(tfmError_t error) {
   if (error != kTfmOK)
     throw std::runtime_error(TfmGetLastErrorMessage());
@@ -179,8 +159,6 @@ std::vector<std::string> Tokenizer::DecodeBatch(std::span<const int32_t> sequenc
   return strings;
 }
 
-#endif
-
 #if USE_CUDA
 // Since Python/Others can and will hold onto a generator object past the model object's lifetime we need to ensure
 // the allocator used is not destroyed until last. This keeps the allocator around until exit, after all other memory

src/models/model.h

@@ -1,8 +1,5 @@
 #pragma once
-#ifndef NO_TOKENIZER
 #include "tfmtok_c.h"
-#endif
-
 #include "captured_graph_pool.h"
 
 #if USE_DML
@@ -36,19 +33,6 @@ struct State {
   void ClearIO();                                             // Clear all inputs/outputs
 };
 
-#ifdef NO_TOKENIZER
-struct TokenizerStream {
-  const std::string& Decode(int32_t token);
-};
-
-struct Tokenizer {
-  Tokenizer(Config& config);
-
-  std::vector<int32_t> Encode(const char* text) const;
-  std::string Decode(std::span<int32_t> tokens) const;
-};
-#else
-
 template <typename T>
 struct TfmPtr {
   ~TfmPtr() { TfmDispose(&p_); }
@@ -94,7 +78,6 @@ struct Tokenizer : std::enable_shared_from_this<Tokenizer> {
  private:
   int32_t pad_token_id_;
 };
-#endif
 
 struct SessionInfo {
   SessionInfo(OrtSession& session);

test/CMakeLists.txt

@@ -43,12 +43,9 @@ if(USE_CUDA AND CMAKE_CUDA_COMPILER)
   target_sources(unit_tests PRIVATE ${cuda_test_srcs})
 endif()
 
-if(NO_TOKENIZER)
-  add_compile_definitions(NO_TOKENIZER=1)
-else()
-  target_include_directories(unit_tests PRIVATE ${TOKENIZER_ROOT})
-  target_link_libraries(unit_tests PRIVATE tokenizer)
-endif()
+target_include_directories(unit_tests PRIVATE ${TOKENIZER_ROOT})
+target_link_libraries(unit_tests PRIVATE tokenizer)
+
 set(TEST_MODEL_SRC_DIR "${CMAKE_CURRENT_SOURCE_DIR}/test_models/")
 set(TEST_MODEL_DES_DIR "$<TARGET_FILE_DIR:unit_tests>/test_models/")
 add_custom_command(TARGET unit_tests POST_BUILD

test/model_tests.cpp

@@ -203,7 +203,6 @@ TEST(ModelTests, BeamSearchGptCuda) {
 
 TEST(ModelTests, TestApiCuda) {
 #if TEST_PHI2
-#ifndef NO_TOKENIZER
 
   auto prompt = R"(
 def print_prime(n):
@@ -234,15 +233,11 @@ Print all primes between 1 and n
   auto result = generator->GetSequence(0);
 
   std::cout << tokenizer->Decode(result.GetCPU()) << "\r\n";
-#else
-  std::cout << "Test skipped - not built with onnxruntime extensions\r\n";
-#endif
 #endif
 }
 
 TEST(ModelTests, TestHighLevelApiCuda) {
 #if TEST_PHI2
-#ifndef NO_TOKENIZER
   auto prompt = R"(
 def print_prime(n):
 '''
@@ -266,9 +261,6 @@ Print all primes between 1 and n
   auto result = Generators::Generate(*model, *params);
 
   std::cout << tokenizer->Decode(result[0]) << "\r\n";
-#else
-  std::cout << "Test skipped - not built with onnxruntime extensions\r\n";
-#endif
 #endif
 }