feat: make --model optional (#668)

CHANGELOG.md

@@ -10,6 +10,7 @@
 * Switch cpu backend to llama.cpp: https://github.com/TabbyML/tabby/pull/638
 * add `server.completion_timeout` to control the code completion interface timeout: https://github.com/TabbyML/tabby/pull/637
 * Switch cuda backend to llama.cpp: https://github.com/TabbyML/tabby/pull/656
+* Make `--model` optional, so users can create a chat only instance.
 
 # v0.4.0
 

crates/tabby/src/serve/health.rs

@@ -9,7 +9,7 @@ use utoipa::ToSchema;
 
 #[derive(Serialize, Deserialize, ToSchema, Clone, Debug)]
 pub struct HealthState {
-    model: String,
+    model: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
     chat_model: Option<String>,
     device: String,

crates/tabby/src/serve/mod.rs

@@ -106,7 +106,7 @@ impl Device {
 pub struct ServeArgs {
     /// Model id for `/completions` API endpoint.
     #[clap(long)]
-    model: String,
+    model: Option<String>,
 
     /// Model id for `/chat/completions` API endpoints.
     #[clap(long)]
@@ -129,7 +129,9 @@ pub async fn main(config: &Config, args: &ServeArgs) {
     valid_args(args);
 
     if args.device != Device::ExperimentalHttp {
-        download_model(&args.model).await;
+        if let Some(model) = &args.model {
+            download_model(model).await;
+        }
         if let Some(chat_model) = &args.chat_model {
             download_model(chat_model).await;
         }
@@ -171,20 +173,22 @@ pub async fn main(config: &Config, args: &ServeArgs) {
 
 fn api_router(args: &ServeArgs, config: &Config) -> Router {
     let index_server = Arc::new(IndexServer::new());
-    let completion_state = {
+    let completion_state = if let Some(model) = &args.model {
         let (
             engine,
             EngineInfo {
                 prompt_template, ..
             },
-        ) = create_engine(&args.model, args);
+        ) = create_engine(model, args);
         let engine = Arc::new(engine);
         let state = completions::CompletionState::new(
             engine.clone(),
             index_server.clone(),
             prompt_template,
         );
-        Arc::new(state)
+        Some(Arc::new(state))
+    } else {
+        None
     };
 
     let chat_state = if let Some(chat_model) = &args.chat_model {
@@ -215,16 +219,18 @@ fn api_router(args: &ServeArgs, config: &Config) -> Router {
             )
     });
 
-    routers.push({
-        Router::new()
-            .route(
-                "/v1/completions",
-                routing::post(completions::completions).with_state(completion_state),
-            )
-            .layer(TimeoutLayer::new(Duration::from_secs(
-                config.server.completion_timeout,
-            )))
-    });
+    if let Some(completion_state) = completion_state {
+        routers.push({
+            Router::new()
+                .route(
+                    "/v1/completions",
+                    routing::post(completions::completions).with_state(completion_state),
+                )
+                .layer(TimeoutLayer::new(Duration::from_secs(
+                    config.server.completion_timeout,
+                )))
+        });
+    }
 
     if let Some(chat_state) = chat_state {
         routers.push({
@@ -279,6 +285,19 @@ trait OpenApiOverride {
 
 impl OpenApiOverride for utoipa::openapi::OpenApi {
     fn override_doc(&mut self, args: &ServeArgs) {
+        if args.model.is_none() {
+            self.paths.paths.remove("/v1/completions");
+            if let Some(components) = self.components.as_mut() {
+                components.schemas.remove("CompletionRequest");
+                components.schemas.remove("CompletionResponse");
+                components.schemas.remove("Choice");
+                components.schemas.remove("DebugData");
+                components.schemas.remove("DebugOptions");
+                components.schemas.remove("Segments");
+                components.schemas.remove("Snippet");
+            }
+        }
+
         if args.chat_model.is_none() {
             self.paths.paths.remove("/v1beta/chat/completions");