Feat: support modelscope

docs/examples/modelscope/model.yaml

@@ -0,0 +1,13 @@
+apiVersion: llmaz.io/v1alpha1
+kind: Model
+metadata:
+  name: llama3-8b
+spec:
+  familyName: llama3
+  dataSource:
+    name: ModelScope
+    modelID: LLM-Research/Meta-Llama-3-8B
+  inferenceFlavors:
+  - name: t4 # usually refer to the model type
+    requests:
+      nvidia.com/gpu: 1

docs/examples/modelscope/playground.yaml

@@ -0,0 +1,8 @@
+apiVersion: inference.llmaz.io/v1alpha1
+kind: Playground
+metadata:
+  name: llama3-8b
+spec:
+  replicas: 1
+  modelClaim:
+    modelName: llama3-8b

llmaz/loader/model_hub/hub_factory.py

@@ -16,11 +16,12 @@
 
 from loader.model_hub.model_hub import ModelHub
 from loader.model_hub.huggingface import HUGGING_FACE, Huggingface
+from loader.model_hub.model_scope import MODEL_SCOPE, ModelScope
 
 
-# TODO: support modelScope.
 SUPPORT_MODEL_HUBS = {
     HUGGING_FACE: Huggingface,
+    MODEL_SCOPE: ModelScope,
 }
 
 

llmaz/loader/model_hub/model_scope.py

@@ -0,0 +1,47 @@
+"""
+Copyright 2024.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import concurrent.futures
+from modelscope import snapshot_download
+from loader.model_hub.model_hub import ModelHub, MODEL_LOCAL_DIR
+
+MODEL_SCOPE = "ModelScope"
+MAX_WORKERS = 4
+
+
+class ModelScope(ModelHub):
+    @classmethod
+    def name(cls) -> str:
+        return MODEL_SCOPE
+
+    @classmethod
+    def load_model(cls, model_id: str) -> None:
+        print(f"Start to download model {model_id}")
+
+        with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
+            futures = []
+            futures.append(
+                executor.submit(
+                    snapshot_download,
+                    model_id=model_id,
+                    local_dir=MODEL_LOCAL_DIR,
+                ).add_done_callback(handle_completion)
+            )
+
+
+def handle_completion(future):
+    filename = future.result()
+    print(f"Download completed for {filename}")

pyproject.toml

@@ -9,6 +9,7 @@ readme = "README.md"
 [tool.poetry.dependencies]
 python = "^3.10"
 huggingface-hub = "^0.23.5"
+modelscope = "^1.17.0"
 
 
 [tool.poetry.group.dev.dependencies]

test/integration/webhook/model_test.go

@@ -58,6 +58,14 @@ var _ = ginkgo.Describe("model default and validation", func() {
 				return wrapper.MakeModel("llama3-8b").DataSourceWithModelID("meta-llama/Meta-Llama-3-8B").DataSourceWithModelHub("Huggingface").FamilyName("llama3").Label(core.ModelFamilyNameLabelKey, "llama3").Obj()
 			},
 		}),
+		ginkgo.Entry("apply modelscope model hub name", &testDefaultingCase{
+			model: func() *core.Model {
+				return wrapper.MakeModel("llama3-8b").FamilyName("llama3").DataSourceWithModelHub("ModelScope").DataSourceWithModelID("LLM-Research/Meta-Llama-3-8B").Obj()
+			},
+			wantModel: func() *core.Model {
+				return wrapper.MakeModel("llama3-8b").DataSourceWithModelID("LLM-Research/Meta-Llama-3-8B").DataSourceWithModelHub("ModelScope").FamilyName("llama3").Label(core.ModelFamilyNameLabelKey, "llama3").Obj()
+			},
+		}),
 	)
 
 	type testValidatingCase struct {
@@ -73,12 +81,18 @@ var _ = ginkgo.Describe("model default and validation", func() {
 				gomega.Expect(k8sClient.Create(ctx, tc.model())).To(gomega.Succeed())
 			}
 		},
-		ginkgo.Entry("normal model creation", &testValidatingCase{
+		ginkgo.Entry("default normal huggingface model creation", &testValidatingCase{
 			model: func() *core.Model {
 				return wrapper.MakeModel("llama3-8b").FamilyName("llama3").DataSourceWithModelID("meta-llama/Meta-Llama-3-8B").Obj()
 			},
 			failed: false,
 		}),
+		ginkgo.Entry("normal modelscope model creation", &testValidatingCase{
+			model: func() *core.Model {
+				return wrapper.MakeModel("llama3-8b").FamilyName("llama3").DataSourceWithModelHub("ModelScope").DataSourceWithModelID("LLM-Research/Meta-Llama-3-8B").Obj()
+			},
+			failed: false,
+		}),
 		// ginkgo.Entry("model creation with URI configured", &testValidatingCase{
 		// 	model: func() *core.Model {
 		// 		return wrapper.MakeModel("llama3-8b").FamilyName("llama3").DataSourceWithURI("image://meta-llama-3-8B").Obj()