Change default model of codegen and codetrans

Fixes issue #503 Signed-off-by: Lianhao Lu <[email protected]>
opea-project · Oct 30, 2024 · 626510a · 626510a
1 parent a5c96ab
commit 626510a
Show file tree

Hide file tree

Showing 8 changed files with 25 additions and 19 deletions.
diff --git a/helm-charts/codegen/README.md b/helm-charts/codegen/README.md
@@ -14,7 +14,7 @@ cd GenAIInfra/helm-charts/
 helm dependency update codegen
 export HFTOKEN="insert-your-huggingface-token-here"
 export MODELDIR="/mnt/opea-models"
-export MODELNAME="meta-llama/CodeLlama-7b-hf"
+export MODELNAME="Qwen/Qwen2.5-Coder-7B-Instruct"
 # To run on Xeon
 helm install codegen codegen --set global.HUGGINGFACEHUB_API_TOKEN=${HFTOKEN} --set global.modelUseHostPath=${MODELDIR} --set tgi.LLM_MODEL_ID=${MODELNAME}
 # To run on Gaudi
@@ -23,7 +23,7 @@ helm install codegen codegen --set global.HUGGINGFACEHUB_API_TOKEN=${HFTOKEN} --
 
 ### IMPORTANT NOTE
 
-1. To use model `meta-llama/CodeLlama-7b-hf`, you should first goto the [huggingface model card](https://huggingface.co/meta-llama/CodeLlama-7b-hf) to apply for the model access first. You need to make sure your huggingface token has at least read access to that model.
+1. To use model `Qwen/Qwen2.5-Coder-7B-Instruct`, you should first goto the [huggingface model card](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct) to apply for the model access first. You need to make sure your huggingface token has at least read access to that model.
 
 2. Make sure your `MODELDIR` exists on the node where your workload is schedueled so you can cache the downloaded model for next time use. Otherwise, set `global.modelUseHostPath` to 'null' if you don't want to cache the model.
 
@@ -58,8 +58,8 @@ Open a browser to access `http://<k8s-node-ip-address>:${port}` to play with the
 
 ## Values
 
-| Key              | Type   | Default                        | Description                                                              |
-| ---------------- | ------ | ------------------------------ | ------------------------------------------------------------------------ |
-| image.repository | string | `"opea/codegen"`               |                                                                          |
-| service.port     | string | `"7778"`                       |                                                                          |
-| tgi.LLM_MODEL_ID | string | `"meta-llama/CodeLlama-7b-hf"` | Models id from https://huggingface.co/, or predownloaded model directory |
+| Key              | Type   | Default                            | Description                                                              |
+| ---------------- | ------ | ---------------------------------- | ------------------------------------------------------------------------ |
+| image.repository | string | `"opea/codegen"`                   |                                                                          |
+| service.port     | string | `"7778"`                           |                                                                          |
+| tgi.LLM_MODEL_ID | string | `"Qwen/Qwen2.5-Coder-7B-Instruct"` | Models id from https://huggingface.co/, or predownloaded model directory |
diff --git a/helm-charts/codegen/values.yaml b/helm-charts/codegen/values.yaml
@@ -41,7 +41,7 @@ affinity: {}
 
 # To override values in subchart tgi
 tgi:
-  LLM_MODEL_ID: meta-llama/CodeLlama-7b-hf
+  LLM_MODEL_ID: Qwen/Qwen2.5-Coder-7B-Instruct
 
 codegen-ui:
   image:

diff --git a/helm-charts/codetrans/README.md b/helm-charts/codetrans/README.md
@@ -14,12 +14,18 @@ cd GenAIInfra/helm-charts/
 helm dependency update codetrans
 export HFTOKEN="insert-your-huggingface-token-here"
 export MODELDIR="/mnt/opea-models"
-export MODELNAME="HuggingFaceH4/mistral-7b-grok"
+export MODELNAME="mistralai/Mistral-7B-Instruct-v0.3"
 helm install codetrans codetrans --set global.HUGGINGFACEHUB_API_TOKEN=${HFTOKEN} --set global.modelUseHostPath=${MODELDIR} --set tgi.LLM_MODEL_ID=${MODELNAME}
 # To use Gaudi device
 # helm install codetrans codetrans --set global.HUGGINGFACEHUB_API_TOKEN=${HFTOKEN} --values codetrans/gaudi-values.yaml
 ```
 
+### IMPORTANT NOTE
+
+1. To use model `mistralai/Mistral-7B-Instruct-v0.3`, you should first goto the [huggingface model card](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3) to apply for the model access first. You need to make sure your huggingface token has at least read access to that model.
+
+2. Make sure your `MODELDIR` exists on the node where your workload is schedueled so you can cache the downloaded model for next time use. Otherwise, set `global.modelUseHostPath` to 'null' if you don't want to cache the model.
+
 ## Verify
 
 To verify the installation, run the command `kubectl get pod` to make sure all pods are running.
@@ -51,8 +57,8 @@ Open a browser to access `http://<k8s-node-ip-address>:${port}` to play with the
 
 ## Values
 
-| Key              | Type   | Default                           | Description                                                              |
-| ---------------- | ------ | --------------------------------- | ------------------------------------------------------------------------ |
-| image.repository | string | `"opea/codetrans"`                |                                                                          |
-| service.port     | string | `"7777"`                          |                                                                          |
-| tgi.LLM_MODEL_ID | string | `"HuggingFaceH4/mistral-7b-grok"` | Models id from https://huggingface.co/, or predownloaded model directory |
+| Key              | Type   | Default                                | Description                                                              |
+| ---------------- | ------ | -------------------------------------- | ------------------------------------------------------------------------ |
+| image.repository | string | `"opea/codetrans"`                     |                                                                          |
+| service.port     | string | `"7777"`                               |                                                                          |
+| tgi.LLM_MODEL_ID | string | `"mistralai/Mistral-7B-Instruct-v0.3"` | Models id from https://huggingface.co/, or predownloaded model directory |
diff --git a/helm-charts/codetrans/values.yaml b/helm-charts/codetrans/values.yaml
@@ -42,7 +42,7 @@ affinity: {}
 
 # To override values in subchart tgi
 tgi:
-  LLM_MODEL_ID: HuggingFaceH4/mistral-7b-grok
+  LLM_MODEL_ID: mistralai/Mistral-7B-Instruct-v0.3
 
 codetrans-ui:
   image:

diff --git a/microservices-connector/config/samples/CodeGen/codegen_gaudi.yaml b/microservices-connector/config/samples/CodeGen/codegen_gaudi.yaml
@@ -29,6 +29,6 @@ spec:
         internalService:
           serviceName: tgi-gaudi-svc
           config:
-            MODEL_ID: meta-llama/CodeLlama-7b-hf
+            MODEL_ID: Qwen/Qwen2.5-Coder-7B-Instruct
             endpoint: /generate
           isDownstreamService: true
diff --git a/microservices-connector/config/samples/CodeGen/codegen_xeon.yaml b/microservices-connector/config/samples/CodeGen/codegen_xeon.yaml
@@ -29,6 +29,6 @@ spec:
         internalService:
           serviceName: tgi-service
           config:
-            MODEL_ID: meta-llama/CodeLlama-7b-hf
+            MODEL_ID: Qwen/Qwen2.5-Coder-7B-Instruct
             endpoint: /generate
           isDownstreamService: true
diff --git a/microservices-connector/config/samples/CodeTrans/codetrans_gaudi.yaml b/microservices-connector/config/samples/CodeTrans/codetrans_gaudi.yaml
@@ -29,6 +29,6 @@ spec:
         internalService:
           serviceName: tgi-gaudi-svc
           config:
-            MODEL_ID: HuggingFaceH4/mistral-7b-grok
+            MODEL_ID: mistralai/Mistral-7B-Instruct-v0.3
             endpoint: /generate
           isDownstreamService: true
diff --git a/microservices-connector/config/samples/CodeTrans/codetrans_xeon.yaml b/microservices-connector/config/samples/CodeTrans/codetrans_xeon.yaml
@@ -29,6 +29,6 @@ spec:
         internalService:
           serviceName: tgi-service
           config:
-            MODEL_ID: HuggingFaceH4/mistral-7b-grok
+            MODEL_ID: mistralai/Mistral-7B-Instruct-v0.3
             endpoint: /generate
           isDownstreamService: true