add image build for new examples (opea-project#802)

Signed-off-by: chensuyue <[email protected]>
ZePan110 · Sep 12, 2024 · 3f2e7b7 · 3f2e7b7
1 parent 264759d
commit 3f2e7b7
Show file tree

Hide file tree

Showing 11 changed files with 81 additions and 64 deletions.
diff --git a/DocIndexRetriever/docker_image_build/build.yaml b/DocIndexRetriever/docker_image_build/build.yaml
@@ -35,3 +35,9 @@ services:
       dockerfile: comps/dataprep/redis/langchain/Dockerfile
     extends: doc-index-retriever
     image: ${REGISTRY:-opea}/dataprep-redis:${TAG:-latest}
+  tei-gaudi:
+    build:
+      context: tei-gaudi
+      dockerfile: Dockerfile-hpu
+    extends: doc-index-retriever
+    image: ${REGISTRY:-opea}/tei-gaudi:${TAG:-latest}
diff --git a/DocIndexRetriever/tests/test_compose_on_gaudi.sh b/DocIndexRetriever/tests/test_compose_on_gaudi.sh
@@ -3,12 +3,12 @@
 # SPDX-License-Identifier: Apache-2.0
 
 set -e
-echo "IMAGE_REPO=${IMAGE_REPO}"
+IMAGE_REPO=${IMAGE_REPO:-"opea"}
 IMAGE_TAG=${IMAGE_TAG:-"latest"}
 echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
 echo "TAG=IMAGE_TAG=${IMAGE_TAG}"
-# export REGISTRY=${IMAGE_REPO}
-# export TAG=${IMAGE_TAG}
+export REGISTRY=${IMAGE_REPO}
+export TAG=${IMAGE_TAG}
 
 WORKPATH=$(dirname "$PWD")
 LOG_PATH="$WORKPATH/tests"
@@ -19,22 +19,18 @@ function build_docker_images() {
     if [ ! -d "GenAIComps" ] ; then
         git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout "${opea_branch:-"main"}" && cd ../
     fi
-    cd GenAIComps
+    if [ ! -d "tei-gaudi" ] ; then
+        git clone https://github.com/huggingface/tei-gaudi
+    fi
 
-    docker build -t opea/embedding-tei:latest -f comps/embeddings/tei/langchain/Dockerfile .
-    docker build -t opea/retriever-redis:latest -f comps/retrievers/redis/langchain/Dockerfile .
-    docker build -t opea/reranking-tei:latest -f comps/reranks/tei/Dockerfile .
-    docker build -t opea/dataprep-redis:latest -f comps/dataprep/redis/langchain/Dockerfile .
+    echo "Build all the images with --no-cache, check docker_image_build.log for details..."
+    docker compose -f build.yaml build --no-cache > ${LOG_PATH}/docker_image_build.log
 
-    docker pull ghcr.io/huggingface/tgi-gaudi:latest
     docker pull redis/redis-stack:7.2.0-v9
-
-    cd $WORKPATH/
-    docker build -t opea/doc-index-retriever:latest -f ./Dockerfile .
+    docker images && sleep 1s
 }
 
 function start_services() {
-    # build tei-gaudi for each test instead of pull from local registry
     cd $WORKPATH/docker_compose/intel/hpu/gaudi
     export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
     export RERANK_MODEL_ID="BAAI/bge-reranker-base"

diff --git a/DocIndexRetriever/tests/test_compose_on_xeon.sh b/DocIndexRetriever/tests/test_compose_on_xeon.sh
@@ -3,12 +3,12 @@
 # SPDX-License-Identifier: Apache-2.0
 
 set -e
-echo "IMAGE_REPO=${IMAGE_REPO}"
+IMAGE_REPO=${IMAGE_REPO:-"opea"}
 IMAGE_TAG=${IMAGE_TAG:-"latest"}
 echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
 echo "TAG=IMAGE_TAG=${IMAGE_TAG}"
-# export REGISTRY=${IMAGE_REPO}
-# export TAG=${IMAGE_TAG}
+export REGISTRY=${IMAGE_REPO}
+export TAG=${IMAGE_TAG}
 
 WORKPATH=$(dirname "$PWD")
 LOG_PATH="$WORKPATH/tests"
@@ -19,18 +19,12 @@ function build_docker_images() {
     if [ ! -d "GenAIComps" ] ; then
         git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout "${opea_branch:-"main"}" && cd ../
     fi
-    cd GenAIComps
-
-    docker build -t opea/embedding-tei:latest -f comps/embeddings/tei/langchain/Dockerfile .
-    docker build -t opea/retriever-redis:latest -f comps/retrievers/redis/langchain/Dockerfile .
-    docker build -t opea/reranking-tei:latest -f comps/reranks/tei/Dockerfile .
-    docker build -t opea/dataprep-redis:latest -f comps/dataprep/redis/langchain/Dockerfile .
+    service_list="dataprep-redis embedding-tei retriever-redis reranking-tei doc-index-retriever"
+    docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
 
     docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
     docker pull redis/redis-stack:7.2.0-v9
-
-    cd $WORKPATH/
-    docker build -t opea/doc-index-retriever:latest -f ./Dockerfile .
+    docker images && sleep 1s
 }
 
 function start_services() {

diff --git a/InstructionTuning/docker_image_build/build.yaml b/InstructionTuning/docker_image_build/build.yaml
@@ -0,0 +1,13 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  finetuning:
+    build:
+      args:
+        http_proxy: ${http_proxy}
+        https_proxy: ${https_proxy}
+        no_proxy: ${no_proxy}
+      context: GenAIComps
+      dockerfile: comps/finetuning/Dockerfile
+    image: ${REGISTRY:-opea}/finetuning:${TAG:-latest}
diff --git a/InstructionTuning/tests/test_compose_on_xeon.sh b/InstructionTuning/tests/test_compose_on_xeon.sh
@@ -3,6 +3,12 @@
 # SPDX-License-Identifier: Apache-2.0
 
 set -x
+IMAGE_REPO=${IMAGE_REPO:-"opea"}
+IMAGE_TAG=${IMAGE_TAG:-"latest"}
+echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
+echo "TAG=IMAGE_TAG=${IMAGE_TAG}"
+export REGISTRY=${IMAGE_REPO}
+export TAG=${IMAGE_TAG}
 
 WORKPATH=$(dirname "$PWD")
 LOG_PATH="$WORKPATH/tests"
@@ -11,24 +17,16 @@ finetuning_service_port=8015
 ray_port=8265
 
 function build_docker_images() {
-    cd $WORKPATH
+    cd $WORKPATH/docker_image_build
     if [ ! -d "GenAIComps" ] ; then
         git clone https://github.com/opea-project/GenAIComps.git
     fi
-    cd GenAIComps
-    echo PWD: $(pwd)
-    docker build -t opea/finetuning:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy --build-arg HF_TOKEN=$HF_TOKEN -f comps/finetuning/Dockerfile .
-    if [ $? -ne 0 ]; then
-        echo "opea/finetuning built fail"
-        exit 1
-    else
-        echo "opea/finetuning built successful"
-    fi
+    docker compose -f build.yaml build --no-cache > ${LOG_PATH}/docker_image_build.log
 }
 
 function start_service() {
     export no_proxy="localhost,127.0.0.1,"${ip_address}
-    docker run -d --name="finetuning-server" -p $finetuning_service_port:$finetuning_service_port -p $ray_port:$ray_port --runtime=runc --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e no_proxy=$no_proxy opea/finetuning:latest
+    docker run -d --name="finetuning-server" -p $finetuning_service_port:$finetuning_service_port -p $ray_port:$ray_port --runtime=runc --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e no_proxy=$no_proxy ${IMAGE_REPO}/finetuning:${IMAGE_TAG}
     sleep 1m
 }
 

diff --git a/RerankFinetuning/docker_image_build/build.yaml b/RerankFinetuning/docker_image_build/build.yaml
@@ -0,0 +1,13 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  finetuning:
+    build:
+      args:
+        http_proxy: ${http_proxy}
+        https_proxy: ${https_proxy}
+        no_proxy: ${no_proxy}
+      context: GenAIComps
+      dockerfile: comps/finetuning/Dockerfile
+    image: ${REGISTRY:-opea}/finetuning:${TAG:-latest}
diff --git a/RerankFinetuning/tests/test_compose_on_xeon.sh b/RerankFinetuning/tests/test_compose_on_xeon.sh
@@ -2,6 +2,12 @@
 # SPDX-License-Identifier: Apache-2.0
 
 set -x
+IMAGE_REPO=${IMAGE_REPO:-"opea"}
+IMAGE_TAG=${IMAGE_TAG:-"latest"}
+echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
+echo "TAG=IMAGE_TAG=${IMAGE_TAG}"
+export REGISTRY=${IMAGE_REPO}
+export TAG=${IMAGE_TAG}
 
 WORKPATH=$(dirname "$PWD")
 LOG_PATH="$WORKPATH/tests"
@@ -10,24 +16,16 @@ finetuning_service_port=8015
 ray_port=8265
 
 function build_docker_images() {
-    cd $WORKPATH
+    cd $WORKPATH/docker_image_build
     if [ ! -d "GenAIComps" ] ; then
         git clone https://github.com/opea-project/GenAIComps.git
     fi
-    cd GenAIComps
-    echo PWD: $(pwd)
-    docker build -t opea/finetuning:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy --build-arg HF_TOKEN=$HF_TOKEN -f comps/finetuning/Dockerfile .
-    if [ $? -ne 0 ]; then
-        echo "opea/finetuning built fail"
-        exit 1
-    else
-        echo "opea/finetuning built successful"
-    fi
+    docker compose -f build.yaml build --no-cache > ${LOG_PATH}/docker_image_build.log
 }
 
 function start_service() {
     export no_proxy="localhost,127.0.0.1,"${ip_address}
-    docker run -d --name="finetuning-server" -p $finetuning_service_port:$finetuning_service_port -p $ray_port:$ray_port --runtime=runc --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e no_proxy=$no_proxy opea/finetuning:latest
+    docker run -d --name="finetuning-server" -p $finetuning_service_port:$finetuning_service_port -p $ray_port:$ray_port --runtime=runc --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e no_proxy=$no_proxy ${IMAGE_REPO}/finetuning:${IMAGE_TAG}
     sleep 1m
 }
 

diff --git a/VideoRAGQnA/docker_compose/intel/cpu/xeon/README.md b/VideoRAGQnA/docker_compose/intel/cpu/xeon/README.md
@@ -56,7 +56,7 @@ cd GenAIComps
 ### 1. Build Embedding Image
 
 ```bash
-docker build -t opea/embedding-multimodal:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/multimodal_clip/Dockerfile .
+docker build -t opea/embedding-multimodal-clip:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/multimodal_clip/Dockerfile .
 ```
 
 ### 2. Build Retriever Image
@@ -83,7 +83,7 @@ docker build -t opea/lvm-video-llama:latest --build-arg https_proxy=$https_proxy
 ### 5. Build Dataprep Image
 
 ```bash
-docker build -t opea/dataprep-vdms:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/vdms/multimodal_langchain/Dockerfile .
+docker build -t opea/dataprep-multimodal-vdms:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/vdms/multimodal_langchain/Dockerfile .
 cd ..
 ```
 
@@ -110,8 +110,8 @@ docker build -t opea/videoragqna-ui:latest --build-arg https_proxy=$https_proxy
 
 Then run the command `docker images`, you will have the following 8 Docker Images:
 
-1. `opea/dataprep-vdms:latest`
-2. `opea/embedding-multimodal:latest`
+1. `opea/dataprep-multimodal-vdms:latest`
+2. `opea/embedding-multimodal-clip:latest`
 3. `opea/retriever-vdms:latest`
 4. `opea/reranking-videoragqna:latest`
 5. `opea/video-llama-lvm-server:latest`

diff --git a/VideoRAGQnA/docker_compose/intel/cpu/xeon/compose.yaml b/VideoRAGQnA/docker_compose/intel/cpu/xeon/compose.yaml
@@ -10,7 +10,7 @@ services:
     ports:
       - "8001:55555"
   dataprep:
-    image: ${REGISTRY:-opea}/dataprep-vdms:${TAG:-latest}
+    image: ${REGISTRY:-opea}/dataprep-multimodal-vdms:${TAG:-latest}
     container_name: dataprep-vdms-server
     depends_on:
       - vdms-vector-db
@@ -28,7 +28,7 @@ services:
       - /home/$USER/.cache/clip:/home/user/.cache/clip
       - /home/$USER/.cache/huggingface/hub:/home/user/.cache/huggingface/hub
   embedding:
-    image: ${REGISTRY:-opea}/embedding-multimodal:${TAG:-latest}
+    image: ${REGISTRY:-opea}/embedding-multimodal-clip:${TAG:-latest}
     container_name: embedding-multimodal-server
     ports:
       - "6000:6000"

diff --git a/VideoRAGQnA/docker_image_build/build.yaml b/VideoRAGQnA/docker_image_build/build.yaml
@@ -11,43 +11,43 @@ services:
       context: ../
       dockerfile: ./Dockerfile
     image: ${REGISTRY:-opea}/videoragqna:${TAG:-latest}
-  videoragqna-xeon-ui-server:
+  videoragqna-ui:
     build:
       context: ../ui
       dockerfile: ./docker/Dockerfile
     extends: videoragqna
     image: ${REGISTRY:-opea}/videoragqna-ui:${TAG:-latest}
-  dataprep:
+  dataprep-multimodal-vdms:
     build:
       context: GenAIComps
       dockerfile: comps/dataprep/vdms/multimodal_langchain/Dockerfile
     extends: videoragqna
-    image: ${REGISTRY:-opea}/dataprep-vdms:${TAG:-latest}
-  embedding:
+    image: ${REGISTRY:-opea}/dataprep-multimodal-vdms:${TAG:-latest}
+  embedding-multimodal-clip:
     build:
       context: GenAIComps
       dockerfile: comps/embeddings/multimodal_clip/Dockerfile
     extends: videoragqna
-    image: ${REGISTRY:-opea}/embedding-multimodal:${TAG:-latest}
-  retriever:
+    image: ${REGISTRY:-opea}/embedding-multimodal-clip:${TAG:-latest}
+  retriever-vdms:
     build:
       context: GenAIComps
       dockerfile: comps/retrievers/vdms/langchain/Dockerfile
     extends: videoragqna
     image: ${REGISTRY:-opea}/retriever-vdms:${TAG:-latest}
-  reranking:
+  reranking-videoragqna:
     build:
       context: GenAIComps
       dockerfile: comps/reranks/video-rag-qna/Dockerfile
     extends: videoragqna
     image: ${REGISTRY:-opea}/reranking-videoragqna:${TAG:-latest}
-  lvm-video-llama:
+  video-llama-lvm-server:
     build:
       context: GenAIComps
       dockerfile: comps/lvms/video-llama/dependency/Dockerfile
     extends: videoragqna
     image: ${REGISTRY:-opea}/video-llama-lvm-server:${TAG:-latest}
-  lvm:
+  lvm-video-llama:
     build:
       context: GenAIComps
       dockerfile: comps/lvms/video-llama/Dockerfile

diff --git a/VideoRAGQnA/tests/test_compose_on_xeon.sh b/VideoRAGQnA/tests/test_compose_on_xeon.sh
@@ -19,8 +19,7 @@ function build_docker_images() {
     git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout "${opea_branch:-"main"}" && cd ../
 
     echo "Build all the images with --no-cache, check docker_image_build.log for details..."
-    service_list="videoragqna videoragqna-xeon-ui-server dataprep embedding retriever reranking lvm-video-llama lvm"
-    docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
+    docker compose -f build.yaml build --no-cache > ${LOG_PATH}/docker_image_build.log
 
     docker pull intellabs/vdms:v2.8.0
     docker images && sleep 1s