From 3f2e7b73aca5d7b938df066c97382c81c1d6f0d0 Mon Sep 17 00:00:00 2001 From: "chen, suyue" Date: Thu, 12 Sep 2024 22:17:12 +0800 Subject: [PATCH] add image build for new examples (#802) Signed-off-by: chensuyue --- .../docker_image_build/build.yaml | 6 +++++ .../tests/test_compose_on_gaudi.sh | 22 ++++++++----------- .../tests/test_compose_on_xeon.sh | 18 +++++---------- .../docker_image_build/build.yaml | 13 +++++++++++ .../tests/test_compose_on_xeon.sh | 20 ++++++++--------- .../docker_image_build/build.yaml | 13 +++++++++++ .../tests/test_compose_on_xeon.sh | 20 ++++++++--------- .../docker_compose/intel/cpu/xeon/README.md | 8 +++---- .../intel/cpu/xeon/compose.yaml | 4 ++-- VideoRAGQnA/docker_image_build/build.yaml | 18 +++++++-------- VideoRAGQnA/tests/test_compose_on_xeon.sh | 3 +-- 11 files changed, 81 insertions(+), 64 deletions(-) create mode 100644 InstructionTuning/docker_image_build/build.yaml create mode 100644 RerankFinetuning/docker_image_build/build.yaml diff --git a/DocIndexRetriever/docker_image_build/build.yaml b/DocIndexRetriever/docker_image_build/build.yaml index 3ed44fa24..b0775069b 100644 --- a/DocIndexRetriever/docker_image_build/build.yaml +++ b/DocIndexRetriever/docker_image_build/build.yaml @@ -35,3 +35,9 @@ services: dockerfile: comps/dataprep/redis/langchain/Dockerfile extends: doc-index-retriever image: ${REGISTRY:-opea}/dataprep-redis:${TAG:-latest} + tei-gaudi: + build: + context: tei-gaudi + dockerfile: Dockerfile-hpu + extends: doc-index-retriever + image: ${REGISTRY:-opea}/tei-gaudi:${TAG:-latest} diff --git a/DocIndexRetriever/tests/test_compose_on_gaudi.sh b/DocIndexRetriever/tests/test_compose_on_gaudi.sh index f5247a191..01cca6a42 100644 --- a/DocIndexRetriever/tests/test_compose_on_gaudi.sh +++ b/DocIndexRetriever/tests/test_compose_on_gaudi.sh @@ -3,12 +3,12 @@ # SPDX-License-Identifier: Apache-2.0 set -e -echo "IMAGE_REPO=${IMAGE_REPO}" +IMAGE_REPO=${IMAGE_REPO:-"opea"} IMAGE_TAG=${IMAGE_TAG:-"latest"} echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" echo "TAG=IMAGE_TAG=${IMAGE_TAG}" -# export REGISTRY=${IMAGE_REPO} -# export TAG=${IMAGE_TAG} +export REGISTRY=${IMAGE_REPO} +export TAG=${IMAGE_TAG} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" @@ -19,22 +19,18 @@ function build_docker_images() { if [ ! -d "GenAIComps" ] ; then git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout "${opea_branch:-"main"}" && cd ../ fi - cd GenAIComps + if [ ! -d "tei-gaudi" ] ; then + git clone https://github.com/huggingface/tei-gaudi + fi - docker build -t opea/embedding-tei:latest -f comps/embeddings/tei/langchain/Dockerfile . - docker build -t opea/retriever-redis:latest -f comps/retrievers/redis/langchain/Dockerfile . - docker build -t opea/reranking-tei:latest -f comps/reranks/tei/Dockerfile . - docker build -t opea/dataprep-redis:latest -f comps/dataprep/redis/langchain/Dockerfile . + echo "Build all the images with --no-cache, check docker_image_build.log for details..." + docker compose -f build.yaml build --no-cache > ${LOG_PATH}/docker_image_build.log - docker pull ghcr.io/huggingface/tgi-gaudi:latest docker pull redis/redis-stack:7.2.0-v9 - - cd $WORKPATH/ - docker build -t opea/doc-index-retriever:latest -f ./Dockerfile . + docker images && sleep 1s } function start_services() { - # build tei-gaudi for each test instead of pull from local registry cd $WORKPATH/docker_compose/intel/hpu/gaudi export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5" export RERANK_MODEL_ID="BAAI/bge-reranker-base" diff --git a/DocIndexRetriever/tests/test_compose_on_xeon.sh b/DocIndexRetriever/tests/test_compose_on_xeon.sh index d061b8d20..4a04030d4 100644 --- a/DocIndexRetriever/tests/test_compose_on_xeon.sh +++ b/DocIndexRetriever/tests/test_compose_on_xeon.sh @@ -3,12 +3,12 @@ # SPDX-License-Identifier: Apache-2.0 set -e -echo "IMAGE_REPO=${IMAGE_REPO}" +IMAGE_REPO=${IMAGE_REPO:-"opea"} IMAGE_TAG=${IMAGE_TAG:-"latest"} echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" echo "TAG=IMAGE_TAG=${IMAGE_TAG}" -# export REGISTRY=${IMAGE_REPO} -# export TAG=${IMAGE_TAG} +export REGISTRY=${IMAGE_REPO} +export TAG=${IMAGE_TAG} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" @@ -19,18 +19,12 @@ function build_docker_images() { if [ ! -d "GenAIComps" ] ; then git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout "${opea_branch:-"main"}" && cd ../ fi - cd GenAIComps - - docker build -t opea/embedding-tei:latest -f comps/embeddings/tei/langchain/Dockerfile . - docker build -t opea/retriever-redis:latest -f comps/retrievers/redis/langchain/Dockerfile . - docker build -t opea/reranking-tei:latest -f comps/reranks/tei/Dockerfile . - docker build -t opea/dataprep-redis:latest -f comps/dataprep/redis/langchain/Dockerfile . + service_list="dataprep-redis embedding-tei retriever-redis reranking-tei doc-index-retriever" + docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 docker pull redis/redis-stack:7.2.0-v9 - - cd $WORKPATH/ - docker build -t opea/doc-index-retriever:latest -f ./Dockerfile . + docker images && sleep 1s } function start_services() { diff --git a/InstructionTuning/docker_image_build/build.yaml b/InstructionTuning/docker_image_build/build.yaml new file mode 100644 index 000000000..9d4797cbd --- /dev/null +++ b/InstructionTuning/docker_image_build/build.yaml @@ -0,0 +1,13 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +services: + finetuning: + build: + args: + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + no_proxy: ${no_proxy} + context: GenAIComps + dockerfile: comps/finetuning/Dockerfile + image: ${REGISTRY:-opea}/finetuning:${TAG:-latest} diff --git a/InstructionTuning/tests/test_compose_on_xeon.sh b/InstructionTuning/tests/test_compose_on_xeon.sh index ce1d8aebb..97a5f3c7e 100644 --- a/InstructionTuning/tests/test_compose_on_xeon.sh +++ b/InstructionTuning/tests/test_compose_on_xeon.sh @@ -3,6 +3,12 @@ # SPDX-License-Identifier: Apache-2.0 set -x +IMAGE_REPO=${IMAGE_REPO:-"opea"} +IMAGE_TAG=${IMAGE_TAG:-"latest"} +echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" +echo "TAG=IMAGE_TAG=${IMAGE_TAG}" +export REGISTRY=${IMAGE_REPO} +export TAG=${IMAGE_TAG} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" @@ -11,24 +17,16 @@ finetuning_service_port=8015 ray_port=8265 function build_docker_images() { - cd $WORKPATH + cd $WORKPATH/docker_image_build if [ ! -d "GenAIComps" ] ; then git clone https://github.com/opea-project/GenAIComps.git fi - cd GenAIComps - echo PWD: $(pwd) - docker build -t opea/finetuning:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy --build-arg HF_TOKEN=$HF_TOKEN -f comps/finetuning/Dockerfile . - if [ $? -ne 0 ]; then - echo "opea/finetuning built fail" - exit 1 - else - echo "opea/finetuning built successful" - fi + docker compose -f build.yaml build --no-cache > ${LOG_PATH}/docker_image_build.log } function start_service() { export no_proxy="localhost,127.0.0.1,"${ip_address} - docker run -d --name="finetuning-server" -p $finetuning_service_port:$finetuning_service_port -p $ray_port:$ray_port --runtime=runc --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e no_proxy=$no_proxy opea/finetuning:latest + docker run -d --name="finetuning-server" -p $finetuning_service_port:$finetuning_service_port -p $ray_port:$ray_port --runtime=runc --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e no_proxy=$no_proxy ${IMAGE_REPO}/finetuning:${IMAGE_TAG} sleep 1m } diff --git a/RerankFinetuning/docker_image_build/build.yaml b/RerankFinetuning/docker_image_build/build.yaml new file mode 100644 index 000000000..9d4797cbd --- /dev/null +++ b/RerankFinetuning/docker_image_build/build.yaml @@ -0,0 +1,13 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +services: + finetuning: + build: + args: + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + no_proxy: ${no_proxy} + context: GenAIComps + dockerfile: comps/finetuning/Dockerfile + image: ${REGISTRY:-opea}/finetuning:${TAG:-latest} diff --git a/RerankFinetuning/tests/test_compose_on_xeon.sh b/RerankFinetuning/tests/test_compose_on_xeon.sh index b75294695..b0a10d506 100644 --- a/RerankFinetuning/tests/test_compose_on_xeon.sh +++ b/RerankFinetuning/tests/test_compose_on_xeon.sh @@ -2,6 +2,12 @@ # SPDX-License-Identifier: Apache-2.0 set -x +IMAGE_REPO=${IMAGE_REPO:-"opea"} +IMAGE_TAG=${IMAGE_TAG:-"latest"} +echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" +echo "TAG=IMAGE_TAG=${IMAGE_TAG}" +export REGISTRY=${IMAGE_REPO} +export TAG=${IMAGE_TAG} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" @@ -10,24 +16,16 @@ finetuning_service_port=8015 ray_port=8265 function build_docker_images() { - cd $WORKPATH + cd $WORKPATH/docker_image_build if [ ! -d "GenAIComps" ] ; then git clone https://github.com/opea-project/GenAIComps.git fi - cd GenAIComps - echo PWD: $(pwd) - docker build -t opea/finetuning:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy --build-arg HF_TOKEN=$HF_TOKEN -f comps/finetuning/Dockerfile . - if [ $? -ne 0 ]; then - echo "opea/finetuning built fail" - exit 1 - else - echo "opea/finetuning built successful" - fi + docker compose -f build.yaml build --no-cache > ${LOG_PATH}/docker_image_build.log } function start_service() { export no_proxy="localhost,127.0.0.1,"${ip_address} - docker run -d --name="finetuning-server" -p $finetuning_service_port:$finetuning_service_port -p $ray_port:$ray_port --runtime=runc --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e no_proxy=$no_proxy opea/finetuning:latest + docker run -d --name="finetuning-server" -p $finetuning_service_port:$finetuning_service_port -p $ray_port:$ray_port --runtime=runc --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e no_proxy=$no_proxy ${IMAGE_REPO}/finetuning:${IMAGE_TAG} sleep 1m } diff --git a/VideoRAGQnA/docker_compose/intel/cpu/xeon/README.md b/VideoRAGQnA/docker_compose/intel/cpu/xeon/README.md index 1802f0ab6..2c8426108 100644 --- a/VideoRAGQnA/docker_compose/intel/cpu/xeon/README.md +++ b/VideoRAGQnA/docker_compose/intel/cpu/xeon/README.md @@ -56,7 +56,7 @@ cd GenAIComps ### 1. Build Embedding Image ```bash -docker build -t opea/embedding-multimodal:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/multimodal_clip/Dockerfile . +docker build -t opea/embedding-multimodal-clip:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/multimodal_clip/Dockerfile . ``` ### 2. Build Retriever Image @@ -83,7 +83,7 @@ docker build -t opea/lvm-video-llama:latest --build-arg https_proxy=$https_proxy ### 5. Build Dataprep Image ```bash -docker build -t opea/dataprep-vdms:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/vdms/multimodal_langchain/Dockerfile . +docker build -t opea/dataprep-multimodal-vdms:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/vdms/multimodal_langchain/Dockerfile . cd .. ``` @@ -110,8 +110,8 @@ docker build -t opea/videoragqna-ui:latest --build-arg https_proxy=$https_proxy Then run the command `docker images`, you will have the following 8 Docker Images: -1. `opea/dataprep-vdms:latest` -2. `opea/embedding-multimodal:latest` +1. `opea/dataprep-multimodal-vdms:latest` +2. `opea/embedding-multimodal-clip:latest` 3. `opea/retriever-vdms:latest` 4. `opea/reranking-videoragqna:latest` 5. `opea/video-llama-lvm-server:latest` diff --git a/VideoRAGQnA/docker_compose/intel/cpu/xeon/compose.yaml b/VideoRAGQnA/docker_compose/intel/cpu/xeon/compose.yaml index a379e08af..488be51ab 100644 --- a/VideoRAGQnA/docker_compose/intel/cpu/xeon/compose.yaml +++ b/VideoRAGQnA/docker_compose/intel/cpu/xeon/compose.yaml @@ -10,7 +10,7 @@ services: ports: - "8001:55555" dataprep: - image: ${REGISTRY:-opea}/dataprep-vdms:${TAG:-latest} + image: ${REGISTRY:-opea}/dataprep-multimodal-vdms:${TAG:-latest} container_name: dataprep-vdms-server depends_on: - vdms-vector-db @@ -28,7 +28,7 @@ services: - /home/$USER/.cache/clip:/home/user/.cache/clip - /home/$USER/.cache/huggingface/hub:/home/user/.cache/huggingface/hub embedding: - image: ${REGISTRY:-opea}/embedding-multimodal:${TAG:-latest} + image: ${REGISTRY:-opea}/embedding-multimodal-clip:${TAG:-latest} container_name: embedding-multimodal-server ports: - "6000:6000" diff --git a/VideoRAGQnA/docker_image_build/build.yaml b/VideoRAGQnA/docker_image_build/build.yaml index cbcf03ccd..1354d378f 100644 --- a/VideoRAGQnA/docker_image_build/build.yaml +++ b/VideoRAGQnA/docker_image_build/build.yaml @@ -11,43 +11,43 @@ services: context: ../ dockerfile: ./Dockerfile image: ${REGISTRY:-opea}/videoragqna:${TAG:-latest} - videoragqna-xeon-ui-server: + videoragqna-ui: build: context: ../ui dockerfile: ./docker/Dockerfile extends: videoragqna image: ${REGISTRY:-opea}/videoragqna-ui:${TAG:-latest} - dataprep: + dataprep-multimodal-vdms: build: context: GenAIComps dockerfile: comps/dataprep/vdms/multimodal_langchain/Dockerfile extends: videoragqna - image: ${REGISTRY:-opea}/dataprep-vdms:${TAG:-latest} - embedding: + image: ${REGISTRY:-opea}/dataprep-multimodal-vdms:${TAG:-latest} + embedding-multimodal-clip: build: context: GenAIComps dockerfile: comps/embeddings/multimodal_clip/Dockerfile extends: videoragqna - image: ${REGISTRY:-opea}/embedding-multimodal:${TAG:-latest} - retriever: + image: ${REGISTRY:-opea}/embedding-multimodal-clip:${TAG:-latest} + retriever-vdms: build: context: GenAIComps dockerfile: comps/retrievers/vdms/langchain/Dockerfile extends: videoragqna image: ${REGISTRY:-opea}/retriever-vdms:${TAG:-latest} - reranking: + reranking-videoragqna: build: context: GenAIComps dockerfile: comps/reranks/video-rag-qna/Dockerfile extends: videoragqna image: ${REGISTRY:-opea}/reranking-videoragqna:${TAG:-latest} - lvm-video-llama: + video-llama-lvm-server: build: context: GenAIComps dockerfile: comps/lvms/video-llama/dependency/Dockerfile extends: videoragqna image: ${REGISTRY:-opea}/video-llama-lvm-server:${TAG:-latest} - lvm: + lvm-video-llama: build: context: GenAIComps dockerfile: comps/lvms/video-llama/Dockerfile diff --git a/VideoRAGQnA/tests/test_compose_on_xeon.sh b/VideoRAGQnA/tests/test_compose_on_xeon.sh index 7dc4df845..fbaba5910 100755 --- a/VideoRAGQnA/tests/test_compose_on_xeon.sh +++ b/VideoRAGQnA/tests/test_compose_on_xeon.sh @@ -19,8 +19,7 @@ function build_docker_images() { git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout "${opea_branch:-"main"}" && cd ../ echo "Build all the images with --no-cache, check docker_image_build.log for details..." - service_list="videoragqna videoragqna-xeon-ui-server dataprep embedding retriever reranking lvm-video-llama lvm" - docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log + docker compose -f build.yaml build --no-cache > ${LOG_PATH}/docker_image_build.log docker pull intellabs/vdms:v2.8.0 docker images && sleep 1s