From a41f29dc829e2ff89347c91cb5db83d902242767 Mon Sep 17 00:00:00 2001 From: pm3310 Date: Fri, 23 Feb 2024 22:01:38 +0000 Subject: [PATCH] Simplify llm gateway command --- docs/index.md | 27 ++++++++++++--------------- sagify/commands/llm.py | 31 ++++++++++++++++++++----------- 2 files changed, 32 insertions(+), 26 deletions(-) diff --git a/docs/index.md b/docs/index.md index 5c6e131..4c00425 100644 --- a/docs/index.md +++ b/docs/index.md @@ -191,7 +191,7 @@ It takes 15 to 30 minutes to deploy all the backend services as Sagemaker endpoi The deployed model names, which are the Sagemaker endpoint names, are printed out and stored in the hidden file `.sagify_llm_infra.json`. You can also access them from the AWS Sagemaker web console. -#### Deploy FastAPI LLM Gateway - Local - Docker +#### Deploy FastAPI LLM Gateway - Docker Once you have set up your backend platform, you can deploy the FastAPI LLM Gateway locally. @@ -213,7 +213,7 @@ In case of using the OpenAI platform, you need to define the following env varia - `OPENAI_EMBEDDINGS_MODEL`: It should have one of values [here](https://platform.openai.com/docs/models/embeddings). - `OPENAI_IMAGE_CREATION_MODEL`: It should have one of values [here](https://platform.openai.com/docs/models/dall-e). -Now, you can run the command `sagify llm start-local-gateway --image sagify-llm-gateway:v0.1.0` to start the LLM Gateway locally. You can change the name of the image via the `--image` argument. +Now, you can run the command `sagify llm gateway --image sagify-llm-gateway:v0.1.0 --start-local` to start the LLM Gateway locally. You can change the name of the image via the `--image` argument. This command will output the Docker container id. You can stop the container by executing `docker stop `. @@ -223,19 +223,14 @@ This command will output the Docker container id. You can stop the container by In the case you want to create a docker image and then run it ```{bash} -sagify llm start-local-gateway --image sagify-llm-gateway:v0.1.0 --dockerfile-dir . +sagify llm gateway --image sagify-llm-gateway:v0.1.0 --start-local ``` - If you want to use an existing container just run + If you want to use just build the image ```{bash} - sagify llm start-local-gateway --image sagify-llm-gateway:v0.1.0 + sagify llm gateway --image sagify-llm-gateway:v0.1.0 ``` -Or manually by using `docker build` and `docker run` -- Build the Docker image `docker build -t sagify-llm-gateway .` -- Run the Docker image for OpenAI `docker run -p 8000:8000 -e OPENAI_API_KEY=sk-... -e OPEN_AI_CHAT_COMPLETIONS_MODEL=gpt-3.5-turbo -e OPEN_AI_EMBEDDINGS_MODEL=text-embedding-3-small -e OPEN_AI_IMAGE_CREATION_MODEL=dall-e-3 --name my-fastapi-container-13 sagify-llm-gateway` -- Run the Docker image for AWS Sagemaker `docker run -p 8000:8000 -e AWS_ACCESS_KEY_ID=... -e AWS_SECRET_ACCESS_KEY=... -e AWS_REGION_NAME=... -e S3_BUCKET_NAME=... -e IMAGE_URL_TTL_IN_SECONDS=... -e SM_CHAT_COMPLETIONS_MODEL=... -e SM_EMBEDDINGS_MODEL=... -e SM_IMAGE_CREATION_MODEL=...` - If you want to support both platforms (OpenAI and AWS Sagemaker), then pass all the env variables for both platforms. #### Deploy FastAPI LLM Gateway - AWS Fargate @@ -1483,20 +1478,20 @@ It stop all or some of the services that are running. `--external-id EXTERNAL_ID` or `-x EXTERNAL_ID`: Optional external id used when using an IAM role -### LLM Start Gateway Locally +### LLM Gateway #### Name -Command to start LLM gateway locally +Command to build gateway docker image and start the gateway locally #### Synopsis ```sh -sagify llm start --image IMAGE_NAME [--dockerfile-dir DOCKERFILE_DIR] [--platform PLATFORM] +sagify llm gateway --image IMAGE_NAME [--dockerfile-dir DOCKERFILE_DIR] [--platform PLATFORM] [--start-local] ``` #### Description -It spins up the LLM gateway locally by building a Docker image and running it. +It builds gateway docker image and starts the gateway locally. #### Required Flags @@ -1504,4 +1499,6 @@ It spins up the LLM gateway locally by building a Docker image and running it. #### Optional Flags -`--platform PLATFORM`: Operating system. Platform in the format `os[/arch[/variant]]`. \ No newline at end of file +`--platform PLATFORM`: Operating system. Platform in the format `os[/arch[/variant]]`. + +`--start-local`: Flag to indicate if to start the gateway locally. \ No newline at end of file diff --git a/sagify/commands/llm.py b/sagify/commands/llm.py index 526690a..f8a2920 100644 --- a/sagify/commands/llm.py +++ b/sagify/commands/llm.py @@ -599,15 +599,22 @@ def stop( required=True, help="The docker image to run" ) +@click.option( + '--start-local', + is_flag=True, + show_default=True, + default=False, + help='Start gateway locally' +) @click.option( u"--platform", default="linux/amd64", required=False, help="The platform to use for the docker build" ) -def start_local_gateway(image, platform): +def gateway(image, start_local, platform): """ - Command to start local gateway + Command to build gateway docker image and start the gateway locally """ logger.info(ASCII_LOGO) environment_vars = { @@ -630,7 +637,7 @@ def start_local_gateway(image, platform): try: client.images.get(image) build_image = False - logger.info(f"Using existing docker image: {image}") + logger.info(f"Docker image: {image} exists. Skipping build...") except docker.errors.ImageNotFound: build_image = True logger.info(f"Docker image: {image} was not found") @@ -647,17 +654,19 @@ def start_local_gateway(image, platform): for log in build_logs: logger.info(log) - logger.info("Starting local gateway...\n") - container = client.containers.run( - image=image, - environment=environment_vars, - ports={'8000/tcp': PORT}, - detach=True) - logger.info(f"Local gateway started successfully. Container ID: {container.short_id}") - logger.info(f"Access service docs: http://localhost:{PORT}/docs") + if start_local: + logger.info("Starting local gateway...\n") + container = client.containers.run( + image=image, + environment=environment_vars, + ports={'8000/tcp': PORT}, + detach=True) + logger.info(f"Local gateway started successfully. Container ID: {container.short_id}") + logger.info(f"Access service docs: http://localhost:{PORT}/docs") llm.add_command(platforms) llm.add_command(models) llm.add_command(start) llm.add_command(stop) +llm.add_command(gateway)