New provider Anthropic added

docs/index.md

@@ -127,6 +127,18 @@ And for embeddings:
 
 All these lists of supported models on Openai can be retrieved by running the command `sagify llm models --all --provider openai`. If you want to focus only on chat completions models, then run `sagify llm models --chat-completions --provider openai`. For image creations and embeddings, `sagify llm models --image-creations --provider openai` and `sagify llm models --embeddings --provider openai`, respectively.
 
+
+### Anthropic
+
+The following models are offered for chat completions:
+
+| Model Name | URL |
+|:------------:|:-----:|
+|claude-2.1|https://docs.anthropic.com/claude/reference/models|
+|claude-2.0|https://docs.anthropic.com/claude/reference/models|
+|claude-instant-1.2|https://docs.anthropic.com/claude/reference/models|
+
+
 #### Open-Source
 
 The following open-source models are offered for chat completions:
@@ -229,6 +241,11 @@ In case of using the OpenAI platform, you need to define the following env varia
 - `OPENAI_EMBEDDINGS_MODEL`: It should have one of values [here](https://platform.openai.com/docs/models/embeddings).
 - `OPENAI_IMAGE_CREATION_MODEL`: It should have one of values [here](https://platform.openai.com/docs/models/dall-e).
 
+
+In case of using the Anthropic platform, you need to define the following env variables before you start the LLM Gateway server:
+- `ANTHROPIC_API_KEY`: Your Anthropic API key. Example: `export ANTHROPIC_API_KEY=...`.
+- `ANTHROPIC_CHAT_COMPLETIONS_MODEL`: It should have one of values [here](https://docs.anthropic.com/claude/reference/models).
+
 Now, you can run the command `sagify llm gateway --image sagify-llm-gateway:v0.1.0 --start-local` to start the LLM Gateway locally. You can change the name of the image via the `--image` argument.
 
 This command will output the Docker container id. You can stop the container by executing `docker stop <CONTAINER_ID>`.
@@ -296,6 +313,8 @@ Resources:
               Value: "value10"
             - Name: OPENAI_IMAGE_CREATION_MODEL
               Value: "value11"
+            - Name: ANTHROPIC_CHAT_COMPLETIONS_MODEL
+              Value: "value11"
 
   MyFargateService:
     Type: AWS::ECS::Service

sagify/Dockerfile

@@ -4,7 +4,7 @@ FROM python:3.9-slim
 WORKDIR /app
 
 # Install dependencies
-RUN pip install --no-cache-dir fastapi pydantic==1.10.13 python-dotenv structlog uvicorn openai sagemaker Pillow
+RUN pip install --no-cache-dir fastapi pydantic==1.10.13 python-dotenv structlog uvicorn openai sagemaker Pillow anthropic
 
 # Copy the rest of the application code into the container
 COPY ./ /app/sagify/

sagify/commands/llm.py

@@ -28,6 +28,11 @@
 OPENAI_MODELS = 'models'
 OPENAI_URL = f'{OPENAI_BASE_URL}/{OPENAI_DOCS}/{OPENAI_MODELS}'
 
+ANTHROPIC_BASE_URL = 'https://docs.anthropic.com'
+ANTHROPIC_DOCS = 'claude/reference'
+ANTHROPIC_MODELS = 'models'
+ANTHROPIC_URL = f'{ANTHROPIC_BASE_URL}/{ANTHROPIC_DOCS}/{ANTHROPIC_MODELS}'
+
 HF_BASE_URL = 'https://huggingface.co'
 HF_LLAMA = 'meta-llama'
 HF_STABILITYAI = 'stabilityai'
@@ -50,6 +55,11 @@
         'llama-2-7b': ('meta-textgeneration-llama-2-7b-f', f'{HF_LLAMA_URL}/Llama-2-7b'),
         'llama-2-13b': ('meta-textgeneration-llama-2-13b-f', f'{HF_LLAMA_URL}/Llama-2-13b'),
         'llama-2-70b': ('meta-textgeneration-llama-2-70b-f', f'{HF_LLAMA_URL}/Llama-2-70b'),
+    },
+    'anthropic': {
+        'claude-2.1': ('claude-2.1', ANTHROPIC_URL),
+        'claude-2.0': ('claude-2.0', ANTHROPIC_URL),
+        'claude-instant-1.2': ('claude-instant-1.2', ANTHROPIC_URL)
     }
 }
 
@@ -90,6 +100,9 @@
             'model-txt2img-stabilityai-stable-diffusion-v2-fp16',
             f'{HF_STABILITY_URL}/stable-diffusion-2/tree/fp16'
         )
+    },
+    'anthropic': {
+        'NOT SUPPORTED': ('Learn more', ANTHROPIC_BASE_URL)
     }
 }
 
@@ -132,6 +145,9 @@
         'e5-small-v2': ('huggingface-sentencesimilarity-e5-small-v2', 'https://huggingface.co/intfloat/e5-small-v2'),
         'multilingual-e5-base': ('huggingface-sentencesimilarity-multilingual-e5-base', 'https://huggingface.co/intfloat/multilingual-e5-base'),
         'all-MiniLM-L6-v2': ('huggingface-sentencesimilarity-all-MiniLM-L6-v2', 'https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2'),
+    },
+    'anthropic': {
+        'NOT SUPPORTED': ('Learn more', ANTHROPIC_BASE_URL)
     }
 
 }
@@ -618,6 +634,8 @@ def gateway(image, start_local, platform):
     """
     logger.info(ASCII_LOGO)
     environment_vars = {
+        'ANTHROPIC_API_KEY': os.environ.get('ANTHROPIC_API_KEY'),
+        'ANTHROPIC_CHAT_COMPLETIONS_MODEL': os.environ.get('ANTHROPIC_CHAT_COMPLETIONS_MODEL'),
         'OPENAI_API_KEY': os.environ.get('OPENAI_API_KEY'),
         'OPENAI_CHAT_COMPLETIONS_MODEL': os.environ.get('OPENAI_CHAT_COMPLETIONS_MODEL'),
         'OPENAI_EMBEDDINGS_MODEL': os.environ.get('OPENAI_EMBEDDINGS_MODEL'),

sagify/llm_gateway/providers/anthropic/client.py

@@ -0,0 +1,60 @@
+import structlog
+import anthropic
+import os
+import time
+
+from sagify.llm_gateway.api.v1.exceptions import InternalServerError
+from sagify.llm_gateway.schemas.chat import CreateCompletionDTO, ResponseCompletionDTO
+from sagify.llm_gateway.schemas.embeddings import CreateEmbeddingDTO
+from sagify.llm_gateway.schemas.images import CreateImageDTO
+
+
+logger = structlog.get_logger()
+
+
+class AnthropicClient:
+    def __init__(self):
+        self.client = anthropic.Anthropic()
+        self._chat_completions_model = os.environ.get("ANTHROPIC_CHAT_COMPLETIONS_MODEL")
+
+    async def completions(self, message: CreateCompletionDTO):
+        request = {
+            "model": message.model if message.model else self._chat_completions_model,
+            "messages": message.messages,
+            "temperature": message.temperature,
+            "max_tokens": message.max_tokens,
+            "top_p": message.top_p,
+            "stream": False
+        }
+        try:
+            response = self.client.messages.create(**request)
+            response_dict = response.model_dump()
+            response_dict["provider"] = message.provider
+            response_dict["created"] = int(time.time())
+            response_dict["usage"] = {
+                 "prompt_tokens": response_dict["usage"]["input_tokens"],
+                 "total_tokens": response_dict["usage"]["input_tokens"] + response_dict["usage"]["output_tokens"]
+            }
+            response_dict["object"] = response_dict["type"]
+            response_dict["choices"] = []
+            for i, message in enumerate(response_dict["content"]):
+                response_dict["choices"].append(
+                    {
+                        "index": i,
+                        "message": {
+                            "role": response_dict["role"],
+                            "content": message["text"]
+                        },
+                        "finish_reason": response_dict["stop_reason"]
+                    }
+                )
+            return ResponseCompletionDTO(**response_dict)
+        except Exception as e:
+            logger.error(e)
+            raise InternalServerError(str(e))
+
+    async def embeddings(self, embedding_input: CreateEmbeddingDTO):
+        raise InternalServerError("Not supported")
+
+    async def generations(self, image_input: CreateImageDTO):
+        raise InternalServerError("Not supported")

sagify/llm_gateway/providers/client_factory.py

@@ -1,10 +1,11 @@
 from sagify.llm_gateway.providers.openai.client import OpenAIClient
 from sagify.llm_gateway.providers.aws.sagemaker import SageMakerClient
+from sagify.llm_gateway.providers.anthropic.client import AnthropicClient
 
 
 class LLMClientFactory:
     def __init__(self, provider):
-        self._providers = ["openai", "sagemaker"]
+        self._providers = ["openai", "sagemaker", "anthropic"]
         if provider not in self._providers:
             raise ValueError(f"Invalid provider name {provider}")
         self.provider = provider
@@ -14,3 +15,5 @@ async def create_client(self):
             return OpenAIClient()
         if self.provider == "sagemaker":
             return SageMakerClient()
+        if self.provider == "anthropic":
+            return AnthropicClient()