doc use its own pyproject.toml in deploy

.github/workflows/documentation.yml

@@ -29,22 +29,23 @@ jobs:
           curl -sSL https://install.python-poetry.org | python3 -
           echo "$HOME/.local/bin" >> $GITHUB_PATH  # Ensure Poetry's bin directory is in PATH
 
-      - name: Install dependencies using Poetry
+      - name: Install documentation dependencies using Poetry
         run: |
+          cd docs
           poetry config virtualenvs.create false  # Avoid creating a virtual environment
-          poetry install --only doc  # Install only the doc dependencies as specified in pyproject.toml
+          poetry install   # Install only the doc dependencies as specified in pyproject.toml
 
       - name: Build documentation using Makefile
         run: |
           echo "Building documentation from: $(pwd)"
           ls -l  # Debug: List current directory contents
-          poetry run make -C docs html  # Run Makefile in docs directory to build HTML docs
-        working-directory: ${{ github.workspace }}
+          poetry run make html  # Run Makefile in docs directory to build HTML docs
+        working-directory: ${{ github.workspace }}/docs
 
       - name: List built documentation
         run: |
-          find ./build/ -type f  # List all files in the build directory
-        working-directory: ${{ github.workspace }}/docs
+          find ./docs/build/ -type f  # List all files in the build directory
+        working-directory: ${{ github.workspace }}
 
       - name: Create .nojekyll file
         run: |
@@ -53,7 +54,8 @@ jobs:
 
       - name: Copy CNAME file
         run: |
-          cp ${{ github.workspace }}/CNAME ${{ github.workspace }}/docs/build/CNAME
+          cp ${{ github.workspace }}/CNAME ${{ github.workspace }}/docs/build/CNAME || true
+        working-directory: ${{ github.workspace }}
 
       - name: Deploy to GitHub Pages
         uses: peaceiris/actions-gh-pages@v3

docs/pyproject.toml

@@ -1,106 +1,40 @@
-# [tool.poetry]
-# name = "lightrag"
-
-# packages = [
-#     { include = "core", from = "." },
-#     { include = "components", from = "." },
-#     { include = "prompts", from = "." },
-#     { include = "eval", from = "." },
-#     { include = "utils", from = "." },
-#     { include = "tracing", from = "." },
-# ]
-# version = "0.1.0"
-# description = "The 'PyTorch' library for LLM applications. RAG=Retriever-Agent-Generator."
-# authors = ["Li Yin <[email protected]>"]
-# readme = "README.md"
-# license = "MIT"
-# classifiers = [
-#     "Topic :: Software Development :: Build Tools",
-#     "Topic :: Software Development :: Libraries :: Python Modules",
-# ]
-
-# # [[tool.poetry.packages]]
-# # include = "lightrag"
-
-# [tool.poetry.dependencies]
-# python = ">=3.11, <4.0"
-# python-dotenv = "^1.0.1"
-# backoff = "^2.2.1"
-# jinja2 = "^3.1.3"
-# # TODO: decide if we need people to install faiss, or openai, or groq separately
-# openai = "^1.12.0"
-# groq = "^0.5.0"       # should only be installed if groq client is used
-# faiss-cpu = "^1.8.0"
-# torchviz = "^0.0.2"
-# matplotlib = "^3.8.4"
-# colorama = "^0.4.6"
-# jsonlines = "^4.0.0"
-
-
-# [tool.poetry.group.test.dependencies]
-# pytest = "^8.1.1"
-# pytest-mock = "^3.14.0"
-
-# [tool.poetry.group.typing.dependencies]
-# mypy = "^1"
-
-# [tool.poetry.group.doc.dependencies]
-# datasets = ">=2.14.6, <=2.19.1"
-# sphinx = "^7.3.7"
-# sphinx-rtd-theme = "^2.0.0"
-# pydata-sphinx-theme = "0.15.2"
-# sphinx-design = "^0.6.0"
-
-
-# [tool.poetry.group.dev.dependencies]
-# langchain = "^0.1.16"
-# llama-index = "^0.10.30"
-# pre-commit = "^3.7.0"
-# litellm = "^1.35.34"
-# haystack-ai = "^2.0.1"
-# torchvision = "^0.18.0"
-# torch = "^2.3.0"
-# langsmith = "^0.1.56"
-# langchain-openai = "^0.1.6"
-# pyvis = "^0.3.2"
-# llama-index-llms-ollama = "^0.1.3"
-# anthropic = "^0.26.0"
-# google-generativeai = "^0.5.4"
-# transformers = "^4.41.0"
-# torchmetrics = "^1.4.0.post0"
-# lightning = "^2.2.4"
-# dspy-ai = "^2.4.9"
-# jupyter = "^1.0.0"
-# ipykernel = "^6.29.4"
-# colorama = "^0.4.6"
-
-
-# [tool.ruff]
-# exclude = ["images"]
-
-
-# [build-system]
-# requires = ["poetry-core"]
-# build-backend = "poetry.core.masonry.api"
-
 [tool.poetry]
 name = "lightrag-docs"
 version = "0.1.0"
-description = "A project to develop and test the lightrag library"
+description = "Documentation for the lightrag project"
 authors = ["Your Name <[email protected]>"]
 license = "MIT"
-
-packages = [{ include = "../docs", from = "." }]
-
+packages = [{ from = "_dummy", include = "dummy" }]    # empty packages
 
 [tool.poetry.dependencies]
 python = ">=3.11, <4.0"
-lightrag = { path = "../lightrag", develop = true }
-
+lightrag = "^0.0.0a13"  # Assuming this is the main library you are developing
 
-[tool.poetry.group.dev.dependencies]
-torch = "^2.3.0"
+# [tool.poetry.group.dev.dependencies]
+# pre-commit = "^3.7.0"
+# anthropic = "^0.26.0"
+# torchmetrics = "^1.4.0.post0"
+# jupyter = "^1.0.0"
+# ipykernel = "^6.29.4"
+# dataclasses-jsonschema = "^2.16.0"
+# groq = "^0.5.0"
+# pgvector = "^0.2.5"
+# cohere = "^5.5.8"
+# pydot = "^2.0.0"
+# matplotlib = "^3.9.0"
+# pyvis = "^0.3.2"
 
+[tool.poetry.group.doc.dependencies]
+pydata-sphinx-theme = "^0.15.3"
+sphinx-design = "^0.6.0"
+sphinx-copybutton = "^0.5.2"
+sphinx = "^7.3.7"
+nbsphinx = "^0.9.4"
+nbconvert = "^7.16.4"
+pandoc = "^2.3"
+readthedocs-sphinx-search = "^0.3.2"
+sqlalchemy = "^2.0.31"
+google-generativeai = "^0.7.1"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

lightrag/lightrag/components/model_client/anthropic_client.py

@@ -5,7 +5,17 @@
 import backoff
 import logging
 
-import anthropic
+
+from lightrag.core.model_client import ModelClient
+from lightrag.core.types import ModelType
+
+# optional import
+from lightrag.utils.lazy_import import safe_import, OptionalPackages
+
+anthropic = safe_import(
+    OptionalPackages.ANTHROPIC.value[0], OptionalPackages.ANTHROPIC.value[1]
+)
+
 from anthropic import (
     RateLimitError,
     APITimeoutError,
@@ -15,15 +25,8 @@
 )
 from anthropic.types import Message
 
-
-from lightrag.core.model_client import ModelClient
-from lightrag.core.types import ModelType
-from lightrag.utils.lazy_import import safe_import, OptionalPackages
-
 log = logging.getLogger(__name__)
 
-safe_import(OptionalPackages.ANTHROPIC.value[0], OptionalPackages.ANTHROPIC.value[1])
-
 
 def get_first_message_content(completion: Message) -> str:
     r"""When we only need the content of the first message.

lightrag/lightrag/components/model_client/cohere_client.py

@@ -5,8 +5,7 @@
 import backoff
 from lightrag.utils.lazy_import import safe_import, OptionalPackages
 
-safe_import(OptionalPackages.COHERE.value[0], OptionalPackages.COHERE.value[1])
-import cohere
+cohere = safe_import(OptionalPackages.COHERE.value[0], OptionalPackages.COHERE.value[1])
 from cohere import (
     BadRequestError,
     InternalServerError,

lightrag/lightrag/components/model_client/google_client.py

@@ -5,24 +5,23 @@
 import backoff
 
 
-import google.generativeai as genai
-from google.api_core.exceptions import (
-    InternalServerError,
-    BadRequest,
-    GoogleAPICallError,
-)
-from google.generativeai.types import GenerateContentResponse
-
-
 from lightrag.core.model_client import ModelClient
 from lightrag.core.types import ModelType
 
 from lightrag.utils.lazy_import import safe_import, OptionalPackages
 
-safe_import(
+# optional import
+google = safe_import(
     OptionalPackages.GOOGLE_GENERATIVEAI.value[0],
     OptionalPackages.GOOGLE_GENERATIVEAI.value[1],
 )
+import google.generativeai as genai
+from google.api_core.exceptions import (
+    InternalServerError,
+    BadRequest,
+    GoogleAPICallError,
+)
+from google.generativeai.types import GenerateContentResponse
 
 
 class GoogleGenAIClient(ModelClient):

lightrag/lightrag/components/model_client/groq_client.py

@@ -3,7 +3,14 @@
 import os
 from typing import Dict, Sequence, Optional, Any
 import backoff
+from lightrag.core.model_client import ModelClient
+from lightrag.core.types import ModelType
+
 
+from lightrag.utils.lazy_import import safe_import, OptionalPackages
+
+# optional import
+groq = safe_import(OptionalPackages.GROQ.value[0], OptionalPackages.GROQ.value[1])
 
 from groq import Groq, AsyncGroq
 from groq import (
@@ -14,13 +21,6 @@
 )
 
 
-from lightrag.core.model_client import ModelClient
-from lightrag.core.types import ModelType
-from lightrag.utils.lazy_import import safe_import, OptionalPackages
-
-safe_import(OptionalPackages.GROQ.value[0], OptionalPackages.GROQ.value[1])
-
-
 class GroqAPIClient(ModelClient):
     __doc__ = r"""A component wrapper for the Groq API client.
 

lightrag/lightrag/components/model_client/openai_client.py

@@ -4,32 +4,30 @@
 from typing import Dict, Sequence, Optional, List, Any, TypeVar, Callable
 
 import logging
-
-from openai import OpenAI, AsyncOpenAI
-from openai import (
-    APITimeoutError,
-    InternalServerError,
-    RateLimitError,
-    UnprocessableEntityError,
-    BadRequestError,
-)
-
-from lightrag.utils.lazy_import import safe_import, OptionalPackages
+import backoff
 
 
 from lightrag.core.model_client import ModelClient
 from lightrag.core.types import ModelType, EmbedderOutput, TokenLogProb
 from lightrag.components.model_client.utils import parse_embedding_response
 
+# optional import
+from lightrag.utils.lazy_import import safe_import, OptionalPackages
 
-safe_import(OptionalPackages.OPENAI.value[0], OptionalPackages.OPENAI.value[1])
 
+openai = safe_import(OptionalPackages.OPENAI.value[0], OptionalPackages.OPENAI.value[1])
 
+from openai import OpenAI, AsyncOpenAI
+from openai import (
+    APITimeoutError,
+    InternalServerError,
+    RateLimitError,
+    UnprocessableEntityError,
+    BadRequestError,
+)
 from openai.types import Completion, CreateEmbeddingResponse
 
 
-import backoff
-
 log = logging.getLogger(__name__)
 T = TypeVar("T")
 

lightrag/lightrag/components/model_client/transformers_client.py

@@ -4,28 +4,30 @@
 import logging
 from functools import lru_cache
 
+
+from lightrag.core.model_client import ModelClient
+from lightrag.core.types import ModelType, Embedding, EmbedderOutput
+from lightrag.core.functional import get_top_k_indices_scores
+
+# optional import
+from lightrag.utils.lazy_import import safe_import, OptionalPackages
+
+
+transformers = safe_import(
+    OptionalPackages.TRANSFORMERS.value[0], OptionalPackages.TRANSFORMERS.value[1]
+)
+torch = safe_import(OptionalPackages.TORCH.value[0], OptionalPackages.TORCH.value[1])
+
 import torch.nn.functional as F
-import torch
 from torch import Tensor
 
-
 from transformers import (
     AutoTokenizer,
     AutoModel,
     AutoModelForSequenceClassification,
     AutoModelForCausalLM,
 )
 
-from lightrag.core.model_client import ModelClient
-from lightrag.core.types import ModelType, Embedding, EmbedderOutput
-from lightrag.core.functional import get_top_k_indices_scores
-from lightrag.utils.lazy_import import safe_import, OptionalPackages
-
-
-safe_import(
-    OptionalPackages.TRANSFORMERS.value[0], OptionalPackages.TRANSFORMERS.value[1]
-)
-safe_import(OptionalPackages.TORCH.value[0], OptionalPackages.TORCH.value[1])
 
 log = logging.getLogger(__name__)