[Docs] Resolve mypy type-checking issues in docs generation files (#621)

* WIP * updated mako template * WIP * Update meko template * Update meko template * Use pdoc3 for generating the API docs * Fix the generated docs/reference directory structure * Polishing * polishing * Remove setup.py * Remove docs-requirement file * Polishing * Polishing * Docs test * Docs test * Fix broken links * Remove docs extra from dev * Add missing packages * Remove redundant command * Add returns section to docs * Update docs * Update python version in workflow file * Pass OPENAI_API_KEY in CI * Fix review comments * Remove docs from dev extra * Add quotes around each packahe name * Polishing * WIP * WIP * Fix missed import in file * Fix missed third party imports in arxiv files * Add jinja in docs dependency * Cleanup * Polishing * wip * WIP * Fix mypy issues * fix typos * pre-commit fixed --------- Co-authored-by: Kumaran Rajendhiran <[email protected]> Co-authored-by: Davor Runje <[email protected]>
ag2ai · Jan 22, 2025 · a3c7216 · a3c7216
1 parent 90be743
commit a3c7216
Show file tree

Hide file tree

Showing 5 changed files with 91 additions and 79 deletions.
diff --git a/pyproject.toml b/pyproject.toml
@@ -270,8 +270,22 @@ exclude = "(.eggs|.git|.hg|.mypy_cache|.venv|_build|buck-out|build|dist)"
 fix = true
 line-length = 120
 target-version = 'py39'
-#include = ["autogen", "test", "docs"]
-exclude = ["setup_*.py"]
+# Exclude a variety of commonly ignored directories.
+exclude = [
+    ".eggs",
+    ".git",
+    ".mypy_cache",
+    ".ruff_cache",
+    "__pypackages__",
+    "_build",
+    "build",
+    "dist",
+    "docs",
+    # This file needs to be either upgraded or removed and therefore should be
+    # ignore from type checking for now
+    "math_utils\\.py$",
+    "setup_*.py",
+]
 
 [tool.ruff.lint]
 # Enable Pyflakes `E` and `F` codes by default.
@@ -305,21 +319,6 @@ ignore = ["E501", "F403", "C901",
     "D100", "D101", "D102", "D103", "D104",
     "C901",  # too complex
 ]
-# Exclude a variety of commonly ignored directories.
-exclude = [
-    ".eggs",
-    ".git",
-    ".mypy_cache",
-    ".ruff_cache",
-    "__pypackages__",
-    "_build",
-    "build",
-    "dist",
-    "docs",
-    # This file needs to be either upgraded or removed and therefore should be
-    # ignore from type checking for now
-    "math_utils\\.py$",
-]
 
 [tool.ruff.lint.mccabe]
 # Unlike Flake8, default to a complexity level of 10.
@@ -348,6 +347,7 @@ files = [
     "autogen/agentchat/realtime_agent",
     "autogen/messages",
     "autogen/import_utils.py",
+    "website/*.py",
     "test/test_pydantic.py",
     "test/io",
     "test/tools",
@@ -357,6 +357,7 @@ files = [
     "test/conftest.py",
     "test/test_import_utils.py",
     "test/test_import.py",
+    "test/website",
 ]
 
 exclude = [

diff --git a/test/website/test_process_api_reference.py b/test/website/test_process_api_reference.py
@@ -6,15 +6,21 @@
 import sys
 import tempfile
 from pathlib import Path
+from typing import Generator
 
 import pytest
 
 # Add the ../../website directory to sys.path
-sys.path.append(str(Path(__file__).resolve().parent.parent.parent / "website"))
+website_path = Path(__file__).resolve().parents[2] / "website"
+assert website_path.exists()
+assert website_path.is_dir()
+sys.path.append(str(website_path))
+
 from process_api_reference import generate_mint_json_from_template, move_files_excluding_index
 
 
-def create_test_directory_structure(tmp_path):
+@pytest.fixture
+def api_dir(tmp_path: Path) -> Path:
     """Helper function to create test directory structure"""
     # Create autogen directory
     autogen_dir = tmp_path / "autogen"
@@ -48,11 +54,8 @@ def create_test_directory_structure(tmp_path):
     return tmp_path
 
 
-def test_move_files_excluding_index(tmp_path):
+def test_move_files_excluding_index(api_dir: Path) -> None:
     """Test that files are moved correctly excluding index.md"""
-    # Setup the test directory structure
-    api_dir = create_test_directory_structure(tmp_path)
-
     # Call the function under test
     move_files_excluding_index(api_dir)
 
@@ -79,7 +82,7 @@ def test_move_files_excluding_index(tmp_path):
 
 
 @pytest.fixture
-def template_content():
+def template_content() -> str:
     """Fixture providing the template JSON content."""
     template = """
     {
@@ -103,14 +106,14 @@ def template_content():
 
 
 @pytest.fixture
-def temp_dir():
+def temp_dir() -> Generator[Path, None, None]:
     """Fixture providing a temporary directory."""
     with tempfile.TemporaryDirectory() as tmp_dir:
         yield Path(tmp_dir)
 
 
 @pytest.fixture
-def template_file(temp_dir, template_content):
+def template_file(temp_dir: Path, template_content: str) -> Path:
     """Fixture creating a template file in a temporary directory."""
     template_path = temp_dir / "mint-json-template.json.jinja"
     with open(template_path, "w") as f:
@@ -119,12 +122,12 @@ def template_file(temp_dir, template_content):
 
 
 @pytest.fixture
-def target_file(temp_dir):
+def target_file(temp_dir: Path) -> Path:
     """Fixture providing the target mint.json path."""
     return temp_dir / "mint.json"
 
 
-def test_generate_mint_json_from_template(template_file, target_file, template_content):
+def test_generate_mint_json_from_template(template_file: Path, target_file: Path, template_content: str) -> None:
     """Test that mint.json is generated correctly from template."""
     # Run the function
     generate_mint_json_from_template(template_file, target_file)
@@ -140,7 +143,7 @@ def test_generate_mint_json_from_template(template_file, target_file, template_c
     assert actual == expected
 
 
-def test_generate_mint_json_existing_file(template_file, target_file, template_content):
+def test_generate_mint_json_existing_file(template_file: Path, target_file: Path, template_content: str) -> None:
     """Test that function works when mint.json already exists."""
     # Create an existing mint.json with different content
     existing_content = {"name": "existing"}
@@ -158,7 +161,7 @@ def test_generate_mint_json_existing_file(template_file, target_file, template_c
     assert actual == expected
 
 
-def test_generate_mint_json_missing_template(target_file):
+def test_generate_mint_json_missing_template(target_file: Path) -> None:
     """Test handling of missing template file."""
     with tempfile.TemporaryDirectory() as tmp_dir:
         nonexistent_template = Path(tmp_dir) / "nonexistent.template"

diff --git a/test/website/test_process_notebooks.py b/test/website/test_process_notebooks.py
@@ -7,6 +7,7 @@
 import tempfile
 import textwrap
 from pathlib import Path
+from typing import Generator, Optional, Union
 
 import pytest
 
@@ -24,7 +25,7 @@
 )
 
 
-def test_ensure_mint_json():
+def test_ensure_mint_json() -> None:
     # Test with empty temp directory - should raise SystemExit
     with tempfile.TemporaryDirectory() as tmp_dir:
         tmp_path = Path(tmp_dir)
@@ -36,7 +37,7 @@ def test_ensure_mint_json():
         ensure_mint_json_exists(tmp_path)  # Should not raise any exception
 
 
-def test_cleanup_tmp_dirs_if_no_metadata():
+def test_cleanup_tmp_dirs_if_no_metadata() -> None:
     # Test without the tmp_dir / "snippets" / "data" / "NotebooksMetadata.mdx"
     # the tmp_dir / "notebooks" should be removed.
     with tempfile.TemporaryDirectory() as tmp_dir:
@@ -68,8 +69,8 @@ def test_cleanup_tmp_dirs_if_no_metadata():
 
 
 class TestAddFrontMatterToMetadataMdx:
-    def test_without_metadata_mdx(self):
-        front_matter_dict = {
+    def test_without_metadata_mdx(self) -> None:
+        front_matter_dict: dict[str, Union[str, Optional[Union[list[str]]]]] = {
             "title": "some title",
             "link": "/notebooks/some-title",
             "description": "some description",
@@ -118,8 +119,8 @@ def test_without_metadata_mdx(self):
 """
             )
 
-    def test_with_metadata_mdx(self):
-        front_matter_dict = {
+    def test_with_metadata_mdx(self) -> None:
+        front_matter_dict: dict[str, Optional[Union[str, Union[list[str]]]]] = {
             "title": "some title",
             "link": "/notebooks/some-title",
             "description": "some description",
@@ -203,7 +204,7 @@ def test_with_metadata_mdx(self):
 
 class TestAddBlogsToNavigation:
     @pytest.fixture
-    def test_dir(self):
+    def test_dir(self) -> Generator[Path, None, None]:
         """Create a temporary directory with test files."""
         with tempfile.TemporaryDirectory() as tmp_dir:
             tmp_path = Path(tmp_dir)
@@ -231,7 +232,7 @@ def test_dir(self):
             yield tmp_path
 
     @pytest.fixture
-    def expected(self):
+    def expected(self) -> list[str]:
         return [
             "blog/2024-12-20-Tools-interoperability/index",
             "blog/2024-12-20-RetrieveChat/index",
@@ -246,11 +247,11 @@ def expected(self):
             "blog/2023-04-21-LLM-tuning-math/index",
         ]
 
-    def test_get_sorted_files(self, test_dir, expected):
+    def test_get_sorted_files(self, test_dir: Path, expected: list[str]) -> None:
         actual = get_sorted_files(test_dir, "blog")
         assert actual == expected, actual
 
-    def test_add_blogs_to_navigation(self):
+    def test_add_blogs_to_navigation(self) -> None:
         with tempfile.TemporaryDirectory() as tmp_dir:
             website_dir = Path(tmp_dir)
             blog_dir = website_dir / "blog"
@@ -374,7 +375,7 @@ def setup(self, temp_dir: Path) -> None:
         with open(metadata_path, "w", encoding="utf-8") as f:
             f.write(notebooks_metadata_content)
 
-    def test_extract_example_group(self):
+    def test_extract_example_group(self) -> None:
         with tempfile.TemporaryDirectory() as tmp_dir:
             tmp_path = Path(tmp_dir)
             self.setup(tmp_path)
@@ -403,7 +404,7 @@ def test_extract_example_group(self):
 
 class TestAddAuthorsAndSocialImgToBlogPosts:
     @pytest.fixture
-    def test_dir(self):
+    def test_dir(self) -> Generator[Path, None, None]:
         """Create temporary test directory with blog posts and authors file."""
         with tempfile.TemporaryDirectory() as tmp_dir:
             website_dir = Path(tmp_dir)
@@ -534,7 +535,7 @@ def test_dir(self):
 
             yield website_dir
 
-    def test_add_authors_and_social_img(self, test_dir):
+    def test_add_authors_and_social_img(self, test_dir: Path) -> None:
         # Run the function
         add_authors_and_social_img_to_blog_posts(test_dir)
 
@@ -682,6 +683,6 @@ def expected(self) -> str:
             This is a conclusion.
             """)
 
-    def test_convert_callout_blocks(self, content: str, expected: str) -> str:
+    def test_convert_callout_blocks(self, content: str, expected: str) -> None:
         actual = convert_callout_blocks(content)
         assert actual == expected, actual
diff --git a/website/process_api_reference.py b/website/process_api_reference.py
@@ -15,7 +15,7 @@
 import subprocess
 import sys
 from pathlib import Path
-from typing import Any
+from typing import Any, Optional
 
 from jinja2 import Template
 
@@ -55,7 +55,7 @@ def run_pdoc3(api_dir: Path) -> None:
         sys.exit(1)
 
 
-def read_file_content(file_path: str) -> str:
+def read_file_content(file_path: Path) -> str:
     """Read content from a file.
 
     Args:
@@ -108,15 +108,15 @@ def get_mdx_files(directory: Path) -> list[str]:
     return [f"{p.relative_to(directory).with_suffix('')!s}".replace("\\", "/") for p in directory.rglob("*.mdx")]
 
 
-def add_prefix(path: str, parent_groups: list[str] = None) -> str:
+def add_prefix(path: str, parent_groups: Optional[list[str]] = None) -> str:
     """Create full path with prefix and parent groups."""
     groups = parent_groups or []
     return f"docs/reference/{'/'.join(groups + [path])}"
 
 
-def create_nav_structure(paths: list[str], parent_groups: list[str] = None) -> list[Any]:
+def create_nav_structure(paths: list[str], parent_groups: Optional[list[str]] = None) -> list[Any]:
     """Convert list of file paths into nested navigation structure."""
-    groups = {}
+    groups: dict[str, list[str]] = {}
     pages = []
     parent_groups = parent_groups or []