diff --git a/pyproject.toml b/pyproject.toml index ee5472aae1..3684cafa09 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -270,8 +270,22 @@ exclude = "(.eggs|.git|.hg|.mypy_cache|.venv|_build|buck-out|build|dist)" fix = true line-length = 120 target-version = 'py39' -#include = ["autogen", "test", "docs"] -exclude = ["setup_*.py"] +# Exclude a variety of commonly ignored directories. +exclude = [ + ".eggs", + ".git", + ".mypy_cache", + ".ruff_cache", + "__pypackages__", + "_build", + "build", + "dist", + "docs", + # This file needs to be either upgraded or removed and therefore should be + # ignore from type checking for now + "math_utils\\.py$", + "setup_*.py", +] [tool.ruff.lint] # Enable Pyflakes `E` and `F` codes by default. @@ -305,21 +319,6 @@ ignore = ["E501", "F403", "C901", "D100", "D101", "D102", "D103", "D104", "C901", # too complex ] -# Exclude a variety of commonly ignored directories. -exclude = [ - ".eggs", - ".git", - ".mypy_cache", - ".ruff_cache", - "__pypackages__", - "_build", - "build", - "dist", - "docs", - # This file needs to be either upgraded or removed and therefore should be - # ignore from type checking for now - "math_utils\\.py$", -] [tool.ruff.lint.mccabe] # Unlike Flake8, default to a complexity level of 10. @@ -348,6 +347,7 @@ files = [ "autogen/agentchat/realtime_agent", "autogen/messages", "autogen/import_utils.py", + "website/*.py", "test/test_pydantic.py", "test/io", "test/tools", @@ -357,6 +357,7 @@ files = [ "test/conftest.py", "test/test_import_utils.py", "test/test_import.py", + "test/website", ] exclude = [ diff --git a/test/website/test_process_api_reference.py b/test/website/test_process_api_reference.py index 53629a7777..3550f905e4 100644 --- a/test/website/test_process_api_reference.py +++ b/test/website/test_process_api_reference.py @@ -6,15 +6,21 @@ import sys import tempfile from pathlib import Path +from typing import Generator import pytest # Add the ../../website directory to sys.path -sys.path.append(str(Path(__file__).resolve().parent.parent.parent / "website")) +website_path = Path(__file__).resolve().parents[2] / "website" +assert website_path.exists() +assert website_path.is_dir() +sys.path.append(str(website_path)) + from process_api_reference import generate_mint_json_from_template, move_files_excluding_index -def create_test_directory_structure(tmp_path): +@pytest.fixture +def api_dir(tmp_path: Path) -> Path: """Helper function to create test directory structure""" # Create autogen directory autogen_dir = tmp_path / "autogen" @@ -48,11 +54,8 @@ def create_test_directory_structure(tmp_path): return tmp_path -def test_move_files_excluding_index(tmp_path): +def test_move_files_excluding_index(api_dir: Path) -> None: """Test that files are moved correctly excluding index.md""" - # Setup the test directory structure - api_dir = create_test_directory_structure(tmp_path) - # Call the function under test move_files_excluding_index(api_dir) @@ -79,7 +82,7 @@ def test_move_files_excluding_index(tmp_path): @pytest.fixture -def template_content(): +def template_content() -> str: """Fixture providing the template JSON content.""" template = """ { @@ -103,14 +106,14 @@ def template_content(): @pytest.fixture -def temp_dir(): +def temp_dir() -> Generator[Path, None, None]: """Fixture providing a temporary directory.""" with tempfile.TemporaryDirectory() as tmp_dir: yield Path(tmp_dir) @pytest.fixture -def template_file(temp_dir, template_content): +def template_file(temp_dir: Path, template_content: str) -> Path: """Fixture creating a template file in a temporary directory.""" template_path = temp_dir / "mint-json-template.json.jinja" with open(template_path, "w") as f: @@ -119,12 +122,12 @@ def template_file(temp_dir, template_content): @pytest.fixture -def target_file(temp_dir): +def target_file(temp_dir: Path) -> Path: """Fixture providing the target mint.json path.""" return temp_dir / "mint.json" -def test_generate_mint_json_from_template(template_file, target_file, template_content): +def test_generate_mint_json_from_template(template_file: Path, target_file: Path, template_content: str) -> None: """Test that mint.json is generated correctly from template.""" # Run the function generate_mint_json_from_template(template_file, target_file) @@ -140,7 +143,7 @@ def test_generate_mint_json_from_template(template_file, target_file, template_c assert actual == expected -def test_generate_mint_json_existing_file(template_file, target_file, template_content): +def test_generate_mint_json_existing_file(template_file: Path, target_file: Path, template_content: str) -> None: """Test that function works when mint.json already exists.""" # Create an existing mint.json with different content existing_content = {"name": "existing"} @@ -158,7 +161,7 @@ def test_generate_mint_json_existing_file(template_file, target_file, template_c assert actual == expected -def test_generate_mint_json_missing_template(target_file): +def test_generate_mint_json_missing_template(target_file: Path) -> None: """Test handling of missing template file.""" with tempfile.TemporaryDirectory() as tmp_dir: nonexistent_template = Path(tmp_dir) / "nonexistent.template" diff --git a/test/website/test_process_notebooks.py b/test/website/test_process_notebooks.py index 18cf4f104a..5cd6342085 100644 --- a/test/website/test_process_notebooks.py +++ b/test/website/test_process_notebooks.py @@ -7,6 +7,7 @@ import tempfile import textwrap from pathlib import Path +from typing import Generator, Optional, Union import pytest @@ -24,7 +25,7 @@ ) -def test_ensure_mint_json(): +def test_ensure_mint_json() -> None: # Test with empty temp directory - should raise SystemExit with tempfile.TemporaryDirectory() as tmp_dir: tmp_path = Path(tmp_dir) @@ -36,7 +37,7 @@ def test_ensure_mint_json(): ensure_mint_json_exists(tmp_path) # Should not raise any exception -def test_cleanup_tmp_dirs_if_no_metadata(): +def test_cleanup_tmp_dirs_if_no_metadata() -> None: # Test without the tmp_dir / "snippets" / "data" / "NotebooksMetadata.mdx" # the tmp_dir / "notebooks" should be removed. with tempfile.TemporaryDirectory() as tmp_dir: @@ -68,8 +69,8 @@ def test_cleanup_tmp_dirs_if_no_metadata(): class TestAddFrontMatterToMetadataMdx: - def test_without_metadata_mdx(self): - front_matter_dict = { + def test_without_metadata_mdx(self) -> None: + front_matter_dict: dict[str, Union[str, Optional[Union[list[str]]]]] = { "title": "some title", "link": "/notebooks/some-title", "description": "some description", @@ -118,8 +119,8 @@ def test_without_metadata_mdx(self): """ ) - def test_with_metadata_mdx(self): - front_matter_dict = { + def test_with_metadata_mdx(self) -> None: + front_matter_dict: dict[str, Optional[Union[str, Union[list[str]]]]] = { "title": "some title", "link": "/notebooks/some-title", "description": "some description", @@ -203,7 +204,7 @@ def test_with_metadata_mdx(self): class TestAddBlogsToNavigation: @pytest.fixture - def test_dir(self): + def test_dir(self) -> Generator[Path, None, None]: """Create a temporary directory with test files.""" with tempfile.TemporaryDirectory() as tmp_dir: tmp_path = Path(tmp_dir) @@ -231,7 +232,7 @@ def test_dir(self): yield tmp_path @pytest.fixture - def expected(self): + def expected(self) -> list[str]: return [ "blog/2024-12-20-Tools-interoperability/index", "blog/2024-12-20-RetrieveChat/index", @@ -246,11 +247,11 @@ def expected(self): "blog/2023-04-21-LLM-tuning-math/index", ] - def test_get_sorted_files(self, test_dir, expected): + def test_get_sorted_files(self, test_dir: Path, expected: list[str]) -> None: actual = get_sorted_files(test_dir, "blog") assert actual == expected, actual - def test_add_blogs_to_navigation(self): + def test_add_blogs_to_navigation(self) -> None: with tempfile.TemporaryDirectory() as tmp_dir: website_dir = Path(tmp_dir) blog_dir = website_dir / "blog" @@ -374,7 +375,7 @@ def setup(self, temp_dir: Path) -> None: with open(metadata_path, "w", encoding="utf-8") as f: f.write(notebooks_metadata_content) - def test_extract_example_group(self): + def test_extract_example_group(self) -> None: with tempfile.TemporaryDirectory() as tmp_dir: tmp_path = Path(tmp_dir) self.setup(tmp_path) @@ -403,7 +404,7 @@ def test_extract_example_group(self): class TestAddAuthorsAndSocialImgToBlogPosts: @pytest.fixture - def test_dir(self): + def test_dir(self) -> Generator[Path, None, None]: """Create temporary test directory with blog posts and authors file.""" with tempfile.TemporaryDirectory() as tmp_dir: website_dir = Path(tmp_dir) @@ -534,7 +535,7 @@ def test_dir(self): yield website_dir - def test_add_authors_and_social_img(self, test_dir): + def test_add_authors_and_social_img(self, test_dir: Path) -> None: # Run the function add_authors_and_social_img_to_blog_posts(test_dir) @@ -682,6 +683,6 @@ def expected(self) -> str: This is a conclusion. """) - def test_convert_callout_blocks(self, content: str, expected: str) -> str: + def test_convert_callout_blocks(self, content: str, expected: str) -> None: actual = convert_callout_blocks(content) assert actual == expected, actual diff --git a/website/process_api_reference.py b/website/process_api_reference.py index e5ec83b2ed..bf7919b274 100755 --- a/website/process_api_reference.py +++ b/website/process_api_reference.py @@ -15,7 +15,7 @@ import subprocess import sys from pathlib import Path -from typing import Any +from typing import Any, Optional from jinja2 import Template @@ -55,7 +55,7 @@ def run_pdoc3(api_dir: Path) -> None: sys.exit(1) -def read_file_content(file_path: str) -> str: +def read_file_content(file_path: Path) -> str: """Read content from a file. Args: @@ -108,15 +108,15 @@ def get_mdx_files(directory: Path) -> list[str]: return [f"{p.relative_to(directory).with_suffix('')!s}".replace("\\", "/") for p in directory.rglob("*.mdx")] -def add_prefix(path: str, parent_groups: list[str] = None) -> str: +def add_prefix(path: str, parent_groups: Optional[list[str]] = None) -> str: """Create full path with prefix and parent groups.""" groups = parent_groups or [] return f"docs/reference/{'/'.join(groups + [path])}" -def create_nav_structure(paths: list[str], parent_groups: list[str] = None) -> list[Any]: +def create_nav_structure(paths: list[str], parent_groups: Optional[list[str]] = None) -> list[Any]: """Convert list of file paths into nested navigation structure.""" - groups = {} + groups: dict[str, list[str]] = {} pages = [] parent_groups = parent_groups or [] diff --git a/website/process_notebooks.py b/website/process_notebooks.py index 41f6a312cf..c9cc0afe84 100755 --- a/website/process_notebooks.py +++ b/website/process_notebooks.py @@ -25,7 +25,7 @@ from multiprocessing import current_process from pathlib import Path from textwrap import dedent, indent -from typing import Dict, List, Tuple, Union +from typing import Any, Collection, Dict, List, Sequence, Tuple, Union from termcolor import colored @@ -37,10 +37,10 @@ try: import nbclient # noqa: F401 - from nbclient.client import ( + from nbclient.client import NotebookClient + from nbclient.exceptions import ( CellExecutionError, CellTimeoutError, - NotebookClient, ) except ImportError: if current_process().name == "MainProcess": @@ -66,8 +66,8 @@ def __init__(self, returncode: int, stdout: str, stderr: str): def check_quarto_bin(quarto_bin: str = "quarto") -> None: """Check if quarto is installed.""" try: - version = subprocess.check_output([quarto_bin, "--version"], text=True).strip() - version = tuple(map(int, version.split("."))) + version_str = subprocess.check_output([quarto_bin, "--version"], text=True).strip() + version = tuple(map(int, version_str.split("."))) if version < (1, 5, 23): print("Quarto version is too old. Please upgrade to 1.5.23 or later.") sys.exit(1) @@ -82,12 +82,13 @@ def notebooks_target_dir(website_directory: Path) -> Path: return website_directory / "notebooks" -def load_metadata(notebook: Path) -> dict: +def load_metadata(notebook: Path) -> dict[str, dict[str, Union[str, list[str], None]]]: content = json.load(notebook.open(encoding="utf-8")) - return content["metadata"] + metadata: dict[str, dict[str, Union[str, list[str], None]]] = content.get("metadata", {}) + return metadata -def skip_reason_or_none_if_ok(notebook: Path) -> str | None: +def skip_reason_or_none_if_ok(notebook: Path) -> Union[str, None, dict[str, Any]]: """Return a reason to skip the notebook, or None if it should not be skipped.""" if notebook.suffix != ".ipynb": return "not a notebook" @@ -129,8 +130,9 @@ def skip_reason_or_none_if_ok(notebook: Path) -> str | None: return "description is not in front matter" # Make sure tags is a list of strings - if not all([isinstance(tag, str) for tag in front_matter["tags"]]): - return "tags must be a list of strings" + if front_matter["tags"] is not None: + if not all([isinstance(tag, str) for tag in front_matter["tags"]]): + return "tags must be a list of strings" # Make sure description is a string if not isinstance(front_matter["description"], str): @@ -151,7 +153,7 @@ def extract_title(notebook: Path) -> str | None: # find the # title for line in first_cell["source"]: if line.startswith("# "): - title = line[2:].strip() + title: str = line[2:].strip() # Strip off the { if it exists if "{" in title: title = title[: title.find("{")].strip() @@ -249,7 +251,7 @@ class NotebookSkip: def test_notebook(notebook_path: Path, timeout: int = 300) -> tuple[Path, NotebookError | NotebookSkip | None]: - nb = nbformat.read(str(notebook_path), NB_VERSION) + nb = nbformat.read(str(notebook_path), NB_VERSION) # type: ignore if "skip_test" in nb.metadata: return notebook_path, NotebookSkip(reason=nb.metadata.skip_test) @@ -313,7 +315,7 @@ def get_error_info(nb: NotebookNode) -> NotebookError | None: def add_front_matter_to_metadata_mdx( - front_matter: dict[str, str | list[str]], website_dir: Path, rendered_mdx: Path + front_matter: dict[str, Union[str, list[str], None]], website_dir: Path, rendered_mdx: Path ) -> None: source = front_matter.get("source_notebook") if isinstance(source, str) and source.startswith("/website/docs/"): @@ -427,7 +429,7 @@ def convert_callout_blocks(content: str) -> str: r")" ) - def replace_callout(m: re.Match) -> str: + def replace_callout(m: re.Match[str]) -> str: # Determine the matched alternative and extract the corresponding groups. ctype = m.group("callout_type_backtick") or m.group("callout_type_no_backtick") inner = m.group("inner_backtick") or m.group("inner_no_backtick") or "" @@ -458,7 +460,7 @@ def convert_mdx_image_blocks(content: str, rendered_mdx: Path, website_dir: Path str: The converted markdown content with standard image syntax """ - def resolve_path(match): + def resolve_path(match: re.Match[str]) -> str: img_pattern = r"!\[(.*?)\]\((.*?)\)" img_match = re.search(img_pattern, match.group(1)) if not img_match: @@ -473,7 +475,9 @@ def resolve_path(match): # rendered_notebook is the final mdx file -def post_process_mdx(rendered_mdx: Path, source_notebooks: Path, front_matter: dict, website_dir: Path) -> None: +def post_process_mdx( + rendered_mdx: Path, source_notebooks: Path, front_matter: dict[str, Union[str, list[str], None]], website_dir: Path +) -> None: with open(rendered_mdx, encoding="utf-8") as f: content = f.read() @@ -536,7 +540,7 @@ def post_process_mdx(rendered_mdx: Path, source_notebooks: Path, front_matter: d add_front_matter_to_metadata_mdx(front_matter, website_dir, rendered_mdx) # Dump front_matter to ysaml - front_matter = yaml.dump(front_matter, default_flow_style=False) + front_matter_str = yaml.dump(front_matter, default_flow_style=False) # Convert callout blocks content = convert_callout_blocks(content) @@ -546,10 +550,10 @@ def post_process_mdx(rendered_mdx: Path, source_notebooks: Path, front_matter: d # Rewrite the content as # --- - # front_matter + # front_matter_str # --- # content - new_content = f"---\n{front_matter}---\n{content}" + new_content = f"---\n{front_matter_str}---\n{content}" with open(rendered_mdx, "w", encoding="utf-8") as f: f.write(new_content) @@ -582,7 +586,7 @@ def fmt_error(notebook: Path, error: NotebookError | str) -> str: raise ValueError("error must be a string or a NotebookError") -def start_thread_to_terminate_when_parent_process_dies(ppid: int): +def start_thread_to_terminate_when_parent_process_dies(ppid: int) -> None: pid = os.getpid() def f() -> None: @@ -602,12 +606,12 @@ def copy_examples_mdx_files(website_dir: str) -> None: example_section_mdx_files = ["Gallery", "Notebooks"] # Create notebooks directory if it doesn't exist - website_dir = Path(website_dir) - notebooks_dir = website_dir / "notebooks" + website_dir_path = Path(website_dir) + notebooks_dir = website_dir_path / "notebooks" notebooks_dir.mkdir(parents=True, exist_ok=True) for mdx_file in example_section_mdx_files: - src_mdx_file_path = (website_dir / "docs" / f"{mdx_file}.mdx").resolve() + src_mdx_file_path = (website_dir_path / "docs" / f"{mdx_file}.mdx").resolve() dest_mdx_file_path = (notebooks_dir / f"{mdx_file}.mdx").resolve() # Copy mdx file to notebooks directory shutil.copy(src_mdx_file_path, dest_mdx_file_path) @@ -619,7 +623,7 @@ def get_sorted_files(input_dir: Path, prefix: str) -> list[str]: raise FileNotFoundError(f"Directory not found: {input_dir}") # Sort files by parent directory date (if exists) and name - def sort_key(file_path): + def sort_key(file_path: Path) -> Tuple[datetime, str]: dirname = file_path.parent.name try: # Extract date from directory name (first 3 parts) @@ -648,7 +652,7 @@ def generate_nav_group(input_dir: Path, group_header: str, prefix: str) -> Dict[ return {"group": group_header, "pages": sorted_dir_files} -def extract_example_group(metadata_path): +def extract_example_group(metadata_path: Path) -> dict[str, Sequence[Collection[str]]]: # Read NotebooksMetadata.mdx and extract metadata links with open(metadata_path, encoding="utf-8") as f: content = f.read() @@ -657,7 +661,7 @@ def extract_example_group(metadata_path): end = content.rfind("]") if start == -1 or end == -1: print("Could not find notebooksMetadata in the file") - return + return {} metadata_str = content[start + 32 : end + 1] notebooks_metadata = json.loads(metadata_str) @@ -705,10 +709,13 @@ def update_navigation_with_notebooks(website_dir: Path) -> None: # add talks to navigation talks_dir = website_dir / "talks" talks_section = generate_nav_group(talks_dir, "Talks", "talks") + talks_section_pages = ( + [talks_section["pages"]] if isinstance(talks_section["pages"], str) else talks_section["pages"] + ) # Add "talks/future_talks/index" item at the beginning of the list - future_talks_index = talks_section["pages"].pop() - talks_section["pages"].insert(0, future_talks_index) + future_talks_index = talks_section_pages.pop() + talks_section_pages.insert(0, future_talks_index) mint_config["navigation"].append(talks_section) # add blogs to navigation @@ -737,7 +744,7 @@ def fix_internal_references(content: str, root_path: Path, current_file_path: Pa current_file_path: Path of the current file being processed """ - def resolve_link(match): + def resolve_link(match: re.Match[str]) -> str: display_text, raw_path = match.groups() try: path_parts = raw_path.split("#") @@ -960,7 +967,7 @@ def main() -> None: filtered_notebooks = [] for notebook in collected_notebooks: reason = skip_reason_or_none_if_ok(notebook) - if reason: + if reason and isinstance(reason, str): print(fmt_skip(notebook, reason)) else: filtered_notebooks.append(notebook)