From 5db34fe707960530f2d4ff92f330230ea3672d56 Mon Sep 17 00:00:00 2001 From: Emmie Maeda Date: Sun, 14 Jan 2024 23:32:47 -0500 Subject: [PATCH] Fix mypy issues. --- yellowstone/requests/site_home_raw.py | 3 ++- yellowstone/requests/site_members.py | 4 ++-- yellowstone/requests/user.py | 10 +++++++++- yellowstone/s3.py | 2 +- yellowstone/utils.py | 2 +- 5 files changed, 15 insertions(+), 6 deletions(-) diff --git a/yellowstone/requests/site_home_raw.py b/yellowstone/requests/site_home_raw.py index 1399abd..952cb92 100644 --- a/yellowstone/requests/site_home_raw.py +++ b/yellowstone/requests/site_home_raw.py @@ -2,6 +2,7 @@ Scrape site and page data from the home page of a site. """ +import logging import re from dataclasses import dataclass from typing import Optional @@ -16,7 +17,7 @@ PAGE_CATEGORY_ID_REGEX = re.compile(r"WIKIREQUEST\.info\.categoryId = (\d+);") FORUM_POST_ID_REGEX = re.compile(r"/forum/t-(\d+)/.*") -logger = logging.getName(__name__) +logger = logging.getLogger(__name__) @dataclass diff --git a/yellowstone/requests/site_members.py b/yellowstone/requests/site_members.py index 5dbeea7..20399b0 100644 --- a/yellowstone/requests/site_members.py +++ b/yellowstone/requests/site_members.py @@ -11,7 +11,7 @@ from ..scraper import find_element, make_soup, regex_extract, get_entity_date from ..wikidot import Wikidot -from .user import USER_SLUG_REGEX +from .user import get_user_slug USER_ID_REGEX = re.compile(r"WIKIDOT\.page\.listeners\.userInfo\((\d+)\).*") @@ -50,7 +50,7 @@ def process_row(row: Tag) -> SiteMemberData: # Extract user information element = row.find_all("a")[1] name = element.text - slug = regex_extract(source, element["href"], USER_SLUG_REGEX)[1] + slug = get_user_slug(source, element) id = int(regex_extract(source, element["onclick"], USER_ID_REGEX)[1]) # Extract membership join date diff --git a/yellowstone/requests/user.py b/yellowstone/requests/user.py index 81ce45f..c69bfa7 100644 --- a/yellowstone/requests/user.py +++ b/yellowstone/requests/user.py @@ -57,7 +57,7 @@ def get(user_id: int, *, wikidot: Wikidot) -> UserData: # Get name-like fields name = find_element(source, soup, "h1").text element = find_element(source, soup, "a.btn-primary") - slug = regex_extract(source, element["href"], USER_SLUG_REGEX)[1] + slug = get_user_slug(source, element) # Process user details created_at = None @@ -127,6 +127,14 @@ def get(user_id: int, *, wikidot: Wikidot) -> UserData: ) +def get_user_slug(source: str, element: Tag) -> str: + href = element["href"] + assert isinstance(href, str), "multiple href attributes found" + match = regex_extract(source, href, USER_SLUG_REGEX) + assert isinstance(match[1], str), "match group is not a string" + return match[1] + + def split_user_detail(columns: tuple[Tag, Tag]) -> tuple[str, str, Tag]: field, element = columns assert "active" in field.attrs["class"], "field lacks 'active' class" diff --git a/yellowstone/s3.py b/yellowstone/s3.py index ff16f47..0a16042 100644 --- a/yellowstone/s3.py +++ b/yellowstone/s3.py @@ -34,7 +34,7 @@ def __init__(self, config: Config) -> None: ) def upload_avatar(self, blob: bytes) -> None: - self.upload_avatar(blob, AVATAR_DIRECTORY) + self.upload_blob(blob, AVATAR_DIRECTORY) def upload_file(self, blob: bytes) -> None: self.upload_blob(blob, FILE_DIRECTORY) diff --git a/yellowstone/utils.py b/yellowstone/utils.py index fea7573..06e4da1 100644 --- a/yellowstone/utils.py +++ b/yellowstone/utils.py @@ -8,7 +8,7 @@ T = TypeVar("T") -def chunks(it: Iterable[T], size: int) -> Iterable[tuple[T]]: +def chunks(it: Iterable[T], size: int) -> Iterable[tuple[T, ...]]: iterator = iter(it) while chunk := tuple(islice(iterator, size)): yield chunk