Skip to content

Commit

Permalink
Fix mypy issues.
Browse files Browse the repository at this point in the history
  • Loading branch information
emmiegit committed Jan 15, 2024
1 parent 9ac6370 commit 5db34fe
Show file tree
Hide file tree
Showing 5 changed files with 15 additions and 6 deletions.
3 changes: 2 additions & 1 deletion yellowstone/requests/site_home_raw.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
Scrape site and page data from the home page of a site.
"""

import logging
import re
from dataclasses import dataclass
from typing import Optional
Expand All @@ -16,7 +17,7 @@
PAGE_CATEGORY_ID_REGEX = re.compile(r"WIKIREQUEST\.info\.categoryId = (\d+);")
FORUM_POST_ID_REGEX = re.compile(r"/forum/t-(\d+)/.*")

logger = logging.getName(__name__)
logger = logging.getLogger(__name__)


@dataclass
Expand Down
4 changes: 2 additions & 2 deletions yellowstone/requests/site_members.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@

from ..scraper import find_element, make_soup, regex_extract, get_entity_date
from ..wikidot import Wikidot
from .user import USER_SLUG_REGEX
from .user import get_user_slug

USER_ID_REGEX = re.compile(r"WIKIDOT\.page\.listeners\.userInfo\((\d+)\).*")

Expand Down Expand Up @@ -50,7 +50,7 @@ def process_row(row: Tag) -> SiteMemberData:
# Extract user information
element = row.find_all("a")[1]
name = element.text
slug = regex_extract(source, element["href"], USER_SLUG_REGEX)[1]
slug = get_user_slug(source, element)
id = int(regex_extract(source, element["onclick"], USER_ID_REGEX)[1])

# Extract membership join date
Expand Down
10 changes: 9 additions & 1 deletion yellowstone/requests/user.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ def get(user_id: int, *, wikidot: Wikidot) -> UserData:
# Get name-like fields
name = find_element(source, soup, "h1").text
element = find_element(source, soup, "a.btn-primary")
slug = regex_extract(source, element["href"], USER_SLUG_REGEX)[1]
slug = get_user_slug(source, element)

# Process user details
created_at = None
Expand Down Expand Up @@ -127,6 +127,14 @@ def get(user_id: int, *, wikidot: Wikidot) -> UserData:
)


def get_user_slug(source: str, element: Tag) -> str:
href = element["href"]
assert isinstance(href, str), "multiple href attributes found"
match = regex_extract(source, href, USER_SLUG_REGEX)
assert isinstance(match[1], str), "match group is not a string"
return match[1]


def split_user_detail(columns: tuple[Tag, Tag]) -> tuple[str, str, Tag]:
field, element = columns
assert "active" in field.attrs["class"], "field lacks 'active' class"
Expand Down
2 changes: 1 addition & 1 deletion yellowstone/s3.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ def __init__(self, config: Config) -> None:
)

def upload_avatar(self, blob: bytes) -> None:
self.upload_avatar(blob, AVATAR_DIRECTORY)
self.upload_blob(blob, AVATAR_DIRECTORY)

def upload_file(self, blob: bytes) -> None:
self.upload_blob(blob, FILE_DIRECTORY)
Expand Down
2 changes: 1 addition & 1 deletion yellowstone/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
T = TypeVar("T")


def chunks(it: Iterable[T], size: int) -> Iterable[tuple[T]]:
def chunks(it: Iterable[T], size: int) -> Iterable[tuple[T, ...]]:
iterator = iter(it)
while chunk := tuple(islice(iterator, size)):
yield chunk

0 comments on commit 5db34fe

Please sign in to comment.