diff --git a/.env.dev b/.env.dev deleted file mode 100644 index 6d8bdb8..0000000 --- a/.env.dev +++ /dev/null @@ -1,6 +0,0 @@ -DB_USERNAME=${DB_USERNAME} -DB_PASSWORD=${DB_PASSWORD} -DB_NAME=${DB_NAME} -DB_HOST=${DB_HOST} -DB_PORT=${DB_PORT} -SLACK_API_TOKEN=${SLACK_API_TOKEN} diff --git a/.env.prod b/.env.prod deleted file mode 100644 index 6d8bdb8..0000000 --- a/.env.prod +++ /dev/null @@ -1,6 +0,0 @@ -DB_USERNAME=${DB_USERNAME} -DB_PASSWORD=${DB_PASSWORD} -DB_NAME=${DB_NAME} -DB_HOST=${DB_HOST} -DB_PORT=${DB_PORT} -SLACK_API_TOKEN=${SLACK_API_TOKEN} diff --git a/.gitignore b/.gitignore index d0b256c..c7d090a 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,6 @@ node_modules *.pickle .DS_Store /install-pyenv-win.ps1 + +*.json +*.csv diff --git a/README.md b/README.md index af7b513..c07868b 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,18 @@ docker run --name waffledotcom-test \ -d mysql:latest ``` +To run locally: + +``` +docker run -d \ + -e MYSQL_ROOT_PASSWORD=local \ + -e MYSQL_DATABASE=waffledotcom_local \ + -e MYSQL_USER=waffledotcom-local \ + -e MYSQL_PASSWORD=local \ + -p 3306:3306 \ + --name waffledotcom-local \ + mysql:latest +``` ## Convention diff --git a/waffledotcom/src/apps/user/models.py b/waffledotcom/src/apps/user/models.py index 8b7fe63..d11b75a 100644 --- a/waffledotcom/src/apps/user/models.py +++ b/waffledotcom/src/apps/user/models.py @@ -46,6 +46,8 @@ class User(DeclarativeBase): department: Mapped[str50_default_none] college: Mapped[str50_default_none] + generation: Mapped[str30 | None] + phone_number: Mapped[str30 | None] github_id: Mapped[str50_default_none] diff --git a/waffledotcom/src/apps/user/repositories.py b/waffledotcom/src/apps/user/repositories.py index 13c2136..b6bc924 100644 --- a/waffledotcom/src/apps/user/repositories.py +++ b/waffledotcom/src/apps/user/repositories.py @@ -23,6 +23,9 @@ def get_user_by_id(self, user_id: int) -> User | None: def get_user_by_sso_id(self, sso_id: str) -> User | None: return self.session.query(User).filter(User.sso_id == sso_id).first() + def get_user_by_slack_id(self, slack_id: str) -> User | None: + return self.session.query(User).filter(User.slack_id == slack_id).first() + def create_user(self, user: User) -> User: with self.transaction: self.session.add(user) diff --git a/waffledotcom/src/apps/user/services.py b/waffledotcom/src/apps/user/services.py index f2631f6..ce113e5 100644 --- a/waffledotcom/src/apps/user/services.py +++ b/waffledotcom/src/apps/user/services.py @@ -12,7 +12,7 @@ UserCreateUpdateRequest, UserDetailResponse, ) -from waffledotcom.src.external.slack.schema import SlackMember +from waffledotcom.src.batch.slack.schema import SlackMember class UserService: @@ -55,16 +55,27 @@ def create_users_from_slack(self, slack_members: list[SlackMember]) -> None: slack_id=slack_member.id, slack_email=slack_member.profile.email, first_name=slack_member.real_name or "", + phone_number=slack_member.profile.phone or None, last_name="", is_member=True, ) for slack_member in slack_members ] - try: - self.user_repository.create_users(users) - except IntegrityError as exc: - raise UserAlreadyExistsException from exc + for user in users: + assert isinstance(user.slack_id, str) + if ( + created_user := self.user_repository.get_user_by_slack_id(user.slack_id) + ) is None: + self.user_repository.create_user(user) + continue + + created_user.slack_email = user.slack_email + created_user.phone_number = user.phone_number + created_user.image_url = user.image_url + created_user.first_name = user.first_name + created_user.last_name = user.last_name + self.user_repository.update_user(created_user) def update_user( self, user_id: int, request: UserCreateUpdateRequest diff --git a/waffledotcom/src/batch/slack/config.py b/waffledotcom/src/batch/slack/config.py new file mode 100644 index 0000000..fe1e2a5 --- /dev/null +++ b/waffledotcom/src/batch/slack/config.py @@ -0,0 +1,21 @@ +from pydantic_settings import BaseSettings, SettingsConfigDict + +from waffledotcom.src.secrets import AWSSecretManager +from waffledotcom.src.settings import settings + + +class SlackConfig(BaseSettings): + token: str = "" + + model_config = SettingsConfigDict( + case_sensitive=False, env_prefix="SLACK_", env_file=settings.env_files + ) + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + aws_secret = AWSSecretManager() + if aws_secret.is_available(): + self.token = aws_secret.get_secret("slack_token") + + +slack_config = SlackConfig() diff --git a/waffledotcom/src/batch/slack/main.py b/waffledotcom/src/batch/slack/main.py new file mode 100644 index 0000000..8a38ea5 --- /dev/null +++ b/waffledotcom/src/batch/slack/main.py @@ -0,0 +1,49 @@ +import asyncio + +from fastapi import Depends +from loguru import logger +from slack_sdk import WebClient +from slack_sdk.errors import SlackApiError + +from waffledotcom.src.apps.user.services import UserService +from waffledotcom.src.batch.slack.config import slack_config +from waffledotcom.src.batch.slack.schema import SlackMember +from waffledotcom.src.utils.dependency_solver import DependencySolver + + +async def create_users_from_slack(user_service: UserService = Depends()): + client = WebClient(token=slack_config.token) + data = client.users_list().data + + assert isinstance(data, dict) + + if not data.get("ok", False): + raise SlackApiError("Slack API Error", data) + + members_to_create = [] + for member in data.get("members", []): + if member["is_bot"] or member["deleted"] or member["id"] == "USLACKBOT": + continue + + member = SlackMember(**member) + if member.profile.phone is not None: + phone = ( + member.profile.phone.replace("-", "") + .replace(" ", "") + .replace("+82", "") + ) + member.profile.phone = phone + + members_to_create.append(member) + + user_service.create_users_from_slack(members_to_create) + logger.debug(f"Created {len(members_to_create)} users from slack") + + +def main(): + solver = DependencySolver() + asyncio.run(solver.run(create_users_from_slack)) + + +if __name__ == "__main__": + main() diff --git a/waffledotcom/src/batch/slack/schema.py b/waffledotcom/src/batch/slack/schema.py new file mode 100644 index 0000000..921b11f --- /dev/null +++ b/waffledotcom/src/batch/slack/schema.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from pydantic import BaseModel + + +class SlackMember(BaseModel): + id: str + real_name: str | None = None + profile: SlackMemberProfile + deleted: bool + is_bot: bool + is_email_confirmed: bool | None = None + is_admin: bool | None = None + + +class SlackMemberProfile(BaseModel): + first_name: str | None = None + last_name: str | None = None + email: str | None = None + phone: str | None = None + image_192: str | None = None + + +SlackMember.model_rebuild() diff --git a/waffledotcom/src/database/__init__.py b/waffledotcom/src/database/__init__.py new file mode 100644 index 0000000..d582bc0 --- /dev/null +++ b/waffledotcom/src/database/__init__.py @@ -0,0 +1,2 @@ +import waffledotcom.src.apps.team.models # noqa +import waffledotcom.src.apps.user.models # noqa diff --git a/waffledotcom/src/database/config.py b/waffledotcom/src/database/config.py index e069571..09c3013 100644 --- a/waffledotcom/src/database/config.py +++ b/waffledotcom/src/database/config.py @@ -17,8 +17,8 @@ class DBConfig(BaseSettings): case_sensitive=False, env_prefix="DB_", env_file=settings.env_files ) - @property - def url(self) -> str: + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) aws_secrets = AWSSecretManager() if aws_secrets.is_available(): self.username = aws_secrets.get_secret("db_username") @@ -26,6 +26,9 @@ def url(self) -> str: self.host = aws_secrets.get_secret("db_host") self.port = int(aws_secrets.get_secret("db_port")) self.name = aws_secrets.get_secret("db_name") + + @property + def url(self) -> str: return ( f"mysql+mysqldb://{self.username}:{self.password}" f"@{self.host}:{self.port}/{self.name}" diff --git a/waffledotcom/src/database/migrations/versions/1696590278_initial_migration_f2b7ce2c8874.py b/waffledotcom/src/database/migrations/versions/1696590278_initial_migration_f2b7ce2c8874.py new file mode 100644 index 0000000..5762013 --- /dev/null +++ b/waffledotcom/src/database/migrations/versions/1696590278_initial_migration_f2b7ce2c8874.py @@ -0,0 +1,112 @@ +"""Initial Migration + +Revision ID: f2b7ce2c8874 +Revises: +Create Date: 2023-10-06 20:04:38.514945 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "f2b7ce2c8874" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "position", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("name", sa.String(length=50), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("name"), + ) + op.create_table( + "team", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("name", sa.String(length=30), nullable=False), + sa.Column("introduction", sa.String(length=1000), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "user", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("sso_id", sa.String(length=50), nullable=True), + sa.Column("username", sa.String(length=50), nullable=True), + sa.Column("image_url", sa.String(length=200), nullable=True), + sa.Column("first_name", sa.String(length=30), nullable=False), + sa.Column("last_name", sa.String(length=30), nullable=False), + sa.Column("department", sa.String(length=50), nullable=True), + sa.Column("college", sa.String(length=50), nullable=True), + sa.Column("phone_number", sa.String(length=30), nullable=True), + sa.Column("github_id", sa.String(length=50), nullable=True), + sa.Column("github_email", sa.String(length=50), nullable=True), + sa.Column("slack_id", sa.String(length=50), nullable=True), + sa.Column("slack_email", sa.String(length=50), nullable=True), + sa.Column("notion_email", sa.String(length=50), nullable=True), + sa.Column("apple_email", sa.String(length=50), nullable=True), + sa.Column("is_rookie", sa.Boolean(), nullable=False), + sa.Column("is_member", sa.Boolean(), nullable=False), + sa.Column("is_active", sa.Boolean(), nullable=False), + sa.Column("is_admin", sa.Boolean(), nullable=False), + sa.Column("introduction", sa.String(length=1000), nullable=True), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("slack_id"), + sa.UniqueConstraint("sso_id"), + sa.UniqueConstraint("username"), + ) + op.create_table( + "position_user_association", + sa.Column("position_id", sa.Integer(), nullable=False), + sa.Column("user_id", sa.Integer(), nullable=False), + sa.ForeignKeyConstraint( + ["position_id"], + ["position.id"], + ), + sa.ForeignKeyConstraint( + ["user_id"], + ["user.id"], + ), + sa.PrimaryKeyConstraint("position_id", "user_id"), + ) + op.create_table( + "sns_account", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("user_id", sa.Integer(), nullable=False), + sa.Column("name", sa.String(length=30), nullable=False), + sa.Column("url", sa.String(length=200), nullable=False), + sa.ForeignKeyConstraint( + ["user_id"], + ["user.id"], + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "team_user_association", + sa.Column("team_id", sa.Integer(), nullable=False), + sa.Column("user_id", sa.Integer(), nullable=False), + sa.ForeignKeyConstraint( + ["team_id"], + ["team.id"], + ), + sa.ForeignKeyConstraint( + ["user_id"], + ["user.id"], + ), + sa.PrimaryKeyConstraint("team_id", "user_id"), + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("team_user_association") + op.drop_table("sns_account") + op.drop_table("position_user_association") + op.drop_table("user") + op.drop_table("team") + op.drop_table("position") + # ### end Alembic commands ### diff --git a/waffledotcom/src/database/migrations/versions/1698755947_add_generation_field_c8dea6766159.py b/waffledotcom/src/database/migrations/versions/1698755947_add_generation_field_c8dea6766159.py new file mode 100644 index 0000000..3d3a29d --- /dev/null +++ b/waffledotcom/src/database/migrations/versions/1698755947_add_generation_field_c8dea6766159.py @@ -0,0 +1,27 @@ +"""Add generation field + +Revision ID: c8dea6766159 +Revises: f2b7ce2c8874 +Create Date: 2023-10-31 21:39:07.729126 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "c8dea6766159" +down_revision = "f2b7ce2c8874" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column("user", sa.Column("generation", sa.String(length=30), nullable=True)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("user", "generation") + # ### end Alembic commands ### diff --git a/waffledotcom/src/external/github/teams.py b/waffledotcom/src/external/github/teams.py deleted file mode 100644 index 8b49644..0000000 --- a/waffledotcom/src/external/github/teams.py +++ /dev/null @@ -1,14 +0,0 @@ -# from github import Github - -# # FIXME: This is just a draft. -# g = Github(login_or_token="ghp_token") - -# waffle_org = g.get_organization("wafflestudio") - -# print("teams count", len(list(waffle_org.get_teams()))) -# for team in waffle_org.get_teams(): -# print(team.name, team.members_count, team.description, team.parent) - -# print("members count", len(list(waffle_org.get_members()))) -# for member in waffle_org.get_members(): -# print(member) diff --git a/waffledotcom/src/external/slack/batch.py b/waffledotcom/src/external/slack/batch.py deleted file mode 100644 index fab73c3..0000000 --- a/waffledotcom/src/external/slack/batch.py +++ /dev/null @@ -1,43 +0,0 @@ -# import asyncio -# import pickle - -# from fastapi import Depends -# from loguru import logger -# from slack_sdk.errors import SlackApiError - -# from waffledotcom.src.apis.user.services import UserService -# from waffledotcom.src.external.slack.schema import SlackMember -# from waffledotcom.src.utils.dependency_solver import DependencySolver - - -# async def create_users_from_slack(user_service: UserService = Depends()): -# with open("response.pickle", "rb") as f: -# response = pickle.load(f) -# data = response.data - -# # Uncomment this to get data from slack -# # client = WebClient(token="") -# # assert isinstance(data := client.users_list().data, dict) - -# if not data.get("ok", False): -# raise SlackApiError("Slack API Error", response) - -# members_to_create = [] -# for member in data.get("members", []): -# member = SlackMember(**member) - -# if member.is_bot or member.deleted or member.id == "USLACKBOT": -# continue - -# members_to_create.append(member) -# user_service.create_users_from_slack(members_to_create) -# logger.debug(f"Created {len(members_to_create)} users from slack") - - -# def main(): -# solver = DependencySolver() -# asyncio.run(solver.run(create_users_from_slack)) - - -# if __name__ == "__main__": -# main() diff --git a/waffledotcom/src/external/slack/config.py b/waffledotcom/src/external/slack/config.py deleted file mode 100644 index c5fd824..0000000 --- a/waffledotcom/src/external/slack/config.py +++ /dev/null @@ -1,13 +0,0 @@ -# from pydantic_settings import BaseSettings - -# from waffledotcom.src.settings import settings - - -# class SlackConfig(BaseSettings): -# api_token: str = "" - -# class Config: -# case_sensitive = False -# env_prefix = "SLACK_" - -# env_file = settings.env_files diff --git a/waffledotcom/src/external/slack/schema.py b/waffledotcom/src/external/slack/schema.py deleted file mode 100644 index 923f88f..0000000 --- a/waffledotcom/src/external/slack/schema.py +++ /dev/null @@ -1,23 +0,0 @@ -from __future__ import annotations - -from pydantic import BaseModel - - -class SlackMember(BaseModel): - id: str - real_name: str | None - profile: SlackMemberProfile - deleted: bool - is_bot: bool - is_email_confirmed: bool | None - is_admin: bool | None - - -class SlackMemberProfile(BaseModel): - first_name: str | None - last_name: str | None - email: str | None - image_192: str | None - - -SlackMember.update_forward_refs() diff --git a/waffledotcom/src/settings.py b/waffledotcom/src/settings.py index 06997a5..7cdeb71 100644 --- a/waffledotcom/src/settings.py +++ b/waffledotcom/src/settings.py @@ -30,10 +30,8 @@ def env_files(self) -> tuple[Path, ...]: if self.env in ["local", "test"]: return (ROOT_PATH / f".env.{self.env}",) - return ( - ROOT_PATH / f".env.{self.env}", - ROOT_PATH / f".env.{self.env}.local", - ) + # Get from AWS Secret Manager + return () settings = Settings()