Merge branch 'main' into configure-socks-proxies

.github/workflows/ansible-lint.yaml

@@ -1,5 +1,5 @@
 name: ansible-lint
-on: 
+on:
  push:
  paths:
  - 'ansible/**'
@@ -20,14 +20,14 @@ jobs:
  - name: Set up Python
  uses: actions/setup-python@v4
  with:
- python-version: '3.10'
+ python-version: '3.12'
 
  - name: Install dependencies
  run: |
  cd ansible/
  python -m pip install --upgrade pip
  pip install pipenv
- pipenv install --skip-lock
+ pipenv sync
 
  - name: Lint
  run: |

.github/workflows/k3d-ci.yaml

@@ -81,9 +81,9 @@ jobs:
  helm upgrade --install -f ./chart/values.yaml -f ./chart/test/test.yaml btrix ./chart/
 
  - name: Install Python
- uses: actions/setup-python@v3
+ uses: actions/setup-python@v5
  with:
- python-version: '3.9'
+ python-version: 3.x
 
  - name: Install Python Libs
  run: pip install -r ./backend/test-requirements.txt

ansible/Pipfile

@@ -4,12 +4,13 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-ansible = "==6.2.0"
-molecule = {extras = ["docker"], version = "==4.0.1"}
+ansible = "*"
+molecule = {extras = ["docker"], version = "*"}
 certifi = "*"
-ansible-lint = "==6.16.0"
-yamllint = "==1.30.0"
-ansible-core = "==2.13.2"
+ansible-lint = "*"
+wcmatch = "*" # workaround ubuntu shipping a broken wcmatch package
+yamllint = "*"
+ansible-core = "*"
 docker = "*"
 boto3 = "*"
 jmespath = "*"
@@ -18,4 +19,4 @@ mkdocs-material = "*"
 [dev-packages]
 
 [requires]
-python_version = "3.10"
+python_version = "3.12"

backend/.pylintrc

@@ -1,3 +1,4 @@
-[MASTER]
+[MAIN]
 extension-pkg-whitelist=pydantic
 ignore-paths=btrixcloud/migrations/migration_0028_page_files_errors.py
+disable=too-many-positional-arguments,invalid-name

backend/btrixcloud/invites.py

@@ -13,6 +13,7 @@
 
 from .pagination import DEFAULT_PAGE_SIZE
 from .models import (
+ EmailStr,
  UserRole,
  InvitePending,
  InviteRequest,
@@ -133,7 +134,10 @@ async def add_existing_user_invite(
  )
 
  async def get_valid_invite(
- self, invite_token: UUID, email: Optional[str], userid: Optional[UUID] = None
+ self,
+ invite_token: UUID,
+ email: Optional[EmailStr],
+ userid: Optional[UUID] = None,
  ) -> InvitePending:
  """Retrieve a valid invite data from db, or throw if invalid"""
  token_hash = get_hash(invite_token)
@@ -156,7 +160,7 @@ async def remove_invite(self, invite_token: UUID) -> None:
  await self.invites.delete_one({"_id": invite_token})
 
  async def remove_invite_by_email(
- self, email: str, oid: Optional[UUID] = None
+ self, email: EmailStr, oid: Optional[UUID] = None
  ) -> Any:
  """remove invite from invite list by email"""
  query: dict[str, object] = {"email": email}
@@ -255,14 +259,23 @@ async def get_invite_out(
  self, invite: InvitePending, users: UserManager, include_first_org_admin=False
  ) -> InviteOut:
  """format an InvitePending to return via api, resolve name of inviter"""
- inviter = await users.get_by_email(invite.inviterEmail)
- if not inviter:
- raise HTTPException(status_code=400, detail="invalid_invite")
+ from_superuser = invite.fromSuperuser
+ inviter_name = None
+ inviter_email = None
+ inviter = None
+ if not from_superuser:
+ inviter = await users.get_by_email(invite.inviterEmail)
+ if not inviter:
+ raise HTTPException(status_code=400, detail="invalid_invite")
+
+ inviter_name = inviter.name
+ inviter_email = invite.inviterEmail
 
  invite_out = InviteOut(
  created=invite.created,
- inviterEmail=invite.inviterEmail,
- inviterName=inviter.name,
+ inviterEmail=inviter_email,
+ inviterName=inviter_name,
+ fromSuperuser=from_superuser,
  oid=invite.oid,
  role=invite.role,
  email=invite.email,

backend/btrixcloud/models.py

@@ -15,7 +15,8 @@
  Field,
  HttpUrl as HttpUrlNonStr,
  AnyHttpUrl as AnyHttpUrlNonStr,
- EmailStr,
+ EmailStr as CasedEmailStr,
+ validate_email,
  RootModel,
  BeforeValidator,
  TypeAdapter,
@@ -47,6 +48,15 @@
 ]
 
 
+# pylint: disable=too-few-public-methods
+class EmailStr(CasedEmailStr):
+ """EmailStr type that lowercases the full email"""
+
+ @classmethod
+ def _validate(cls, value: CasedEmailStr, /) -> CasedEmailStr:
+ return validate_email(value)[1].lower()
+
+
 # pylint: disable=invalid-name, too-many-lines
 # ============================================================================
 class UserRole(IntEnum):
@@ -70,11 +80,11 @@ class InvitePending(BaseMongoModel):
  id: UUID
  created: datetime
  tokenHash: str
- inviterEmail: str
+ inviterEmail: EmailStr
  fromSuperuser: Optional[bool] = False
  oid: Optional[UUID] = None
  role: UserRole = UserRole.VIEWER
- email: Optional[str] = ""
+ email: Optional[EmailStr] = None
  # set if existing user
  userid: Optional[UUID] = None
 
@@ -84,21 +94,22 @@ class InviteOut(BaseModel):
  """Single invite output model"""
 
  created: datetime
- inviterEmail: str
- inviterName: str
+ inviterEmail: Optional[EmailStr] = None
+ inviterName: Optional[str] = None
+ fromSuperuser: bool
  oid: Optional[UUID] = None
  orgName: Optional[str] = None
  orgSlug: Optional[str] = None
  role: UserRole = UserRole.VIEWER
- email: Optional[str] = ""
+ email: Optional[EmailStr] = None
  firstOrgAdmin: Optional[bool] = None
 
 
 # ============================================================================
 class InviteRequest(BaseModel):
  """Request to invite another user"""
 
- email: str
+ email: EmailStr
 
 
 # ============================================================================
@@ -1218,7 +1229,7 @@ class SubscriptionCreate(BaseModel):
  status: str
  planId: str
 
- firstAdminInviteEmail: str
+ firstAdminInviteEmail: EmailStr
  quotas: Optional[OrgQuotas] = None
 
 

backend/btrixcloud/orgs.py

@@ -8,7 +8,6 @@
 import math
 import os
 import time
-import urllib.parse
 
 from uuid import UUID, uuid4
 from tempfile import NamedTemporaryFile
@@ -258,11 +257,11 @@ async def get_orgs_for_user(
  return [Organization.from_dict(data) for data in items], total
 
  async def get_org_for_user_by_id(
- self, oid: UUID, user: User, role: UserRole = UserRole.VIEWER
+ self, oid: UUID, user: Optional[User], role: UserRole = UserRole.VIEWER
  ) -> Optional[Organization]:
  """Get an org for user by unique id"""
  query: dict[str, object]
- if user.is_superuser:
+ if not user or user.is_superuser:
  query = {"_id": oid}
  else:
  query = {f"users.{user.id}": {"$gte": role.value}, "_id": oid}
@@ -1640,9 +1639,7 @@ async def get_pending_org_invites(
  async def delete_invite(
  invite: RemovePendingInvite, org: Organization = Depends(org_owner_dep)
  ):
- # URL decode email just in case
- email = urllib.parse.unquote(invite.email)
- result = await user_manager.invites.remove_invite_by_email(email, org.id)
+ result = await user_manager.invites.remove_invite_by_email(invite.email, org.id)
  if result.deleted_count > 0:
  return {
  "removed": True,

backend/btrixcloud/users.py

@@ -8,8 +8,6 @@
 
 from typing import Optional, List, TYPE_CHECKING, cast, Callable
 
-from pydantic import EmailStr
-
 from fastapi import (
  Request,
  HTTPException,
@@ -22,6 +20,7 @@
 from pymongo.collation import Collation
 
 from .models import (
+ EmailStr,
  UserCreate,
  UserUpdateEmailName,
  UserUpdatePassword,
@@ -685,7 +684,7 @@ async def get_existing_user_invite_info(
  return await user_manager.invites.get_invite_out(invite, user_manager, True)
 
  @users_router.get("/invite/{token}", tags=["invites"], response_model=InviteOut)
- async def get_invite_info(token: UUID, email: str):
+ async def get_invite_info(token: UUID, email: EmailStr):
  invite = await user_manager.invites.get_valid_invite(token, email)
 
  return await user_manager.invites.get_invite_out(invite, user_manager, True)

backend/test/test_org.py

@@ -360,16 +360,18 @@ def test_get_pending_org_invites(
  ("[email protected]", "[email protected]"),
  # URL encoded email address with comments
  (
- "user%2Bcomment-encoded-org%40example.com",
+ "user%2Bcomment-encoded-org@example.com",
  "[email protected]",
  ),
  # User email with diacritic characters
  ("diacritic-té[email protected]", "diacritic-té[email protected]"),
  # User email with encoded diacritic characters
  (
- "diacritic-t%C3%A9st-encoded-org%40example.com",
+ "diacritic-t%C3%A9st-encoded-org@example.com",
  "diacritic-té[email protected]",
  ),
+ # User email with upper case characters, stored as all lowercase
+ ("[email protected]", "[email protected]"),
  ],
 )
 def test_send_and_accept_org_invite(

backend/test/test_org_subs.py

@@ -12,7 +12,7 @@
 
 VALID_PASSWORD = "ValidPassW0rd!"
 
-invite_email = "test-user@example.com"
+invite_email = "test-User@EXample.com"
 
 
 def test_create_sub_org_invalid_auth(crawler_auth_headers):
@@ -206,6 +206,9 @@ def test_login_existing_user_for_invite():
  assert data["firstOrgAdmin"] == True
  assert data["orgName"] == data["oid"]
  assert data["orgName"] == data["orgSlug"]
+ assert data["fromSuperuser"] == True
+ assert not data["inviterEmail"]
+ assert not data["inviterName"]
 
  # Accept existing user invite
  r = requests.post(