diff --git a/docs/changelog/next_release/187.feature.rst b/docs/changelog/next_release/187.feature.rst new file mode 100644 index 00000000..b2d5964c --- /dev/null +++ b/docs/changelog/next_release/187.feature.rst @@ -0,0 +1 @@ +Add API schemas for SFTP, FTP, FTPS, WebDAV, Samba file sources \ No newline at end of file diff --git a/syncmaster/db/models/connection.py b/syncmaster/db/models/connection.py index e4696ffe..1af2805d 100644 --- a/syncmaster/db/models/connection.py +++ b/syncmaster/db/models/connection.py @@ -24,6 +24,11 @@ class ConnectionType(StrEnum): MYSQL = "mysql" S3 = "s3" HDFS = "hdfs" + SFTP = "sftp" + FTP = "ftp" + FTPS = "ftps" + WEBDAV = "webdav" + SAMBA = "samba" class Connection(Base, ResourceMixin, TimestampMixin): diff --git a/syncmaster/schemas/v1/auth/__init__.py b/syncmaster/schemas/v1/auth/__init__.py new file mode 100644 index 00000000..65afcced --- /dev/null +++ b/syncmaster/schemas/v1/auth/__init__.py @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: 2023-2024 MTS PJSC +# SPDX-License-Identifier: Apache-2.0 +from syncmaster.schemas.v1.auth.basic import ( + BasicAuthSchema, + CreateBasicAuthSchema, + ReadBasicAuthSchema, + UpdateBasicAuthSchema, +) +from syncmaster.schemas.v1.auth.s3 import ( + CreateS3AuthSchema, + ReadS3AuthSchema, + S3AuthSchema, + UpdateS3AuthSchema, +) +from syncmaster.schemas.v1.auth.samba import ( + CreateSambaAuthSchema, + ReadSambaAuthSchema, + SambaAuthSchema, + UpdateSambaAuthSchema, +) +from syncmaster.schemas.v1.auth.token import AuthTokenSchema, TokenPayloadSchema diff --git a/syncmaster/schemas/v1/auth.py b/syncmaster/schemas/v1/auth/basic.py similarity index 50% rename from syncmaster/schemas/v1/auth.py rename to syncmaster/schemas/v1/auth/basic.py index a33c2692..e3658b77 100644 --- a/syncmaster/schemas/v1/auth.py +++ b/syncmaster/schemas/v1/auth/basic.py @@ -5,16 +5,6 @@ from pydantic import BaseModel, SecretStr -class TokenPayloadSchema(BaseModel): - user_id: int - - -class AuthTokenSchema(BaseModel): - access_token: str - token_type: str - expires_at: float - - class BasicAuthSchema(BaseModel): type: Literal["basic"] @@ -31,21 +21,3 @@ class ReadBasicAuthSchema(BasicAuthSchema): class UpdateBasicAuthSchema(BasicAuthSchema): user: str | None = None # noqa: F722 password: SecretStr | None = None - - -class S3AuthSchema(BaseModel): - type: Literal["s3"] - - -class CreateS3AuthSchema(S3AuthSchema): - access_key: str - secret_key: SecretStr - - -class ReadS3AuthSchema(S3AuthSchema): - access_key: str - - -class UpdateS3AuthSchema(S3AuthSchema): - access_key: str | None = None - secret_key: SecretStr | None = None diff --git a/syncmaster/schemas/v1/auth/s3.py b/syncmaster/schemas/v1/auth/s3.py new file mode 100644 index 00000000..fbd94fb2 --- /dev/null +++ b/syncmaster/schemas/v1/auth/s3.py @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: 2023-2024 MTS PJSC +# SPDX-License-Identifier: Apache-2.0 +from typing import Literal + +from pydantic import BaseModel, SecretStr + + +class S3AuthSchema(BaseModel): + type: Literal["s3"] + + +class CreateS3AuthSchema(S3AuthSchema): + access_key: str + secret_key: SecretStr + + +class ReadS3AuthSchema(S3AuthSchema): + access_key: str + + +class UpdateS3AuthSchema(S3AuthSchema): + access_key: str | None = None + secret_key: SecretStr | None = None diff --git a/syncmaster/schemas/v1/auth/samba.py b/syncmaster/schemas/v1/auth/samba.py new file mode 100644 index 00000000..85657edf --- /dev/null +++ b/syncmaster/schemas/v1/auth/samba.py @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: 2023-2024 MTS PJSC +# SPDX-License-Identifier: Apache-2.0 +from typing import Literal + +from pydantic import BaseModel, SecretStr + + +class SambaAuthSchema(BaseModel): + type: Literal["samba"] + + +class CreateSambaAuthSchema(SambaAuthSchema): + user: str + password: SecretStr + auth_type: Literal["NTLMv1", "NTLMv2"] = "NTLMv2" + + +class ReadSambaAuthSchema(SambaAuthSchema): + user: str + auth_type: Literal["NTLMv1", "NTLMv2"] + + +class UpdateSambaAuthSchema(SambaAuthSchema): + user: str | None = None + password: SecretStr | None = None + auth_type: Literal["NTLMv1", "NTLMv2"] | None = None diff --git a/syncmaster/schemas/v1/auth/token.py b/syncmaster/schemas/v1/auth/token.py new file mode 100644 index 00000000..7777035e --- /dev/null +++ b/syncmaster/schemas/v1/auth/token.py @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: 2023-2024 MTS PJSC +# SPDX-License-Identifier: Apache-2.0 +from pydantic import BaseModel + + +class TokenPayloadSchema(BaseModel): + user_id: int + + +class AuthTokenSchema(BaseModel): + access_token: str + token_type: str + expires_at: float diff --git a/syncmaster/schemas/v1/connection_types.py b/syncmaster/schemas/v1/connection_types.py index 335179da..b9827d1f 100644 --- a/syncmaster/schemas/v1/connection_types.py +++ b/syncmaster/schemas/v1/connection_types.py @@ -10,3 +10,8 @@ MYSQL_TYPE = Literal["mysql"] S3_TYPE = Literal["s3"] HDFS_TYPE = Literal["hdfs"] +SFTP_TYPE = Literal["sftp"] +FTP_TYPE = Literal["ftp"] +FTPS_TYPE = Literal["ftps"] +WEBDAV_TYPE = Literal["webdav"] +SAMBA_TYPE = Literal["samba"] diff --git a/syncmaster/schemas/v1/connections/connection.py b/syncmaster/schemas/v1/connections/connection.py index a3391e39..d74b22e8 100644 --- a/syncmaster/schemas/v1/connections/connection.py +++ b/syncmaster/schemas/v1/connections/connection.py @@ -10,6 +10,16 @@ ReadClickhouseConnectionSchema, UpdateClickhouseConnectionSchema, ) +from syncmaster.schemas.v1.connections.ftp import ( + CreateFTPConnectionSchema, + ReadFTPConnectionSchema, + UpdateFTPConnectionSchema, +) +from syncmaster.schemas.v1.connections.ftps import ( + CreateFTPSConnectionSchema, + ReadFTPSConnectionSchema, + UpdateFTPSConnectionSchema, +) from syncmaster.schemas.v1.connections.hdfs import ( CreateHDFSConnectionSchema, ReadHDFSConnectionSchema, @@ -45,6 +55,21 @@ ReadS3ConnectionSchema, UpdateS3ConnectionSchema, ) +from syncmaster.schemas.v1.connections.samba import ( + CreateSambaConnectionSchema, + ReadSambaConnectionSchema, + UpdateSambaConnectionSchema, +) +from syncmaster.schemas.v1.connections.sftp import ( + CreateSFTPConnectionSchema, + ReadSFTPConnectionSchema, + UpdateSFTPConnectionSchema, +) +from syncmaster.schemas.v1.connections.webdav import ( + CreateWebDAVConnectionSchema, + ReadWebDAVConnectionSchema, + UpdateWebDAVConnectionSchema, +) from syncmaster.schemas.v1.page import PageSchema from syncmaster.schemas.v1.types import NameConstr @@ -56,7 +81,12 @@ | CreateClickhouseConnectionSchema | CreateHiveConnectionSchema | CreateHDFSConnectionSchema - | CreateS3ConnectionSchema, + | CreateS3ConnectionSchema + | CreateSFTPConnectionSchema + | CreateFTPConnectionSchema + | CreateFTPSConnectionSchema + | CreateWebDAVConnectionSchema + | CreateSambaConnectionSchema, Field(discriminator="type"), ] ReadConnectionSchema = Annotated[ @@ -67,7 +97,12 @@ | ReadClickhouseConnectionSchema | ReadHiveConnectionSchema | ReadHDFSConnectionSchema - | ReadS3ConnectionSchema, + | ReadS3ConnectionSchema + | ReadSFTPConnectionSchema + | ReadFTPConnectionSchema + | ReadFTPSConnectionSchema + | ReadWebDAVConnectionSchema + | ReadSambaConnectionSchema, Field(discriminator="type"), ] UpdateConnectionSchema = Annotated[ @@ -78,7 +113,12 @@ | UpdateClickhouseConnectionSchema | UpdateHiveConnectionSchema | UpdateHDFSConnectionSchema - | UpdateS3ConnectionSchema, + | UpdateS3ConnectionSchema + | UpdateSFTPConnectionSchema + | UpdateFTPConnectionSchema + | UpdateFTPSConnectionSchema + | UpdateWebDAVConnectionSchema + | UpdateSambaConnectionSchema, Field(discriminator="type"), ] diff --git a/syncmaster/schemas/v1/connections/ftp.py b/syncmaster/schemas/v1/connections/ftp.py new file mode 100644 index 00000000..fe8ba1f1 --- /dev/null +++ b/syncmaster/schemas/v1/connections/ftp.py @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: 2023-2024 MTS PJSC +# SPDX-License-Identifier: Apache-2.0 + +from pydantic import BaseModel, Field + +from syncmaster.schemas.v1.auth import ( + CreateBasicAuthSchema, + ReadBasicAuthSchema, + UpdateBasicAuthSchema, +) +from syncmaster.schemas.v1.connection_types import FTP_TYPE +from syncmaster.schemas.v1.connections.connection_base import ( + CreateConnectionBaseSchema, + ReadConnectionBaseSchema, + UpdateConnectionBaseSchema, +) + + +class CreateFTPConnectionDataSchema(BaseModel): + host: str + port: int + + +class ReadFTPConnectionDataSchema(BaseModel): + host: str + port: int + + +class UpdateFTPConnectionDataSchema(BaseModel): + host: str | None = None + port: int | None = None + + +class CreateFTPConnectionSchema(CreateConnectionBaseSchema): + type: FTP_TYPE = Field(..., description="Connection type") + data: CreateFTPConnectionDataSchema = Field( + ..., + alias="connection_data", + description=( + "Data required to connect to the remote server. These are the parameters that are specified in the URL request." + ), + ) + auth_data: CreateBasicAuthSchema = Field( + description="Credentials for authorization", + ) + + +class ReadFTPConnectionSchema(ReadConnectionBaseSchema): + type: FTP_TYPE + data: ReadFTPConnectionDataSchema = Field(alias="connection_data") + auth_data: ReadBasicAuthSchema | None = None + + +class UpdateFTPConnectionSchema(UpdateConnectionBaseSchema): + type: FTP_TYPE + data: UpdateFTPConnectionDataSchema | None = Field(alias="connection_data", default=None) + auth_data: UpdateBasicAuthSchema | None = None diff --git a/syncmaster/schemas/v1/connections/ftps.py b/syncmaster/schemas/v1/connections/ftps.py new file mode 100644 index 00000000..9a466887 --- /dev/null +++ b/syncmaster/schemas/v1/connections/ftps.py @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: 2023-2024 MTS PJSC +# SPDX-License-Identifier: Apache-2.0 + +from pydantic import BaseModel, Field + +from syncmaster.schemas.v1.auth import ( + CreateBasicAuthSchema, + ReadBasicAuthSchema, + UpdateBasicAuthSchema, +) +from syncmaster.schemas.v1.connection_types import FTPS_TYPE +from syncmaster.schemas.v1.connections.connection_base import ( + CreateConnectionBaseSchema, + ReadConnectionBaseSchema, + UpdateConnectionBaseSchema, +) + + +class CreateFTPSConnectionDataSchema(BaseModel): + host: str + port: int + + +class ReadFTPSConnectionDataSchema(BaseModel): + host: str + port: int + + +class UpdateFTPSConnectionDataSchema(BaseModel): + host: str | None = None + port: int | None = None + + +class CreateFTPSConnectionSchema(CreateConnectionBaseSchema): + type: FTPS_TYPE = Field(..., description="Connection type") + data: CreateFTPSConnectionDataSchema = Field( + ..., + alias="connection_data", + description=( + "Data required to connect to the remote server. These are the parameters that are specified in the URL request." + ), + ) + auth_data: CreateBasicAuthSchema = Field( + description="Credentials for authorization", + ) + + +class ReadFTPSConnectionSchema(ReadConnectionBaseSchema): + type: FTPS_TYPE + data: ReadFTPSConnectionDataSchema = Field(alias="connection_data") + auth_data: ReadBasicAuthSchema | None = None + + +class UpdateFTPSConnectionSchema(UpdateConnectionBaseSchema): + type: FTPS_TYPE + data: UpdateFTPSConnectionDataSchema | None = Field(alias="connection_data", default=None) + auth_data: UpdateBasicAuthSchema | None = None diff --git a/syncmaster/schemas/v1/connections/hdfs.py b/syncmaster/schemas/v1/connections/hdfs.py index 3368bb70..c2acddb0 100644 --- a/syncmaster/schemas/v1/connections/hdfs.py +++ b/syncmaster/schemas/v1/connections/hdfs.py @@ -34,7 +34,7 @@ class CreateHDFSConnectionSchema(CreateConnectionBaseSchema): ..., alias="connection_data", description=( - "Data required to connect to the database. These are the parameters that are specified in the URL request." + "Data required to connect to the HDFS cluster. These are the parameters that are specified in the URL request." ), ) auth_data: CreateBasicAuthSchema = Field( diff --git a/syncmaster/schemas/v1/connections/s3.py b/syncmaster/schemas/v1/connections/s3.py index d65b0103..9ac796ed 100644 --- a/syncmaster/schemas/v1/connections/s3.py +++ b/syncmaster/schemas/v1/connections/s3.py @@ -61,7 +61,7 @@ class CreateS3ConnectionSchema(CreateConnectionBaseSchema): ..., alias="connection_data", description=( - "Data required to connect to the database. These are the parameters that are specified in the URL request." + "Data required to connect to the S3 bucket. These are the parameters that are specified in the URL request." ), ) auth_data: CreateS3AuthSchema = Field( diff --git a/syncmaster/schemas/v1/connections/samba.py b/syncmaster/schemas/v1/connections/samba.py new file mode 100644 index 00000000..73f6b096 --- /dev/null +++ b/syncmaster/schemas/v1/connections/samba.py @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: 2023-2024 MTS PJSC +# SPDX-License-Identifier: Apache-2.0 +from typing import Literal + +from pydantic import BaseModel, Field + +from syncmaster.schemas.v1.auth import ( + CreateSambaAuthSchema, + ReadSambaAuthSchema, + UpdateSambaAuthSchema, +) +from syncmaster.schemas.v1.connection_types import SAMBA_TYPE +from syncmaster.schemas.v1.connections.connection_base import ( + CreateConnectionBaseSchema, + ReadConnectionBaseSchema, + UpdateConnectionBaseSchema, +) + + +class CreateSambaConnectionDataSchema(BaseModel): + host: str + share: str + port: int | None = None + protocol: Literal["SMB", "NetBIOS"] = "SMB" + domain: str = "" + + +class ReadSambaConnectionDataSchema(BaseModel): + host: str + share: str + port: int | None + protocol: Literal["SMB", "NetBIOS"] + domain: str + + +class UpdateSambaConnectionDataSchema(BaseModel): + host: str | None = None + share: str | None = None + port: int | None = None + protocol: Literal["SMB", "NetBIOS"] | None = None + domain: str | None = None + + +class CreateSambaConnectionSchema(CreateConnectionBaseSchema): + type: SAMBA_TYPE = Field(..., description="Connection type") + data: CreateSambaConnectionDataSchema = Field( + ..., + alias="connection_data", + description=( + "Data required to connect to the remote server. These are the parameters that are specified in the URL request." + ), + ) + auth_data: CreateSambaAuthSchema = Field( + description="Credentials for authorization", + ) + + +class ReadSambaConnectionSchema(ReadConnectionBaseSchema): + type: SAMBA_TYPE + data: ReadSambaConnectionDataSchema = Field(alias="connection_data") + auth_data: ReadSambaAuthSchema | None = None + + +class UpdateSambaConnectionSchema(UpdateConnectionBaseSchema): + type: SAMBA_TYPE + data: UpdateSambaConnectionDataSchema | None = Field(alias="connection_data", default=None) + auth_data: UpdateSambaAuthSchema | None = None diff --git a/syncmaster/schemas/v1/connections/sftp.py b/syncmaster/schemas/v1/connections/sftp.py new file mode 100644 index 00000000..7ad59aa7 --- /dev/null +++ b/syncmaster/schemas/v1/connections/sftp.py @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: 2023-2024 MTS PJSC +# SPDX-License-Identifier: Apache-2.0 + +from pydantic import BaseModel, Field + +from syncmaster.schemas.v1.auth.basic import ( + CreateBasicAuthSchema, + ReadBasicAuthSchema, + UpdateBasicAuthSchema, +) +from syncmaster.schemas.v1.connection_types import SFTP_TYPE +from syncmaster.schemas.v1.connections.connection_base import ( + CreateConnectionBaseSchema, + ReadConnectionBaseSchema, + UpdateConnectionBaseSchema, +) + + +class CreateSFTPConnectionDataSchema(BaseModel): + host: str + port: int + + +class ReadSFTPConnectionDataSchema(BaseModel): + host: str + port: int + + +class UpdateSFTPConnectionDataSchema(BaseModel): + host: str | None = None + port: int | None = None + + +class CreateSFTPConnectionSchema(CreateConnectionBaseSchema): + type: SFTP_TYPE = Field(..., description="Connection type") + data: CreateSFTPConnectionDataSchema = Field( + ..., + alias="connection_data", + description=( + "Data required to connect to the remote server. These are the parameters that are specified in the URL request." + ), + ) + auth_data: CreateBasicAuthSchema = Field( + description="Credentials for authorization", + ) + + +class ReadSFTPConnectionSchema(ReadConnectionBaseSchema): + type: SFTP_TYPE + data: ReadSFTPConnectionDataSchema = Field(alias="connection_data") + auth_data: ReadBasicAuthSchema | None = None + + +class UpdateSFTPConnectionSchema(UpdateConnectionBaseSchema): + type: SFTP_TYPE + data: UpdateSFTPConnectionDataSchema | None = Field(alias="connection_data", default=None) + auth_data: UpdateBasicAuthSchema | None = None diff --git a/syncmaster/schemas/v1/connections/webdav.py b/syncmaster/schemas/v1/connections/webdav.py new file mode 100644 index 00000000..391f881a --- /dev/null +++ b/syncmaster/schemas/v1/connections/webdav.py @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: 2023-2024 MTS PJSC +# SPDX-License-Identifier: Apache-2.0 + +from typing import Literal + +from pydantic import BaseModel, Field + +from syncmaster.schemas.v1.auth import ( + CreateBasicAuthSchema, + ReadBasicAuthSchema, + UpdateBasicAuthSchema, +) +from syncmaster.schemas.v1.connection_types import WEBDAV_TYPE +from syncmaster.schemas.v1.connections.connection_base import ( + CreateConnectionBaseSchema, + ReadConnectionBaseSchema, + UpdateConnectionBaseSchema, +) + + +class CreateWebDAVConnectionDataSchema(BaseModel): + host: str + port: int | None = None + protocol: Literal["http", "https"] = "https" + + +class ReadWebDAVConnectionDataSchema(BaseModel): + host: str + port: int | None + protocol: Literal["http", "https"] + + +class UpdateWebDAVConnectionDataSchema(BaseModel): + host: str | None = None + port: int | None = None + protocol: Literal["http", "https"] | None = None + + +class CreateWebDAVConnectionSchema(CreateConnectionBaseSchema): + type: WEBDAV_TYPE = Field(..., description="Connection type") + data: CreateWebDAVConnectionDataSchema = Field( + ..., + alias="connection_data", + description=( + "Data required to connect to the remote server. These are the parameters that are specified in the URL request." + ), + ) + auth_data: CreateBasicAuthSchema = Field( + description="Credentials for authorization", + ) + + +class ReadWebDAVConnectionSchema(ReadConnectionBaseSchema): + type: WEBDAV_TYPE + data: ReadWebDAVConnectionDataSchema = Field(alias="connection_data") + auth_data: ReadBasicAuthSchema | None = None + + +class UpdateWebDAVConnectionSchema(UpdateConnectionBaseSchema): + type: WEBDAV_TYPE + data: UpdateWebDAVConnectionDataSchema | None = Field(alias="connection_data", default=None) + auth_data: UpdateBasicAuthSchema | None = None diff --git a/syncmaster/schemas/v1/transfers/__init__.py b/syncmaster/schemas/v1/transfers/__init__.py index 147ae18a..2cdda4b5 100644 --- a/syncmaster/schemas/v1/transfers/__init__.py +++ b/syncmaster/schemas/v1/transfers/__init__.py @@ -16,6 +16,18 @@ OracleReadTransferSourceAndTarget, PostgresReadTransferSourceAndTarget, ) +from syncmaster.schemas.v1.transfers.file.ftp import ( + FTPCreateTransferSource, + FTPCreateTransferTarget, + FTPReadTransferSource, + FTPReadTransferTarget, +) +from syncmaster.schemas.v1.transfers.file.ftps import ( + FTPSCreateTransferSource, + FTPSCreateTransferTarget, + FTPSReadTransferSource, + FTPSReadTransferTarget, +) from syncmaster.schemas.v1.transfers.file.hdfs import ( HDFSCreateTransferSource, HDFSCreateTransferTarget, @@ -28,6 +40,24 @@ S3ReadTransferSource, S3ReadTransferTarget, ) +from syncmaster.schemas.v1.transfers.file.samba import ( + SambaCreateTransferSource, + SambaCreateTransferTarget, + SambaReadTransferSource, + SambaReadTransferTarget, +) +from syncmaster.schemas.v1.transfers.file.sftp import ( + SFTPCreateTransferSource, + SFTPCreateTransferTarget, + SFTPReadTransferSource, + SFTPReadTransferTarget, +) +from syncmaster.schemas.v1.transfers.file.webdav import ( + WebDAVCreateTransferSource, + WebDAVCreateTransferTarget, + WebDAVReadTransferSource, + WebDAVReadTransferTarget, +) from syncmaster.schemas.v1.transfers.strategy import FullStrategy, IncrementalStrategy from syncmaster.schemas.v1.transfers.transformations.dataframe_columns_filter import ( DataframeColumnsFilter, @@ -46,6 +76,11 @@ | MSSQLReadTransferSourceAndTarget | MySQLReadTransferSourceAndTarget | S3ReadTransferSource + | SFTPReadTransferSource + | FTPReadTransferSource + | FTPSReadTransferSource + | WebDAVReadTransferSource + | SambaReadTransferSource ) ReadTransferSchemaTarget = ( @@ -57,6 +92,11 @@ | MSSQLReadTransferSourceAndTarget | MySQLReadTransferSourceAndTarget | S3ReadTransferTarget + | SFTPReadTransferTarget + | FTPReadTransferTarget + | FTPSReadTransferTarget + | WebDAVReadTransferTarget + | SambaReadTransferTarget ) CreateTransferSchemaSource = ( @@ -68,6 +108,11 @@ | MSSQLReadTransferSourceAndTarget | MySQLReadTransferSourceAndTarget | S3CreateTransferSource + | SFTPCreateTransferSource + | FTPCreateTransferSource + | FTPSCreateTransferSource + | WebDAVCreateTransferSource + | SambaCreateTransferSource ) CreateTransferSchemaTarget = ( @@ -79,6 +124,11 @@ | MSSQLReadTransferSourceAndTarget | MySQLReadTransferSourceAndTarget | S3CreateTransferTarget + | SFTPCreateTransferTarget + | FTPCreateTransferTarget + | FTPSCreateTransferTarget + | WebDAVCreateTransferTarget + | SambaCreateTransferTarget ) UpdateTransferSchemaSource = ( @@ -89,7 +139,12 @@ | ClickhouseReadTransferSourceAndTarget | MSSQLReadTransferSourceAndTarget | MySQLReadTransferSourceAndTarget - | S3CreateTransferSource + | S3ReadTransferSource + | SFTPReadTransferSource + | FTPReadTransferSource + | FTPSReadTransferSource + | WebDAVReadTransferSource + | SambaReadTransferSource | None ) @@ -101,7 +156,12 @@ | ClickhouseReadTransferSourceAndTarget | MSSQLReadTransferSourceAndTarget | MySQLReadTransferSourceAndTarget - | S3CreateTransferTarget + | S3ReadTransferTarget + | SFTPReadTransferTarget + | FTPReadTransferTarget + | FTPSReadTransferTarget + | WebDAVReadTransferTarget + | SambaReadTransferTarget | None ) diff --git a/syncmaster/schemas/v1/transfers/file/ftp.py b/syncmaster/schemas/v1/transfers/file/ftp.py new file mode 100644 index 00000000..414d261e --- /dev/null +++ b/syncmaster/schemas/v1/transfers/file/ftp.py @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: 2023-2024 MTS PJSC +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +from syncmaster.schemas.v1.connection_types import FTP_TYPE +from syncmaster.schemas.v1.transfers.file.base import ( + CreateFileTransferSource, + CreateFileTransferTarget, + ReadFileTransferSource, + ReadFileTransferTarget, +) + + +class FTPReadTransferSource(ReadFileTransferSource): + type: FTP_TYPE + + +class FTPReadTransferTarget(ReadFileTransferTarget): + type: FTP_TYPE + + +class FTPCreateTransferSource(CreateFileTransferSource): + type: FTP_TYPE + + +class FTPCreateTransferTarget(CreateFileTransferTarget): + type: FTP_TYPE diff --git a/syncmaster/schemas/v1/transfers/file/ftps.py b/syncmaster/schemas/v1/transfers/file/ftps.py new file mode 100644 index 00000000..7069a995 --- /dev/null +++ b/syncmaster/schemas/v1/transfers/file/ftps.py @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: 2023-2024 MTS PJSC +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +from syncmaster.schemas.v1.connection_types import FTPS_TYPE +from syncmaster.schemas.v1.transfers.file.base import ( + CreateFileTransferSource, + CreateFileTransferTarget, + ReadFileTransferSource, + ReadFileTransferTarget, +) + + +class FTPSReadTransferSource(ReadFileTransferSource): + type: FTPS_TYPE + + +class FTPSReadTransferTarget(ReadFileTransferTarget): + type: FTPS_TYPE + + +class FTPSCreateTransferSource(CreateFileTransferSource): + type: FTPS_TYPE + + +class FTPSCreateTransferTarget(CreateFileTransferTarget): + type: FTPS_TYPE diff --git a/syncmaster/schemas/v1/transfers/file/samba.py b/syncmaster/schemas/v1/transfers/file/samba.py new file mode 100644 index 00000000..331e89c3 --- /dev/null +++ b/syncmaster/schemas/v1/transfers/file/samba.py @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: 2023-2024 MTS PJSC +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +from syncmaster.schemas.v1.connection_types import SAMBA_TYPE +from syncmaster.schemas.v1.transfers.file.base import ( + CreateFileTransferSource, + CreateFileTransferTarget, + ReadFileTransferSource, + ReadFileTransferTarget, +) + + +class SambaReadTransferSource(ReadFileTransferSource): + type: SAMBA_TYPE + + +class SambaReadTransferTarget(ReadFileTransferTarget): + type: SAMBA_TYPE + + +class SambaCreateTransferSource(CreateFileTransferSource): + type: SAMBA_TYPE + + +class SambaCreateTransferTarget(CreateFileTransferTarget): + type: SAMBA_TYPE diff --git a/syncmaster/schemas/v1/transfers/file/sftp.py b/syncmaster/schemas/v1/transfers/file/sftp.py new file mode 100644 index 00000000..77dd96e5 --- /dev/null +++ b/syncmaster/schemas/v1/transfers/file/sftp.py @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: 2023-2024 MTS PJSC +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +from syncmaster.schemas.v1.connection_types import SFTP_TYPE +from syncmaster.schemas.v1.transfers.file.base import ( + CreateFileTransferSource, + CreateFileTransferTarget, + ReadFileTransferSource, + ReadFileTransferTarget, +) + + +class SFTPReadTransferSource(ReadFileTransferSource): + type: SFTP_TYPE + + +class SFTPReadTransferTarget(ReadFileTransferTarget): + type: SFTP_TYPE + + +class SFTPCreateTransferSource(CreateFileTransferSource): + type: SFTP_TYPE + + +class SFTPCreateTransferTarget(CreateFileTransferTarget): + type: SFTP_TYPE diff --git a/syncmaster/schemas/v1/transfers/file/webdav.py b/syncmaster/schemas/v1/transfers/file/webdav.py new file mode 100644 index 00000000..729501c3 --- /dev/null +++ b/syncmaster/schemas/v1/transfers/file/webdav.py @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: 2023-2024 MTS PJSC +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +from syncmaster.schemas.v1.connection_types import WEBDAV_TYPE +from syncmaster.schemas.v1.transfers.file.base import ( + CreateFileTransferSource, + CreateFileTransferTarget, + ReadFileTransferSource, + ReadFileTransferTarget, +) + + +class WebDAVReadTransferSource(ReadFileTransferSource): + type: WEBDAV_TYPE + + +class WebDAVReadTransferTarget(ReadFileTransferTarget): + type: WEBDAV_TYPE + + +class WebDAVCreateTransferSource(CreateFileTransferSource): + type: WEBDAV_TYPE + + +class WebDAVCreateTransferTarget(CreateFileTransferTarget): + type: WEBDAV_TYPE diff --git a/tests/test_unit/conftest.py b/tests/test_unit/conftest.py index 7f0d0888..2e74ef94 100644 --- a/tests/test_unit/conftest.py +++ b/tests/test_unit/conftest.py @@ -22,8 +22,6 @@ create_user, ) -ALLOWED_SOURCES = "'hive', 'oracle', 'postgres', 'hdfs', 's3'" - async def create_group_member( username: str, diff --git a/tests/test_unit/test_connections/connection_fixtures/group_connections_fixture.py b/tests/test_unit/test_connections/connection_fixtures/group_connections_fixture.py index 53900a43..523a5154 100644 --- a/tests/test_unit/test_connections/connection_fixtures/group_connections_fixture.py +++ b/tests/test_unit/test_connections/connection_fixtures/group_connections_fixture.py @@ -37,6 +37,20 @@ async def group_connections( "bucket": "bucket", }, ) + elif conn_type == ConnectionType.SAMBA: + new_data.update( + { + "share": "folder", + "protocol": "SMB", + "domain": "domain", + }, + ) + elif conn_type == ConnectionType.WEBDAV: + new_data.update( + { + "protocol": "http", + }, + ) elif conn_type in [ ConnectionType.POSTGRES, ConnectionType.CLICKHOUSE, diff --git a/tests/test_unit/test_connections/test_create_connection.py b/tests/test_unit/test_connections/test_create_connection.py index 477e627f..a0c2640b 100644 --- a/tests/test_unit/test_connections/test_create_connection.py +++ b/tests/test_unit/test_connections/test_create_connection.py @@ -276,7 +276,7 @@ async def test_check_fields_validation_on_create_connection( assert result.status_code == 422 assert ( result.json()["error"]["details"][0]["message"] - == "Input tag 'POSTGRESQL' found using 'type' does not match any of the expected tags: 'oracle', 'postgres', 'mysql', 'mssql', 'clickhouse', 'hive', 'hdfs', 's3'" + == "Input tag 'POSTGRESQL' found using 'type' does not match any of the expected tags: 'oracle', 'postgres', 'mysql', 'mssql', 'clickhouse', 'hive', 'hdfs', 's3', 'sftp', 'ftp', 'ftps', 'webdav', 'samba'" ) diff --git a/tests/test_unit/test_connections/test_file_connection/test_create_ftp_connection.py b/tests/test_unit/test_connections/test_file_connection/test_create_ftp_connection.py new file mode 100644 index 00000000..86218b87 --- /dev/null +++ b/tests/test_unit/test_connections/test_file_connection/test_create_ftp_connection.py @@ -0,0 +1,77 @@ +import pytest +from httpx import AsyncClient +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from syncmaster.db.models import AuthData, Connection +from syncmaster.db.repositories.utils import decrypt_auth_data +from syncmaster.server.settings import ServerAppSettings as Settings +from tests.mocks import MockGroup, UserTestRoles + +pytestmark = [pytest.mark.asyncio, pytest.mark.server, pytest.mark.ftp] + + +async def test_developer_plus_can_create_ftp_connection( + client: AsyncClient, + group: MockGroup, + session: AsyncSession, + settings: Settings, + role_developer_plus: UserTestRoles, +): + # Arrange + user = group.get_member_of_role(role_developer_plus) + + # Act + result = await client.post( + "v1/connections", + headers={"Authorization": f"Bearer {user.token}"}, + json={ + "group_id": group.id, + "name": "New connection", + "description": "", + "type": "ftp", + "connection_data": { + "host": "some_host", + "port": 80, + }, + "auth_data": { + "type": "basic", + "user": "user", + "password": "secret", + }, + }, + ) + connection = ( + await session.scalars( + select(Connection).filter_by( + name="New connection", + ), + ) + ).first() + + creds = ( + await session.scalars( + select(AuthData).filter_by( + connection_id=connection.id, + ), + ) + ).one() + + # Assert + decrypted = decrypt_auth_data(creds.value, settings=settings) + assert result.status_code == 200 + assert result.json() == { + "id": connection.id, + "group_id": connection.group_id, + "name": connection.name, + "description": connection.description, + "type": connection.type, + "connection_data": { + "host": connection.data["host"], + "port": connection.data["port"], + }, + "auth_data": { + "type": decrypted["type"], + "user": decrypted["user"], + }, + } diff --git a/tests/test_unit/test_connections/test_file_connection/test_create_ftps_connection.py b/tests/test_unit/test_connections/test_file_connection/test_create_ftps_connection.py new file mode 100644 index 00000000..8dde2a4e --- /dev/null +++ b/tests/test_unit/test_connections/test_file_connection/test_create_ftps_connection.py @@ -0,0 +1,77 @@ +import pytest +from httpx import AsyncClient +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from syncmaster.db.models import AuthData, Connection +from syncmaster.db.repositories.utils import decrypt_auth_data +from syncmaster.server.settings import ServerAppSettings as Settings +from tests.mocks import MockGroup, UserTestRoles + +pytestmark = [pytest.mark.asyncio, pytest.mark.server, pytest.mark.ftps] + + +async def test_developer_plus_can_create_ftps_connection( + client: AsyncClient, + group: MockGroup, + session: AsyncSession, + settings: Settings, + role_developer_plus: UserTestRoles, +): + # Arrange + user = group.get_member_of_role(role_developer_plus) + + # Act + result = await client.post( + "v1/connections", + headers={"Authorization": f"Bearer {user.token}"}, + json={ + "group_id": group.id, + "name": "New connection", + "description": "", + "type": "ftps", + "connection_data": { + "host": "some_host", + "port": 80, + }, + "auth_data": { + "type": "basic", + "user": "user", + "password": "secret", + }, + }, + ) + connection = ( + await session.scalars( + select(Connection).filter_by( + name="New connection", + ), + ) + ).first() + + creds = ( + await session.scalars( + select(AuthData).filter_by( + connection_id=connection.id, + ), + ) + ).one() + + # Assert + decrypted = decrypt_auth_data(creds.value, settings=settings) + assert result.status_code == 200 + assert result.json() == { + "id": connection.id, + "group_id": connection.group_id, + "name": connection.name, + "description": connection.description, + "type": connection.type, + "connection_data": { + "host": connection.data["host"], + "port": connection.data["port"], + }, + "auth_data": { + "type": decrypted["type"], + "user": decrypted["user"], + }, + } diff --git a/tests/test_unit/test_connections/test_db_connection/test_create_hdfs_connection.py b/tests/test_unit/test_connections/test_file_connection/test_create_hdfs_connection.py similarity index 100% rename from tests/test_unit/test_connections/test_db_connection/test_create_hdfs_connection.py rename to tests/test_unit/test_connections/test_file_connection/test_create_hdfs_connection.py diff --git a/tests/test_unit/test_connections/test_db_connection/test_create_s3_connection.py b/tests/test_unit/test_connections/test_file_connection/test_create_s3_connection.py similarity index 100% rename from tests/test_unit/test_connections/test_db_connection/test_create_s3_connection.py rename to tests/test_unit/test_connections/test_file_connection/test_create_s3_connection.py diff --git a/tests/test_unit/test_connections/test_file_connection/test_create_samba_connection.py b/tests/test_unit/test_connections/test_file_connection/test_create_samba_connection.py new file mode 100644 index 00000000..65b7d0df --- /dev/null +++ b/tests/test_unit/test_connections/test_file_connection/test_create_samba_connection.py @@ -0,0 +1,84 @@ +import pytest +from httpx import AsyncClient +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from syncmaster.db.models import AuthData, Connection +from syncmaster.db.repositories.utils import decrypt_auth_data +from syncmaster.server.settings import ServerAppSettings as Settings +from tests.mocks import MockGroup, UserTestRoles + +pytestmark = [pytest.mark.asyncio, pytest.mark.server, pytest.mark.samba] + + +async def test_developer_plus_can_create_samba_connection( + client: AsyncClient, + group: MockGroup, + session: AsyncSession, + settings: Settings, + role_developer_plus: UserTestRoles, +): + # Arrange + user = group.get_member_of_role(role_developer_plus) + + # Act + result = await client.post( + "v1/connections", + headers={"Authorization": f"Bearer {user.token}"}, + json={ + "group_id": group.id, + "name": "New connection", + "description": "", + "type": "samba", + "connection_data": { + "host": "some_host", + "port": 80, + "share": "some_folder", + "protocol": "SMB", + }, + "auth_data": { + "type": "samba", + "auth_type": "NTLMv2", + "user": "user", + "password": "secret", + }, + }, + ) + connection = ( + await session.scalars( + select(Connection).filter_by( + name="New connection", + ), + ) + ).first() + + creds = ( + await session.scalars( + select(AuthData).filter_by( + connection_id=connection.id, + ), + ) + ).one() + + # Assert + decrypted = decrypt_auth_data(creds.value, settings=settings) + assert result.status_code == 200 + assert result.json() == { + "id": connection.id, + "group_id": connection.group_id, + "name": connection.name, + "description": connection.description, + "type": connection.type, + "connection_data": { + "host": connection.data["host"], + "port": connection.data["port"], + "share": connection.data["share"], + "protocol": connection.data["protocol"], + "domain": "", + }, + "auth_data": { + "type": decrypted["type"], + "auth_type": decrypted["auth_type"], + "user": decrypted["user"], + }, + } diff --git a/tests/test_unit/test_connections/test_file_connection/test_create_sftp_connection.py b/tests/test_unit/test_connections/test_file_connection/test_create_sftp_connection.py new file mode 100644 index 00000000..c240ce52 --- /dev/null +++ b/tests/test_unit/test_connections/test_file_connection/test_create_sftp_connection.py @@ -0,0 +1,77 @@ +import pytest +from httpx import AsyncClient +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from syncmaster.db.models import AuthData, Connection +from syncmaster.db.repositories.utils import decrypt_auth_data +from syncmaster.server.settings import ServerAppSettings as Settings +from tests.mocks import MockGroup, UserTestRoles + +pytestmark = [pytest.mark.asyncio, pytest.mark.server, pytest.mark.sftp] + + +async def test_developer_plus_can_create_sftp_connection( + client: AsyncClient, + group: MockGroup, + session: AsyncSession, + settings: Settings, + role_developer_plus: UserTestRoles, +): + # Arrange + user = group.get_member_of_role(role_developer_plus) + + # Act + result = await client.post( + "v1/connections", + headers={"Authorization": f"Bearer {user.token}"}, + json={ + "group_id": group.id, + "name": "New connection", + "description": "", + "type": "sftp", + "connection_data": { + "host": "some_host", + "port": 80, + }, + "auth_data": { + "type": "basic", + "user": "user", + "password": "secret", + }, + }, + ) + connection = ( + await session.scalars( + select(Connection).filter_by( + name="New connection", + ), + ) + ).first() + + creds = ( + await session.scalars( + select(AuthData).filter_by( + connection_id=connection.id, + ), + ) + ).one() + + # Assert + decrypted = decrypt_auth_data(creds.value, settings=settings) + assert result.status_code == 200 + assert result.json() == { + "id": connection.id, + "group_id": connection.group_id, + "name": connection.name, + "description": connection.description, + "type": connection.type, + "connection_data": { + "host": connection.data["host"], + "port": connection.data["port"], + }, + "auth_data": { + "type": decrypted["type"], + "user": decrypted["user"], + }, + } diff --git a/tests/test_unit/test_connections/test_file_connection/test_create_webdav_connection.py b/tests/test_unit/test_connections/test_file_connection/test_create_webdav_connection.py new file mode 100644 index 00000000..17cc0e9a --- /dev/null +++ b/tests/test_unit/test_connections/test_file_connection/test_create_webdav_connection.py @@ -0,0 +1,79 @@ +import pytest +from httpx import AsyncClient +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from syncmaster.db.models import AuthData, Connection +from syncmaster.db.repositories.utils import decrypt_auth_data +from syncmaster.server.settings import ServerAppSettings as Settings +from tests.mocks import MockGroup, UserTestRoles + +pytestmark = [pytest.mark.asyncio, pytest.mark.server, pytest.mark.webdav] + + +async def test_developer_plus_can_create_webdav_connection( + client: AsyncClient, + group: MockGroup, + session: AsyncSession, + settings: Settings, + role_developer_plus: UserTestRoles, +): + # Arrange + user = group.get_member_of_role(role_developer_plus) + + # Act + result = await client.post( + "v1/connections", + headers={"Authorization": f"Bearer {user.token}"}, + json={ + "group_id": group.id, + "name": "New connection", + "description": "", + "type": "webdav", + "connection_data": { + "host": "some_host", + "port": 443, + "protocol": "https", + }, + "auth_data": { + "type": "basic", + "user": "user", + "password": "secret", + }, + }, + ) + connection = ( + await session.scalars( + select(Connection).filter_by( + name="New connection", + ), + ) + ).first() + + creds = ( + await session.scalars( + select(AuthData).filter_by( + connection_id=connection.id, + ), + ) + ).one() + + # Assert + decrypted = decrypt_auth_data(creds.value, settings=settings) + assert result.status_code == 200 + assert result.json() == { + "id": connection.id, + "group_id": connection.group_id, + "name": connection.name, + "description": connection.description, + "type": connection.type, + "connection_data": { + "host": connection.data["host"], + "port": connection.data["port"], + "protocol": connection.data["protocol"], + }, + "auth_data": { + "type": decrypted["type"], + "user": decrypted["user"], + }, + } diff --git a/tests/test_unit/test_connections/test_file_connection/test_update_ftp_connection.py b/tests/test_unit/test_connections/test_file_connection/test_update_ftp_connection.py new file mode 100644 index 00000000..5a1ee06a --- /dev/null +++ b/tests/test_unit/test_connections/test_file_connection/test_update_ftp_connection.py @@ -0,0 +1,57 @@ +import pytest +from httpx import AsyncClient + +from tests.mocks import MockConnection, UserTestRoles + +pytestmark = [pytest.mark.asyncio, pytest.mark.server, pytest.mark.ftp] + + +@pytest.mark.parametrize( + "connection_type,create_connection_data,create_connection_auth_data", + [ + ( + "ftp", + { + "host": "some_host", + "port": 80, + }, + { + "type": "basic", + "user": "user", + "password": "password", + }, + ), + ], + indirect=["create_connection_data", "create_connection_auth_data"], +) +async def test_developer_plus_can_update_ftp_connection( + client: AsyncClient, + group_connection: MockConnection, + role_developer_plus: UserTestRoles, + create_connection_data: dict, + create_connection_auth_data: dict, +): + # Arrange + user = group_connection.owner_group.get_member_of_role(role_developer_plus) + + # Act + result = await client.patch( + f"v1/connections/{group_connection.id}", + headers={"Authorization": f"Bearer {user.token}"}, + json={"type": "ftp", "connection_data": {"host": "new_host"}}, + ) + + # Assert + assert result.json() == { + "id": group_connection.id, + "name": group_connection.connection.name, + "description": group_connection.description, + "type": group_connection.type, + "group_id": group_connection.group_id, + "connection_data": {"host": "new_host", "port": 80}, + "auth_data": { + "type": group_connection.credentials.value["type"], + "user": group_connection.credentials.value["user"], + }, + } + assert result.status_code == 200 diff --git a/tests/test_unit/test_connections/test_file_connection/test_update_ftps_connection.py b/tests/test_unit/test_connections/test_file_connection/test_update_ftps_connection.py new file mode 100644 index 00000000..5e6d001a --- /dev/null +++ b/tests/test_unit/test_connections/test_file_connection/test_update_ftps_connection.py @@ -0,0 +1,57 @@ +import pytest +from httpx import AsyncClient + +from tests.mocks import MockConnection, UserTestRoles + +pytestmark = [pytest.mark.asyncio, pytest.mark.server, pytest.mark.ftps] + + +@pytest.mark.parametrize( + "connection_type,create_connection_data,create_connection_auth_data", + [ + ( + "ftps", + { + "host": "some_host", + "port": 80, + }, + { + "type": "basic", + "user": "user", + "password": "password", + }, + ), + ], + indirect=["create_connection_data", "create_connection_auth_data"], +) +async def test_developer_plus_can_update_ftps_connection( + client: AsyncClient, + group_connection: MockConnection, + role_developer_plus: UserTestRoles, + create_connection_data: dict, + create_connection_auth_data: dict, +): + # Arrange + user = group_connection.owner_group.get_member_of_role(role_developer_plus) + + # Act + result = await client.patch( + f"v1/connections/{group_connection.id}", + headers={"Authorization": f"Bearer {user.token}"}, + json={"type": "ftps", "connection_data": {"host": "new_host"}}, + ) + + # Assert + assert result.json() == { + "id": group_connection.id, + "name": group_connection.connection.name, + "description": group_connection.description, + "type": group_connection.type, + "group_id": group_connection.group_id, + "connection_data": {"host": "new_host", "port": 80}, + "auth_data": { + "type": group_connection.credentials.value["type"], + "user": group_connection.credentials.value["user"], + }, + } + assert result.status_code == 200 diff --git a/tests/test_unit/test_connections/test_file_connection/test_update_samba_connection.py b/tests/test_unit/test_connections/test_file_connection/test_update_samba_connection.py new file mode 100644 index 00000000..f5c173ee --- /dev/null +++ b/tests/test_unit/test_connections/test_file_connection/test_update_samba_connection.py @@ -0,0 +1,67 @@ +import pytest +from httpx import AsyncClient + +from tests.mocks import MockConnection, UserTestRoles + +pytestmark = [pytest.mark.asyncio, pytest.mark.server, pytest.mark.samba] + + +@pytest.mark.parametrize( + "connection_type,create_connection_data,create_connection_auth_data", + [ + ( + "samba", + { + "host": "some_host", + "share": "some_folder", + "protocol": "NetBIOS", + "domain": "domain", + }, + { + "type": "samba", + "user": "user", + "password": "password", + "auth_type": "NTLMv2", + }, + ), + ], + indirect=["create_connection_data", "create_connection_auth_data"], +) +async def test_developer_plus_can_update_samba_connection( + client: AsyncClient, + group_connection: MockConnection, + role_developer_plus: UserTestRoles, + create_connection_data: dict, + create_connection_auth_data: dict, +): + # Arrange + user = group_connection.owner_group.get_member_of_role(role_developer_plus) + + # Act + result = await client.patch( + f"v1/connections/{group_connection.id}", + headers={"Authorization": f"Bearer {user.token}"}, + json={"type": "samba", "connection_data": {"host": "new_host"}}, + ) + + # Assert + assert result.json() == { + "id": group_connection.id, + "name": group_connection.connection.name, + "description": group_connection.description, + "type": group_connection.type, + "group_id": group_connection.group_id, + "connection_data": { + "host": "new_host", + "share": "some_folder", + "protocol": "NetBIOS", + "domain": "domain", + "port": None, + }, + "auth_data": { + "type": group_connection.credentials.value["type"], + "user": group_connection.credentials.value["user"], + "auth_type": group_connection.credentials.value["auth_type"], + }, + } + assert result.status_code == 200 diff --git a/tests/test_unit/test_connections/test_file_connection/test_update_sftp_connection.py b/tests/test_unit/test_connections/test_file_connection/test_update_sftp_connection.py new file mode 100644 index 00000000..4f0ddba5 --- /dev/null +++ b/tests/test_unit/test_connections/test_file_connection/test_update_sftp_connection.py @@ -0,0 +1,57 @@ +import pytest +from httpx import AsyncClient + +from tests.mocks import MockConnection, UserTestRoles + +pytestmark = [pytest.mark.asyncio, pytest.mark.server, pytest.mark.sftp] + + +@pytest.mark.parametrize( + "connection_type,create_connection_data,create_connection_auth_data", + [ + ( + "sftp", + { + "host": "some_host", + "port": 80, + }, + { + "type": "basic", + "user": "user", + "password": "password", + }, + ), + ], + indirect=["create_connection_data", "create_connection_auth_data"], +) +async def test_developer_plus_can_update_sftp_connection( + client: AsyncClient, + group_connection: MockConnection, + role_developer_plus: UserTestRoles, + create_connection_data: dict, + create_connection_auth_data: dict, +): + # Arrange + user = group_connection.owner_group.get_member_of_role(role_developer_plus) + + # Act + result = await client.patch( + f"v1/connections/{group_connection.id}", + headers={"Authorization": f"Bearer {user.token}"}, + json={"type": "sftp", "connection_data": {"host": "new_host"}}, + ) + + # Assert + assert result.json() == { + "id": group_connection.id, + "name": group_connection.connection.name, + "description": group_connection.description, + "type": group_connection.type, + "group_id": group_connection.group_id, + "connection_data": {"host": "new_host", "port": 80}, + "auth_data": { + "type": group_connection.credentials.value["type"], + "user": group_connection.credentials.value["user"], + }, + } + assert result.status_code == 200 diff --git a/tests/test_unit/test_connections/test_file_connection/test_update_webdav_connection.py b/tests/test_unit/test_connections/test_file_connection/test_update_webdav_connection.py new file mode 100644 index 00000000..6fd0b048 --- /dev/null +++ b/tests/test_unit/test_connections/test_file_connection/test_update_webdav_connection.py @@ -0,0 +1,58 @@ +import pytest +from httpx import AsyncClient + +from tests.mocks import MockConnection, UserTestRoles + +pytestmark = [pytest.mark.asyncio, pytest.mark.server, pytest.mark.webdav] + + +@pytest.mark.parametrize( + "connection_type,create_connection_data,create_connection_auth_data", + [ + ( + "webdav", + { + "host": "some_host", + "port": 443, + "protocol": "https", + }, + { + "type": "basic", + "user": "user", + "password": "password", + }, + ), + ], + indirect=["create_connection_data", "create_connection_auth_data"], +) +async def test_developer_plus_can_update_webdav_connection( + client: AsyncClient, + group_connection: MockConnection, + role_developer_plus: UserTestRoles, + create_connection_data: dict, + create_connection_auth_data: dict, +): + # Arrange + user = group_connection.owner_group.get_member_of_role(role_developer_plus) + + # Act + result = await client.patch( + f"v1/connections/{group_connection.id}", + headers={"Authorization": f"Bearer {user.token}"}, + json={"type": "webdav", "connection_data": {"host": "new_host"}}, + ) + + # Assert + assert result.json() == { + "id": group_connection.id, + "name": group_connection.connection.name, + "description": group_connection.description, + "type": group_connection.type, + "group_id": group_connection.group_id, + "connection_data": {"host": "new_host", "port": 443, "protocol": "https"}, + "auth_data": { + "type": group_connection.credentials.value["type"], + "user": group_connection.credentials.value["user"], + }, + } + assert result.status_code == 200 diff --git a/tests/test_unit/test_connections/test_read_connections.py b/tests/test_unit/test_connections/test_read_connections.py index 27391822..232aa923 100644 --- a/tests/test_unit/test_connections/test_read_connections.py +++ b/tests/test_unit/test_connections/test_read_connections.py @@ -307,10 +307,29 @@ async def test_search_connections_with_nonexistent_query( @pytest.mark.parametrize( "filter_params, expected_total", [ - ({}, 8), # No filters applied, expecting all connections + ({}, 13), # No filters applied, expecting all connections ({"type": ["oracle"]}, 1), ({"type": ["postgres", "hive"]}, 2), - ({"type": ["postgres", "hive", "oracle", "clickhouse", "mssql", "mysql", "hdfs", "s3"]}, 8), + ( + { + "type": [ + "postgres", + "hive", + "oracle", + "clickhouse", + "mssql", + "mysql", + "hdfs", + "s3", + "sftp", + "ftp", + "ftps", + "webdav", + "samba", + ], + }, + 13, + ), ], ids=[ "no_filters", diff --git a/tests/test_unit/test_transfers/test_create_transfer.py b/tests/test_unit/test_transfers/test_create_transfer.py index 770bdca0..785f5c9e 100644 --- a/tests/test_unit/test_transfers/test_create_transfer.py +++ b/tests/test_unit/test_transfers/test_create_transfer.py @@ -425,12 +425,12 @@ async def test_superuser_can_create_transfer( "location": ["body", "source_params"], "message": ( "Input tag 'new some connection type' found using 'type' " - "does not match any of the expected tags: 'postgres', 'hdfs', 'hive', 'oracle', 'clickhouse', 'mssql', 'mysql', 's3'" + "does not match any of the expected tags: 'postgres', 'hdfs', 'hive', 'oracle', 'clickhouse', 'mssql', 'mysql', 's3', 'sftp', 'ftp', 'ftps', 'webdav', 'samba'" ), "code": "union_tag_invalid", "context": { "discriminator": "'type'", - "expected_tags": "'postgres', 'hdfs', 'hive', 'oracle', 'clickhouse', 'mssql', 'mysql', 's3'", + "expected_tags": "'postgres', 'hdfs', 'hive', 'oracle', 'clickhouse', 'mssql', 'mysql', 's3', 'sftp', 'ftp', 'ftps', 'webdav', 'samba'", "tag": "new some connection type", }, "input": {