From e08cc4fff06488e09e9b3366b4c291a6222402fa Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Thu, 14 Sep 2023 12:55:10 -0400 Subject: [PATCH 01/55] network mode --- CHANGES.rst | 4 +- config/magpie.ini | 4 + docs/authentication.rst | 81 ++++++ docs/configuration.rst | 77 ++++++ docs/glossary.rst | 10 + magpie/adapter/magpieowssecurity.py | 9 +- ...3-08-25_2cfe144538e8_add_network_tables.py | 54 ++++ magpie/api/exception.py | 18 +- magpie/api/login/__init__.py | 4 + magpie/api/login/login.py | 257 +++++++++++++----- magpie/api/management/__init__.py | 1 + .../api/management/network_node/__init__.py | 13 + .../network_node/network_node_utils.py | 54 ++++ .../network_node/network_node_views.py | 89 ++++++ magpie/api/management/user/user_formats.py | 6 +- magpie/api/management/user/user_utils.py | 17 +- magpie/api/schemas.py | 183 +++++++++++++ magpie/constants.py | 53 ++++ magpie/models.py | 77 ++++++ magpie/ui/utils.py | 68 +++-- requirements.txt | 1 + 21 files changed, 981 insertions(+), 99 deletions(-) create mode 100644 magpie/alembic/versions/2023-08-25_2cfe144538e8_add_network_tables.py create mode 100644 magpie/api/management/network_node/__init__.py create mode 100644 magpie/api/management/network_node/network_node_utils.py create mode 100644 magpie/api/management/network_node/network_node_views.py diff --git a/CHANGES.rst b/CHANGES.rst index 676e58b11..93293fd37 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -11,7 +11,9 @@ Changes Features / Changes ~~~~~~~~~~~~~~~~~~~~~ -* n/a +* Introduce "Network Mode" which allows other Magpie instances to act as external authentication providers using JSON + Web Tokens (JWT). This allows users registered across multiple Magpie instances in a network to more easily gain + access to the resources within the network, without requiring the duplication of user credentials across the network. Bug Fixes ~~~~~~~~~~~~~~~~~~~~~ diff --git a/config/magpie.ini b/config/magpie.ini index b2750e877..e722bd1bf 100644 --- a/config/magpie.ini +++ b/config/magpie.ini @@ -28,6 +28,10 @@ pyramid.includes = magpie.port = 2001 magpie.url = http://localhost:2001 +# Enable network mode which allows different instances of Magpie to authenticate users for each other. +# magpie.network_mode = true +# magpie.default_token_expiry = 86400 + # magpie.config_path = # --- cookie definition --- (defaults below if omitted) diff --git a/docs/authentication.rst b/docs/authentication.rst index 05fb4ec22..05bfac950 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -422,3 +422,84 @@ Furthermore, as described in the `procedure`_, :envvar:`MAGPIE_USER_REGISTRATION specify whether administrator approval is required or not. This additional step is purely up to the developers and server managers that use `Magpie` to decide if they desire more control over which individuals can join and access their services. + +.. _Network Mode: + +Network Mode +------------ + +If the :envvar:`MAGPIE_NETWORK_MODE` is enabled, an additional external authentication provider is added to Magpie which +allows networked instances of Magpie to authenticate users for each other. + +Users can then log in to any Magpie instance where they have an account and request a personal network token in the form +of a `_JSON Web Token `_ which can be used to authenticate this user on +any Magpie instance in the network. + +Managing the Network +~~~~~~~~~~~~~~~~~~~~ + +In order for Magpie instances to authenticate each other's users, each instance must be made aware of the existence of +the others so that it knows where to send authentication verification requests. + +In order to register another Magpie instance as part of the same network, an admin user can create a +:term:`Network Node` with a request to ``POST /network-nodes``. The parameters given to that request include a ``name`` +and a ``url``. The ``name`` is a the name of that other Magpie instance in the network and should correspond to the +same value as the :envvar:`MAGPIE_INSTANCE_NAME` value set by the other Magpie instance. The ``url`` is a the root URL +of the other Magpie instance. + +Once a :term:`Network Node` is registered, Magpie can use that other instance to authenticate users as long as the other +instance also has :envvar:`MAGPIE_NETWORK_MODE` enabled. + +Managing Personal JWTs +~~~~~~~~~~~~~~~~~~~~~~ + +A :term:`User` can request a new network token with a request to the ``PATCH /token`` route. This route takes one +optional parameter ``expires`` which is an integer indicating how long (in seconds) until that token expires, the +default expiry for this token is :envvar:`MAGPIE_DEFAULT_TOKEN_EXPIRY`. + +Every time a :term:`User` makes a request to the ``PATCH /token`` route a new token is generated for them. This +effectively cancels all previously created tokens for that user. If a user wishes to cancel all tokens, they can provide +an ``expires`` value of ``0`` when making the request. + +Authentication +~~~~~~~~~~~~~~ + +Once a :term:`User` gets a personal network token, they can use that token to authenticate with any Magpie instance in +the same network. When a user makes a request, they should set the ``provider_name`` parameter to the value of +:envvar:`MAGPIE_NETWORK_PROVIDER` and provide the network token in the Authorization header in the following format: + +.. code-block:: http + + Authorization: Bearer + +When using the :ref:`Magpie Adapter `, the token can also be passed as a parameter to the request, +where the parameter name set by :envvar:`MAGPIE_NETWORK_TOKEN_NAME` and the value is the personal network token. + +Authorization +~~~~~~~~~~~~~ + +Managing authorization for :term:`Users` who authenticate using personal network tokens is complicated by the fact that +a :term:`User` is not required to have a full account on both Magpie instances in order to using this authentication +mechanism. This means that a :term:`User` may be logged in as a node-specific "anonymous" user. + +When another Magpie instance is registered as a :term:`Network Node`, a few additional entities are created: + +#. a group used to manage the permissions of all users who authenticate using the new :term:`Network Node`. + * this group's name will be the :envvar:`MAGPIE_NETWORK_NAME_PREFIX` followed by the :term:`Network Node` name +#. a group used to manage the permissions of all users who authenticate using *any* other instance in the network + * this group's name will be the :envvar:`MAGPIE_NETWORK_GROUP_NAME` + * this group will only be created once, when the first :term:`Network Node` is registered +#. an anonymous user that belongs to the two groups that were just created. + * this user name will be the :envvar:`MAGPIE_NETWORK_NAME_PREFIX` followed by the :term:`Network Node` name + +Here is an example to illustrate this point: + +* There are 3 Magpie instances in the network named A, B, and C +* There is a :term:`User` named ``"toto"`` registered on instance A +* There is no :term:`User` named ``"toto"`` who belongs to the ``"anonymous_network_A"`` group registered on instance B +* There is a :term:`User` named ``"toto"`` who belongs to the ``"anonymous_network_A"`` group registered on instance C +* Instance A is registered as a :term:`Network Node` on instances B and C +* when ``"toto"`` gets a personal network token from instance A and uses it to log in on instance B they log in as the + the temporary ``"anonymous_network_A"`` user. +* when ``"toto"`` gets a personal network token from instance A and uses it to log in on instance C they log in as the + ``"toto"`` user on instance C. diff --git a/docs/configuration.rst b/docs/configuration.rst index 22578d64c..0b8d3490f 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -969,6 +969,83 @@ remain available as described at the start of the :ref:`Configuration` section. Name of the :term:`Provider` used for login. This represents the identifier that is set to define how to differentiate between a local sign-in procedure and a dispatched one some known :ref:`authn_providers`. +Network Mode Settings +~~~~~~~~~~~~~~~~~~~~~ + +The following configuration parameters are related to Magpie's "Network Mode" which allows networked instances of Magpie +to authenticate users for each other. All variables defined in this section are only used if +:envvar:`MAGPIE_NETWORK_MODE` is enabled. + +.. envvar:: MAGPIE_NETWORK_MODE + + [:class:`bool`] + (Default: ``False``) + + .. versionadded:: 3.37 + + Enable "Network Mode" which enables all functionality to authenticate users using other Magpie instances as + external authentication providers. + +.. envvar:: MAGPIE_INSTANCE_NAME + + [:class:`str`] + + .. versionadded:: 3.37 + + The name of this Magpie instance in the network. This variable is used to determine if an authentication token was + issued by this instance of Magpie, or another instance in the network. + + This variable is required if :envvar:`MAGPIE_NETWORK_MODE` is ``True``. + +.. envvar:: MAGPIE_DEFAULT_TOKEN_EXPIRY + + [:class:`int`] + (Default: ``86400``) + + .. versionadded:: 3.37 + + The default expiry time (in seconds) for an authentication token issued for the purpose of network authentication. + +.. envvar:: MAGPIE_NETWORK_TOKEN_NAME + + [|constant|_] + (Value: ``"magpie_token"``) + + .. versionadded:: 3.37 + + The name of the request parameter key whose value is the authentication token issued for the purpose of network + authentication. + +.. envvar:: MAGPIE_NETWORK_PROVIDER + + [|constant|_] + (Value: ``"magpie_network"``) + + .. versionadded:: 3.37 + + The name of the external provider that authenticates users using other Magpie instances as external authentication + providers. + +.. envvar:: MAGPIE_NETWORK_NAME_PREFIX + + [|constant|_] + (Value: ``"anonymous_network_"``) + + .. versionadded:: 3.37 + + A prefix added to the anonymous network user and network group names. These names are constructed by prepending the + remote Magpie instance name with this prefix. For example, a Magpie instance named ``"example123"`` will have a + corresponding user and group named ``"anonymous_network_example123"``. + +.. envvar:: MAGPIE_NETWORK_GROUP_NAME + + [|constant|_] + (Value: ``"magpie_network"``) + + .. versionadded:: 3.37 + + The name of the group created to manage permissions for all users authenticated using Magpie instances as external + authentication providers. .. _config_phoenix: diff --git a/docs/glossary.rst b/docs/glossary.rst index 1059109ef..405f047db 100644 --- a/docs/glossary.rst +++ b/docs/glossary.rst @@ -138,6 +138,16 @@ Glossary :py:data:`magpie.constants.MAGPIE_ANONYMOUS_USER`. Otherwise, it is whoever the :term:`Authentication` mechanism identifies with token extracted from request :term:`Cookies`. + Network Node + A reference to an instance of the Magpie software within a network of Magpie instances. Each Magpie instance + within the network is registered in the database as a row in the ``network_nodes`` table. Each node is + represented by a name that is unique across all nodes in the network, and a url that is used to send http + requests to that specific node. + + Network Token + A unique random string that can be used to authenticate a user as part of the :ref:`Network Mode` authentication + procedure. + OpenAPI OAS The |OpenAPI-spec|_ (`OAS`) defines a standard, programming language-agnostic interface description for diff --git a/magpie/adapter/magpieowssecurity.py b/magpie/adapter/magpieowssecurity.py index a3184a1ac..8401de19c 100644 --- a/magpie/adapter/magpieowssecurity.py +++ b/magpie/adapter/magpieowssecurity.py @@ -269,11 +269,18 @@ def update_request_cookies(self, request): """ settings = get_settings(request) token_name = get_constant("MAGPIE_COOKIE_NAME", settings_container=settings) + network_mode = get_constant("MAGPIE_NETWORK_MODE", settings_container=settings, + settings_name="magpie.network_mode") + headers = dict(request.headers) + network_token_name = get_constant("MAGPIE_NETWORK_TOKEN_NAME", settings_container=settings) + if network_mode and "Authorization" not in headers and network_token_name in request.params: + headers["Authorization"] = "Bearer {}".format(request.params[network_token_name]) + request.params["provider_name"] = request.params.get("provider_name", + get_constant("MAGPIE_NETWORK_PROVIDER", settings)) if "Authorization" in request.headers and token_name not in request.cookies: magpie_prov = request.params.get("provider_name", get_constant("MAGPIE_DEFAULT_PROVIDER", settings)) magpie_path = ProviderSigninAPI.path.format(provider_name=magpie_prov) magpie_auth = "{}{}".format(self.magpie_url, magpie_path) - headers = dict(request.headers) headers.update({"Homepage-Route": "/session", "Accept": CONTENT_TYPE_JSON}) session_resp = requests.get(magpie_auth, headers=headers, verify=self.twitcher_ssl_verify) if session_resp.status_code != HTTPOk.code: diff --git a/magpie/alembic/versions/2023-08-25_2cfe144538e8_add_network_tables.py b/magpie/alembic/versions/2023-08-25_2cfe144538e8_add_network_tables.py new file mode 100644 index 000000000..dea28d86d --- /dev/null +++ b/magpie/alembic/versions/2023-08-25_2cfe144538e8_add_network_tables.py @@ -0,0 +1,54 @@ +""" +Add Network_Tokens Table + +Revision ID: 2cfe144538e8 +Revises: 5e5acc33adce +Create Date: 2023-08-25 13:36:16.930374 +""" +import uuid + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm.session import sessionmaker +from sqlalchemy_utils import URLType + +# Revision identifiers, used by Alembic. +# pylint: disable=C0103,invalid-name # revision control variables not uppercase +revision = "2cfe144538e8" +down_revision = "5e5acc33adce" +branch_labels = None +depends_on = None + +Session = sessionmaker() + + +def upgrade(): + op.create_table("network_tokens", + sa.Column("token", UUID(as_uuid=True), + primary_key=True, default=uuid.uuid4, unique=True), + sa.Column("user_id", sa.Integer, + sa.ForeignKey("users.id", onupdate="CASCADE", ondelete="CASCADE"), unique=True) + ) + op.create_table("network_nodes", + sa.Column("id", sa.Integer, primary_key=True, autoincrement=True), + sa.Column("name", sa.Unicode(128), nullable=False, unique=True), + sa.Column("url", URLType(), nullable=False) + ) + op.add_column("users", sa.Column("network_node_id", sa.Integer, + sa.ForeignKey("network_nodes.id", onupdate="CASCADE", ondelete="CASCADE"), + nullable=True)) + op.drop_constraint("uq_users_user_name", "users") + op.create_unique_constraint("uq_users_user_name_network_node_id", "users", ["user_name", "network_node_id"]) + op.drop_constraint("uq_users_email", "users") + op.create_unique_constraint("uq_users_email_network_node_id", "users", ["email", "network_node_id"]) + + +def downgrade(): + op.drop_constraint("uq_users_user_name_network_node_id", "users") + op.create_unique_constraint("uq_users_user_name", "users", ["user_name"]) + op.drop_constraint("uq_users_email_network_node_id", "users") + op.create_unique_constraint("uq_users_email", "users", ["email"]) + op.drop_table("network_tokens") + op.drop_table("network_nodes") + op.drop_column("users", "network_node_id") diff --git a/magpie/api/exception.py b/magpie/api/exception.py index 86179e7fc..587092c14 100644 --- a/magpie/api/exception.py +++ b/magpie/api/exception.py @@ -85,6 +85,7 @@ def verify_param( # noqa: E126 # pylint: disable=R0913,too-many-arguments is_equal=False, # type: bool is_type=False, # type: bool matches=False, # type: bool + not_matches=False, # type: bool ): # type: (...) -> None # noqa: E123,E126 # pylint: disable=R0912,R0914 """ @@ -123,12 +124,13 @@ def verify_param( # noqa: E126 # pylint: disable=R0913,too-many-arguments :param is_equal: test that :paramref:`param` equals :paramref:`param_compare` value :param is_type: test that :paramref:`param` is of same type as specified by :paramref:`param_compare` type :param matches: test that :paramref:`param` matches the regex specified by :paramref:`param_compare` value + :param not_matches: test that :paramref:`param` doesn't match the regex specified by :paramref:`param_compare` value :raises HTTPError: if tests fail, specified exception is raised (default: :class:`HTTPBadRequest`) :raises HTTPInternalServerError: for evaluation error :return: nothing if all tests passed """ content = {} if content is None else content - needs_compare = is_type or is_in or not_in or is_equal or not_equal or matches + needs_compare = is_type or is_in or not_in or is_equal or not_equal or matches or not_matches needs_iterable = is_in or not_in # precondition evaluation of input parameters @@ -159,9 +161,11 @@ def verify_param( # noqa: E126 # pylint: disable=R0913,too-many-arguments raise TypeError("'is_type' is not a 'bool'") if not isinstance(matches, bool): raise TypeError("'matches' is not a 'bool'") + if not isinstance(not_matches, bool): + raise TypeError("'not_matches' is not a 'bool'") # error if none of the flags specified if not any([not_none, not_empty, not_in, not_equal, - is_none, is_empty, is_in, is_equal, is_true, is_false, is_type, matches]): + is_none, is_empty, is_in, is_equal, is_true, is_false, is_type, matches, not_matches]): raise ValueError("no comparison flag specified for verification") if param_compare is None and needs_compare: raise TypeError("'param_compare' cannot be 'None' with specified test flags") @@ -178,11 +182,11 @@ def verify_param( # noqa: E126 # pylint: disable=R0913,too-many-arguments is_str_cmp = isinstance(param, six.string_types) ok_str_cmp = isinstance(param_compare, six.string_types) eq_typ_cmp = type(param) is type(param_compare) - is_pattern = matches and isinstance(param_compare, Pattern) + is_pattern = (matches or not_matches) and isinstance(param_compare, Pattern) if is_type and not (is_str_typ or is_cmp_typ): LOGGER.debug("[param: %s] invalid type compare with [param_compare: %s]", type(param), param_compare) raise TypeError("'param_compare' cannot be of non-type with specified verification flags") - if matches and not isinstance(param_compare, (six.string_types, Pattern)): + if (matches or not_matches) and not isinstance(param_compare, (six.string_types, Pattern)): LOGGER.debug("[param_compare: %s] invalid type is not a regex string or pattern", type(param_compare)) raise TypeError("'param_compare' for matching verification must be a string or compile regex pattern") if not is_type and not ((is_str_cmp and ok_str_cmp) or (not is_str_cmp and eq_typ_cmp) or is_pattern): @@ -252,6 +256,12 @@ def verify_param( # noqa: E126 # pylint: disable=R0913,too-many-arguments param_compare_regex = re.compile(param_compare, re.I | re.X) fail_conditions.update({"matches": bool(re.match(param_compare_regex, param))}) fail_verify = fail_verify or not fail_conditions["matches"] + if not_matches: + param_compare_regex = param_compare + if isinstance(param_compare, six.string_types): + param_compare_regex = re.compile(param_compare, re.I | re.X) + fail_conditions.update({"not_matches": not re.match(param_compare_regex, param)}) + fail_verify = fail_verify or not fail_conditions["not_matches"] if fail_verify: content = apply_param_content(content, param, param_compare, param_name, with_param, param_content, needs_compare, needs_iterable, is_type, fail_conditions) diff --git a/magpie/api/login/__init__.py b/magpie/api/login/__init__.py index 0aaba355a..aadf7778e 100644 --- a/magpie/api/login/__init__.py +++ b/magpie/api/login/__init__.py @@ -1,3 +1,4 @@ +from magpie.constants import get_constant from magpie.utils import get_logger LOGGER = get_logger(__name__) @@ -11,4 +12,7 @@ def includeme(config): config.add_route(**s.service_api_route_info(s.SigninAPI)) config.add_route(**s.service_api_route_info(s.ProvidersAPI)) config.add_route(**s.service_api_route_info(s.ProviderSigninAPI)) + if get_constant("MAGPIE_NETWORK_MODE", settings_name="magpie.network_mode"): + config.add_route(**s.service_api_route_info(s.TokenAPI)) + config.add_route(**s.service_api_route_info(s.TokenValidateAPI)) config.scan() diff --git a/magpie/api/login/login.py b/magpie/api/login/login.py index 143a805ac..abfae0f54 100644 --- a/magpie/api/login/login.py +++ b/magpie/api/login/login.py @@ -1,6 +1,10 @@ import json +import uuid +from datetime import datetime, timedelta from typing import TYPE_CHECKING +import jwt +import requests from authomatic.adapters import WebObAdapter from authomatic.core import Credentials, LoginResult, resolve_provider_class from authomatic.exceptions import OAuth2Error @@ -34,7 +38,7 @@ from magpie.api import schemas as s from magpie.api.management.user.user_formats import format_user from magpie.api.management.user.user_utils import create_user -from magpie.constants import get_constant +from magpie.constants import get_constant, protected_user_name_regex from magpie.security import authomatic_setup, get_providers from magpie.utils import ( CONTENT_TYPE_JSON, @@ -47,14 +51,23 @@ if TYPE_CHECKING: from magpie.typedefs import Session, Str + from typing import Optional LOGGER = get_logger(__name__) +MAGPIE_NETWORK_MODE = get_constant("MAGPIE_NETWORK_MODE", settings_name="magpie.network_mode") # dictionaries of {'provider_id': 'provider_display_name'} MAGPIE_DEFAULT_PROVIDER = get_constant("MAGPIE_DEFAULT_PROVIDER") MAGPIE_INTERNAL_PROVIDERS = {MAGPIE_DEFAULT_PROVIDER: MAGPIE_DEFAULT_PROVIDER.capitalize()} MAGPIE_EXTERNAL_PROVIDERS = get_providers() + +if MAGPIE_NETWORK_MODE: + MAGPIE_NETWORK_TOKEN_NAME = get_constant("MAGPIE_NETWORK_TOKEN_NAME") + MAGPIE_NETWORK_PROVIDER = get_constant("MAGPIE_NETWORK_PROVIDER") + MAGPIE_INSTANCE_NAME = get_constant("MAGPIE_INSTANCE_NAME") + MAGPIE_EXTERNAL_PROVIDERS[MAGPIE_NETWORK_PROVIDER] = MAGPIE_NETWORK_PROVIDER + MAGPIE_PROVIDER_KEYS = frozenset(set(MAGPIE_INTERNAL_PROVIDERS) | set(MAGPIE_EXTERNAL_PROVIDERS)) @@ -130,8 +143,8 @@ def sign_in_view(request): # request handler. Catch that specific exception and return it to bypass the EXCVIEW tween that result in that # automatic convert to return 403 directly. try: - anonymous = get_constant("MAGPIE_ANONYMOUS_USER", request) - ax.verify_param(user_name, not_equal=True, param_compare=anonymous, param_name="user_name", + anonymous_regex = protected_user_name_regex(include_admin=False, settings_container=request) + ax.verify_param(user_name, not_matches=True, param_compare=anonymous_regex, param_name="user_name", http_error=HTTPForbidden, msg_on_fail=s.Signin_POST_ForbiddenResponseSchema.description) except HTTPForbidden as http_error: return http_error @@ -239,17 +252,12 @@ def new_user_external(external_user_name, external_id, email, provider_name, db_ return user -def login_success_external(request, external_user_name, external_id, email, provider_name): - # type: (Request, Str, Str, Str, Str) -> HTTPException +def login_success_external(request, user): + # type: (Request, models.User) -> HTTPException """ Generates the login response in case of successful external provider identification. """ - # find possibly already registered user by external_id/provider - user = ExternalIdentityService.user_by_external_id_and_provider(external_id, provider_name, request.db) - if user is None: - # create new user with an External Identity - user = new_user_external(external_user_name=external_user_name, external_id=external_id, - email=email, provider_name=provider_name, db_session=request.db) + # set a header to remember user (set-cookie) headers = remember(request, user.id) @@ -271,10 +279,27 @@ def login_success_external(request, external_user_name, external_id, email, prov http_kwargs={"location": homepage_route, "headers": headers}) +def user_from_token(request, token): + # type: (Request, Str) -> Optional[models.User] + """ + Return the ``User`` associated with the token as long as the token is valid and issued by this Magpie instance. + """ + magpie_secret = get_constant("MAGPIE_SECRET", request, settings_name="magpie.secret") + decoded_token = jwt.decode(token, magpie_secret, algorithms="HS256", issuer=MAGPIE_INSTANCE_NAME, + options={"require": ["exp", "token"]}) + + network_token = (request.db.query(models.NetworkToken) + .join(models.User) + .filter(models.NetworkToken.token == decoded_token["token"]) + .first()) + if network_token: + return network_token.user + + @s.ProviderSigninAPI.get(schema=s.ProviderSignin_GET_RequestSchema, tags=[s.SessionTag], response_schemas=s.ProviderSignin_GET_responses) @view_config(route_name=s.ProviderSigninAPI.name, permission=NO_PERMISSION_REQUIRED) -def authomatic_login_view(request): +def external_login_view(request): """ Signs in a user session using an external provider. """ @@ -282,58 +307,99 @@ def authomatic_login_view(request): response = Response() verify_provider(provider_name) try: - authomatic_handler = authomatic_setup(request) - - # if we directly have the Authorization header, bypass authomatic login and retrieve 'userinfo' to signin - if "Authorization" in request.headers and "authomatic" not in request.cookies: - provider_config = authomatic_handler.config.get(provider_name, {}) - provider_class = resolve_provider_class(provider_config.get("class_")) - provider = provider_class(authomatic_handler, adapter=None, provider_name=provider_name) - # provide the token user data, let the external provider update it on login afterwards - token_type, access_token = request.headers.get("Authorization").split() - data = {"access_token": access_token, "token_type": token_type} - cred = Credentials(authomatic_handler.config, token=access_token, token_type=token_type, provider=provider) - provider.credentials = cred - result = LoginResult(provider) - # pylint: disable=W0212 - result.provider.user = result.provider._update_or_create_user(data, credentials=cred) # noqa: W0212 - - # otherwise, use the standard login procedure - else: - result = authomatic_handler.login(WebObAdapter(request, response), provider_name) - if result is None: - if response.location is not None: - return HTTPTemporaryRedirect(location=response.location, headers=response.headers) - return response - - if result: - if result.error: - # Login procedure finished with an error. - error = result.error.to_dict() if hasattr(result.error, "to_dict") else result.error - LOGGER.debug("Login failure with error. [%r]", error) - return login_failure_view(request, reason=result.error.message) - if result.user: - # OAuth 2.0 and OAuth 1.0a provide only limited user data on login, - # update the user to get more info. - if not (result.user.name and result.user.id): - try: - response = result.user.update() - # this error can happen if providing incorrectly formed authorization header - except OAuth2Error as exc: - LOGGER.debug("Login failure with Authorization header.") - ax.raise_http(http_error=HTTPBadRequest, content={"reason": str(exc.message)}, - detail=s.ProviderSignin_GET_BadRequestResponseSchema.description) - # verify that the update procedure succeeded with provided token - if 400 <= response.status < 500: - LOGGER.debug("Login failure with invalid token.") + if provider_name == MAGPIE_NETWORK_PROVIDER: + if "Authorization" in request.headers: + token_type, token = request.headers.get("Authorization").split() + if token_type != "Bearer": + ax.raise_http(http_error=HTTPBadRequest, + detail=s.ProviderSignin_GET_BadRequestResponseSchema.description) + authenticated_user = None + try: + authenticated_user = user_from_token(request, token) + except jwt.exceptions.InvalidIssuerError: + decoded_token = jwt.decode(token, options={"verify_signature": False}) + issuer = decoded_token.get("iss") + node = request.db.query(models.NetworkNode).filter(models.NetworkNode.name == issuer).first() + if node: + response = requests.get("{}{}".format(node.url, s.TokenValidateAPI.path), + headers={"Accept": CONTENT_TYPE_JSON}) + + if response.status_code != HTTPOk.code: + ax.raise_http(http_error=HTTPUnauthorized, + detail=s.ProviderSignin_GET_UnauthorizedTokenSchema.description) + user_name = response.json().get("user_name") + authenticated_user = (node.users.filter(models.User.name == user_name).first() or + node.anonymous_user) + else: ax.raise_http(http_error=HTTPUnauthorized, - detail=s.ProviderSignin_GET_UnauthorizedResponseSchema.description) - # create/retrieve the user using found details from login provider - return login_success_external(request, - external_id=result.user.username or result.user.id, - email=result.user.email, - provider_name=result.provider.name, - external_user_name=result.user.name) + detail=s.ProviderSignin_GET_UnauthorizedTokenSchema.description) + except jwt.exceptions.PyJWTError as exc: + ax.raise_http(http_error=HTTPUnauthorized, content={"reason": str(exc)}, + detail=s.ProviderSignin_GET_UnauthorizedTokenSchema.description) + if authenticated_user: + return login_success_external(request, authenticated_user) + else: + ax.raise_http(http_error=HTTPBadRequest, + detail=s.ProviderSignin_GET_BadRequestResponseSchema.description) + else: + authomatic_handler = authomatic_setup(request) + + # if we directly have the Authorization header, bypass authomatic login and retrieve 'userinfo' to signin + if "Authorization" in request.headers and "authomatic" not in request.cookies: + provider_config = authomatic_handler.config.get(provider_name, {}) + provider_class = resolve_provider_class(provider_config.get("class_")) + provider = provider_class(authomatic_handler, adapter=None, provider_name=provider_name) + # provide the token user data, let the external provider update it on login afterwards + token_type, access_token = request.headers.get("Authorization").split() + data = {"access_token": access_token, "token_type": token_type} + cred = Credentials(authomatic_handler.config, token=access_token, token_type=token_type, + provider=provider) + provider.credentials = cred + result = LoginResult(provider) + # pylint: disable=W0212 + result.provider.user = result.provider._update_or_create_user(data, credentials=cred) # noqa: W0212 + + # otherwise, use the standard login procedure + else: + result = authomatic_handler.login(WebObAdapter(request, response), provider_name) + if result is None: + if response.location is not None: + return HTTPTemporaryRedirect(location=response.location, headers=response.headers) + return response + + if result: + if result.error: + # Login procedure finished with an error. + error = result.error.to_dict() if hasattr(result.error, "to_dict") else result.error + LOGGER.debug("Login failure with error. [%r]", error) + return login_failure_view(request, reason=result.error.message) + if result.user: + # OAuth 2.0 and OAuth 1.0a provide only limited user data on login, + # update the user to get more info. + if not (result.user.name and result.user.id): + try: + response = result.user.update() + # this error can happen if providing incorrectly formed authorization header + except OAuth2Error as exc: + LOGGER.debug("Login failure with Authorization header.") + ax.raise_http(http_error=HTTPBadRequest, content={"reason": str(exc.message)}, + detail=s.ProviderSignin_GET_BadRequestResponseSchema.description) + # verify that the update procedure succeeded with provided token + if 400 <= response.status < 500: + LOGGER.debug("Login failure with invalid token.") + ax.raise_http(http_error=HTTPUnauthorized, + detail=s.ProviderSignin_GET_UnauthorizedResponseSchema.description) + # create/retrieve the user using found details from login provider + # find possibly already registered user by external_id/provider + external_id = result.user.username or result.user.id + user = ExternalIdentityService.user_by_external_id_and_provider(external_id, provider_name, + request.db) + if user is None: + # create new user with an External Identity + user = new_user_external(external_user_name=result.user.name, external_id=external_id, + email=result.user.email, provider_name=result.provider.name, + db_session=request.db) + return login_success_external(request, user) except Exception as exc: exc_msg = "Unhandled error during external provider '{}' login. [{!s}]".format(provider_name, exc) LOGGER.exception(exc_msg, exc_info=True) @@ -373,6 +439,71 @@ def _get_session(req): return ax.valid_http(http_success=HTTPOk, detail=s.Session_GET_OkResponseSchema.description, content=session_json) +if MAGPIE_NETWORK_MODE: + @s.TokenAPI.patch(schema=s.Token_PATCH_RequestBodySchema, tags=[s.SessionTag], + response_schemas=s.Token_PATCH_responses) + @view_config(route_name=s.TokenAPI.name, request_method="PATCH", permission=Authenticated) + def token_view(request): + """ + Get a token. Generates a new token every time this route is accessed so this can also + be used to invalidate a previously generated token. + """ + magpie_secret = get_constant("MAGPIE_SECRET", request, settings_name="magpie.secret") + default_expiry = get_constant("MAGPIE_DEFAULT_TOKEN_EXPIRY", request, + settings_name="magpie.default_token_expiry") + magpie_instance_name = get_constant("MAGPIE_INSTANCE_NAME", + request, + settings_name="magpie.instance_name") + + expires = request.GET.get("expires", default_expiry) + try: + expires = int(expires) + except ValueError: + expires = int(default_expiry) + + expiry = datetime.utcnow() + timedelta(seconds=expires) + + def upsert_token(): + network_token = request.db.query(models.NetworkToken).filter( + models.NetworkToken.user_id == request.user.id).first() + if network_token: + network_token.token = uuid.uuid4() + else: + network_token = models.NetworkToken(user_id=request.user.id) + request.db.add(network_token) + return network_token.token + + token_value = ax.evaluate_call(lambda: upsert_token(), fallback=lambda: request.db.rollback(), + http_error=HTTPInternalServerError) + + token = jwt.encode({"token": str(token_value), + "exp": expiry, + "iss": magpie_instance_name}, magpie_secret, algorithm="HS256") + + return ax.valid_http(http_success=HTTPOk, detail=s.Token_PATCH_OkResponseSchema.description, + content={"token": token}) + + + @s.TokenValidateAPI.get(schema=s.TokenValidate_GET_RequestBodySchema, tags=[s.SessionTag], + response_schemas=s.TokenValidate_GET_responses) + @view_config(route_name=s.TokenValidateAPI.name, permission=NO_PERMISSION_REQUIRED) + def validate_token_view(request): + """ Validate a token """ + token = request.GET.get("token") + try: + authenticated_user = user_from_token(request, token) + if authenticated_user: + return ax.valid_http(http_success=HTTPOk, + detail=s.TokenValidate_GET_OkResponseSchema.description, + content={"user_name": authenticated_user.user_name}) + else: + ax.raise_http(http_error=HTTPUnauthorized, + detail=s.TokenValidate_GET_BadRequestResponseSchema.description) + except jwt.exceptions.PyJWTError as exc: + ax.raise_http(http_error=HTTPUnauthorized, content={"reason": str(exc)}, + detail=s.TokenValidate_GET_BadRequestResponseSchema.description) + + @s.ProvidersAPI.get(tags=[s.SessionTag], response_schemas=s.Providers_GET_responses) @view_config(route_name=s.ProvidersAPI.name, request_method="GET", permission=NO_PERMISSION_REQUIRED) def get_providers_view(request): # noqa: F811 diff --git a/magpie/api/management/__init__.py b/magpie/api/management/__init__.py index fb09062db..6226e2dd2 100644 --- a/magpie/api/management/__init__.py +++ b/magpie/api/management/__init__.py @@ -10,4 +10,5 @@ def includeme(config): config.include("magpie.api.management.service") config.include("magpie.api.management.resource") config.include("magpie.api.management.register") + config.include("magpie.api.management.network_node") config.scan() diff --git a/magpie/api/management/network_node/__init__.py b/magpie/api/management/network_node/__init__.py new file mode 100644 index 000000000..1a1b2479a --- /dev/null +++ b/magpie/api/management/network_node/__init__.py @@ -0,0 +1,13 @@ +from magpie.api import schemas as s +from magpie.constants import get_constant +from magpie.utils import get_logger + +LOGGER = get_logger(__name__) + + +def includeme(config): + if get_constant("MAGPIE_NETWORK_MODE", settings_name="magpie.network_mode"): + LOGGER.info("Adding API network node...") + config.add_route(**s.service_api_route_info(s.NetworkNodeAPI)) + config.add_route(**s.service_api_route_info(s.NetworkNodesAPI)) + config.scan() diff --git a/magpie/api/management/network_node/network_node_utils.py b/magpie/api/management/network_node/network_node_utils.py new file mode 100644 index 000000000..4877861da --- /dev/null +++ b/magpie/api/management/network_node/network_node_utils.py @@ -0,0 +1,54 @@ +from typing import TYPE_CHECKING + +from magpie import models +from magpie.cli.register_defaults import register_user_with_group +from magpie.constants import get_constant + +if TYPE_CHECKING: + from magpie.typedefs import Str + from pyramid.request import Request + + +def create_network_node(request, name, url): + # type: (Request, Str, Str) -> None + """ + Create a NetworkNode with the given name and url. + """ + network_node = models.NetworkNode(name=name, url=url) + name = network_node.anonymous_user_name + + # create an anonymous user and group for this network node + register_user_with_group(user_name=name, + group_name=name, + email=get_constant("MAGPIE_ANONYMOUS_EMAIL"), + password=None, # autogen, value doesn't matter as no login applicable, just make it valid + db_session=request.db) + + group = models.GroupService.by_group_name(name, db_session=request.db) + group.description = "Group for users who have accounts on the networked Magpie instance named '{}'.".format(name) + group.discoverable = False + + # add the anonymous user to a group for all users in the network (from nodes other than this one). + register_user_with_group(user_name=name, + group_name=get_constant("MAGPIE_NETWORK_GROUP_NAME"), + email=get_constant("MAGPIE_ANONYMOUS_EMAIL"), + password=None, + db_session=request.db) + + group = models.GroupService.by_group_name(get_constant("MAGPIE_NETWORK_GROUP_NAME"), db_session=request.db) + group.description = "Group for users who have accounts on a different Magpie instance on this network.".format(name) + group.discoverable = False + + request.tm.commit() + + +def delete_network_node(request, node): + # type: (Request, Str) -> None + """ + Delete a NetworkNode and the associated anonymous user. + """ + anonymous_user = node.anonymous_user + if anonymous_user: + anonymous_user.delete() + node.delete() + request.tm.commit() diff --git a/magpie/api/management/network_node/network_node_views.py b/magpie/api/management/network_node/network_node_views.py new file mode 100644 index 000000000..f0db0dcd7 --- /dev/null +++ b/magpie/api/management/network_node/network_node_views.py @@ -0,0 +1,89 @@ +from pyramid.httpexceptions import ( + HTTPBadRequest, + HTTPConflict, + HTTPNotFound, + HTTPOk, + HTTPCreated, + HTTPInternalServerError +) + +from pyramid.view import view_config + +from magpie import models +from magpie.api import exception as ax +from magpie.api import requests as ar +from magpie.api import schemas as s +from magpie.api.management.network_node.network_node_utils import create_network_node, delete_network_node +from magpie.constants import get_constant + + +if get_constant("MAGPIE_NETWORK_MODE", settings_name="magpie.network_mode"): + @s.NetworkNodesAPI.get(tags=[s.NetworkNodeTag], response_schemas=s.NetworkNodes_GET_responses) + @view_config(route_name=s.NetworkNodesAPI.name, request_method="GET") + def get_network_nodes_view(request): + nodes = [{"name": n.name, "url": n.url} for n in request.db.query(models.NetworkNode).all()] + return ax.valid_http(http_success=HTTPOk, detail=s.NetworkNode_GET_OkResponseSchema.description, + content={"nodes": nodes}) + + + @s.NetworkNodeAPI.get(tags=[s.NetworkNodeTag], response_schemas=s.NetworkNode_GET_responses) + @view_config(route_name=s.NetworkNodeAPI.name, request_method="GET") + def get_network_node_view(request): + node_name = ar.get_value_matchdict_checked(request, "node") + node = ax.evaluate_call( + lambda: request.db.query(models.NetworkNode).filter(models.NetworkNode.name == node_name).one(), + http_error=HTTPNotFound, + msg_on_fail=s.NetworkNode_GET_NotFoundResponseSchema.description) + return ax.valid_http(http_success=HTTPOk, detail=s.NetworkNode_GET_OkResponseSchema.description, + content={"name": node.name, "url": node.url}) + + + @s.NetworkNodesAPI.post(schema=s.NetworkNode_POST_RequestBodySchema, tags=[s.NetworkNodeTag], + response_schemas=s.NetworkNodes_POST_responses) + @view_config(route_name=s.NetworkNodesAPI.name, request_method="POST") + def create_network_node_view(request): + node_name = request.POST.get("name") + node_url = request.POST.get("url") + ax.verify_param(all([node_name, node_url]), is_true=True, + http_error=HTTPBadRequest, msg_on_fail=s.BadRequestResponseSchema.description) + ax.evaluate_call(lambda: create_network_node(request, node_name, node_url), + http_error=HTTPConflict, + msg_on_fail=s.NetworkNodes_POST_ConflictResponseSchema.description) + return ax.valid_http(http_success=HTTPCreated, detail=s.NetworkNode_GET_OkResponseSchema.description) + + + @s.NetworkNodeAPI.put(schema=s.NetworkNode_PUT_RequestBodySchema, + tags=[s.NetworkNodeTag], response_schemas=s.NetworkNode_PUT_responses) + @view_config(route_name=s.NetworkNodeAPI.name, request_method="PUT") + def update_network_node_view(request): + node_name = ar.get_value_matchdict_checked(request, "node") + node = ax.evaluate_call( + lambda: request.db.query(models.NetworkNode).filter(models.NetworkNode.name == node_name).one(), + http_error=HTTPNotFound, + msg_on_fail=s.NetworkNode_GET_NotFoundResponseSchema.description) + new_name = request.POST.get("name") + new_url = request.POST.get("url") + ax.verify_param(any([new_name, new_url]), is_true=True, + http_error=HTTPBadRequest, msg_on_fail=s.BadRequestResponseSchema.description) + node.name = new_name or node.name + node.url = new_url or node.url + ax.evaluate_call(lambda: request.tm.commit(), + http_error=HTTPConflict, + fallback=lambda: request.db.rollback(), + msg_on_fail=s.NetworkNode_PUT_ConflictResponseSchema.description) + return ax.valid_http(http_success=HTTPOk, detail=s.NetworkNode_GET_OkResponseSchema.description) + + + @s.NetworkNodeAPI.delete(tags=[s.NetworkNodeTag], response_schemas=s.NetworkNode_DELETE_responses) + @view_config(route_name=s.NetworkNodeAPI.name, request_method="DELETE") + def delete_network_node_view(request): + node_name = ar.get_value_matchdict_checked(request, "node") + node = ax.evaluate_call( + lambda: request.db.query(models.NetworkNode).filter(models.NetworkNode.name == node_name).one(), + http_error=HTTPNotFound, + msg_on_fail=s.NetworkNode_GET_NotFoundResponseSchema.description) + ax.evaluate_call(lambda: delete_network_node(request, node), + http_error=HTTPInternalServerError, + fallback=lambda: request.db.rollback(), + msg_on_fail=s.InternalServerErrorResponseSchema.description) + return ax.valid_http(http_success=HTTPOk, detail=s.NetworkNode_DELETE_OkResponseSchema.description) diff --git a/magpie/api/management/user/user_formats.py b/magpie/api/management/user/user_formats.py index 08fdaed3b..804e5acbf 100644 --- a/magpie/api/management/user/user_formats.py +++ b/magpie/api/management/user/user_formats.py @@ -1,9 +1,10 @@ +import re from typing import TYPE_CHECKING from pyramid.httpexceptions import HTTPInternalServerError from magpie.api.exception import evaluate_call -from magpie.constants import get_constant +from magpie.constants import get_constant, protected_user_name_regex from magpie.models import UserGroupStatus, UserStatuses if TYPE_CHECKING: @@ -47,7 +48,8 @@ def fmt_usr(): user_info["has_pending_group"] = bool(user.get_groups_by_status(UserGroupStatus.PENDING)) # special users not meant to be used as valid "accounts" marked as without an ID - if user.user_name != get_constant("MAGPIE_ANONYMOUS_USER") and status != UserStatuses.Pending: + anonymous_regex = protected_user_name_regex(include_admin=False) + if not re.search(anonymous_regex, user.user_name) and status != UserStatuses.Pending: user_info["user{}id".format(sep)] = int(user.id) return user_info diff --git a/magpie/api/management/user/user_utils.py b/magpie/api/management/user/user_utils.py index 93e3b6cc5..aa0c42834 100644 --- a/magpie/api/management/user/user_utils.py +++ b/magpie/api/management/user/user_utils.py @@ -33,7 +33,7 @@ get_permission_update_params, process_webhook_requests ) -from magpie.constants import get_constant +from magpie.constants import get_constant, protected_user_name_regex from magpie.models import TemporaryToken, TokenOperation from magpie.permissions import PermissionSet, PermissionType, format_permissions from magpie.services import SERVICE_TYPE_DICT, service_factory @@ -226,14 +226,12 @@ def update_user(user, request, new_user_name=None, new_password=None, new_email= # logged user updating itself is forbidden if it corresponds to special users # cannot edit reserved keywords nor apply them to another user - forbidden_user_names = [ - get_constant("MAGPIE_ADMIN_USER", request), - get_constant("MAGPIE_ANONYMOUS_USER", request), - get_constant("MAGPIE_LOGGED_USER", request), - ] + forbidden_user_names_regex = protected_user_name_regex( + additional_patterns=[get_constant("MAGPIE_LOGGED_USER", request)], settings_container=request + ) check_user_name_cases = [user.user_name, new_user_name] if update_username else [user.user_name] for check_user_name in check_user_name_cases: - ax.verify_param(check_user_name, not_in=True, param_compare=forbidden_user_names, + ax.verify_param(check_user_name, not_matches=True, param_compare=forbidden_user_names_regex, param_name="user_name", with_param=False, # don't leak the user names http_error=HTTPForbidden, content={"user_name": str(check_user_name)}, msg_on_fail=s.User_PATCH_ForbiddenResponseSchema.description) @@ -915,8 +913,9 @@ def check_user_editable(user, container): :raises HTTPForbidden: When user is not allowed to be edited. :return: Nothing if allowed edition. """ - ax.verify_param(user.user_name, not_equal=True, with_param=False, # avoid leaking username details - param_compare=get_constant("MAGPIE_ANONYMOUS_USER", container), + forbidden_user_names_regex = protected_user_name_regex(include_admin=False, settings_container=container) + ax.verify_param(user.user_name, not_matches=True, with_param=False, # avoid leaking username details + param_compare=forbidden_user_names_regex, http_error=HTTPForbidden, msg_on_fail=s.User_CheckAnonymous_ForbiddenResponseSchema.description) diff --git a/magpie/api/schemas.py b/magpie/api/schemas.py index cb0fc870e..33df4a735 100644 --- a/magpie/api/schemas.py +++ b/magpie/api/schemas.py @@ -276,6 +276,18 @@ def service_api_route_info(service_api, **kwargs): TemporaryUrlAPI = Service( path="/tmp/{token}", # nosec: B108 name="temporary_url") +TokenAPI = Service( + path="/token", + name="Token") +TokenValidateAPI = Service( + path="/token/validate", + name="TokenValidate") +NetworkNodeAPI = Service( + path="/network-nodes/{node}", + name="NetworkNode") +NetworkNodesAPI = Service( + path="/network-nodes", + name="NetworkNodes") # Path parameters @@ -316,6 +328,19 @@ def service_api_route_info(service_api, **kwargs): colander.String(), description="Temporary URL token.", example=str(uuid.uuid4())) +NetworkTokenParameter = colander.SchemaNode( + colander.String(), + description="Network token", + example=str(uuid.uuid4())) +NetworkNodeNameParameter = colander.SchemaNode( + colander.String(), + description="Network Node name.", + example="node") +NetworkNodeUrlParameter = colander.SchemaNode( + colander.String(), + description="Public URL of remote Magpie instance.", + example="http://node.example.com/magpie", + validator=colander.url) class ServiceType_RequestPathSchema(colander.MappingSchema): @@ -414,6 +439,7 @@ class TemporaryURL_RequestPathSchema(colander.MappingSchema): RegisterTag = "Register" ResourcesTag = "Resource" ServicesTag = "Service" +NetworkNodeTag = "Network Node" TAG_DESCRIPTIONS = { APITag: "General information about the API.", @@ -435,6 +461,9 @@ class TemporaryURL_RequestPathSchema(colander.MappingSchema): ServicesTag: "Management of service definitions, children resources and their applicable permissions.", } +if get_constant("MAGPIE_NETWORK_MODE", settings_name="magpie.network_mode"): + TAG_DESCRIPTIONS[NetworkNodeTag] = "Management of references to other Magpie instances in the network." + # Header definitions @@ -3312,6 +3341,117 @@ class Session_GET_OkResponseSchema(BaseResponseSchemaAPI): body = Session_GET_ResponseBodySchema(code=HTTPOk.code, description=description) +class Token_PATCH_OkResponseBodySchema(BaseResponseBodySchema): + token = NetworkTokenParameter + + +class Token_RequestBodySchema(colander.MappingSchema): + expires = colander.SchemaNode( + colander.Integer(), + description="Token Expiry (in seconds)", + example=2000, + default=get_constant("MAGPIE_DEFAULT_TOKEN_EXPIRY") + ) + + +class Token_PATCH_RequestBodySchema(BaseRequestSchemaAPI): + body = Token_RequestBodySchema() + + +class Token_PATCH_OkResponseSchema(BaseResponseSchemaAPI): + description = "Get token successful." + body = Token_PATCH_OkResponseBodySchema(code=HTTPOk.code, description=description) + + +class TokenValidate_GET_BadRequestResponseSchema(BaseResponseSchemaAPI): + description = "Invalid Token." + body = BaseResponseBodySchema(code=HTTPBadRequest.code, description=description) + + +class TokenValidate_BodySchema(colander.MappingSchema): + token = NetworkTokenParameter + + +class TokenValidate_GET_RequestBodySchema(BaseRequestSchemaAPI): + body = TokenValidate_BodySchema() + + +class TokenValidate_GET_OkResponseBodySchema(BaseResponseBodySchema): + user_name = UserNameParameter + + +class TokenValidate_GET_OkResponseSchema(BaseResponseSchemaAPI): + description = "Validate token successful." + body = TokenValidate_GET_OkResponseBodySchema(code=HTTPOk.code, description=description) + + +class NetworkNode_GET_NotFoundResponseSchema(BaseResponseSchemaAPI): + description = "Network Node could not be found." + body = ErrorResponseBodySchema(code=HTTPNotFound.code, description=description) + + +class NetworkNodes_GET_NotFoundResponseSchema(BaseResponseSchemaAPI): + description = "Network Nodes could not be found." + body = ErrorResponseBodySchema(code=HTTPNotFound.code, description=description) + + +class NetworkNode_GET_OkResponseBodySchema(BaseResponseBodySchema): + name = NetworkNodeNameParameter + url = NetworkNodeUrlParameter + + +class NetworkNode_GET_OkResponseSchema(BaseResponseSchemaAPI): + description = "Network Node found." + body = NetworkNode_GET_OkResponseBodySchema(code=HTTPOk.code, description=description) + + +class NetworkNode_BodySchema(colander.MappingSchema): + name = NetworkNodeNameParameter + url = NetworkNodeUrlParameter + + +class NetworkNodesSequence(colander.SequenceSchema): + node = NetworkNode_BodySchema() + +class NetworkNodes_GET_OkResponseBodySchema(BaseResponseBodySchema): + nodes = NetworkNodesSequence() + + +class NetworkNodes_GET_OkResponseSchema(BaseResponseSchemaAPI): + description = "Network Nodes found." + body = NetworkNodes_GET_OkResponseBodySchema(code=HTTPOk.code, description=description) + + +class NetworkNode_POST_RequestBodySchema(BaseRequestSchemaAPI): + body = NetworkNode_BodySchema() + + +NetworkNode_PUT_RequestBodySchema = NetworkNode_POST_RequestBodySchema + + +class NetworkNodes_POST_CreatedResponseSchema(BaseResponseSchemaAPI): + description = "Network Node created." + body = BaseResponseBodySchema(code=HTTPCreated.code, description=description) + + +class NetworkNodes_POST_ConflictResponseSchema(BaseResponseSchemaAPI): + description = "Network Node already exists with conflicting attributes." + body = ErrorResponseBodySchema(code=HTTPNotFound.code, description=description) + + +NetworkNode_PUT_ConflictResponseSchema = NetworkNodes_POST_ConflictResponseSchema + + +class NetworkNode_PUT_OkResponseSchema(BaseResponseSchemaAPI): + description = "Network Node updated." + body = BaseResponseBodySchema(code=HTTPOk.code, description=description) + + +class NetworkNode_DELETE_OkResponseSchema(BaseResponseSchemaAPI): + description = "Network Node deleted." + body = BaseResponseBodySchema(code=HTTPOk.code, description=description) + + class Session_GET_InternalServerErrorResponseSchema(BaseResponseSchemaAPI): description = "Failed to get session details." body = InternalServerErrorResponseSchema() @@ -3377,6 +3517,11 @@ class ProviderSignin_GET_UnauthorizedResponseSchema(BaseResponseSchemaAPI): body = ErrorResponseBodySchema(code=HTTPUnauthorized.code, description=description) +class ProviderSignin_GET_UnauthorizedTokenSchema(BaseResponseSchemaAPI): + description = "Unauthorized token in Authorization headers." + body = ErrorResponseBodySchema(code=HTTPUnauthorized.code, description=description) + + class ProviderSignin_GET_ForbiddenResponseSchema(BaseResponseSchemaAPI): description = "Forbidden 'Homepage-Route' host not matching Magpie refused for security reasons." body = ErrorResponseBodySchema(code=HTTPForbidden.code, description=description) @@ -4220,6 +4365,44 @@ class SwaggerAPI_GET_OkResponseSchema(colander.MappingSchema): "406": NotAcceptableResponseSchema(), "500": Session_GET_InternalServerErrorResponseSchema(), } +Token_PATCH_responses = { + "200": Token_PATCH_OkResponseSchema(), + "401": UnauthorizedResponseSchema(), + "406": NotAcceptableResponseSchema(), + "500": InternalServerErrorResponseSchema(), +} +TokenValidate_GET_responses = { + "200": TokenValidate_GET_OkResponseSchema(), + "400": TokenValidate_GET_BadRequestResponseSchema(), + "500": InternalServerErrorResponseSchema(), +} +NetworkNode_GET_responses = { + "200": NetworkNode_GET_OkResponseSchema(), + "404": NetworkNode_GET_NotFoundResponseSchema(), + "500": InternalServerErrorResponseSchema(), +} +NetworkNodes_GET_responses = { + "200": NetworkNodes_GET_OkResponseSchema(), + "404": NetworkNodes_GET_NotFoundResponseSchema(), + "500": InternalServerErrorResponseSchema(), +} +NetworkNodes_POST_responses = { + "201": NetworkNodes_POST_CreatedResponseSchema(), + "400": BadRequestResponseSchema(), + "409": NetworkNodes_POST_ConflictResponseSchema(), + "500": InternalServerErrorResponseSchema(), +} +NetworkNode_PUT_responses = { + "200": NetworkNode_PUT_OkResponseSchema(), + "400": BadRequestResponseSchema(), + "409": NetworkNode_PUT_ConflictResponseSchema(), + "500": InternalServerErrorResponseSchema(), +} +NetworkNode_DELETE_responses = { + "200": NetworkNode_DELETE_OkResponseSchema(), + "400": BadRequestResponseSchema(), + "500": InternalServerErrorResponseSchema(), +} Version_GET_responses = { "200": Version_GET_OkResponseSchema(), "406": NotAcceptableResponseSchema(), diff --git a/magpie/constants.py b/magpie/constants.py index fc183b2c1..1fbf0a55d 100644 --- a/magpie/constants.py +++ b/magpie/constants.py @@ -105,6 +105,9 @@ def _get_default_log_level(): MAGPIE_CRON_LOG = os.getenv("MAGPIE_CRON_LOG", "~/magpie-cron.log") MAGPIE_DB_MIGRATION = asbool(os.getenv("MAGPIE_DB_MIGRATION", True)) # run db migration on startup MAGPIE_DB_MIGRATION_ATTEMPTS = int(os.getenv("MAGPIE_DB_MIGRATION_ATTEMPTS", 5)) +MAGPIE_NETWORK_MODE = os.getenv("MAGPIE_NETWORK_MODE", False) +MAGPIE_INSTANCE_NAME = os.getenv("MAGPIE_INSTANCE_NAME") +MAGPIE_DEFAULT_TOKEN_EXPIRY = int(os.getenv("MAGPIE_DEFAULT_TOKEN_EXPIRY", 86400)) MAGPIE_LOG_LEVEL = os.getenv("MAGPIE_LOG_LEVEL", _get_default_log_level()) # log level to apply to the loggers MAGPIE_LOG_PRINT = asbool(os.getenv("MAGPIE_LOG_PRINT", False)) # log also forces print to the console MAGPIE_LOG_REQUEST = asbool(os.getenv("MAGPIE_LOG_REQUEST", True)) # log detail of every incoming request @@ -146,6 +149,10 @@ def _get_default_log_level(): MAGPIE_CONTEXT_PERMISSION = "MAGPIE_CONTEXT_USER" # path user must be itself, MAGPIE_LOGGED_USER or unauthenticated MAGPIE_LOGGED_USER = "current" MAGPIE_DEFAULT_PROVIDER = "ziggurat" +MAGPIE_NETWORK_TOKEN_NAME = "magpie_token" +MAGPIE_NETWORK_PROVIDER = "magpie_network" +MAGPIE_NETWORK_NAME_PREFIX = "anonymous_network_" +MAGPIE_NETWORK_GROUP_NAME = "magpie_network" # above this length is considered a token, # refuse longer username creation @@ -162,6 +169,10 @@ def _get_default_log_level(): "MAGPIE_DEFAULT_PROVIDER", "MAGPIE_USER_NAME_MAX_LENGTH", "MAGPIE_GROUP_NAME_MAX_LENGTH", + "MAGPIE_NETWORK_TOKEN_NAME", + "MAGPIE_NETWORK_PROVIDER", + "MAGPIE_NETWORK_NAME_PREFIX", + "MAGPIE_NETWORK_GROUP_NAME" ] # =========================== @@ -171,6 +182,48 @@ def _get_default_log_level(): _REGEX_ASCII_ONLY = re.compile(r"\W|^(?=\d)") +def protected_user_name_regex(include_admin=True, + include_anonymous=True, + include_network=True, + additional_patterns=None, + settings_container=None): + # type: (bool, bool, bool, Optional[list], Optional[AnySettingsContainer]) -> Str + """ + Return a regular expression that matches all user names that are protected, meaning that they are generated + by Magpie itself and no regular user account should be created with these user names. + """ + patterns = additional_patterns or [] + if include_admin: + patterns.append(get_constant("MAGPIE_ADMIN_USER", settings_container=settings_container)) + if include_anonymous: + patterns.append(get_constant("MAGPIE_ANONYMOUS_USER", settings_container=settings_container)) + if include_network and get_constant("MAGPIE_NETWORK_MODE", settings_container=settings_container): + patterns.append( + "{}.*".format(get_constant("MAGPIE_NETWORK_NAME_PREFIX", settings_container=settings_container)) + ) + return "^{}$".format("|".join(patterns)) + + +def protected_group_name_regex(include_admin=True, + include_anonymous=True, + include_network=True, + settings_container=None): + # type: (bool, bool, bool, Optional[AnySettingsContainer]) -> Str + """ + Return a regular expression that matches all group names that are protected, meaning that they are generated + by Magpie itself and no regular user account should be created with these group names. + """ + patterns = [] + if include_admin: + patterns.append(get_constant("MAGPIE_ADMIN_GROUP", settings_container=settings_container)) + if include_anonymous: + patterns.append(get_constant("MAGPIE_ANONYMOUS_GROUP", settings_container=settings_container)) + if include_network and get_constant("MAGPIE_NETWORK_MODE", settings_container=settings_container): + patterns.append( + "{}.*".format(get_constant("MAGPIE_NETWORK_NAME_PREFIX", settings_container=settings_container)) + ) + return "^{}$".format("|".join(patterns)) + def get_constant_setting_name(name): # type: (Str) -> Str """ diff --git a/magpie/models.py b/magpie/models.py index e15a5fe35..7d64747cd 100644 --- a/magpie/models.py +++ b/magpie/models.py @@ -1,16 +1,19 @@ import datetime import math import uuid +from collections.abc import Mapping from typing import TYPE_CHECKING import sqlalchemy as sa from pyramid.httpexceptions import HTTPInternalServerError from pyramid.security import ALL_PERMISSIONS, Allow, Authenticated, Everyone +from sqlalchemy import UniqueConstraint from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.ext.declarative import declarative_base, declared_attr from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import relationship from sqlalchemy.sql import func +from sqlalchemy_utils import URLType from ziggurat_foundations import ziggurat_model_init from ziggurat_foundations.models.base import BaseModel, get_db_session from ziggurat_foundations.models.external_identity import ExternalIdentityMixin @@ -182,6 +185,41 @@ class User(UserMixin, Base): def __str__(self): return "".format(self.user_name, self.id) + network_node_id = sa.Column("network_node_id", sa.Integer, + sa.ForeignKey("network_nodes.id", onupdate="CASCADE", ondelete="CASCADE"), + nullable=True) + + # Includes unique constraint to enforce uniqueness over network_node_id + __table_args__ = ( + UniqueConstraint('user_name', 'network_node_id'), + UniqueConstraint('email', 'network_node_id'), + *((UserMixin.__table_args__,) if isinstance(UserMixin.__table_args__, Mapping) else UserMixin.__table_args__) + ) + + @declared_attr + def user_name(self): + """ + User name for user object. + + Overrides function in UserMixin to set unique to False. + This allows us to enforce a uniqueness constraint scoped over the network_node_id. + """ + column = UserMixin.user_name + column.unique = False + return column + + @declared_attr + def email(self): + """ + Email for user object. + + Overrides function in UserMixin to set unique to False. + This allows us to enforce a uniqueness constraint scoped over the network_node_id. + """ + column = UserMixin.email + column.unique = False + return column + def get_groups_by_status(self, status, db_session=None): # type: (UserGroupStatus, Session) -> Set[Str] """ @@ -996,6 +1034,45 @@ def json(self): return {"token": str(self.token), "operation": str(self.operation.value)} +class NetworkToken(BaseModel, Base): + """ + Model that defines a token for authentication across a network of Magpie instances. + """ + __tablename__ = "network_tokens" + + def __init__(self, *_, **__): + super(NetworkToken, self).__init__(*_, **__) + if not self.token: + self.token = uuid.uuid4() + + token = sa.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4, unique=True) + user_id = sa.Column(sa.Integer, + sa.ForeignKey("users.id", onupdate="CASCADE", ondelete="CASCADE"), unique=True) + user = relationship("User", foreign_keys=[user_id]) + + +class NetworkNode(BaseModel, Base): + """ + Model that defines a node in a network of Magpie instances. + """ + __tablename__ = "network_nodes" + + id = sa.Column(sa.Integer(), primary_key=True, nullable=False, autoincrement=True) + name = sa.Column(sa.Unicode(128), nullable=False, unique=True) + url = sa.Column(URLType(), nullable=False) + users = relationship("User", backref="network_nodes", lazy="dynamic") + + @property + def anonymous_user_name(self): + # type: () -> Str + return "{}{}".format(get_constant("MAGPIE_NETWORK_NAME_PREFIX"), self.name) + + @property + def anonymous_user(self): + # type: () -> Optional[User] + return self.users.filter(User.user_name == self.anonymous_user_name).first() + + ziggurat_model_init(User, Group, UserGroup, GroupPermission, UserPermission, UserResourcePermission, GroupResourcePermission, Resource, ExternalIdentity, passwordmanager=None) diff --git a/magpie/ui/utils.py b/magpie/ui/utils.py index 123687c90..a14f47a47 100644 --- a/magpie/ui/utils.py +++ b/magpie/ui/utils.py @@ -1,4 +1,5 @@ import json +import re from secrets import compare_digest # noqa 'python2-secrets' from typing import TYPE_CHECKING @@ -20,7 +21,7 @@ from magpie.api import schemas from magpie.api.generic import get_exception_info, get_request_info from magpie.api.requests import get_logged_user -from magpie.constants import get_constant +from magpie.constants import get_constant, protected_user_name_regex, protected_group_name_regex from magpie.models import UserGroupStatus from magpie.security import mask_credentials from magpie.utils import CONTENT_TYPE_JSON, get_header, get_json, get_logger, get_magpie_url @@ -175,39 +176,61 @@ def wrap(*args, **kwargs): return wrap +class _ReContainsWrapper(object): + """ + Class that wraps the re.search function with the __contains__ method. + + Used in BaseViews so that code in mako templates can continue to use syntax like: + + .. code-block:: python + + user_name in MAGPIE_FIXED_USERS + """ + + def __init__(self, regex="x^"): # Note that the default "x^" matches nothing + self.regex = regex + + def __contains__(self, item): + # type: (Any) -> bool + try: + return bool(re.search(self.regex, item)) + except TypeError: + return False + + @view_defaults(decorator=handle_errors) class BaseViews(object): """ Base methods for Magpie UI pages. """ - MAGPIE_FIXED_GROUP_MEMBERSHIPS = [] + MAGPIE_FIXED_GROUP_MEMBERSHIPS = _ReContainsWrapper() """ Special :term:`Group` memberships that cannot be edited. """ - MAGPIE_FIXED_GROUP_EDITS = [] + MAGPIE_FIXED_GROUP_EDITS = _ReContainsWrapper() """ Special :term:`Group` details that cannot be edited. """ - MAGPIE_FIXED_USERS = [] + MAGPIE_FIXED_USERS = _ReContainsWrapper() """ Special :term:`User` details that cannot be edited. """ - MAGPIE_FIXED_USERS_REFS = [] + MAGPIE_FIXED_USERS_REFS = _ReContainsWrapper() """ Special :term:`User` that cannot have any relationship edited. This includes both :term:`Group` memberships and :term:`Permission` references. """ - MAGPIE_USER_PWD_LOCKED = [] + MAGPIE_USER_PWD_LOCKED = _ReContainsWrapper() """ Special :term:`User` that *could* self-edit themselves, but is disabled since conflicting with other policies. """ - MAGPIE_USER_PWD_DISABLED = [] + MAGPIE_USER_PWD_DISABLED = _ReContainsWrapper() """ Special :term:`User` where password cannot be edited (managed by `Magpie` configuration settings). """ @@ -223,22 +246,29 @@ def __init__(self, request): self.ui_theme = get_constant("MAGPIE_UI_THEME", self.request) self.logged_user = get_logged_user(self.request) - anonym_grp = get_constant("MAGPIE_ANONYMOUS_GROUP", settings_container=self.request) - admin_grp = get_constant("MAGPIE_ADMIN_GROUP", settings_container=self.request) - self.__class__.MAGPIE_FIXED_GROUP_MEMBERSHIPS = [anonym_grp] - self.__class__.MAGPIE_FIXED_GROUP_EDITS = [anonym_grp, admin_grp] - # special users that cannot be deleted - anonym_usr = get_constant("MAGPIE_ANONYMOUS_USER", self.request) - admin_usr = get_constant("MAGPIE_ADMIN_USER", self.request) - self.__class__.MAGPIE_FIXED_USERS_REFS = [anonym_usr] - self.__class__.MAGPIE_FIXED_USERS = [admin_usr, anonym_usr] - self.__class__.MAGPIE_USER_PWD_LOCKED = [admin_usr] - self.__class__.MAGPIE_USER_PWD_DISABLED = [anonym_usr, admin_usr] + self.__class__.MAGPIE_FIXED_GROUP_MEMBERSHIPS = _ReContainsWrapper( + protected_group_name_regex(include_admin=False, include_network=False, settings_container=self.request) + ) + self.__class__.MAGPIE_FIXED_GROUP_EDITS = _ReContainsWrapper( + protected_group_name_regex(settings_container=self.request) + ) + self.__class__.MAGPIE_FIXED_USERS_REFS = _ReContainsWrapper( + protected_user_name_regex(include_admin=False, settings_container=self.request) + ) + self.__class__.MAGPIE_FIXED_USERS = _ReContainsWrapper( + protected_user_name_regex(settings_container=self.request) + ) + self.__class__.MAGPIE_USER_PWD_LOCKED = _ReContainsWrapper( + protected_user_name_regex(include_anonymous=False, include_network=False, settings_container=self.request) + ) + self.__class__.MAGPIE_USER_PWD_DISABLED = _ReContainsWrapper( + protected_user_name_regex(settings_container=self.request) + ) self.__class__.MAGPIE_USER_REGISTRATION_ENABLED = asbool( get_constant("MAGPIE_USER_REGISTRATION_ENABLED", self.request, default_value=False, print_missing=True, raise_missing=False, raise_not_set=False) ) - self.__class__.MAGPIE_ANONYMOUS_GROUP = anonym_grp + self.__class__.MAGPIE_ANONYMOUS_GROUP = get_constant("MAGPIE_ANONYMOUS_GROUP", settings_container=self.request) def add_template_data(self, data=None): # type: (Optional[Dict[Str, Any]]) -> Dict[Str, Any] diff --git a/requirements.txt b/requirements.txt index 8cbd357bb..c3a3c6d3f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,6 +32,7 @@ paste pastedeploy pluggy psycopg2-binary>=2.7.1 +PyJWT==2.8.0 pyramid>=1.10.2,<2 pyramid_beaker==0.8 pyramid_chameleon>=0.3 From 758b15f48147b625fe7572613d8fc1026ef8ac51 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Thu, 12 Oct 2023 16:03:24 -0400 Subject: [PATCH 02/55] review comment updates --- config/magpie.ini | 5 +- docs/authentication.rst | 39 +++--- docs/configuration.rst | 10 +- magpie/adapter/magpieowssecurity.py | 4 +- ...3-08-25_2cfe144538e8_add_network_tables.py | 33 +++-- magpie/api/login/__init__.py | 2 +- magpie/api/login/login.py | 122 ++++++++--------- magpie/api/management/group/group_utils.py | 6 + magpie/api/management/group/group_views.py | 7 +- .../api/management/network_node/__init__.py | 2 +- .../network_node/network_node_utils.py | 6 +- .../network_node/network_node_views.py | 123 +++++++++--------- magpie/api/management/user/user_utils.py | 7 +- magpie/api/schemas.py | 19 +-- magpie/constants.py | 23 ++-- magpie/models.py | 70 ++++------ magpie/ui/utils.py | 7 +- 17 files changed, 227 insertions(+), 258 deletions(-) diff --git a/config/magpie.ini b/config/magpie.ini index e722bd1bf..bd42359b2 100644 --- a/config/magpie.ini +++ b/config/magpie.ini @@ -29,8 +29,9 @@ magpie.port = 2001 magpie.url = http://localhost:2001 # Enable network mode which allows different instances of Magpie to authenticate users for each other. -# magpie.network_mode = true -# magpie.default_token_expiry = 86400 +# magpie.network_enabled = true +# magpie.network_default_token_expiry = 86400 +# magpie.network_instance_name # magpie.config_path = diff --git a/docs/authentication.rst b/docs/authentication.rst index 05bfac950..34a1173cf 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -428,43 +428,40 @@ their services. Network Mode ------------ -If the :envvar:`MAGPIE_NETWORK_MODE` is enabled, an additional external authentication provider is added to Magpie which -allows networked instances of Magpie to authenticate users for each other. +If the :envvar:`MAGPIE_NETWORK_ENABLED` is enabled, an additional external authentication provider is added to `Magpie` +which allows networked instances of `Magpie` to authenticate users for each other. -Users can then log in to any Magpie instance where they have an account and request a personal network token in the form -of a `_JSON Web Token `_ which can be used to authenticate this user on -any Magpie instance in the network. +Users can then log in to any `Magpie` instance where they have an account and request a personal network token in the +form of a `JSON Web Token `_ which can be used to authenticate this user +on any `Magpie` instance in the network. Managing the Network ~~~~~~~~~~~~~~~~~~~~ -In order for Magpie instances to authenticate each other's users, each instance must be made aware of the existence of +In order for `Magpie` instances to authenticate each other's users, each instance must be made aware of the existence of the others so that it knows where to send authentication verification requests. -In order to register another Magpie instance as part of the same network, an admin user can create a +In order to register another `Magpie` instance as part of the same network, an admin user can create a :term:`Network Node` with a request to ``POST /network-nodes``. The parameters given to that request include a ``name`` -and a ``url``. The ``name`` is a the name of that other Magpie instance in the network and should correspond to the -same value as the :envvar:`MAGPIE_INSTANCE_NAME` value set by the other Magpie instance. The ``url`` is a the root URL -of the other Magpie instance. +and a ``url``. The ``name`` is a the name of that other `Magpie` instance in the network and should correspond to the +same value as the :envvar:`MAGPIE_NETWORK_INSTANCE_NAME` value set by the other `Magpie` instance. The ``url`` is a the root URL +of the other `Magpie` instance. -Once a :term:`Network Node` is registered, Magpie can use that other instance to authenticate users as long as the other -instance also has :envvar:`MAGPIE_NETWORK_MODE` enabled. +Once a :term:`Network Node` is registered, `Magpie` can use that other instance to authenticate users as long as the +other instance also has :envvar:`MAGPIE_NETWORK_ENABLED` enabled. Managing Personal JWTs ~~~~~~~~~~~~~~~~~~~~~~ -A :term:`User` can request a new network token with a request to the ``PATCH /token`` route. This route takes one -optional parameter ``expires`` which is an integer indicating how long (in seconds) until that token expires, the -default expiry for this token is :envvar:`MAGPIE_DEFAULT_TOKEN_EXPIRY`. +A :term:`User` can request a new network token with a request to the ``PATCH /token`` route. Every time a :term:`User` makes a request to the ``PATCH /token`` route a new token is generated for them. This -effectively cancels all previously created tokens for that user. If a user wishes to cancel all tokens, they can provide -an ``expires`` value of ``0`` when making the request. +effectively cancels all previously created tokens for that user. Authentication ~~~~~~~~~~~~~~ -Once a :term:`User` gets a personal network token, they can use that token to authenticate with any Magpie instance in +Once a :term:`User` gets a personal network token, they can use that token to authenticate with any `Magpie` instance in the same network. When a user makes a request, they should set the ``provider_name`` parameter to the value of :envvar:`MAGPIE_NETWORK_PROVIDER` and provide the network token in the Authorization header in the following format: @@ -479,10 +476,10 @@ Authorization ~~~~~~~~~~~~~ Managing authorization for :term:`Users` who authenticate using personal network tokens is complicated by the fact that -a :term:`User` is not required to have a full account on both Magpie instances in order to using this authentication +a :term:`User` is not required to have a full account on both `Magpie` instances in order to using this authentication mechanism. This means that a :term:`User` may be logged in as a node-specific "anonymous" user. -When another Magpie instance is registered as a :term:`Network Node`, a few additional entities are created: +When another `Magpie` instance is registered as a :term:`Network Node`, a few additional entities are created: #. a group used to manage the permissions of all users who authenticate using the new :term:`Network Node`. * this group's name will be the :envvar:`MAGPIE_NETWORK_NAME_PREFIX` followed by the :term:`Network Node` name @@ -494,7 +491,7 @@ When another Magpie instance is registered as a :term:`Network Node`, a few addi Here is an example to illustrate this point: -* There are 3 Magpie instances in the network named A, B, and C +* There are 3 `Magpie` instances in the network named A, B, and C * There is a :term:`User` named ``"toto"`` registered on instance A * There is no :term:`User` named ``"toto"`` who belongs to the ``"anonymous_network_A"`` group registered on instance B * There is a :term:`User` named ``"toto"`` who belongs to the ``"anonymous_network_A"`` group registered on instance C diff --git a/docs/configuration.rst b/docs/configuration.rst index 0b8d3490f..94ee44e73 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -974,9 +974,9 @@ Network Mode Settings The following configuration parameters are related to Magpie's "Network Mode" which allows networked instances of Magpie to authenticate users for each other. All variables defined in this section are only used if -:envvar:`MAGPIE_NETWORK_MODE` is enabled. +:envvar:`MAGPIE_NETWORK_ENABLED` is enabled. -.. envvar:: MAGPIE_NETWORK_MODE +.. envvar:: MAGPIE_NETWORK_ENABLED [:class:`bool`] (Default: ``False``) @@ -986,7 +986,7 @@ to authenticate users for each other. All variables defined in this section are Enable "Network Mode" which enables all functionality to authenticate users using other Magpie instances as external authentication providers. -.. envvar:: MAGPIE_INSTANCE_NAME +.. envvar:: MAGPIE_NETWORK_INSTANCE_NAME [:class:`str`] @@ -995,9 +995,9 @@ to authenticate users for each other. All variables defined in this section are The name of this Magpie instance in the network. This variable is used to determine if an authentication token was issued by this instance of Magpie, or another instance in the network. - This variable is required if :envvar:`MAGPIE_NETWORK_MODE` is ``True``. + This variable is required if :envvar:`MAGPIE_NETWORK_ENABLED` is ``True``. -.. envvar:: MAGPIE_DEFAULT_TOKEN_EXPIRY +.. envvar:: MAGPIE_NETWORK_DEFAULT_TOKEN_EXPIRY [:class:`int`] (Default: ``86400``) diff --git a/magpie/adapter/magpieowssecurity.py b/magpie/adapter/magpieowssecurity.py index 8401de19c..491c02962 100644 --- a/magpie/adapter/magpieowssecurity.py +++ b/magpie/adapter/magpieowssecurity.py @@ -269,8 +269,8 @@ def update_request_cookies(self, request): """ settings = get_settings(request) token_name = get_constant("MAGPIE_COOKIE_NAME", settings_container=settings) - network_mode = get_constant("MAGPIE_NETWORK_MODE", settings_container=settings, - settings_name="magpie.network_mode") + network_mode = get_constant("MAGPIE_NETWORK_ENABLED", settings_container=settings, + settings_name="magpie.network_enabled") headers = dict(request.headers) network_token_name = get_constant("MAGPIE_NETWORK_TOKEN_NAME", settings_container=settings) if network_mode and "Authorization" not in headers and network_token_name in request.params: diff --git a/magpie/alembic/versions/2023-08-25_2cfe144538e8_add_network_tables.py b/magpie/alembic/versions/2023-08-25_2cfe144538e8_add_network_tables.py index dea28d86d..6a3dba119 100644 --- a/magpie/alembic/versions/2023-08-25_2cfe144538e8_add_network_tables.py +++ b/magpie/alembic/versions/2023-08-25_2cfe144538e8_add_network_tables.py @@ -28,27 +28,32 @@ def upgrade(): sa.Column("token", UUID(as_uuid=True), primary_key=True, default=uuid.uuid4, unique=True), sa.Column("user_id", sa.Integer, - sa.ForeignKey("users.id", onupdate="CASCADE", ondelete="CASCADE"), unique=True) + sa.ForeignKey("users.id", onupdate="CASCADE", ondelete="CASCADE"), unique=True, + nullable=False) ) op.create_table("network_nodes", - sa.Column("id", sa.Integer, primary_key=True, autoincrement=True), + sa.Column("id", sa.Integer, primary_key=True, nullable=False, autoincrement=True), sa.Column("name", sa.Unicode(128), nullable=False, unique=True), sa.Column("url", URLType(), nullable=False) ) - op.add_column("users", sa.Column("network_node_id", sa.Integer, - sa.ForeignKey("network_nodes.id", onupdate="CASCADE", ondelete="CASCADE"), - nullable=True)) - op.drop_constraint("uq_users_user_name", "users") - op.create_unique_constraint("uq_users_user_name_network_node_id", "users", ["user_name", "network_node_id"]) - op.drop_constraint("uq_users_email", "users") - op.create_unique_constraint("uq_users_email_network_node_id", "users", ["email", "network_node_id"]) + op.create_table("network_users", + sa.Column("id", sa.Integer, primary_key=True, autoincrement=True), + sa.Column("user_id", sa.Integer, sa.ForeignKey("users.id", onupdate="CASCADE", ondelete="CASCADE"), + nullable=False), + sa.Column("network_node_id", sa.Integer, + sa.ForeignKey("network_nodes.id", onupdate="CASCADE", ondelete="CASCADE"), + nullable=False), + sa.Column("network_user_name", sa.Unicode(128)), + ) + op.create_unique_constraint("uq_network_users_user_id_network_node_id", "network_users", + ["user_id", "network_node_id"]) + op.create_unique_constraint("uq_network_users_network_user_name_network_node_id", "network_users", + ["network_user_name", "network_node_id"]) def downgrade(): - op.drop_constraint("uq_users_user_name_network_node_id", "users") - op.create_unique_constraint("uq_users_user_name", "users", ["user_name"]) - op.drop_constraint("uq_users_email_network_node_id", "users") - op.create_unique_constraint("uq_users_email", "users", ["email"]) + op.drop_constraint("uq_network_users_user_id_network_node_id", "network_users") + op.drop_constraint("uq_network_users_network_user_name_network_node_id", "network_users") op.drop_table("network_tokens") op.drop_table("network_nodes") - op.drop_column("users", "network_node_id") + op.drop_table("network_users") diff --git a/magpie/api/login/__init__.py b/magpie/api/login/__init__.py index aadf7778e..a849de880 100644 --- a/magpie/api/login/__init__.py +++ b/magpie/api/login/__init__.py @@ -12,7 +12,7 @@ def includeme(config): config.add_route(**s.service_api_route_info(s.SigninAPI)) config.add_route(**s.service_api_route_info(s.ProvidersAPI)) config.add_route(**s.service_api_route_info(s.ProviderSigninAPI)) - if get_constant("MAGPIE_NETWORK_MODE", settings_name="magpie.network_mode"): + if get_constant("MAGPIE_NETWORK_ENABLED", config, settings_name="magpie.network_enabled"): config.add_route(**s.service_api_route_info(s.TokenAPI)) config.add_route(**s.service_api_route_info(s.TokenValidateAPI)) config.scan() diff --git a/magpie/api/login/login.py b/magpie/api/login/login.py index abfae0f54..6f3a52f5b 100644 --- a/magpie/api/login/login.py +++ b/magpie/api/login/login.py @@ -20,7 +20,7 @@ HTTPOk, HTTPTemporaryRedirect, HTTPUnauthorized, - HTTPUnprocessableEntity + HTTPUnprocessableEntity, HTTPNotImplemented ) from pyramid.request import Request from pyramid.response import Response @@ -55,19 +55,11 @@ LOGGER = get_logger(__name__) -MAGPIE_NETWORK_MODE = get_constant("MAGPIE_NETWORK_MODE", settings_name="magpie.network_mode") - # dictionaries of {'provider_id': 'provider_display_name'} MAGPIE_DEFAULT_PROVIDER = get_constant("MAGPIE_DEFAULT_PROVIDER") MAGPIE_INTERNAL_PROVIDERS = {MAGPIE_DEFAULT_PROVIDER: MAGPIE_DEFAULT_PROVIDER.capitalize()} MAGPIE_EXTERNAL_PROVIDERS = get_providers() -if MAGPIE_NETWORK_MODE: - MAGPIE_NETWORK_TOKEN_NAME = get_constant("MAGPIE_NETWORK_TOKEN_NAME") - MAGPIE_NETWORK_PROVIDER = get_constant("MAGPIE_NETWORK_PROVIDER") - MAGPIE_INSTANCE_NAME = get_constant("MAGPIE_INSTANCE_NAME") - MAGPIE_EXTERNAL_PROVIDERS[MAGPIE_NETWORK_PROVIDER] = MAGPIE_NETWORK_PROVIDER - MAGPIE_PROVIDER_KEYS = frozenset(set(MAGPIE_INTERNAL_PROVIDERS) | set(MAGPIE_EXTERNAL_PROVIDERS)) @@ -285,7 +277,9 @@ def user_from_token(request, token): Return the ``User`` associated with the token as long as the token is valid and issued by this Magpie instance. """ magpie_secret = get_constant("MAGPIE_SECRET", request, settings_name="magpie.secret") - decoded_token = jwt.decode(token, magpie_secret, algorithms="HS256", issuer=MAGPIE_INSTANCE_NAME, + decoded_token = jwt.decode(token, magpie_secret, algorithms="HS256", + issuer=get_constant("MAGPIE_NETWORK_INSTANCE_NAME", request, + settings_name='magpie.network_instance_name'), options={"require": ["exp", "token"]}) network_token = (request.db.query(models.NetworkToken) @@ -307,7 +301,7 @@ def external_login_view(request): response = Response() verify_provider(provider_name) try: - if provider_name == MAGPIE_NETWORK_PROVIDER: + if provider_name == get_constant("MAGPIE_NETWORK_PROVIDER", request): if "Authorization" in request.headers: token_type, token = request.headers.get("Authorization").split() if token_type != "Bearer": @@ -439,69 +433,61 @@ def _get_session(req): return ax.valid_http(http_success=HTTPOk, detail=s.Session_GET_OkResponseSchema.description, content=session_json) -if MAGPIE_NETWORK_MODE: - @s.TokenAPI.patch(schema=s.Token_PATCH_RequestBodySchema, tags=[s.SessionTag], - response_schemas=s.Token_PATCH_responses) - @view_config(route_name=s.TokenAPI.name, request_method="PATCH", permission=Authenticated) - def token_view(request): - """ - Get a token. Generates a new token every time this route is accessed so this can also - be used to invalidate a previously generated token. - """ - magpie_secret = get_constant("MAGPIE_SECRET", request, settings_name="magpie.secret") - default_expiry = get_constant("MAGPIE_DEFAULT_TOKEN_EXPIRY", request, - settings_name="magpie.default_token_expiry") - magpie_instance_name = get_constant("MAGPIE_INSTANCE_NAME", - request, - settings_name="magpie.instance_name") - - expires = request.GET.get("expires", default_expiry) - try: - expires = int(expires) - except ValueError: - expires = int(default_expiry) - - expiry = datetime.utcnow() + timedelta(seconds=expires) - - def upsert_token(): - network_token = request.db.query(models.NetworkToken).filter( - models.NetworkToken.user_id == request.user.id).first() - if network_token: - network_token.token = uuid.uuid4() - else: - network_token = models.NetworkToken(user_id=request.user.id) - request.db.add(network_token) - return network_token.token +@s.TokenAPI.patch(tags=[s.SessionTag], response_schemas=s.Token_PATCH_responses) +@view_config(route_name=s.TokenAPI.name, request_method="PATCH", permission=Authenticated) +def token_view(request): + """ + Get a token. Generates a new token every time this route is accessed so this can also + be used to invalidate a previously generated token. + """ + magpie_secret = get_constant("MAGPIE_SECRET", request, settings_name="magpie.secret") + default_expiry = get_constant("MAGPIE_NETWORK_DEFAULT_TOKEN_EXPIRY", request, + settings_name="magpie.network_default_token_expiry") + magpie_instance_name = get_constant("MAGPIE_NETWORK_INSTANCE_NAME", + request, + settings_name="magpie.network_instance_name") + + expiry = datetime.utcnow() + timedelta(seconds=default_expiry) + + def upsert_token(): + network_token = request.db.query(models.NetworkToken).filter( + models.NetworkToken.user_id == request.user.id).first() + if network_token: + network_token.token = uuid.uuid4() + else: + network_token = models.NetworkToken(user_id=request.user.id) + request.db.add(network_token) + return network_token.token - token_value = ax.evaluate_call(lambda: upsert_token(), fallback=lambda: request.db.rollback(), - http_error=HTTPInternalServerError) + token_value = ax.evaluate_call(lambda: upsert_token(), fallback=lambda: request.db.rollback(), + http_error=HTTPInternalServerError) - token = jwt.encode({"token": str(token_value), - "exp": expiry, - "iss": magpie_instance_name}, magpie_secret, algorithm="HS256") + token = jwt.encode({"token": str(token_value), + "exp": expiry, + "iss": magpie_instance_name}, magpie_secret, algorithm="HS256") - return ax.valid_http(http_success=HTTPOk, detail=s.Token_PATCH_OkResponseSchema.description, - content={"token": token}) + return ax.valid_http(http_success=HTTPOk, detail=s.Token_PATCH_OkResponseSchema.description, + content={"token": token}) - @s.TokenValidateAPI.get(schema=s.TokenValidate_GET_RequestBodySchema, tags=[s.SessionTag], - response_schemas=s.TokenValidate_GET_responses) - @view_config(route_name=s.TokenValidateAPI.name, permission=NO_PERMISSION_REQUIRED) - def validate_token_view(request): - """ Validate a token """ - token = request.GET.get("token") - try: - authenticated_user = user_from_token(request, token) - if authenticated_user: - return ax.valid_http(http_success=HTTPOk, - detail=s.TokenValidate_GET_OkResponseSchema.description, - content={"user_name": authenticated_user.user_name}) - else: - ax.raise_http(http_error=HTTPUnauthorized, - detail=s.TokenValidate_GET_BadRequestResponseSchema.description) - except jwt.exceptions.PyJWTError as exc: - ax.raise_http(http_error=HTTPUnauthorized, content={"reason": str(exc)}, +@s.TokenValidateAPI.get(schema=s.TokenValidate_GET_RequestBodySchema, tags=[s.SessionTag], + response_schemas=s.TokenValidate_GET_responses) +@view_config(route_name=s.TokenValidateAPI.name, permission=NO_PERMISSION_REQUIRED) +def validate_token_view(request): + """ Validate a token """ + token = request.GET.get("token") + try: + authenticated_user = user_from_token(request, token) + if authenticated_user: + return ax.valid_http(http_success=HTTPOk, + detail=s.TokenValidate_GET_OkResponseSchema.description, + content={"user_name": authenticated_user.user_name}) + else: + ax.raise_http(http_error=HTTPUnauthorized, detail=s.TokenValidate_GET_BadRequestResponseSchema.description) + except jwt.exceptions.PyJWTError as exc: + ax.raise_http(http_error=HTTPUnauthorized, content={"reason": str(exc)}, + detail=s.TokenValidate_GET_BadRequestResponseSchema.description) @s.ProvidersAPI.get(tags=[s.SessionTag], response_schemas=s.Providers_GET_responses) diff --git a/magpie/api/management/group/group_utils.py b/magpie/api/management/group/group_utils.py index 0776ed138..bced69cd7 100644 --- a/magpie/api/management/group/group_utils.py +++ b/magpie/api/management/group/group_utils.py @@ -22,6 +22,7 @@ from magpie.api.management.resource.resource_utils import check_valid_service_or_resource_permission from magpie.api.management.service import service_formats as sf from magpie.api.webhooks import WebhookAction, get_permission_update_params, process_webhook_requests +from magpie.constants import protected_group_name_regex, get_constant from magpie.permissions import PermissionSet, PermissionType, format_permissions from magpie.services import SERVICE_TYPE_DICT @@ -94,6 +95,11 @@ def create_group(group_name, description, discoverable, terms, db_session): ax.verify_param(group_name, matches=True, param_compare=ax.PARAM_REGEX, param_name="group_name", http_error=HTTPBadRequest, content=group_content_error, msg_on_fail=s.Groups_POST_BadRequestResponseSchema.description) + if get_constant("MAGPIE_NETWORK_ENABLED", settings_name="magpie.network_enabled"): + anonymous_regex = protected_group_name_regex(include_admin=False) + ax.verify_param(group_name, not_matches=True, param_compare=anonymous_regex, param_name="group_name", + http_error=HTTPBadRequest, content=group_content_error, + msg_on_fail=s.Groups_POST_BadRequestResponseSchema.description) if description: ax.verify_param(description, matches=True, param_compare=ax.PARAM_REGEX, param_name="description", http_error=HTTPBadRequest, content=group_content_error, diff --git a/magpie/api/management/group/group_views.py b/magpie/api/management/group/group_views.py index c3749523c..f84f61089 100644 --- a/magpie/api/management/group/group_views.py +++ b/magpie/api/management/group/group_views.py @@ -9,7 +9,7 @@ from magpie.api.management.group import group_formats as gf from magpie.api.management.group import group_utils as gu from magpie.api.management.service import service_utils as su -from magpie.constants import get_constant +from magpie.constants import get_constant, protected_group_name_regex from magpie.models import TemporaryToken, TokenOperation, UserGroupStatus @@ -85,6 +85,11 @@ def edit_group_view(request): ax.verify_param(GroupService.by_group_name(new_group_name, db_session=request.db), is_none=True, http_error=HTTPConflict, with_param=False, # don't return group as value msg_on_fail=s.Group_PATCH_ConflictResponseSchema.description) + if get_constant("MAGPIE_NETWORK_ENABLED", settings_name="magpie.network_enabled"): + anonymous_regex = protected_group_name_regex(include_admin=False) + ax.verify_param(new_group_name, not_matches=True, param_compare=anonymous_regex, + http_error=HTTPBadRequest, + msg_on_fail=s.Group_PATCH_Size_BadRequestResponseSchema.description) group.group_name = new_group_name if update_desc: group.description = new_description diff --git a/magpie/api/management/network_node/__init__.py b/magpie/api/management/network_node/__init__.py index 1a1b2479a..c3d86e86d 100644 --- a/magpie/api/management/network_node/__init__.py +++ b/magpie/api/management/network_node/__init__.py @@ -6,7 +6,7 @@ def includeme(config): - if get_constant("MAGPIE_NETWORK_MODE", settings_name="magpie.network_mode"): + if get_constant("MAGPIE_NETWORK_ENABLED", config, settings_name="magpie.network_enabled"): LOGGER.info("Adding API network node...") config.add_route(**s.service_api_route_info(s.NetworkNodeAPI)) config.add_route(**s.service_api_route_info(s.NetworkNodesAPI)) diff --git a/magpie/api/management/network_node/network_node_utils.py b/magpie/api/management/network_node/network_node_utils.py index 4877861da..6955cef81 100644 --- a/magpie/api/management/network_node/network_node_utils.py +++ b/magpie/api/management/network_node/network_node_utils.py @@ -14,8 +14,8 @@ def create_network_node(request, name, url): """ Create a NetworkNode with the given name and url. """ - network_node = models.NetworkNode(name=name, url=url) - name = network_node.anonymous_user_name + models.NetworkNode(name=name, url=url) + name = "{}{}".format(get_constant("MAGPIE_NETWORK_NAME_PREFIX"), name) # create an anonymous user and group for this network node register_user_with_group(user_name=name, @@ -39,8 +39,6 @@ def create_network_node(request, name, url): group.description = "Group for users who have accounts on a different Magpie instance on this network.".format(name) group.discoverable = False - request.tm.commit() - def delete_network_node(request, node): # type: (Request, Str) -> None diff --git a/magpie/api/management/network_node/network_node_views.py b/magpie/api/management/network_node/network_node_views.py index f0db0dcd7..8b84535e3 100644 --- a/magpie/api/management/network_node/network_node_views.py +++ b/magpie/api/management/network_node/network_node_views.py @@ -17,73 +17,72 @@ from magpie.constants import get_constant -if get_constant("MAGPIE_NETWORK_MODE", settings_name="magpie.network_mode"): - @s.NetworkNodesAPI.get(tags=[s.NetworkNodeTag], response_schemas=s.NetworkNodes_GET_responses) - @view_config(route_name=s.NetworkNodesAPI.name, request_method="GET") - def get_network_nodes_view(request): - nodes = [{"name": n.name, "url": n.url} for n in request.db.query(models.NetworkNode).all()] - return ax.valid_http(http_success=HTTPOk, detail=s.NetworkNode_GET_OkResponseSchema.description, - content={"nodes": nodes}) +@s.NetworkNodesAPI.get(tags=[s.NetworkNodeTag], response_schemas=s.NetworkNodes_GET_responses) +@view_config(route_name=s.NetworkNodesAPI.name, request_method="GET") +def get_network_nodes_view(request): + nodes = [{"name": n.name, "url": n.url} for n in request.db.query(models.NetworkNode).all()] + return ax.valid_http(http_success=HTTPOk, detail=s.NetworkNode_GET_OkResponseSchema.description, + content={"nodes": nodes}) - @s.NetworkNodeAPI.get(tags=[s.NetworkNodeTag], response_schemas=s.NetworkNode_GET_responses) - @view_config(route_name=s.NetworkNodeAPI.name, request_method="GET") - def get_network_node_view(request): - node_name = ar.get_value_matchdict_checked(request, "node") - node = ax.evaluate_call( - lambda: request.db.query(models.NetworkNode).filter(models.NetworkNode.name == node_name).one(), - http_error=HTTPNotFound, - msg_on_fail=s.NetworkNode_GET_NotFoundResponseSchema.description) - return ax.valid_http(http_success=HTTPOk, detail=s.NetworkNode_GET_OkResponseSchema.description, - content={"name": node.name, "url": node.url}) +@s.NetworkNodeAPI.get(tags=[s.NetworkNodeTag], response_schemas=s.NetworkNode_GET_responses) +@view_config(route_name=s.NetworkNodeAPI.name, request_method="GET") +def get_network_node_view(request): + node_name = ar.get_value_matchdict_checked(request, "node_name") + node = ax.evaluate_call( + lambda: request.db.query(models.NetworkNode).filter(models.NetworkNode.name == node_name).one(), + http_error=HTTPNotFound, + msg_on_fail=s.NetworkNode_GET_NotFoundResponseSchema.description) + return ax.valid_http(http_success=HTTPOk, detail=s.NetworkNode_GET_OkResponseSchema.description, + content={"name": node.name, "url": node.url}) - @s.NetworkNodesAPI.post(schema=s.NetworkNode_POST_RequestBodySchema, tags=[s.NetworkNodeTag], - response_schemas=s.NetworkNodes_POST_responses) - @view_config(route_name=s.NetworkNodesAPI.name, request_method="POST") - def create_network_node_view(request): - node_name = request.POST.get("name") - node_url = request.POST.get("url") - ax.verify_param(all([node_name, node_url]), is_true=True, - http_error=HTTPBadRequest, msg_on_fail=s.BadRequestResponseSchema.description) - ax.evaluate_call(lambda: create_network_node(request, node_name, node_url), - http_error=HTTPConflict, - msg_on_fail=s.NetworkNodes_POST_ConflictResponseSchema.description) - return ax.valid_http(http_success=HTTPCreated, detail=s.NetworkNode_GET_OkResponseSchema.description) +@s.NetworkNodesAPI.post(schema=s.NetworkNode_POST_RequestBodySchema, tags=[s.NetworkNodeTag], + response_schemas=s.NetworkNodes_POST_responses) +@view_config(route_name=s.NetworkNodesAPI.name, request_method="POST") +def create_network_node_view(request): + node_name = request.POST.get("name") + node_url = request.POST.get("url") + ax.verify_param(node_url, matches=True, param_compare=r'[\w-]+', http_error=HTTPBadRequest, param_name="name") + ax.verify_param(node_url, matches=True, param_compare=ax.URL_REGEX, http_error=HTTPBadRequest, param_name="url") + ax.evaluate_call(lambda: create_network_node(request, node_name, node_url), + http_error=HTTPConflict, + msg_on_fail=s.NetworkNodes_POST_ConflictResponseSchema.description) + return ax.valid_http(http_success=HTTPCreated, detail=s.NetworkNode_GET_OkResponseSchema.description) - @s.NetworkNodeAPI.put(schema=s.NetworkNode_PUT_RequestBodySchema, - tags=[s.NetworkNodeTag], response_schemas=s.NetworkNode_PUT_responses) - @view_config(route_name=s.NetworkNodeAPI.name, request_method="PUT") - def update_network_node_view(request): - node_name = ar.get_value_matchdict_checked(request, "node") - node = ax.evaluate_call( - lambda: request.db.query(models.NetworkNode).filter(models.NetworkNode.name == node_name).one(), - http_error=HTTPNotFound, - msg_on_fail=s.NetworkNode_GET_NotFoundResponseSchema.description) - new_name = request.POST.get("name") - new_url = request.POST.get("url") - ax.verify_param(any([new_name, new_url]), is_true=True, - http_error=HTTPBadRequest, msg_on_fail=s.BadRequestResponseSchema.description) - node.name = new_name or node.name - node.url = new_url or node.url - ax.evaluate_call(lambda: request.tm.commit(), - http_error=HTTPConflict, - fallback=lambda: request.db.rollback(), - msg_on_fail=s.NetworkNode_PUT_ConflictResponseSchema.description) - return ax.valid_http(http_success=HTTPOk, detail=s.NetworkNode_GET_OkResponseSchema.description) +@s.NetworkNodeAPI.put(schema=s.NetworkNode_PUT_RequestBodySchema, + tags=[s.NetworkNodeTag], response_schemas=s.NetworkNode_PUT_responses) +@view_config(route_name=s.NetworkNodeAPI.name, request_method="PUT") +def update_network_node_view(request): + node_name = ar.get_value_matchdict_checked(request, "node_name") + node = ax.evaluate_call( + lambda: request.db.query(models.NetworkNode).filter(models.NetworkNode.name == node_name).one(), + http_error=HTTPNotFound, + msg_on_fail=s.NetworkNode_GET_NotFoundResponseSchema.description) + new_name = request.POST.get("name") + new_url = request.POST.get("url") + ax.verify_param(any([new_name, new_url]), is_true=True, + http_error=HTTPBadRequest, msg_on_fail=s.BadRequestResponseSchema.description) + node.name = new_name or node.name + node.url = new_url or node.url + ax.evaluate_call(lambda: request.tm.commit(), + http_error=HTTPConflict, + fallback=lambda: request.db.rollback(), + msg_on_fail=s.NetworkNode_PUT_ConflictResponseSchema.description) + return ax.valid_http(http_success=HTTPOk, detail=s.NetworkNode_GET_OkResponseSchema.description) - @s.NetworkNodeAPI.delete(tags=[s.NetworkNodeTag], response_schemas=s.NetworkNode_DELETE_responses) - @view_config(route_name=s.NetworkNodeAPI.name, request_method="DELETE") - def delete_network_node_view(request): - node_name = ar.get_value_matchdict_checked(request, "node") - node = ax.evaluate_call( - lambda: request.db.query(models.NetworkNode).filter(models.NetworkNode.name == node_name).one(), - http_error=HTTPNotFound, - msg_on_fail=s.NetworkNode_GET_NotFoundResponseSchema.description) - ax.evaluate_call(lambda: delete_network_node(request, node), - http_error=HTTPInternalServerError, - fallback=lambda: request.db.rollback(), - msg_on_fail=s.InternalServerErrorResponseSchema.description) - return ax.valid_http(http_success=HTTPOk, detail=s.NetworkNode_DELETE_OkResponseSchema.description) +@s.NetworkNodeAPI.delete(tags=[s.NetworkNodeTag], response_schemas=s.NetworkNode_DELETE_responses) +@view_config(route_name=s.NetworkNodeAPI.name, request_method="DELETE") +def delete_network_node_view(request): + node_name = ar.get_value_matchdict_checked(request, "node_name") + node = ax.evaluate_call( + lambda: request.db.query(models.NetworkNode).filter(models.NetworkNode.name == node_name).one(), + http_error=HTTPNotFound, + msg_on_fail=s.NetworkNode_GET_NotFoundResponseSchema.description) + ax.evaluate_call(lambda: delete_network_node(request, node), + http_error=HTTPInternalServerError, + fallback=lambda: request.db.rollback(), + msg_on_fail=s.InternalServerErrorResponseSchema.description) + return ax.valid_http(http_success=HTTPOk, detail=s.NetworkNode_DELETE_OkResponseSchema.description) diff --git a/magpie/api/management/user/user_utils.py b/magpie/api/management/user/user_utils.py index aa0c42834..9862ff810 100644 --- a/magpie/api/management/user/user_utils.py +++ b/magpie/api/management/user/user_utils.py @@ -206,7 +206,12 @@ def update_user(user, request, new_user_name=None, new_password=None, new_email= with_param=False, # params are not useful in response for this case content={"user_name": user.user_name}, http_error=HTTPBadRequest, msg_on_fail=s.User_PATCH_BadRequestResponseSchema.description) - + if get_constant("MAGPIE_NETWORK_ENABLED", request, settings_name="magpie.network_enabled"): + anonymous_regex = protected_user_name_regex(include_admin=False, settings_container=request) + ax.verify_param(new_user_name, not_matches=True, param_compare=anonymous_regex, + with_param=False, # params are not useful in response for this case + content={"user_name": user.user_name}, + http_error=HTTPBadRequest, msg_on_fail=s.User_PATCH_BadRequestResponseSchema.description) # FIXME: disable email edit when self-registration is enabled to avoid not having any confirmation of new email # (see https://github.com/Ouranosinc/Magpie/issues/436) update_email_admin_only = False diff --git a/magpie/api/schemas.py b/magpie/api/schemas.py index 33df4a735..c528e62b6 100644 --- a/magpie/api/schemas.py +++ b/magpie/api/schemas.py @@ -283,7 +283,7 @@ def service_api_route_info(service_api, **kwargs): path="/token/validate", name="TokenValidate") NetworkNodeAPI = Service( - path="/network-nodes/{node}", + path="/network-nodes/{node_name}", name="NetworkNode") NetworkNodesAPI = Service( path="/network-nodes", @@ -459,11 +459,9 @@ class TemporaryURL_RequestPathSchema(colander.MappingSchema): RegisterTag: "Registration paths for operations available to users (including non-administrators).", ResourcesTag: "Management of resources that reside under a given service and their applicable permissions.", ServicesTag: "Management of service definitions, children resources and their applicable permissions.", + NetworkNodeTag: "Management of references to other Magpie instances in the network." } -if get_constant("MAGPIE_NETWORK_MODE", settings_name="magpie.network_mode"): - TAG_DESCRIPTIONS[NetworkNodeTag] = "Management of references to other Magpie instances in the network." - # Header definitions @@ -3345,19 +3343,6 @@ class Token_PATCH_OkResponseBodySchema(BaseResponseBodySchema): token = NetworkTokenParameter -class Token_RequestBodySchema(colander.MappingSchema): - expires = colander.SchemaNode( - colander.Integer(), - description="Token Expiry (in seconds)", - example=2000, - default=get_constant("MAGPIE_DEFAULT_TOKEN_EXPIRY") - ) - - -class Token_PATCH_RequestBodySchema(BaseRequestSchemaAPI): - body = Token_RequestBodySchema() - - class Token_PATCH_OkResponseSchema(BaseResponseSchemaAPI): description = "Get token successful." body = Token_PATCH_OkResponseBodySchema(code=HTTPOk.code, description=description) diff --git a/magpie/constants.py b/magpie/constants.py index 1fbf0a55d..cc4b86486 100644 --- a/magpie/constants.py +++ b/magpie/constants.py @@ -23,7 +23,7 @@ if TYPE_CHECKING: # pylint: disable=W0611,unused-import - from typing import Optional + from typing import Optional, List from magpie.typedefs import AnySettingsContainer, SettingValue, Str @@ -105,9 +105,9 @@ def _get_default_log_level(): MAGPIE_CRON_LOG = os.getenv("MAGPIE_CRON_LOG", "~/magpie-cron.log") MAGPIE_DB_MIGRATION = asbool(os.getenv("MAGPIE_DB_MIGRATION", True)) # run db migration on startup MAGPIE_DB_MIGRATION_ATTEMPTS = int(os.getenv("MAGPIE_DB_MIGRATION_ATTEMPTS", 5)) -MAGPIE_NETWORK_MODE = os.getenv("MAGPIE_NETWORK_MODE", False) -MAGPIE_INSTANCE_NAME = os.getenv("MAGPIE_INSTANCE_NAME") -MAGPIE_DEFAULT_TOKEN_EXPIRY = int(os.getenv("MAGPIE_DEFAULT_TOKEN_EXPIRY", 86400)) +MAGPIE_NETWORK_ENABLED = asbool(os.getenv("MAGPIE_NETWORK_ENABLED", False)) +MAGPIE_NETWORK_INSTANCE_NAME = os.getenv("MAGPIE_NETWORK_INSTANCE_NAME") +MAGPIE_NETWORK_DEFAULT_TOKEN_EXPIRY = int(os.getenv("MAGPIE_NETWORK_DEFAULT_TOKEN_EXPIRY", 86400)) MAGPIE_LOG_LEVEL = os.getenv("MAGPIE_LOG_LEVEL", _get_default_log_level()) # log level to apply to the loggers MAGPIE_LOG_PRINT = asbool(os.getenv("MAGPIE_LOG_PRINT", False)) # log also forces print to the console MAGPIE_LOG_REQUEST = asbool(os.getenv("MAGPIE_LOG_REQUEST", True)) # log detail of every incoming request @@ -187,7 +187,7 @@ def protected_user_name_regex(include_admin=True, include_network=True, additional_patterns=None, settings_container=None): - # type: (bool, bool, bool, Optional[list], Optional[AnySettingsContainer]) -> Str + # type: (bool, bool, bool, Optional[List[Str]], Optional[AnySettingsContainer]) -> Str """ Return a regular expression that matches all user names that are protected, meaning that they are generated by Magpie itself and no regular user account should be created with these user names. @@ -197,11 +197,13 @@ def protected_user_name_regex(include_admin=True, patterns.append(get_constant("MAGPIE_ADMIN_USER", settings_container=settings_container)) if include_anonymous: patterns.append(get_constant("MAGPIE_ANONYMOUS_USER", settings_container=settings_container)) - if include_network and get_constant("MAGPIE_NETWORK_MODE", settings_container=settings_container): + if include_network and get_constant("MAGPIE_NETWORK_ENABLED", + settings_name="magpie.network_enabled", + settings_container=settings_container): patterns.append( "{}.*".format(get_constant("MAGPIE_NETWORK_NAME_PREFIX", settings_container=settings_container)) ) - return "^{}$".format("|".join(patterns)) + return "^({})$".format("|".join(patterns)) def protected_group_name_regex(include_admin=True, @@ -218,11 +220,14 @@ def protected_group_name_regex(include_admin=True, patterns.append(get_constant("MAGPIE_ADMIN_GROUP", settings_container=settings_container)) if include_anonymous: patterns.append(get_constant("MAGPIE_ANONYMOUS_GROUP", settings_container=settings_container)) - if include_network and get_constant("MAGPIE_NETWORK_MODE", settings_container=settings_container): + if include_network and get_constant("MAGPIE_NETWORK_ENABLED", + settings_name="magpie.network_enabled", + settings_container=settings_container): patterns.append( "{}.*".format(get_constant("MAGPIE_NETWORK_NAME_PREFIX", settings_container=settings_container)) ) - return "^{}$".format("|".join(patterns)) + return "^({})$".format("|".join(patterns)) + def get_constant_setting_name(name): # type: (Str) -> Str diff --git a/magpie/models.py b/magpie/models.py index 7d64747cd..1f94929e9 100644 --- a/magpie/models.py +++ b/magpie/models.py @@ -1,7 +1,6 @@ import datetime import math import uuid -from collections.abc import Mapping from typing import TYPE_CHECKING import sqlalchemy as sa @@ -185,41 +184,6 @@ class User(UserMixin, Base): def __str__(self): return "".format(self.user_name, self.id) - network_node_id = sa.Column("network_node_id", sa.Integer, - sa.ForeignKey("network_nodes.id", onupdate="CASCADE", ondelete="CASCADE"), - nullable=True) - - # Includes unique constraint to enforce uniqueness over network_node_id - __table_args__ = ( - UniqueConstraint('user_name', 'network_node_id'), - UniqueConstraint('email', 'network_node_id'), - *((UserMixin.__table_args__,) if isinstance(UserMixin.__table_args__, Mapping) else UserMixin.__table_args__) - ) - - @declared_attr - def user_name(self): - """ - User name for user object. - - Overrides function in UserMixin to set unique to False. - This allows us to enforce a uniqueness constraint scoped over the network_node_id. - """ - column = UserMixin.user_name - column.unique = False - return column - - @declared_attr - def email(self): - """ - Email for user object. - - Overrides function in UserMixin to set unique to False. - This allows us to enforce a uniqueness constraint scoped over the network_node_id. - """ - column = UserMixin.email - column.unique = False - return column - def get_groups_by_status(self, status, db_session=None): # type: (UserGroupStatus, Session) -> Set[Str] """ @@ -1034,6 +998,27 @@ def json(self): return {"token": str(self.token), "operation": str(self.operation.value)} +class NetworkUser(BaseModel, Base): + """ + Model that defines a relationship between a User and a User that is authenticated on a different NetworkNode. + """ + __tablename__ = "network_users" + + id = sa.Column(sa.Integer(), primary_key=True, nullable=False, autoincrement=True) + user_id = sa.Column(sa.Integer, + sa.ForeignKey("users.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=False) + user = relationship("User", foreign_keys=[user_id]) + + network_node_id = sa.Column(sa.Integer, + sa.ForeignKey("network_nodes.id", onupdate="CASCADE", ondelete="CASCADE"), + nullable=False) + network_node = relationship("NetworkNode", foreign_keys=[network_node_id]) + network_user_name = sa.Column(sa.Unicode(128)) + + __table_args__ = (UniqueConstraint('user_id', 'network_node_id'), + UniqueConstraint('network_user_name', 'network_node_id')) + + class NetworkToken(BaseModel, Base): """ Model that defines a token for authentication across a network of Magpie instances. @@ -1047,7 +1032,7 @@ def __init__(self, *_, **__): token = sa.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4, unique=True) user_id = sa.Column(sa.Integer, - sa.ForeignKey("users.id", onupdate="CASCADE", ondelete="CASCADE"), unique=True) + sa.ForeignKey("users.id", onupdate="CASCADE", ondelete="CASCADE"), unique=True, nullable=False) user = relationship("User", foreign_keys=[user_id]) @@ -1060,17 +1045,6 @@ class NetworkNode(BaseModel, Base): id = sa.Column(sa.Integer(), primary_key=True, nullable=False, autoincrement=True) name = sa.Column(sa.Unicode(128), nullable=False, unique=True) url = sa.Column(URLType(), nullable=False) - users = relationship("User", backref="network_nodes", lazy="dynamic") - - @property - def anonymous_user_name(self): - # type: () -> Str - return "{}{}".format(get_constant("MAGPIE_NETWORK_NAME_PREFIX"), self.name) - - @property - def anonymous_user(self): - # type: () -> Optional[User] - return self.users.filter(User.user_name == self.anonymous_user_name).first() ziggurat_model_init(User, Group, UserGroup, GroupPermission, UserPermission, diff --git a/magpie/ui/utils.py b/magpie/ui/utils.py index a14f47a47..10e4e1721 100644 --- a/magpie/ui/utils.py +++ b/magpie/ui/utils.py @@ -592,8 +592,11 @@ def create_user(self, data): if (user_email or "").lower() in [usr["email"].lower() for usr in user_details]: data["invalid_user_email"] = True data["reason_user_email"] = "Conflict" - if user_email == "": - data["invalid_user_email"] = True + if get_constant("MAGPIE_NETWORK_ENABLED", self.request, settings_name="magpie.network_enabled"): + anonymous_regex = protected_user_name_regex(include_admin=False, settings_container=self.request) + if re.match(anonymous_regex, user_name): + data["invalid_user_name"] = True + data["reason_user_name"] = "Conflict" if len(user_name) > get_constant("MAGPIE_USER_NAME_MAX_LENGTH", self.request): data["invalid_user_name"] = True data["reason_user_name"] = "Too Long" From 799ad2349a0dceeba7fda44ff3d8d7bac19ff3a8 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Wed, 1 Nov 2023 16:13:29 -0400 Subject: [PATCH 03/55] network mode v2, code only --- .gitignore | 3 + config/magpie.ini | 3 +- magpie/adapter/magpieowssecurity.py | 3 +- ...3-08-25_2cfe144538e8_add_network_tables.py | 34 +- magpie/api/login/__init__.py | 4 - magpie/api/login/login.py | 140 +---- magpie/api/management/__init__.py | 4 +- magpie/api/management/group/group_utils.py | 2 +- magpie/api/management/group/group_views.py | 2 +- magpie/api/management/network/__init__.py | 13 + .../api/management/network/network_utils.py | 169 +++++ .../api/management/network/network_views.py | 49 ++ .../api/management/network/node/__init__.py | 15 + .../network/node/network_node_utils.py | 96 +++ .../network/node/network_node_views.py | 180 ++++++ .../network/remote_user/__init__.py | 13 + .../network/remote_user/remote_user_utils.py | 47 ++ .../network/remote_user/remote_user_views.py | 120 ++++ .../api/management/network_node/__init__.py | 13 - .../network_node/network_node_utils.py | 52 -- .../network_node/network_node_views.py | 88 --- magpie/api/management/user/user_formats.py | 2 +- magpie/api/management/user/user_utils.py | 23 +- magpie/api/schemas.py | 589 +++++++++++++----- magpie/cli/purge_expired_network_tokens.py | 63 ++ magpie/cli/register_defaults.py | 6 +- magpie/constants.py | 33 +- magpie/models.py | 88 ++- magpie/ui/__init__.py | 1 + magpie/ui/home/static/style.css | 16 + magpie/ui/management/templates/edit_user.mako | 51 ++ magpie/ui/management/views.py | 20 +- magpie/ui/network/__init__.py | 11 + magpie/ui/network/templates/authorize.mako | 32 + magpie/ui/network/views.py | 46 ++ .../ui/user/templates/edit_current_user.mako | 45 ++ magpie/ui/user/views.py | 12 +- magpie/ui/utils.py | 2 +- requirements.txt | 3 +- setup.py | 1 + tests/test_webhooks.py | 4 +- 41 files changed, 1629 insertions(+), 469 deletions(-) create mode 100644 magpie/api/management/network/__init__.py create mode 100644 magpie/api/management/network/network_utils.py create mode 100644 magpie/api/management/network/network_views.py create mode 100644 magpie/api/management/network/node/__init__.py create mode 100644 magpie/api/management/network/node/network_node_utils.py create mode 100644 magpie/api/management/network/node/network_node_views.py create mode 100644 magpie/api/management/network/remote_user/__init__.py create mode 100644 magpie/api/management/network/remote_user/remote_user_utils.py create mode 100644 magpie/api/management/network/remote_user/remote_user_views.py delete mode 100644 magpie/api/management/network_node/__init__.py delete mode 100644 magpie/api/management/network_node/network_node_utils.py delete mode 100644 magpie/api/management/network_node/network_node_views.py create mode 100644 magpie/cli/purge_expired_network_tokens.py create mode 100644 magpie/ui/network/__init__.py create mode 100644 magpie/ui/network/templates/authorize.mako create mode 100644 magpie/ui/network/views.py diff --git a/.gitignore b/.gitignore index 4d189a647..79605d7f2 100644 --- a/.gitignore +++ b/.gitignore @@ -56,3 +56,6 @@ magpie_delete_users*.txt requirements-all.txt gunicorn.app.wsgiapp error_log.txt + +# Secrets +key.pem diff --git a/config/magpie.ini b/config/magpie.ini index bd42359b2..7f6c6c9d3 100644 --- a/config/magpie.ini +++ b/config/magpie.ini @@ -31,7 +31,8 @@ magpie.url = http://localhost:2001 # Enable network mode which allows different instances of Magpie to authenticate users for each other. # magpie.network_enabled = true # magpie.network_default_token_expiry = 86400 -# magpie.network_instance_name +# magpie.network_instance_name = +# magpie.network_pem_files = key.pem # magpie.config_path = diff --git a/magpie/adapter/magpieowssecurity.py b/magpie/adapter/magpieowssecurity.py index 491c02962..a34688412 100644 --- a/magpie/adapter/magpieowssecurity.py +++ b/magpie/adapter/magpieowssecurity.py @@ -269,8 +269,7 @@ def update_request_cookies(self, request): """ settings = get_settings(request) token_name = get_constant("MAGPIE_COOKIE_NAME", settings_container=settings) - network_mode = get_constant("MAGPIE_NETWORK_ENABLED", settings_container=settings, - settings_name="magpie.network_enabled") + network_mode = get_constant("MAGPIE_NETWORK_ENABLED", settings_container=settings) headers = dict(request.headers) network_token_name = get_constant("MAGPIE_NETWORK_TOKEN_NAME", settings_container=settings) if network_mode and "Authorization" not in headers and network_token_name in request.params: diff --git a/magpie/alembic/versions/2023-08-25_2cfe144538e8_add_network_tables.py b/magpie/alembic/versions/2023-08-25_2cfe144538e8_add_network_tables.py index 6a3dba119..7812423d8 100644 --- a/magpie/alembic/versions/2023-08-25_2cfe144538e8_add_network_tables.py +++ b/magpie/alembic/versions/2023-08-25_2cfe144538e8_add_network_tables.py @@ -5,11 +5,10 @@ Revises: 5e5acc33adce Create Date: 2023-08-25 13:36:16.930374 """ -import uuid +import datetime import sqlalchemy as sa from alembic import op -from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.orm.session import sessionmaker from sqlalchemy_utils import URLType @@ -25,35 +24,40 @@ def upgrade(): op.create_table("network_tokens", - sa.Column("token", UUID(as_uuid=True), - primary_key=True, default=uuid.uuid4, unique=True), + sa.Column("id", sa.Integer, primary_key=True, nullable=False, autoincrement=True), + sa.Column("token", sa.LargeBinary, nullable=False, unique=True), + sa.Column("created", sa.DateTime, default=datetime.datetime.utcnow), sa.Column("user_id", sa.Integer, - sa.ForeignKey("users.id", onupdate="CASCADE", ondelete="CASCADE"), unique=True, - nullable=False) + sa.ForeignKey("users.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=False) ) op.create_table("network_nodes", sa.Column("id", sa.Integer, primary_key=True, nullable=False, autoincrement=True), sa.Column("name", sa.Unicode(128), nullable=False, unique=True), - sa.Column("url", URLType(), nullable=False) + sa.Column("jwks_url", URLType(), nullable=False), + sa.Column("token_url", URLType(), nullable=False), + sa.Column("authorization_url", URLType(), nullable=False), + sa.Column("redirect_uris", sa.String) ) - op.create_table("network_users", + op.create_table("network_remote_users", sa.Column("id", sa.Integer, primary_key=True, autoincrement=True), sa.Column("user_id", sa.Integer, sa.ForeignKey("users.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=False), sa.Column("network_node_id", sa.Integer, sa.ForeignKey("network_nodes.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=False), - sa.Column("network_user_name", sa.Unicode(128)), + sa.Column("name", sa.Unicode(128)), + sa.Column("network_token_id", sa.Integer, + sa.ForeignKey("network_tokens.id", onupdate="CASCADE", ondelete="CASCADE")) ) - op.create_unique_constraint("uq_network_users_user_id_network_node_id", "network_users", + op.create_unique_constraint("uq_network_remote_users_user_id_network_node_id", "network_remote_users", ["user_id", "network_node_id"]) - op.create_unique_constraint("uq_network_users_network_user_name_network_node_id", "network_users", - ["network_user_name", "network_node_id"]) + op.create_unique_constraint("uq_network_remote_users_name_network_node_id", "network_remote_users", + ["name", "network_node_id"]) def downgrade(): - op.drop_constraint("uq_network_users_user_id_network_node_id", "network_users") - op.drop_constraint("uq_network_users_network_user_name_network_node_id", "network_users") + op.drop_constraint("uq_network_remote_users_user_id_network_node_id", "network_remote_users") + op.drop_constraint("uq_network_remote_users_name_network_node_id", "network_remote_users") op.drop_table("network_tokens") op.drop_table("network_nodes") - op.drop_table("network_users") + op.drop_table("network_remote_users") diff --git a/magpie/api/login/__init__.py b/magpie/api/login/__init__.py index a849de880..0aaba355a 100644 --- a/magpie/api/login/__init__.py +++ b/magpie/api/login/__init__.py @@ -1,4 +1,3 @@ -from magpie.constants import get_constant from magpie.utils import get_logger LOGGER = get_logger(__name__) @@ -12,7 +11,4 @@ def includeme(config): config.add_route(**s.service_api_route_info(s.SigninAPI)) config.add_route(**s.service_api_route_info(s.ProvidersAPI)) config.add_route(**s.service_api_route_info(s.ProviderSigninAPI)) - if get_constant("MAGPIE_NETWORK_ENABLED", config, settings_name="magpie.network_enabled"): - config.add_route(**s.service_api_route_info(s.TokenAPI)) - config.add_route(**s.service_api_route_info(s.TokenValidateAPI)) config.scan() diff --git a/magpie/api/login/login.py b/magpie/api/login/login.py index 6f3a52f5b..8faa051ef 100644 --- a/magpie/api/login/login.py +++ b/magpie/api/login/login.py @@ -1,10 +1,6 @@ import json -import uuid -from datetime import datetime, timedelta from typing import TYPE_CHECKING -import jwt -import requests from authomatic.adapters import WebObAdapter from authomatic.core import Credentials, LoginResult, resolve_provider_class from authomatic.exceptions import OAuth2Error @@ -20,7 +16,7 @@ HTTPOk, HTTPTemporaryRedirect, HTTPUnauthorized, - HTTPUnprocessableEntity, HTTPNotImplemented + HTTPUnprocessableEntity ) from pyramid.request import Request from pyramid.response import Response @@ -38,7 +34,7 @@ from magpie.api import schemas as s from magpie.api.management.user.user_formats import format_user from magpie.api.management.user.user_utils import create_user -from magpie.constants import get_constant, protected_user_name_regex +from magpie.constants import get_constant, protected_user_name_regex, protected_user_email_regex from magpie.security import authomatic_setup, get_providers from magpie.utils import ( CONTENT_TYPE_JSON, @@ -51,7 +47,6 @@ if TYPE_CHECKING: from magpie.typedefs import Session, Str - from typing import Optional LOGGER = get_logger(__name__) @@ -135,7 +130,10 @@ def sign_in_view(request): # request handler. Catch that specific exception and return it to bypass the EXCVIEW tween that result in that # automatic convert to return 403 directly. try: - anonymous_regex = protected_user_name_regex(include_admin=False, settings_container=request) + if pattern == ax.EMAIL_REGEX: + anonymous_regex = protected_user_email_regex(include_admin=False, settings_container=request) + else: + anonymous_regex = protected_user_name_regex(include_admin=False, settings_container=request) ax.verify_param(user_name, not_matches=True, param_compare=anonymous_regex, param_name="user_name", http_error=HTTPForbidden, msg_on_fail=s.Signin_POST_ForbiddenResponseSchema.description) except HTTPForbidden as http_error: @@ -271,23 +269,28 @@ def login_success_external(request, user): http_kwargs={"location": homepage_route, "headers": headers}) -def user_from_token(request, token): - # type: (Request, Str) -> Optional[models.User] +def network_login(request): + # type: (Request) -> HTTPException """ - Return the ``User`` associated with the token as long as the token is valid and issued by this Magpie instance. + Sign in a user authenticating using a `Magpie` network access token. """ - magpie_secret = get_constant("MAGPIE_SECRET", request, settings_name="magpie.secret") - decoded_token = jwt.decode(token, magpie_secret, algorithms="HS256", - issuer=get_constant("MAGPIE_NETWORK_INSTANCE_NAME", request, - settings_name='magpie.network_instance_name'), - options={"require": ["exp", "token"]}) - - network_token = (request.db.query(models.NetworkToken) - .join(models.User) - .filter(models.NetworkToken.token == decoded_token["token"]) - .first()) - if network_token: - return network_token.user + if "Authorization" in request.headers: + token_type, token = request.headers.get("Authorization").split() + if token_type != "Bearer": + ax.raise_http(http_error=HTTPBadRequest, + detail=s.ProviderSignin_GET_BadRequestResponseSchema.description) + network_token = models.NetworkToken.by_decrypted_token(token) + if network_token is None or network_token.expired(): + return login_failure_view(request, reason=s.Signin_POST_UnauthorizedResponseSchema.description) + authenticated_user = network_token.user + # We should never create a token for protected users but just in case + anonymous_regex = protected_user_name_regex(include_admin=False, settings_container=request) + ax.verify_param(authenticated_user.name, not_matches=True, param_compare=anonymous_regex, + http_error=HTTPForbidden, msg_on_fail=s.Signin_POST_ForbiddenResponseSchema.description) + return login_success_external(request, authenticated_user) + else: + ax.raise_http(http_error=HTTPBadRequest, + detail=s.ProviderSignin_GET_BadRequestResponseSchema.description) @s.ProviderSigninAPI.get(schema=s.ProviderSignin_GET_RequestSchema, tags=[s.SessionTag], @@ -302,39 +305,7 @@ def external_login_view(request): verify_provider(provider_name) try: if provider_name == get_constant("MAGPIE_NETWORK_PROVIDER", request): - if "Authorization" in request.headers: - token_type, token = request.headers.get("Authorization").split() - if token_type != "Bearer": - ax.raise_http(http_error=HTTPBadRequest, - detail=s.ProviderSignin_GET_BadRequestResponseSchema.description) - authenticated_user = None - try: - authenticated_user = user_from_token(request, token) - except jwt.exceptions.InvalidIssuerError: - decoded_token = jwt.decode(token, options={"verify_signature": False}) - issuer = decoded_token.get("iss") - node = request.db.query(models.NetworkNode).filter(models.NetworkNode.name == issuer).first() - if node: - response = requests.get("{}{}".format(node.url, s.TokenValidateAPI.path), - headers={"Accept": CONTENT_TYPE_JSON}) - - if response.status_code != HTTPOk.code: - ax.raise_http(http_error=HTTPUnauthorized, - detail=s.ProviderSignin_GET_UnauthorizedTokenSchema.description) - user_name = response.json().get("user_name") - authenticated_user = (node.users.filter(models.User.name == user_name).first() or - node.anonymous_user) - else: - ax.raise_http(http_error=HTTPUnauthorized, - detail=s.ProviderSignin_GET_UnauthorizedTokenSchema.description) - except jwt.exceptions.PyJWTError as exc: - ax.raise_http(http_error=HTTPUnauthorized, content={"reason": str(exc)}, - detail=s.ProviderSignin_GET_UnauthorizedTokenSchema.description) - if authenticated_user: - return login_success_external(request, authenticated_user) - else: - ax.raise_http(http_error=HTTPBadRequest, - detail=s.ProviderSignin_GET_BadRequestResponseSchema.description) + return network_login(request) else: authomatic_handler = authomatic_setup(request) @@ -433,63 +404,6 @@ def _get_session(req): return ax.valid_http(http_success=HTTPOk, detail=s.Session_GET_OkResponseSchema.description, content=session_json) -@s.TokenAPI.patch(tags=[s.SessionTag], response_schemas=s.Token_PATCH_responses) -@view_config(route_name=s.TokenAPI.name, request_method="PATCH", permission=Authenticated) -def token_view(request): - """ - Get a token. Generates a new token every time this route is accessed so this can also - be used to invalidate a previously generated token. - """ - magpie_secret = get_constant("MAGPIE_SECRET", request, settings_name="magpie.secret") - default_expiry = get_constant("MAGPIE_NETWORK_DEFAULT_TOKEN_EXPIRY", request, - settings_name="magpie.network_default_token_expiry") - magpie_instance_name = get_constant("MAGPIE_NETWORK_INSTANCE_NAME", - request, - settings_name="magpie.network_instance_name") - - expiry = datetime.utcnow() + timedelta(seconds=default_expiry) - - def upsert_token(): - network_token = request.db.query(models.NetworkToken).filter( - models.NetworkToken.user_id == request.user.id).first() - if network_token: - network_token.token = uuid.uuid4() - else: - network_token = models.NetworkToken(user_id=request.user.id) - request.db.add(network_token) - return network_token.token - - token_value = ax.evaluate_call(lambda: upsert_token(), fallback=lambda: request.db.rollback(), - http_error=HTTPInternalServerError) - - token = jwt.encode({"token": str(token_value), - "exp": expiry, - "iss": magpie_instance_name}, magpie_secret, algorithm="HS256") - - return ax.valid_http(http_success=HTTPOk, detail=s.Token_PATCH_OkResponseSchema.description, - content={"token": token}) - - -@s.TokenValidateAPI.get(schema=s.TokenValidate_GET_RequestBodySchema, tags=[s.SessionTag], - response_schemas=s.TokenValidate_GET_responses) -@view_config(route_name=s.TokenValidateAPI.name, permission=NO_PERMISSION_REQUIRED) -def validate_token_view(request): - """ Validate a token """ - token = request.GET.get("token") - try: - authenticated_user = user_from_token(request, token) - if authenticated_user: - return ax.valid_http(http_success=HTTPOk, - detail=s.TokenValidate_GET_OkResponseSchema.description, - content={"user_name": authenticated_user.user_name}) - else: - ax.raise_http(http_error=HTTPUnauthorized, - detail=s.TokenValidate_GET_BadRequestResponseSchema.description) - except jwt.exceptions.PyJWTError as exc: - ax.raise_http(http_error=HTTPUnauthorized, content={"reason": str(exc)}, - detail=s.TokenValidate_GET_BadRequestResponseSchema.description) - - @s.ProvidersAPI.get(tags=[s.SessionTag], response_schemas=s.Providers_GET_responses) @view_config(route_name=s.ProvidersAPI.name, request_method="GET", permission=NO_PERMISSION_REQUIRED) def get_providers_view(request): # noqa: F811 diff --git a/magpie/api/management/__init__.py b/magpie/api/management/__init__.py index 6226e2dd2..fb63ad8f7 100644 --- a/magpie/api/management/__init__.py +++ b/magpie/api/management/__init__.py @@ -1,3 +1,4 @@ +from magpie.constants import get_constant from magpie.utils import get_logger LOGGER = get_logger(__name__) @@ -10,5 +11,6 @@ def includeme(config): config.include("magpie.api.management.service") config.include("magpie.api.management.resource") config.include("magpie.api.management.register") - config.include("magpie.api.management.network_node") + if get_constant("MAGPIE_NETWORK_ENABLED", config): + config.include("magpie.api.management.network") config.scan() diff --git a/magpie/api/management/group/group_utils.py b/magpie/api/management/group/group_utils.py index bced69cd7..e7812bfad 100644 --- a/magpie/api/management/group/group_utils.py +++ b/magpie/api/management/group/group_utils.py @@ -95,7 +95,7 @@ def create_group(group_name, description, discoverable, terms, db_session): ax.verify_param(group_name, matches=True, param_compare=ax.PARAM_REGEX, param_name="group_name", http_error=HTTPBadRequest, content=group_content_error, msg_on_fail=s.Groups_POST_BadRequestResponseSchema.description) - if get_constant("MAGPIE_NETWORK_ENABLED", settings_name="magpie.network_enabled"): + if get_constant("MAGPIE_NETWORK_ENABLED"): anonymous_regex = protected_group_name_regex(include_admin=False) ax.verify_param(group_name, not_matches=True, param_compare=anonymous_regex, param_name="group_name", http_error=HTTPBadRequest, content=group_content_error, diff --git a/magpie/api/management/group/group_views.py b/magpie/api/management/group/group_views.py index f84f61089..8a3bf63a6 100644 --- a/magpie/api/management/group/group_views.py +++ b/magpie/api/management/group/group_views.py @@ -85,7 +85,7 @@ def edit_group_view(request): ax.verify_param(GroupService.by_group_name(new_group_name, db_session=request.db), is_none=True, http_error=HTTPConflict, with_param=False, # don't return group as value msg_on_fail=s.Group_PATCH_ConflictResponseSchema.description) - if get_constant("MAGPIE_NETWORK_ENABLED", settings_name="magpie.network_enabled"): + if get_constant("MAGPIE_NETWORK_ENABLED"): anonymous_regex = protected_group_name_regex(include_admin=False) ax.verify_param(new_group_name, not_matches=True, param_compare=anonymous_regex, http_error=HTTPBadRequest, diff --git a/magpie/api/management/network/__init__.py b/magpie/api/management/network/__init__.py new file mode 100644 index 000000000..94ea63cb3 --- /dev/null +++ b/magpie/api/management/network/__init__.py @@ -0,0 +1,13 @@ +from magpie.utils import get_logger + +LOGGER = get_logger(__name__) + + +def includeme(config): + from magpie.api import schemas as s + LOGGER.info("Adding API network ...") + config.add_route(**s.service_api_route_info(s.NetworkTokenAPI)) # /network/token POST and DELETE + config.add_route(**s.service_api_route_info(s.NetworkJSONWebKeySetAPI)) # /network/jwks GET + config.include("magpie.api.management.network.node") + config.include("magpie.api.management.network.remote_user") + config.scan() diff --git a/magpie/api/management/network/network_utils.py b/magpie/api/management/network/network_utils.py new file mode 100644 index 000000000..b9799c22f --- /dev/null +++ b/magpie/api/management/network/network_utils.py @@ -0,0 +1,169 @@ +from datetime import datetime, timedelta +from itertools import zip_longest +from typing import TYPE_CHECKING + +import jwt +from cryptography.hazmat.primitives import serialization +from jwcrypto import jwk + +from pyramid.httpexceptions import HTTPInternalServerError, HTTPNotFound +from magpie import models +from magpie.api import exception as ax +from magpie.api import schemas as s +from magpie.constants import get_constant +from magpie.utils import get_logger + +if TYPE_CHECKING: + from typing import List, Optional, Dict, Tuple + from magpie.typedefs import JSON, Str, AnySettingsContainer + from cryptography.hazmat.primitives.asymmetric.types import PrivateKeyTypes + from pyramid.request import Request + +LOGGER = get_logger(__name__) + +PEM_FILE_DELIMITER = ":" +PEM_PASSWORD_DELIMITER = ":" + + +def _pem_file_content(primary=False): + # type: (bool) -> List[bytes] + """ + Return the content of all PEM files + """ + pem_files = get_constant("MAGPIE_NETWORK_PEM_FILES").split(PEM_FILE_DELIMITER) + content = [] + for pem_file in pem_files: + with open(pem_file, 'rb') as f: + content.append(f.read()) + if primary: + break + return content + + +def _pem_file_passwords(primary=False): + # type: (bool) -> List[Optional[bytes]] + """ + Return the passwords used to encrypt the PEM files. + The passwords will be returned in the same order as the file content from `_pem_file_content`. + + If a file is not encrypted with a password, a ``None`` value will be returned in place of the password. + + For example: if there are 4 PEM files and the second and fourth are not encrypted, this will return + ``["password1", None, "password2"]`` + """ + pem_passwords = get_constant("MAGPIE_NETWORK_PEM_PASSWORDS", raise_missing=False, raise_not_set=False) + passwords = [] + if pem_passwords: + for password in pem_passwords.split(PEM_PASSWORD_DELIMITER): + if password: + passwords.append(password.encode()) + else: + passwords.append(None) + if primary: + break + return passwords + + +def jwks(primary=False): + # type: (bool) -> jwk.JWKSet + """ + Return a JSON Web Key Set containing all JSON Web Keys loaded from the PEM files listed + in ``MAGPIE_NETWORK_PEM_FILES``. + """ + jwks_ = jwk.JWKSet() + for pem_content, pem_password in zip_longest(_pem_file_content(primary), _pem_file_passwords(primary)): + jwks_["keys"].add(jwk.JWK.from_pem(pem_content, password=pem_password)) + return jwks_ + + +def _private_keys(primary=False): + # type: (bool) -> Dict[Str, PrivateKeyTypes] + """ + Return a dictionary containing key ids and private keys from the PEM files listed in ``MAGPIE_NETWORK_PEM_FILES``. + + If the ``primary`` argument is True, only the primary key will be included in the returned list. + """ + keys = {} + for pem_content, pem_password in zip_longest(_pem_file_content(primary), _pem_file_passwords(primary)): + kid = jwk.JWK.from_pem(pem_content, password=pem_password).export(as_dict=True)["kid"] + keys[kid] = serialization.load_pem_private_key(pem_content, password=pem_password) + return keys + + +def encode_jwt(claims, audience_name, settings_container=None): + # type: (JSON, Str, Optional[AnySettingsContainer]) -> Str + """ + Encode claims as a JSON web token. + + Unless overridden by a field in the ``claims`` argument, the ``"iss"`` claim will default to + `MAGPIE_NETWORK_INSTANCE_NAME`, the ``"exp"`` claim will default to `MAGPIE_NETWORK_INTERNAL_TOKEN_EXPIRY`, + and the ``"aud"`` claim will default to ``audience_name``. The JWT will be signed with `Magpie`'s primary private + key (see the `_private_keys` function for details) using the asymmetric RS256 algorithm. + """ + claims_override = {} + kid, secret = ax.evaluate_call(lambda: next(iter(_private_keys().items())), + http_error=HTTPInternalServerError, + msg_on_fail="No private key found. Cannot sign JWT.") + headers = {"kid": kid} + algorithm = "RS256" + if "exp" not in claims: + expiry = int(get_constant("MAGPIE_NETWORK_INTERNAL_TOKEN_EXPIRY", settings_container)) + expiry_time = datetime.utcnow() + timedelta(seconds=expiry) + claims_override["exp"] = expiry_time + if "iss" not in claims: + claims_override["iss"] = get_constant("MAGPIE_NETWORK_INSTANCE_NAME", settings_container) + if "aud" not in claims: + claims_override["aud"] = audience_name + return jwt.encode({**claims, **claims_override}, secret, algorithm=algorithm, headers=headers) + + +def decode_jwt(token, node, settings_container=None): + # type: (Str, models.NetworkNode, Optional[AnySettingsContainer]) -> JSON + """ + Decode a JSON Web Token issued by a node in the network. + + The token must include the ``"exp"``, ``"aud"``, and ``"iss"`` claims. + If the issuer is not the same as ``node.name``, or the audience is not this instance (i.e. the same as + ``MAGPIE_NETWORK_INSTANCE_NAME``), or the token is expired, an error will be raised. + An error will also be raised if the token cannot be verified with the issuer node's public key. + """ + jwks_client = jwt.PyJWKClient(node.jwks_url) + instance_name = get_constant("MAGPIE_NETWORK_INSTANCE_NAME", settings_container) + key = ax.evaluate_call(lambda: jwks_client.get_signing_key_from_jwt(token), + http_error=HTTPInternalServerError, + msg_on_fail="No valid public key found. Cannot decode JWT.") + return ax.evaluate_call(lambda: jwt.decode(token, key.key, + algorithms=["RS256"], + issuer=node.name, + audience=instance_name), + http_error=HTTPInternalServerError, + msg_on_fail="Cannot verify JWT") + + +def get_network_models_from_request_token(request, create_network_remote_user=False): + # type: (Request, bool) -> Tuple[models.NetworkNode, Optional[models.NetworkRemoteUser]] + """ + Return a ``NetworkNode`` and associated ``NetworkRemoteUser`` determined by parsing the claims in the JWT included in the + ``request`` argument. + + If the ``NetworkRemoteUser`` does not exist and ``create_network_remote_user`` is ``True``, this creates a new + ``NetworkRemoteUser`` associated with the anonymous user for the given ``NetworkNode`` and adds it to the current + database transaction. + """ + token = request.POST.get("token") + node_name = jwt.decode(token, options={"verify_signature": False}).get("iss") + node = ax.evaluate_call( + lambda: request.db.query(models.NetworkNode).filter(models.NetworkNode.name == node_name).one(), + http_error=HTTPNotFound, + msg_on_fail=s.NetworkNode_GET_NotFoundResponseSchema.description) + decoded_token = decode_jwt(token, node, request) + user_name = decoded_token.get("user_name") + network_remote_user = (request.db.query(models.NetworkRemoteUser) + .filter(models.NetworkRemoteUser.name == user_name) + .filter(models.NetworkRemoteUser.network_node_id == node.id) + .first()) + if network_remote_user is None and create_network_remote_user: + anonymous_user = node.anonymous_user(request.db) + network_remote_user = models.NetworkRemoteUser(user=anonymous_user, network_node=node, name=user_name) + request.db.add(network_remote_user) + return node, network_remote_user diff --git a/magpie/api/management/network/network_views.py b/magpie/api/management/network/network_views.py new file mode 100644 index 000000000..44e69b685 --- /dev/null +++ b/magpie/api/management/network/network_views.py @@ -0,0 +1,49 @@ +import sqlalchemy +from pyramid.httpexceptions import ( + HTTPNotFound, + HTTPOk, + HTTPCreated, +) +from pyramid.security import NO_PERMISSION_REQUIRED +from pyramid.view import view_config + +from magpie import models +from magpie.api import exception as ax +from magpie.api import schemas as s +from magpie.api.management.network.network_utils import jwks, get_network_models_from_request_token + + +@s.NetworkTokenAPI.post(schema=s.NetworkToken_POST_RequestSchema, tags=[s.NetworkTag], + response_schemas=s.NetworkToken_POST_responses) +@view_config(route_name=s.NetworkTokenAPI.name, request_method="POST") +def post_network_token_view(request): + node, network_remote_user = get_network_models_from_request_token(request, create_network_remote_user=True) + network_token = network_remote_user.network_token + if network_token: + network_token.refresh_token() + else: + network_token = models.NetworkToken(user_id=network_remote_user.user_id) + network_remote_user.network_token = network_token + return ax.valid_http(http_success=HTTPCreated, content={"token": network_token.decrypted_token()}, + detail=s.NetworkToken_POST_CreatedResponseSchema.description) + + +@s.NetworkTokenAPI.delete(schema=s.NetworkToken_DELETE_RequestSchema, tags=[s.NetworkTag], + response_schemas=s.NetworkToken_DELETE_responses) +@view_config(route_name=s.NetworkTokenAPI.name, request_method="DELETE") +def delete_network_token_view(request): + node, network_remote_user = get_network_models_from_request_token(request) + if network_remote_user.network_token: + request.db.delete(network_remote_user.network_token) + if (network_remote_user.user.id == node.anonymous_user(request.db).id and + sqlalchemy.inspect(network_remote_user).persisted): + request.db.delete(network_remote_user) # clean up unused record in the database + return ax.valid_http(http_success=HTTPOk, detail=s.NetworkToken_DELETE_OkResponseSchema.description) + else: + ax.raise_http(http_error=HTTPNotFound, detail=s.NetworkNodeToken_DELETE_NotFoundResponseSchema.description) + + +@s.NetworkJSONWebKeySetAPI.get(tags=[s.NetworkTag], response_schemas=s.NetworkJSONWebKeySet_GET_responses) +@view_config(route_name=s.NetworkJSONWebKeySetAPI.name, request_method="GET", permission=NO_PERMISSION_REQUIRED) +def get_network_jwks_view(_request): + return ax.valid_http(http_success=HTTPOk, content=jwks().export(private_keys=False, as_dict=True)) diff --git a/magpie/api/management/network/node/__init__.py b/magpie/api/management/network/node/__init__.py new file mode 100644 index 000000000..e611a0fe5 --- /dev/null +++ b/magpie/api/management/network/node/__init__.py @@ -0,0 +1,15 @@ +from magpie.utils import get_logger + +LOGGER = get_logger(__name__) + + +def includeme(config): + from magpie.api import schemas as s + LOGGER.info("Adding API network node...") + config.add_route(**s.service_api_route_info(s.NetworkNodesAPI)) # /network/nodes GET and POST + config.add_route(**s.service_api_route_info(s.NetworkNodeAPI)) # /network/nodes/{node_name} GET, DELETE, PATCH + config.add_route(**s.service_api_route_info(s.NetworkNodeTokenAPI)) # /network/nodes/{node_name}/token GET and DELETE + config.add_route(**s.service_api_route_info(s.NetworkNodesLinkAPI)) # /network/nodes/link GET + config.add_route(**s.service_api_route_info(s.NetworkNodeLinkAPI)) # /network/nodes/link POST + + config.scan() diff --git a/magpie/api/management/network/node/network_node_utils.py b/magpie/api/management/network/node/network_node_utils.py new file mode 100644 index 000000000..ef27a332c --- /dev/null +++ b/magpie/api/management/network/node/network_node_utils.py @@ -0,0 +1,96 @@ +from typing import TYPE_CHECKING + +from magpie import models +from magpie.api import exception as ax +from magpie.api import schemas as s +from magpie.api.exception import URL_REGEX +from magpie.cli.register_defaults import register_user_with_group +from magpie.constants import get_constant +from pyramid.httpexceptions import HTTPBadRequest, HTTPConflict + +if TYPE_CHECKING: + from magpie.typedefs import Str, Optional, Session + from pyramid.request import Request + +NAME_REGEX = r"^[\w-]+$" + + +def create_associated_user_groups(new_node, request): + # type: (models.NetworkNode, Request) -> None + """ + Create a NetworkNode with the given name and url. + """ + name = new_node.anonymous_user_name() + + # create an anonymous user and group for this network node + register_user_with_group(user_name=name, + group_name=name, + email=get_constant("MAGPIE_NETWORK_ANONYMOUS_EMAIL_FORMAT").format(new_node.name), + password=None, # autogen, value doesn't matter as no login applicable, just make it valid + db_session=request.db) + + group = models.GroupService.by_group_name(name, db_session=request.db) + group.description = "Group for users who have accounts on the networked Magpie instance named '{}'.".format( + new_node.name) + group.discoverable = False + + # add the anonymous user to a group for all users in the network (from nodes other than this one). + register_user_with_group(user_name=name, + group_name=get_constant("MAGPIE_NETWORK_GROUP_NAME"), + email=get_constant("MAGPIE_NETWORK_ANONYMOUS_EMAIL_FORMAT").format(new_node.name), + password=None, + db_session=request.db) + + group = models.GroupService.by_group_name(get_constant("MAGPIE_NETWORK_GROUP_NAME"), db_session=request.db) + group.description = "Group for users who have accounts on a different Magpie instance on this network." + group.discoverable = False + + +def update_associated_user_groups(node, old_node_name, request): + # type: (models.NetworkNode, Str, Request) -> None + if node.name != old_node_name: + old_anonymous_name = models.NetworkNode.anonymous_user_name_formatter(old_node_name) + anonymous_user = request.db.query(models.User).filter(models.User.user_name == old_anonymous_name).one() + anonymous_group = request.db.query(models.Group).filter(models.Group.group_name == old_anonymous_name).one() + anonymous_user.user_name = node.anonymous_user_name() + anonymous_group.group_name = node.anonymous_user_name() + + +def delete_network_node(request, node): + # type: (Request, Str) -> None + """ + Delete a NetworkNode and the associated anonymous user and group. + """ + request.db.delete(node.anonymous_user(request.db)) + request.db.delete(node.anonymous_group(request.db)) + request.db.delete(node) + + +def check_network_node_info(db_session=None, name=None, jwks_url=None, token_url=None, authorization_url=None, + redirect_uris=None): + # type: (Optional[Session], Optional[Str], Optional[Str], Optional[Str], Optional[Str], Optional[Str]) -> None + if name is not None: + ax.verify_param(name, matches=True, param_name="name", param_compare=NAME_REGEX, + http_error=HTTPBadRequest, + msg_on_fail=s.NetworkNodes_CheckInfo_NameValue_BadRequestResponseSchema.description) + ax.verify_param(name, not_in=True, param_name="name", + param_compare=[n.name for n in db_session.query(models.NetworkNode)], + http_error=HTTPConflict, + msg_on_fail=s.NetworkNodes_CheckInfo_NameValue_ConflictResponseSchema.description) + if jwks_url is not None: + ax.verify_param(jwks_url, matches=True, param_name="jwks_url", param_compare=URL_REGEX, + http_error=HTTPBadRequest, + msg_on_fail=s.NetworkNodes_CheckInfo_JWKSURLValue_BadRequestResponseSchema.description) + if token_url is not None: + ax.verify_param(token_url, matches=True, param_name="token_url", param_compare=URL_REGEX, + http_error=HTTPBadRequest, + msg_on_fail=s.NetworkNodes_CheckInfo_TokenURLValue_BadRequestResponseSchema.description) + if authorization_url is not None: + ax.verify_param(authorization_url, matches=True, param_name="authorization_url", param_compare=URL_REGEX, + http_error=HTTPBadRequest, + msg_on_fail=s.NetworkNodes_CheckInfo_AuthorizationURLValue_BadRequestResponseSchema.description) + if redirect_uris is not None: + for uri in redirect_uris.split(): + ax.verify_param(uri, matches=True, param_name="redirect_uris", param_compare=URL_REGEX, + http_error=HTTPBadRequest, + msg_on_fail=s.NetworkNodes_CheckInfo_RedirectURIsValue_BadRequestResponseSchema.description) diff --git a/magpie/api/management/network/node/network_node_views.py b/magpie/api/management/network/node/network_node_views.py new file mode 100644 index 000000000..54ce461ad --- /dev/null +++ b/magpie/api/management/network/node/network_node_views.py @@ -0,0 +1,180 @@ +import jwt +from pyramid.httpexceptions import ( + HTTPBadRequest, + HTTPNotFound, + HTTPOk, + HTTPCreated, + HTTPInternalServerError, + HTTPForbidden, + HTTPTemporaryRedirect +) +from pyramid.security import Authenticated +from pyramid.view import view_config +from six.moves.urllib import parse as up +import requests + +from magpie import models +from magpie.api import exception as ax +from magpie.api import requests as ar +from magpie.api import schemas as s +from magpie.api.management.network.network_utils import encode_jwt, decode_jwt +from magpie.api.management.network.node.network_node_utils import delete_network_node, \ + check_network_node_info, create_associated_user_groups, update_associated_user_groups + + +@s.NetworkNodesAPI.get(tags=[s.NetworkTag], response_schemas=s.NetworkNodes_GET_responses) +@view_config(route_name=s.NetworkNodesAPI.name, request_method="GET") +def get_network_nodes_view(request): + nodes = [n.as_dict() for n in request.db.query(models.NetworkNode).all()] + return ax.valid_http(http_success=HTTPOk, detail=s.NetworkNodes_GET_OkResponseSchema.description, + content={"nodes": nodes}) + + +@s.NetworkNodeAPI.get(tags=[s.NetworkTag], response_schemas=s.NetworkNode_GET_responses) +@view_config(route_name=s.NetworkNodeAPI.name, request_method="GET") +def get_network_node_view(request): + node_name = ar.get_value_matchdict_checked(request, "node_name") + node = ax.evaluate_call( + lambda: request.db.query(models.NetworkNode).filter(models.NetworkNode.name == node_name).one(), + http_error=HTTPNotFound, + msg_on_fail=s.NetworkNode_GET_NotFoundResponseSchema.description) + return ax.valid_http(http_success=HTTPOk, detail=s.NetworkNode_GET_OkResponseSchema.description, + content=node.as_dict()) + + +@s.NetworkNodesAPI.post(schema=s.NetworkNode_POST_RequestSchema, tags=[s.NetworkTag], + response_schemas=s.NetworkNodes_POST_responses) +@view_config(route_name=s.NetworkNodesAPI.name, request_method="POST") +def post_network_nodes_view(request): + required_params = ("name", "jwks_url", "token_url", "authorization_url") + kwargs = {} + for param in required_params: + if param in request.POST: + kwargs[param] = request.POST[param] + else: + ax.raise_http(http_error=HTTPBadRequest, detail=s.NetworkNodes_POST_BadRequestResponseSchema.description) + if "redirect_uris" in request.POST: + kwargs["redirect_uris"] = request.POST.get("redirect_uris") + check_network_node_info(request.db, **kwargs) + + node = models.NetworkNode(**kwargs) + request.db.add(node) + create_associated_user_groups(node, request) + return ax.valid_http(http_success=HTTPCreated, detail=s.NetworkNodes_POST_CreatedResponseSchema.description) + + +@s.NetworkNodeAPI.patch(schema=s.NetworkNode_PATCH_RequestSchema, + tags=[s.NetworkTag], response_schemas=s.NetworkNode_PATCH_responses) +@view_config(route_name=s.NetworkNodeAPI.name, request_method="PATCH") +def patch_network_node_view(request): + node_name = ar.get_value_matchdict_checked(request, "node_name") + node = ax.evaluate_call( + lambda: request.db.query(models.NetworkNode).filter(models.NetworkNode.name == node_name).one(), + http_error=HTTPNotFound, + msg_on_fail=s.NetworkNode_GET_NotFoundResponseSchema.description) + params = ("name", "jwks_url", "token_url", "authorization_url", "redirect_uris") + kwargs = {} + for param in params: + if param in request.POST: + kwargs[param] = request.POST[param] + if not kwargs: + ax.raise_http(http_error=HTTPBadRequest, detail=s.NetworkNodes_PATCH_BadRequestResponseSchema.description) + + check_network_node_info(request.db, **kwargs) + + for attr, value in kwargs.items(): + setattr(node, attr, value) + + update_associated_user_groups(node, node_name, request) + return ax.valid_http(http_success=HTTPOk, detail=s.NetworkNode_PATCH_OkResponseSchema.description) + + +@s.NetworkNodeAPI.delete(tags=[s.NetworkTag], response_schemas=s.NetworkNode_DELETE_responses) +@view_config(route_name=s.NetworkNodeAPI.name, request_method="DELETE") +def delete_network_node_view(request): + node_name = ar.get_value_matchdict_checked(request, "node_name") + node = ax.evaluate_call( + lambda: request.db.query(models.NetworkNode).filter(models.NetworkNode.name == node_name).one(), + http_error=HTTPNotFound, + msg_on_fail=s.NetworkNode_GET_NotFoundResponseSchema.description) + ax.evaluate_call(lambda: delete_network_node(request, node), + http_error=HTTPInternalServerError, + fallback=lambda: request.db.rollback(), + msg_on_fail=s.InternalServerErrorResponseSchema.description) + return ax.valid_http(http_success=HTTPOk, detail=s.NetworkNode_DELETE_OkResponseSchema.description) + + +@s.NetworkNodeTokenAPI.get(tags=[s.NetworkTag], response_schemas=s.NetworkNodeToken_GET_responses) +@view_config(route_name=s.NetworkNodeTokenAPI.name, request_method="GET", permission=Authenticated) +def get_network_node_token_view(request): + node_name = ar.get_value_matchdict_checked(request, "node_name") + node = ax.evaluate_call( + lambda: request.db.query(models.NetworkNode).filter(models.NetworkNode.name == node_name).one(), + http_error=HTTPNotFound, + msg_on_fail=s.NetworkNode_GET_NotFoundResponseSchema.description) + token = encode_jwt({"user_name": request.user.user_name}, node_name, request) + access_token = ax.evaluate_call(lambda: requests.post(node.token_url, json={"token": token}).json()["token"], + http_error=HTTPInternalServerError, + msg_on_fail=s.NetworkNodeToken_GET_InternalServerErrorResponseSchema.description) + return ax.valid_http(http_success=HTTPOk, content={"token": access_token}, + detail=s.NetworkNodeToken_GET_OkResponseSchema) + + +@s.NetworkNodeTokenAPI.delete(tags=[s.NetworkTag], response_schemas=s.NetworkNodeToken_DELETE_responses) +@view_config(route_name=s.NetworkNodeTokenAPI.name, request_method="DELETE", permission=Authenticated) +def delete_network_node_token_view(request): + node_name = ar.get_value_matchdict_checked(request, "node_name") + node = ax.evaluate_call( + lambda: request.db.query(models.NetworkNode).filter(models.NetworkNode.name == node_name).one(), + http_error=HTTPNotFound, + msg_on_fail=s.NetworkNode_GET_NotFoundResponseSchema.description) + token = encode_jwt({"user_name": request.user.user_name}, node_name, request) + ax.evaluate_call(lambda: requests.delete(node.token_url, json={"token": token}).raise_for_status(), + http_error=HTTPInternalServerError, + msg_on_fail=s.NetworkNodeToken_DELETE_InternalServerErrorResponseSchema.description) + return ax.valid_http(http_success=HTTPOk, detail=s.NetworkNodeToken_DELETE_OkResponseSchema) + + +@s.NetworkNodesLinkAPI.get(schema=s.NetworkNodesLink_GET_RequestSchema, tags=[s.NetworkTag], + response_schemas=s.NetworkNodesLink_GET_responses) +@view_config(route_name=s.NetworkNodesLinkAPI.name, request_method="GET", permission=Authenticated) +def get_network_node_link_view(request): + token = request.POST.get("token") + node_name = jwt.decode(token, options={"verify_signature": False}).get("iss") + node = ax.evaluate_call( + lambda: request.db.query(models.NetworkNode).filter(models.NetworkNode.name == node_name).one(), + http_error=HTTPNotFound, + msg_on_fail=s.NetworkNode_GET_NotFoundResponseSchema.description) + decoded_token = decode_jwt(token, node, request) + remote_user_name = ax.evaluate_call(lambda: decoded_token["user_name"], + http_error=HTTPBadRequest, + msg_on_fail=s.NetworkNodeLink_GET_BadRequestResponseSchema.description) + requesting_user_name = ax.evaluate_call(lambda: decoded_token["requesting_user_name"], + http_error=HTTPBadRequest, + msg_on_fail=s.NetworkNodeLink_GET_BadRequestResponseSchema.description) + if requesting_user_name != request.user.user_name: + ax.raise_http(HTTPForbidden, detail=s.HTTPForbiddenResponseSchema.description) + new_remote_user = models.NetworkRemoteUser(user_id=request.user.id, network_node_id=node.id, + name=remote_user_name) + request.db.add(new_remote_user) + return ax.valid_http(http_success=HTTPOk, detail=s.NetworkNodeLink_GET_OkResponseSchema) + + +@s.NetworkNodeLinkAPI.post(tags=[s.NetworkTag], response_schemas=s.NetworkNodeLink_POST_responses) +@view_config(route_name=s.NetworkNodeLinkAPI.name, request_method="POST", permission=Authenticated) +def post_network_node_link_view(request): + node_name = ar.get_value_matchdict_checked(request, "node_name") + node = ax.evaluate_call( + lambda: request.db.query(models.NetworkNode).filter(models.NetworkNode.name == node_name).one(), + http_error=HTTPNotFound, + msg_on_fail=s.NetworkNode_GET_NotFoundResponseSchema.description + ) + location_tuple = up.urlparse(node.authorization_url) + location_query_list = up.parse_qsl(location_tuple.query) + location_query_list.extend(( + ("token", encode_jwt({"user_name": request.user.user_name}, node.name, request)), + ("response_type", "id_token"), + ("redirect_uri", request.route_url(s.NetworkNodesLinkAPI.name)) + )) + location = up.urlunparse(location_tuple._replace(query=up.urlencode(location_query_list, doseq=True))) + return ax.valid_http(http_success=HTTPTemporaryRedirect, http_kwargs={"location": location}) diff --git a/magpie/api/management/network/remote_user/__init__.py b/magpie/api/management/network/remote_user/__init__.py new file mode 100644 index 000000000..5b1a5ba98 --- /dev/null +++ b/magpie/api/management/network/remote_user/__init__.py @@ -0,0 +1,13 @@ +from magpie.utils import get_logger + +LOGGER = get_logger(__name__) + + +def includeme(config): + from magpie.api import schemas as s + LOGGER.info("Adding API network remote users ...") + config.add_route(**s.service_api_route_info(s.NetworkRemoteUsersAPI)) # /network/remote_users GET and POST + config.add_route(**s.service_api_route_info(s.NetworkRemoteUserAPI)) # /network/remote_users/{remote_user_name} GET, DELETE, PATCH + config.add_route(**s.service_api_route_info(s.NetworkRemoteUsersCurrentAPI)) # /network/remote_users/current GET + + config.scan() diff --git a/magpie/api/management/network/remote_user/remote_user_utils.py b/magpie/api/management/network/remote_user/remote_user_utils.py new file mode 100644 index 000000000..c47b71c1f --- /dev/null +++ b/magpie/api/management/network/remote_user/remote_user_utils.py @@ -0,0 +1,47 @@ +from typing import TYPE_CHECKING + +from pyramid.httpexceptions import HTTPNotFound, HTTPForbidden + +from magpie import models +from magpie.api import exception as ax +from magpie.api import requests as ar +from magpie.api import schemas as s +from magpie.constants import get_constant + +if TYPE_CHECKING: + from typing import Optional + from pyramid.request import Request + from magpie.typedefs import Session, Str + + +def _remote_user_from_names(node_name, remote_user_name, db_session): + # type: (Str, Str, Session) -> models.NetworkRemoteUser + return (db_session.query(models.NetworkRemoteUser) + .join(models.NetworkNode) + .filter(models.NetworkRemoteUser.name == remote_user_name) + .filter(models.NetworkNode.name == node_name) + .one()) + + +def requested_remote_user(request): + # type: (Request) -> models.NetworkRemoteUser + node_name = ar.get_value_matchdict_checked(request, "node_name") + remote_user_name = ar.get_value_matchdict_checked(request, "remote_user_name") + remote_user = ax.evaluate_call( + lambda: _remote_user_from_names(node_name, remote_user_name, request.db), + http_error=HTTPNotFound, + msg_on_fail=s.NetworkRemoteUser_GET_NotFoundResponseSchema.description) + return remote_user + + +def check_remote_user_access_permissions(request, remote_user=None): + # type: (Request, Optional[models.NetworkRemoteUser]) -> None + if remote_user is None: + remote_user = requested_remote_user(request) + admin_group = get_constant("MAGPIE_ADMIN_GROUP", settings_container=request) + is_admin = admin_group in [group.group_name for group in request.user.groups] + is_logged_user = request.user.user_name == remote_user.user.user_name + if not (is_admin or is_logged_user): + # admins can access any remote user, other users can only delete remote users associated with themselves + ax.raise_http(http_error=HTTPForbidden, + detail=s.HTTPForbiddenResponseSchema.description) diff --git a/magpie/api/management/network/remote_user/remote_user_views.py b/magpie/api/management/network/remote_user/remote_user_views.py new file mode 100644 index 000000000..9111c9c8c --- /dev/null +++ b/magpie/api/management/network/remote_user/remote_user_views.py @@ -0,0 +1,120 @@ +from pyramid.httpexceptions import ( + HTTPBadRequest, + HTTPNotFound, + HTTPOk, + HTTPCreated, + HTTPForbidden +) +from pyramid.security import Authenticated + +from pyramid.view import view_config + +from magpie import models +from magpie.api import exception as ax +from magpie.api import schemas as s +from magpie.api.management.network.remote_user.remote_user_utils import requested_remote_user, \ + check_remote_user_access_permissions +from magpie.constants import protected_user_name_regex + + +@s.NetworkRemoteUsersAPI.get(tags=[s.NetworkTag], response_schemas=s.NetworkRemoteUsers_GET_responses) +@view_config(route_name=s.NetworkRemoteUsersAPI.name, request_method="GET") +def get_network_remote_users_view(request): + nodes = [n.as_dict() for n in request.db.query(models.NetworkRemoteUser).all()] + return ax.valid_http(http_success=HTTPOk, detail=s.NetworkRemoteUsers_GET_OkResponseSchema.description, + content={"nodes": nodes}) + + +@s.NetworkRemoteUserAPI.get(tags=[s.NetworkTag], response_schemas=s.NetworkRemoteUser_GET_responses) +@view_config(route_name=s.NetworkRemoteUserAPI.name, request_method="GET", permission=Authenticated) +def get_network_remote_user_view(request): + remote_user = requested_remote_user(request) + check_remote_user_access_permissions(request, remote_user) + return ax.valid_http(http_success=HTTPOk, detail=s.NetworkRemoteUser_GET_OkResponseSchema.description, + content=remote_user.as_dict()) + + +@s.NetworkRemoteUsersAPI.post(schema=s.NetworkRemoteUsers_POST_RequestSchema, tags=[s.NetworkTag], + response_schemas=s.NetworkRemoteUsers_POST_responses) +@view_config(route_name=s.NetworkRemoteUsersAPI.name, request_method="POST") +def post_network_remote_users_view(request): + required_params = ("remote_user_name", "user_name", "node_name") + for param in required_params: + if param not in request.POST: + ax.raise_http(http_error=HTTPBadRequest, + detail=s.NetworkRemoteUsers_POST_BadRequestResponseSchema.description) + node = ax.evaluate_call( + lambda: request.db.query(models.NetworkNode).filter(models.NetworkNode.name == request.POST["node_name"]).one(), + http_error=HTTPNotFound, + msg_on_fail="No network node with name '{}' found".format(request.POST["node_name"]) + ) + forbidden_user_names_regex = protected_user_name_regex(include_admin=False, settings_container=request) + user_name = request.POST["user_name"] + ax.verify_param(user_name, not_matches=True, param_compare=forbidden_user_names_regex, + param_name="user_name", + http_error=HTTPForbidden, content={"user_name": user_name}, + msg_on_fail=s.NetworkRemoteUsers_POST_ForbiddenResponseSchema.description) + user = ax.evaluate_call( + lambda: request.db.query(models.User).filter(models.User.user_name == request.POST["user_name"]).one(), + http_error=HTTPNotFound, + msg_on_fail="No user with user_name '{}' found".format(request.POST["user_name"]) + ) + remote_user_name = request.POST["remote_user_name"] + ax.verify_param(remote_user_name, not_empty=True, + param_name="remote_user_name", + http_error=HTTPForbidden, + msg_on_fail="remote_user_name is empty") + remote_user = models.NetworkRemoteUser(user_id=user.id, network_node_id=node.id, name=remote_user_name) + request.db.add(remote_user) + return ax.valid_http(http_success=HTTPCreated, detail=s.NetworkRemoteUsers_POST_CreatedResponseSchema.description) + + +@s.NetworkRemoteUserAPI.patch(schema=s.NetworkRemoteUser_PATCH_RequestSchema, + tags=[s.NetworkTag], response_schemas=s.NetworkRemoteUser_PATCH_responses) +@view_config(route_name=s.NetworkRemoteUserAPI.name, request_method="PATCH") +def patch_network_remote_user_view(request): + update_params = [p for p in request.POST if p in ("remote_user_name", "user_name", "node_name")] + if not update_params: + ax.raise_http(http_error=HTTPBadRequest, detail=s.NetworkRemoteUser_PATCH_BadRequestResponseSchema.description) + remote_user = requested_remote_user(request) + if "remote_user_name" in request.POST: + remote_user_name = request.POST["remote_user_name"] + ax.verify_param(remote_user_name, not_empty=True, + param_name="remote_user_name", + http_error=HTTPForbidden, + msg_on_fail="remote_user_name is empty") + remote_user.name = remote_user_name + if "user_name" in request.POST: + user = ax.evaluate_call( + lambda: request.db.query(models.User).filter(models.User.user_name == request.POST["user_name"]).one(), + http_error=HTTPNotFound, + msg_on_fail="No user with user_name '{}' found".format(request.POST["user_name"]) + ) + remote_user.user_id = user.id + if "node_name" in request.POST: + node = ax.evaluate_call( + lambda: request.db.query(models.NetworkNode).filter( + models.NetworkNode.name == request.POST["node_name"]).one(), + http_error=HTTPNotFound, + msg_on_fail="No network node with name '{}' found".format(request.POST["node_name"]) + ) + remote_user.network_node_id = node.id + return ax.valid_http(http_success=HTTPOk, detail=s.NetworkRemoteUsers_PATCH_OkResponseSchema.description) + + +@s.NetworkRemoteUserAPI.delete(tags=[s.NetworkTag], response_schemas=s.NetworkRemoteUser_DELETE_responses) +@view_config(route_name=s.NetworkRemoteUserAPI.name, request_method="DELETE", permission=Authenticated) +def delete_network_remote_user_view(request): + remote_user = requested_remote_user(request) + check_remote_user_access_permissions(request, remote_user) + request.db.delete(remote_user) + return ax.valid_http(http_success=HTTPOk, detail=s.NetworkRemoteUsers_DELETE_OkResponseSchema.description) + + +@s.NetworkRemoteUsersCurrentAPI.get(tags=[s.NetworkTag], response_schemas=s.NetworkRemoteUsersCurrent_GET_responses) +@view_config(route_name=s.NetworkRemoteUsersCurrentAPI.name, request_method="GET", permission=Authenticated) +def get_network_remote_users_current_view(request): + nodes = [n.as_dict() for n in + request.db.query(models.NetworkRemoteUser).filter(models.NetworkRemoteUser.user_id == request.user.id)] + return ax.valid_http(http_success=HTTPOk, detail=s.NetworkRemoteUsers_GET_OkResponseSchema.description, + content={"nodes": nodes}) diff --git a/magpie/api/management/network_node/__init__.py b/magpie/api/management/network_node/__init__.py deleted file mode 100644 index c3d86e86d..000000000 --- a/magpie/api/management/network_node/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -from magpie.api import schemas as s -from magpie.constants import get_constant -from magpie.utils import get_logger - -LOGGER = get_logger(__name__) - - -def includeme(config): - if get_constant("MAGPIE_NETWORK_ENABLED", config, settings_name="magpie.network_enabled"): - LOGGER.info("Adding API network node...") - config.add_route(**s.service_api_route_info(s.NetworkNodeAPI)) - config.add_route(**s.service_api_route_info(s.NetworkNodesAPI)) - config.scan() diff --git a/magpie/api/management/network_node/network_node_utils.py b/magpie/api/management/network_node/network_node_utils.py deleted file mode 100644 index 6955cef81..000000000 --- a/magpie/api/management/network_node/network_node_utils.py +++ /dev/null @@ -1,52 +0,0 @@ -from typing import TYPE_CHECKING - -from magpie import models -from magpie.cli.register_defaults import register_user_with_group -from magpie.constants import get_constant - -if TYPE_CHECKING: - from magpie.typedefs import Str - from pyramid.request import Request - - -def create_network_node(request, name, url): - # type: (Request, Str, Str) -> None - """ - Create a NetworkNode with the given name and url. - """ - models.NetworkNode(name=name, url=url) - name = "{}{}".format(get_constant("MAGPIE_NETWORK_NAME_PREFIX"), name) - - # create an anonymous user and group for this network node - register_user_with_group(user_name=name, - group_name=name, - email=get_constant("MAGPIE_ANONYMOUS_EMAIL"), - password=None, # autogen, value doesn't matter as no login applicable, just make it valid - db_session=request.db) - - group = models.GroupService.by_group_name(name, db_session=request.db) - group.description = "Group for users who have accounts on the networked Magpie instance named '{}'.".format(name) - group.discoverable = False - - # add the anonymous user to a group for all users in the network (from nodes other than this one). - register_user_with_group(user_name=name, - group_name=get_constant("MAGPIE_NETWORK_GROUP_NAME"), - email=get_constant("MAGPIE_ANONYMOUS_EMAIL"), - password=None, - db_session=request.db) - - group = models.GroupService.by_group_name(get_constant("MAGPIE_NETWORK_GROUP_NAME"), db_session=request.db) - group.description = "Group for users who have accounts on a different Magpie instance on this network.".format(name) - group.discoverable = False - - -def delete_network_node(request, node): - # type: (Request, Str) -> None - """ - Delete a NetworkNode and the associated anonymous user. - """ - anonymous_user = node.anonymous_user - if anonymous_user: - anonymous_user.delete() - node.delete() - request.tm.commit() diff --git a/magpie/api/management/network_node/network_node_views.py b/magpie/api/management/network_node/network_node_views.py deleted file mode 100644 index 8b84535e3..000000000 --- a/magpie/api/management/network_node/network_node_views.py +++ /dev/null @@ -1,88 +0,0 @@ -from pyramid.httpexceptions import ( - HTTPBadRequest, - HTTPConflict, - HTTPNotFound, - HTTPOk, - HTTPCreated, - HTTPInternalServerError -) - -from pyramid.view import view_config - -from magpie import models -from magpie.api import exception as ax -from magpie.api import requests as ar -from magpie.api import schemas as s -from magpie.api.management.network_node.network_node_utils import create_network_node, delete_network_node -from magpie.constants import get_constant - - -@s.NetworkNodesAPI.get(tags=[s.NetworkNodeTag], response_schemas=s.NetworkNodes_GET_responses) -@view_config(route_name=s.NetworkNodesAPI.name, request_method="GET") -def get_network_nodes_view(request): - nodes = [{"name": n.name, "url": n.url} for n in request.db.query(models.NetworkNode).all()] - return ax.valid_http(http_success=HTTPOk, detail=s.NetworkNode_GET_OkResponseSchema.description, - content={"nodes": nodes}) - - -@s.NetworkNodeAPI.get(tags=[s.NetworkNodeTag], response_schemas=s.NetworkNode_GET_responses) -@view_config(route_name=s.NetworkNodeAPI.name, request_method="GET") -def get_network_node_view(request): - node_name = ar.get_value_matchdict_checked(request, "node_name") - node = ax.evaluate_call( - lambda: request.db.query(models.NetworkNode).filter(models.NetworkNode.name == node_name).one(), - http_error=HTTPNotFound, - msg_on_fail=s.NetworkNode_GET_NotFoundResponseSchema.description) - return ax.valid_http(http_success=HTTPOk, detail=s.NetworkNode_GET_OkResponseSchema.description, - content={"name": node.name, "url": node.url}) - - -@s.NetworkNodesAPI.post(schema=s.NetworkNode_POST_RequestBodySchema, tags=[s.NetworkNodeTag], - response_schemas=s.NetworkNodes_POST_responses) -@view_config(route_name=s.NetworkNodesAPI.name, request_method="POST") -def create_network_node_view(request): - node_name = request.POST.get("name") - node_url = request.POST.get("url") - ax.verify_param(node_url, matches=True, param_compare=r'[\w-]+', http_error=HTTPBadRequest, param_name="name") - ax.verify_param(node_url, matches=True, param_compare=ax.URL_REGEX, http_error=HTTPBadRequest, param_name="url") - ax.evaluate_call(lambda: create_network_node(request, node_name, node_url), - http_error=HTTPConflict, - msg_on_fail=s.NetworkNodes_POST_ConflictResponseSchema.description) - return ax.valid_http(http_success=HTTPCreated, detail=s.NetworkNode_GET_OkResponseSchema.description) - - -@s.NetworkNodeAPI.put(schema=s.NetworkNode_PUT_RequestBodySchema, - tags=[s.NetworkNodeTag], response_schemas=s.NetworkNode_PUT_responses) -@view_config(route_name=s.NetworkNodeAPI.name, request_method="PUT") -def update_network_node_view(request): - node_name = ar.get_value_matchdict_checked(request, "node_name") - node = ax.evaluate_call( - lambda: request.db.query(models.NetworkNode).filter(models.NetworkNode.name == node_name).one(), - http_error=HTTPNotFound, - msg_on_fail=s.NetworkNode_GET_NotFoundResponseSchema.description) - new_name = request.POST.get("name") - new_url = request.POST.get("url") - ax.verify_param(any([new_name, new_url]), is_true=True, - http_error=HTTPBadRequest, msg_on_fail=s.BadRequestResponseSchema.description) - node.name = new_name or node.name - node.url = new_url or node.url - ax.evaluate_call(lambda: request.tm.commit(), - http_error=HTTPConflict, - fallback=lambda: request.db.rollback(), - msg_on_fail=s.NetworkNode_PUT_ConflictResponseSchema.description) - return ax.valid_http(http_success=HTTPOk, detail=s.NetworkNode_GET_OkResponseSchema.description) - - -@s.NetworkNodeAPI.delete(tags=[s.NetworkNodeTag], response_schemas=s.NetworkNode_DELETE_responses) -@view_config(route_name=s.NetworkNodeAPI.name, request_method="DELETE") -def delete_network_node_view(request): - node_name = ar.get_value_matchdict_checked(request, "node_name") - node = ax.evaluate_call( - lambda: request.db.query(models.NetworkNode).filter(models.NetworkNode.name == node_name).one(), - http_error=HTTPNotFound, - msg_on_fail=s.NetworkNode_GET_NotFoundResponseSchema.description) - ax.evaluate_call(lambda: delete_network_node(request, node), - http_error=HTTPInternalServerError, - fallback=lambda: request.db.rollback(), - msg_on_fail=s.InternalServerErrorResponseSchema.description) - return ax.valid_http(http_success=HTTPOk, detail=s.NetworkNode_DELETE_OkResponseSchema.description) diff --git a/magpie/api/management/user/user_formats.py b/magpie/api/management/user/user_formats.py index 804e5acbf..7783468a9 100644 --- a/magpie/api/management/user/user_formats.py +++ b/magpie/api/management/user/user_formats.py @@ -4,7 +4,7 @@ from pyramid.httpexceptions import HTTPInternalServerError from magpie.api.exception import evaluate_call -from magpie.constants import get_constant, protected_user_name_regex +from magpie.constants import protected_user_name_regex from magpie.models import UserGroupStatus, UserStatuses if TYPE_CHECKING: diff --git a/magpie/api/management/user/user_utils.py b/magpie/api/management/user/user_utils.py index 9862ff810..d5cadd75e 100644 --- a/magpie/api/management/user/user_utils.py +++ b/magpie/api/management/user/user_utils.py @@ -33,7 +33,7 @@ get_permission_update_params, process_webhook_requests ) -from magpie.constants import get_constant, protected_user_name_regex +from magpie.constants import get_constant, protected_user_name_regex, protected_user_email_regex from magpie.models import TemporaryToken, TokenOperation from magpie.permissions import PermissionSet, PermissionType, format_permissions from magpie.services import SERVICE_TYPE_DICT, service_factory @@ -206,12 +206,6 @@ def update_user(user, request, new_user_name=None, new_password=None, new_email= with_param=False, # params are not useful in response for this case content={"user_name": user.user_name}, http_error=HTTPBadRequest, msg_on_fail=s.User_PATCH_BadRequestResponseSchema.description) - if get_constant("MAGPIE_NETWORK_ENABLED", request, settings_name="magpie.network_enabled"): - anonymous_regex = protected_user_name_regex(include_admin=False, settings_container=request) - ax.verify_param(new_user_name, not_matches=True, param_compare=anonymous_regex, - with_param=False, # params are not useful in response for this case - content={"user_name": user.user_name}, - http_error=HTTPBadRequest, msg_on_fail=s.User_PATCH_BadRequestResponseSchema.description) # FIXME: disable email edit when self-registration is enabled to avoid not having any confirmation of new email # (see https://github.com/Ouranosinc/Magpie/issues/436) update_email_admin_only = False @@ -219,7 +213,6 @@ def update_user(user, request, new_user_name=None, new_password=None, new_email= update_email_admin_only = asbool(get_constant("MAGPIE_USER_REGISTRATION_ENABLED", request, default_value=False, print_missing=True, raise_missing=False, raise_not_set=False)) - # user name/status change is admin-only operation if update_username or update_status or update_email_admin_only: err_msg = s.User_PATCH_ForbiddenResponseSchema.description @@ -858,8 +851,8 @@ def get_user_service_resources_permissions_dict(user, service, request, def check_user_info(user_name=None, email=None, password=None, group_name=None, # required unless disabled explicitly - check_name=True, check_email=True, check_password=True, check_group=True): - # type: (Str, Str, Str, Str, bool, bool, bool, bool) -> None + check_name=True, check_email=True, check_password=True, check_group=True, check_anonymous=True): + # type: (Str, Str, Str, Str, bool, bool, bool, bool, bool) -> None """ Validates provided user information to ensure they are adequate for user creation. @@ -883,6 +876,11 @@ def check_user_info(user_name=None, email=None, password=None, group_name=None, ax.verify_param(user_name, param_compare=name_logged, not_equal=True, param_name="user_name", http_error=HTTPBadRequest, msg_on_fail=s.Users_CheckInfo_ReservedKeyword_BadRequestResponseSchema.description) + if check_anonymous: + anonymous_user_name_regex = protected_user_name_regex() + ax.verify_param(user_name, not_matches=True, param_compare=anonymous_user_name_regex, param_name="user_name", + http_error=HTTPBadRequest, + msg_on_fail=s.Users_CheckInfo_Email_BadRequestResponseSchema.description) if check_email: ax.verify_param(email, not_none=True, not_empty=True, param_name="email", http_error=HTTPBadRequest, @@ -890,6 +888,11 @@ def check_user_info(user_name=None, email=None, password=None, group_name=None, ax.verify_param(email, matches=True, param_compare=ax.EMAIL_REGEX, param_name="email", http_error=HTTPBadRequest, msg_on_fail=s.Users_CheckInfo_Email_BadRequestResponseSchema.description) + if check_anonymous: + anonymous_email_regex = protected_user_email_regex() + ax.verify_param(email, not_matches=True, param_compare=anonymous_email_regex, param_name="email", + http_error=HTTPBadRequest, + msg_on_fail=s.Users_CheckInfo_Email_BadRequestResponseSchema.description) if check_password: ax.verify_param(password, not_none=True, not_empty=True, param_name="password", is_type=True, param_compare=six.string_types, # no match since it can be any character diff --git a/magpie/api/schemas.py b/magpie/api/schemas.py index c528e62b6..07ce00ecf 100644 --- a/magpie/api/schemas.py +++ b/magpie/api/schemas.py @@ -2,6 +2,7 @@ from typing import TYPE_CHECKING import colander +import jwt import six from cornice import Service from cornice.service import get_services @@ -276,19 +277,37 @@ def service_api_route_info(service_api, **kwargs): TemporaryUrlAPI = Service( path="/tmp/{token}", # nosec: B108 name="temporary_url") -TokenAPI = Service( - path="/token", - name="Token") -TokenValidateAPI = Service( - path="/token/validate", - name="TokenValidate") NetworkNodeAPI = Service( - path="/network-nodes/{node_name}", + path="/network/nodes/{node_name}", name="NetworkNode") NetworkNodesAPI = Service( - path="/network-nodes", + path="/network/nodes", name="NetworkNodes") - +NetworkNodeTokenAPI = Service( + path="/network/nodes/{node_name}/token", + name="NetworkNodeToken") +NetworkNodesLinkAPI = Service( + path="/network/nodes/link", + name="NetworkNodesLink") +NetworkNodeLinkAPI = Service( + path="/network/nodes/{node_name}/link", + name="NetworkNodeLink") +NetworkRemoteUserAPI = Service( + path="/network/nodes/{node_name}/remote_users/{remote_user_name}", + name="NetworkRemoteUser") +NetworkRemoteUsersAPI = Service( + path="/network/remote_users", + name="NetworkRemoteUsers") +NetworkRemoteUsersCurrentAPI = Service( + path="/network/remote_users/current", + name="NetworkRemoteUserCurrent") +NetworkTokenAPI = Service( + path="/network/token", + name="NetworkToken") +NetworkJSONWebKeySetAPI = Service( + path="/network/jwks", + name="NetworkJSONWebKeySet" +) # Path parameters GroupNameParameter = colander.SchemaNode( @@ -328,19 +347,6 @@ def service_api_route_info(service_api, **kwargs): colander.String(), description="Temporary URL token.", example=str(uuid.uuid4())) -NetworkTokenParameter = colander.SchemaNode( - colander.String(), - description="Network token", - example=str(uuid.uuid4())) -NetworkNodeNameParameter = colander.SchemaNode( - colander.String(), - description="Network Node name.", - example="node") -NetworkNodeUrlParameter = colander.SchemaNode( - colander.String(), - description="Public URL of remote Magpie instance.", - example="http://node.example.com/magpie", - validator=colander.url) class ServiceType_RequestPathSchema(colander.MappingSchema): @@ -439,7 +445,7 @@ class TemporaryURL_RequestPathSchema(colander.MappingSchema): RegisterTag = "Register" ResourcesTag = "Resource" ServicesTag = "Service" -NetworkNodeTag = "Network Node" +NetworkTag = "Network" TAG_DESCRIPTIONS = { APITag: "General information about the API.", @@ -459,7 +465,7 @@ class TemporaryURL_RequestPathSchema(colander.MappingSchema): RegisterTag: "Registration paths for operations available to users (including non-administrators).", ResourcesTag: "Management of resources that reside under a given service and their applicable permissions.", ServicesTag: "Management of service definitions, children resources and their applicable permissions.", - NetworkNodeTag: "Management of references to other Magpie instances in the network." + NetworkTag: "Management of references to other Magpie instances, user and access tokens in the network." } # Header definitions @@ -673,6 +679,7 @@ class ErrorVerifyParamConditions(colander.MappingSchema): is_false = colander.SchemaNode(colander.Boolean(), missing=colander.drop) is_type = colander.SchemaNode(colander.Boolean(), missing=colander.drop) matches = colander.SchemaNode(colander.Boolean(), missing=colander.drop) + not_matches = colander.SchemaNode(colander.Boolean(), missing=colander.drop) class ErrorVerifyParamBodySchema(colander.MappingSchema): @@ -3339,104 +3346,6 @@ class Session_GET_OkResponseSchema(BaseResponseSchemaAPI): body = Session_GET_ResponseBodySchema(code=HTTPOk.code, description=description) -class Token_PATCH_OkResponseBodySchema(BaseResponseBodySchema): - token = NetworkTokenParameter - - -class Token_PATCH_OkResponseSchema(BaseResponseSchemaAPI): - description = "Get token successful." - body = Token_PATCH_OkResponseBodySchema(code=HTTPOk.code, description=description) - - -class TokenValidate_GET_BadRequestResponseSchema(BaseResponseSchemaAPI): - description = "Invalid Token." - body = BaseResponseBodySchema(code=HTTPBadRequest.code, description=description) - - -class TokenValidate_BodySchema(colander.MappingSchema): - token = NetworkTokenParameter - - -class TokenValidate_GET_RequestBodySchema(BaseRequestSchemaAPI): - body = TokenValidate_BodySchema() - - -class TokenValidate_GET_OkResponseBodySchema(BaseResponseBodySchema): - user_name = UserNameParameter - - -class TokenValidate_GET_OkResponseSchema(BaseResponseSchemaAPI): - description = "Validate token successful." - body = TokenValidate_GET_OkResponseBodySchema(code=HTTPOk.code, description=description) - - -class NetworkNode_GET_NotFoundResponseSchema(BaseResponseSchemaAPI): - description = "Network Node could not be found." - body = ErrorResponseBodySchema(code=HTTPNotFound.code, description=description) - - -class NetworkNodes_GET_NotFoundResponseSchema(BaseResponseSchemaAPI): - description = "Network Nodes could not be found." - body = ErrorResponseBodySchema(code=HTTPNotFound.code, description=description) - - -class NetworkNode_GET_OkResponseBodySchema(BaseResponseBodySchema): - name = NetworkNodeNameParameter - url = NetworkNodeUrlParameter - - -class NetworkNode_GET_OkResponseSchema(BaseResponseSchemaAPI): - description = "Network Node found." - body = NetworkNode_GET_OkResponseBodySchema(code=HTTPOk.code, description=description) - - -class NetworkNode_BodySchema(colander.MappingSchema): - name = NetworkNodeNameParameter - url = NetworkNodeUrlParameter - - -class NetworkNodesSequence(colander.SequenceSchema): - node = NetworkNode_BodySchema() - -class NetworkNodes_GET_OkResponseBodySchema(BaseResponseBodySchema): - nodes = NetworkNodesSequence() - - -class NetworkNodes_GET_OkResponseSchema(BaseResponseSchemaAPI): - description = "Network Nodes found." - body = NetworkNodes_GET_OkResponseBodySchema(code=HTTPOk.code, description=description) - - -class NetworkNode_POST_RequestBodySchema(BaseRequestSchemaAPI): - body = NetworkNode_BodySchema() - - -NetworkNode_PUT_RequestBodySchema = NetworkNode_POST_RequestBodySchema - - -class NetworkNodes_POST_CreatedResponseSchema(BaseResponseSchemaAPI): - description = "Network Node created." - body = BaseResponseBodySchema(code=HTTPCreated.code, description=description) - - -class NetworkNodes_POST_ConflictResponseSchema(BaseResponseSchemaAPI): - description = "Network Node already exists with conflicting attributes." - body = ErrorResponseBodySchema(code=HTTPNotFound.code, description=description) - - -NetworkNode_PUT_ConflictResponseSchema = NetworkNodes_POST_ConflictResponseSchema - - -class NetworkNode_PUT_OkResponseSchema(BaseResponseSchemaAPI): - description = "Network Node updated." - body = BaseResponseBodySchema(code=HTTPOk.code, description=description) - - -class NetworkNode_DELETE_OkResponseSchema(BaseResponseSchemaAPI): - description = "Network Node deleted." - body = BaseResponseBodySchema(code=HTTPOk.code, description=description) - - class Session_GET_InternalServerErrorResponseSchema(BaseResponseSchemaAPI): description = "Failed to get session details." body = InternalServerErrorResponseSchema() @@ -3502,11 +3411,6 @@ class ProviderSignin_GET_UnauthorizedResponseSchema(BaseResponseSchemaAPI): body = ErrorResponseBodySchema(code=HTTPUnauthorized.code, description=description) -class ProviderSignin_GET_UnauthorizedTokenSchema(BaseResponseSchemaAPI): - description = "Unauthorized token in Authorization headers." - body = ErrorResponseBodySchema(code=HTTPUnauthorized.code, description=description) - - class ProviderSignin_GET_ForbiddenResponseSchema(BaseResponseSchemaAPI): description = "Forbidden 'Homepage-Route' host not matching Magpie refused for security reasons." body = ErrorResponseBodySchema(code=HTTPForbidden.code, description=description) @@ -3618,6 +3522,357 @@ class SwaggerAPI_GET_OkResponseSchema(colander.MappingSchema): body = colander.SchemaNode(colander.String(), example="This page!") +class JWTRequestBodySchema(colander.MappingSchema): + token = colander.SchemaNode( + colander.String(), + description="JSON Web Token.", + example=jwt.encode({"example": "content"}, "example_secret", algorithm="HS256") + ) + + +class NetworkToken_POST_RequestSchema(BaseRequestSchemaAPI): + body = JWTRequestBodySchema() + + +class NetworkToken_DELETE_RequestSchema(BaseRequestSchemaAPI): + body = JWTRequestBodySchema() + + +class NetworkNode_PATCH_RequestBodySchema(colander.MappingSchema): + name = colander.SchemaNode( + colander.String(), + description="Name of another Magpie node (instance) in the network.", + example="NodeA", + missing=colander.drop + ) + jwks_url = colander.SchemaNode( + colander.String(), + description="URL that provides the JWKS data for another Magpie node (instance) in the network.", + example="https://nodea.example.com/jwks.json", + validator=colander.url, + missing=colander.drop + ) + token_url = colander.SchemaNode( + colander.String(), + description="URL that provides the Oauth authorize endpoint for another Magpie node (instance) in the network.", + example="https://nodea.example.com/ui/network/authorize", + validator=colander.url, + missing=colander.drop + ) + authorization_url = colander.SchemaNode( + colander.String(), + description="URL that provides the Oauth token endpoint for another Magpie node (instance) in the network.", + example="https://nodea.example.com/network/token", + validator=colander.url, + missing=colander.drop + ) + redirect_uris = colander.SchemaNode( + colander.String(), + description="Space delimited list of valid redirect URIs for another Magpie node (instance) in the " + "network.", + example="https://node.example.com/network/nodes/link https://node.example.com/some/other/uri", + missing=colander.drop + ) + + +class NetworkNode_PATCH_RequestSchema(BaseRequestSchemaAPI): + body = NetworkNode_PATCH_RequestBodySchema() + + +class NetworkNode_GET_NotFoundResponseSchema(BaseResponseSchemaAPI): + description = "Network Node could not be found." + body = ErrorResponseBodySchema(code=HTTPNotFound.code, description=description) + + +class NetworkNodes_GET_NotFoundResponseSchema(BaseResponseSchemaAPI): + description = "Network Nodes could not be found." + body = ErrorResponseBodySchema(code=HTTPNotFound.code, description=description) + + +class NetworkNode_BodySchema(colander.MappingSchema): + name = colander.SchemaNode( + colander.String(), + description="Name of another Magpie node (instance) in the network.", + example="NodeA" + ) + jwks_url = colander.SchemaNode( + colander.String(), + description="URL that provides the JWKS data for another Magpie node (instance) in the network.", + example="https://nodea.example.com/jwks.json", + validator=colander.url + ) + token_url = colander.SchemaNode( + colander.String(), + description="URL that provides the Oauth authorize endpoint for another Magpie node (instance) in the network.", + example="https://nodea.example.com/ui/network/authorize", + validator=colander.url, + ) + authorization_url = colander.SchemaNode( + colander.String(), + description="URL that provides the Oauth token endpoint for another Magpie node (instance) in the network.", + example="https://nodea.example.com/network/token", + validator=colander.url, + ) + redirect_uris = colander.SchemaNode( + colander.String(), + description="Space delimited list of valid redirect URIs for another Magpie node (instance) in the network", + example="https://node.example.com/network/nodes/link https://node.example.com/some/other/uri", + ) + + +NetworkNode_GET_OkResponseBodySchema = NetworkNode_BodySchema + + +class NetworkNode_GET_OkResponseSchema(BaseResponseSchemaAPI): + description = "Network Node found." + body = NetworkNode_GET_OkResponseBodySchema(code=HTTPOk.code, description=description) + + +class NetworkNodesSequence(colander.SequenceSchema): + node = NetworkNode_BodySchema() + + +class NetworkNodes_GET_OkResponseBodySchema(BaseResponseBodySchema): + nodes = NetworkNodesSequence() + + +class NetworkNodes_GET_OkResponseSchema(BaseResponseSchemaAPI): + description = "Network Nodes found." + body = NetworkNodes_GET_OkResponseBodySchema(code=HTTPOk.code, description=description) + + +class NetworkNode_POST_RequestSchema(BaseRequestSchemaAPI): + body = NetworkNode_BodySchema() + + +class NetworkNodes_POST_CreatedResponseSchema(BaseResponseSchemaAPI): + description = "Network Node created." + body = BaseResponseBodySchema(code=HTTPCreated.code, description=description) + + +class NetworkNodes_POST_BadRequestResponseSchema(BaseResponseSchemaAPI): + description = "Missing required parameter." + body = BaseResponseBodySchema(code=HTTPBadRequest.code, description=description) + + +class NetworkNodes_PATCH_BadRequestResponseSchema(BaseResponseSchemaAPI): + description = "Missing parameters to update network node." + body = BaseResponseBodySchema(code=HTTPBadRequest.code, description=description) + + +class NetworkNodes_CheckInfo_NameValue_ConflictResponseSchema(BaseResponseSchemaAPI): + description = "Network Node already exists with the same name." + body = ErrorResponseBodySchema(code=HTTPConflict.code, description=description) + + +class NetworkNodes_CheckInfo_NameValue_BadRequestResponseSchema(BaseResponseSchemaAPI): + description = "name is not valid." + body = ErrorResponseBodySchema(code=HTTPBadRequest.code, description=description) + + +class NetworkNodes_CheckInfo_JWKSURLValue_BadRequestResponseSchema(BaseResponseSchemaAPI): + description = "jwks_url is not a valid URL." + body = ErrorResponseBodySchema(code=HTTPBadRequest.code, description=description) + + +class NetworkNodes_CheckInfo_TokenURLValue_BadRequestResponseSchema(BaseResponseSchemaAPI): + description = "token_url is not a valid URL." + body = ErrorResponseBodySchema(code=HTTPBadRequest.code, description=description) + + +class NetworkNodes_CheckInfo_AuthorizationURLValue_BadRequestResponseSchema(BaseResponseSchemaAPI): + description = "authorization_url is not a valid URL." + body = ErrorResponseBodySchema(code=HTTPBadRequest.code, description=description) + + +class NetworkNodes_CheckInfo_RedirectURIsValue_BadRequestResponseSchema(BaseResponseSchemaAPI): + description = "redirect_uris contains a URI that is not valid." + body = ErrorResponseBodySchema(code=HTTPBadRequest.code, description=description) + + +class NetworkNode_PATCH_OkResponseSchema(BaseResponseSchemaAPI): + description = "Network Node updated." + body = BaseResponseBodySchema(code=HTTPOk.code, description=description) + + +class NetworkNode_DELETE_OkResponseSchema(BaseResponseSchemaAPI): + description = "Network Node deleted." + body = BaseResponseBodySchema(code=HTTPOk.code, description=description) + + +class NetworkNodesLink_GET_RequestSchema(BaseRequestSchemaAPI): + body = JWTRequestBodySchema() + + +class NetworkNodeLink_GET_OkResponseSchema(BaseResponseSchemaAPI): + description = "User successfully linked with network node." + body = BaseResponseBodySchema(code=HTTPOk.code, description=description) + + +class NetworkRemoteUser_PATCH_RequestBodySchema(colander.MappingSchema): + remote_user_name = colander.SchemaNode( + colander.String(), + description="Name of the associated user a remote Magpie node (instance) in the network.", + example="userAremote", + missing=colander.drop + ) + user_name = colander.SchemaNode( + colander.String(), + description="Name of the associated user on this Magpie node (instance).", + example="userAlocal", + missing=colander.drop + ) + node_name = colander.SchemaNode( + colander.String(), + description="Name of another Magpie node (instance) in the network.", + example="NodeA", + missing=colander.drop + ) + + +class NetworkRemoteUser_PATCH_RequestSchema(BaseRequestSchemaAPI): + body = NetworkRemoteUser_PATCH_RequestBodySchema() + + +class NetworkRemoteUser_PATCH_BadRequestResponseSchema(BaseResponseSchemaAPI): + description = "Missing parameters to update network user." + body = BaseResponseBodySchema(code=HTTPBadRequest.code, description=description) + + +class NetworkRemoteUser_GET_NotFoundResponseSchema(BaseResponseSchemaAPI): + description = "Network User could not be found." + body = ErrorResponseBodySchema(code=HTTPNotFound.code, description=description) + + +class NetworkRemoteUsers_GET_NotFoundResponseSchema(BaseResponseSchemaAPI): + description = "Network User could not be found." + body = ErrorResponseBodySchema(code=HTTPNotFound.code, description=description) + + +class NetworkRemoteUser_BodySchema(colander.MappingSchema): + remote_user_name = colander.SchemaNode( + colander.String(), + description="Name of the associated user a remote Magpie node (instance) in the network.", + example="userAremote" + ) + user_name = colander.SchemaNode( + colander.String(), + description="Name of the associated user on this Magpie node (instance).", + example="userAlocal" + ) + node_name = colander.SchemaNode( + colander.String(), + description="Name of another Magpie node (instance) in the network.", + example="NodeA" + ) + + +NetworkRemoteUser_GET_OkResponseBodySchema = NetworkRemoteUser_BodySchema + + +class NetworkRemoteUser_GET_OkResponseSchema(BaseResponseSchemaAPI): + description = "Network Node found." + body = NetworkRemoteUser_GET_OkResponseBodySchema(code=HTTPOk.code, description=description) + + +class NetworkRemoteUsers_POST_BadRequestResponseSchema(BaseResponseSchemaAPI): + description = "Missing required parameter." + body = BaseResponseBodySchema(code=HTTPBadRequest.code, description=description) + + +class NetworkRemoteUsersSequence(colander.SequenceSchema): + node = NetworkRemoteUser_BodySchema() + + +class NetworkRemoteUsers_GET_OkResponseBodySchema(BaseResponseBodySchema): + nodes = NetworkRemoteUsersSequence() + + +class NetworkRemoteUsers_GET_OkResponseSchema(BaseResponseSchemaAPI): + description = "Network Nodes found." + body = NetworkRemoteUsers_GET_OkResponseBodySchema(code=HTTPOk.code, description=description) + + +class NetworkRemoteUsers_POST_RequestSchema(BaseRequestSchemaAPI): + body = NetworkRemoteUser_BodySchema() + + +class NetworkRemoteUsers_POST_CreatedResponseSchema(BaseResponseSchemaAPI): + description = "Remote user created." + body = BaseResponseBodySchema(code=HTTPCreated.code, description=description) + + +class NetworkRemoteUsers_PATCH_OkResponseSchema(BaseResponseSchemaAPI): + description = "Remote user updated." + body = BaseResponseBodySchema(code=HTTPOk.code, description=description) + + +class NetworkRemoteUsers_DELETE_OkResponseSchema(BaseResponseSchemaAPI): + description = "Remote user deleted." + body = BaseResponseBodySchema(code=HTTPOk.code, description=description) + + +class NetworkRemoteUsers_POST_ConflictResponseSchema(BaseResponseSchemaAPI): + description = "Remote user already exists with conflicting attributes." + body = ErrorResponseBodySchema(code=HTTPNotFound.code, description=description) + + +NetworkRemoteUser_PATCH_ConflictResponseSchema = NetworkRemoteUsers_POST_ConflictResponseSchema + + +class NetworkRemoteUser_PATCH_OkResponseSchema(BaseResponseSchemaAPI): + description = "Network Node updated." + body = BaseResponseBodySchema(code=HTTPOk.code, description=description) + + +class NetworkRemoteUser_DELETE_OkResponseSchema(BaseResponseSchemaAPI): + description = "Network Node deleted." + body = BaseResponseBodySchema(code=HTTPOk.code, description=description) + + +class NetworkNodeToken_GET_OkResponseSchema(BaseResponseSchemaAPI): + description = "Successfully obtained access token." + body = BaseResponseBodySchema(code=HTTPOk.code, description=description) + + +class NetworkNodeToken_GET_InternalServerErrorResponseSchema(BaseResponseSchemaAPI): + description = "Unable to obtain an access token." + body = InternalServerErrorResponseBodySchema(code=HTTPInternalServerError.code, description=description) + + +class NetworkNodeToken_DELETE_OkResponseSchema(BaseResponseSchemaAPI): + description = "Successfully deleted access token." + body = BaseResponseBodySchema(code=HTTPOk.code, description=description) + + +class NetworkNodeToken_DELETE_InternalServerErrorResponseSchema(BaseResponseSchemaAPI): + description = "Unable to delete an access token." + body = InternalServerErrorResponseBodySchema(code=HTTPInternalServerError.code, description=description) + + +class NetworkNodeLink_GET_BadRequestResponseSchema(BaseResponseSchemaAPI): + description = "Unable to link user with network node. Missing parameters." + body = InternalServerErrorResponseBodySchema(code=HTTPBadRequest.code, description=description) + + +class NetworkToken_POST_CreatedResponseSchema(BaseResponseSchemaAPI): + description = "Access token created or refreshed." + body = BaseResponseBodySchema(code=HTTPCreated.code, description=description) + + +NetworkToken_DELETE_OkResponseSchema = NetworkNodeToken_DELETE_OkResponseSchema + + +class NetworkNodeToken_DELETE_NotFoundResponseSchema(BaseResponseSchemaAPI): + description = "Unable to delete an access token. Does not exist." + body = BaseResponseBodySchema(code=HTTPNotFound.code, description=description) + + +class NetworkRemoteUsers_POST_ForbiddenResponseSchema(BaseResponseSchemaAPI): + description = "Targeted user update not allowed by requesting user." + body = ErrorResponseBodySchema(code=HTTPForbidden.code, description=description) + + # Responses for specific views Resource_GET_responses = { "200": Resource_GET_OkResponseSchema(), @@ -4350,16 +4605,30 @@ class SwaggerAPI_GET_OkResponseSchema(colander.MappingSchema): "406": NotAcceptableResponseSchema(), "500": Session_GET_InternalServerErrorResponseSchema(), } -Token_PATCH_responses = { - "200": Token_PATCH_OkResponseSchema(), - "401": UnauthorizedResponseSchema(), +Version_GET_responses = { + "200": Version_GET_OkResponseSchema(), "406": NotAcceptableResponseSchema(), "500": InternalServerErrorResponseSchema(), } -TokenValidate_GET_responses = { - "200": TokenValidate_GET_OkResponseSchema(), - "400": TokenValidate_GET_BadRequestResponseSchema(), +Homepage_GET_responses = { + "200": Homepage_GET_OkResponseSchema(), + "406": NotAcceptableResponseSchema(), "500": InternalServerErrorResponseSchema(), +} +SwaggerAPI_GET_responses = { + "200": SwaggerAPI_GET_OkResponseSchema(), + "500": InternalServerErrorResponseSchema(), +} +NetworkToken_POST_responses = { + "201": NetworkToken_POST_CreatedResponseSchema(), + "404": NetworkNode_GET_NotFoundResponseSchema(), + "500": InternalServerErrorResponseSchema(), +} +NetworkToken_DELETE_responses = { + +} +NetworkJSONWebKeySet_GET_responses = { + } NetworkNode_GET_responses = { "200": NetworkNode_GET_OkResponseSchema(), @@ -4374,13 +4643,13 @@ class SwaggerAPI_GET_OkResponseSchema(colander.MappingSchema): NetworkNodes_POST_responses = { "201": NetworkNodes_POST_CreatedResponseSchema(), "400": BadRequestResponseSchema(), - "409": NetworkNodes_POST_ConflictResponseSchema(), + "409": NetworkNodes_CheckInfo_NameValue_ConflictResponseSchema(), "500": InternalServerErrorResponseSchema(), } -NetworkNode_PUT_responses = { - "200": NetworkNode_PUT_OkResponseSchema(), +NetworkNode_PATCH_responses = { + "200": NetworkNode_PATCH_OkResponseSchema(), "400": BadRequestResponseSchema(), - "409": NetworkNode_PUT_ConflictResponseSchema(), + "409": NetworkNodes_CheckInfo_NameValue_ConflictResponseSchema(), "500": InternalServerErrorResponseSchema(), } NetworkNode_DELETE_responses = { @@ -4388,18 +4657,48 @@ class SwaggerAPI_GET_OkResponseSchema(colander.MappingSchema): "400": BadRequestResponseSchema(), "500": InternalServerErrorResponseSchema(), } -Version_GET_responses = { - "200": Version_GET_OkResponseSchema(), - "406": NotAcceptableResponseSchema(), +NetworkNodeToken_GET_responses = { + "200": NetworkNodeToken_GET_OkResponseSchema(), + "404": NetworkNode_GET_NotFoundResponseSchema(), + "500": NetworkNodeToken_GET_InternalServerErrorResponseSchema() +} +NetworkNodeToken_DELETE_responses = { + "200": NetworkNodeToken_DELETE_OkResponseSchema(), + "404": NetworkNode_GET_NotFoundResponseSchema(), + "500": NetworkNodeToken_DELETE_InternalServerErrorResponseSchema() +} +NetworkNodesLink_GET_responses = { + +} +NetworkNodeLink_POST_responses = { + +} +NetworkRemoteUser_GET_responses = { + "200": NetworkRemoteUser_GET_OkResponseSchema(), + "404": NetworkRemoteUser_GET_NotFoundResponseSchema(), "500": InternalServerErrorResponseSchema(), } -Homepage_GET_responses = { - "200": Homepage_GET_OkResponseSchema(), - "406": NotAcceptableResponseSchema(), +NetworkRemoteUsersCurrent_GET_responses = NetworkRemoteUser_GET_responses +NetworkRemoteUsers_GET_responses = { + "200": NetworkRemoteUsers_GET_OkResponseSchema(), + "404": NetworkRemoteUsers_GET_NotFoundResponseSchema(), "500": InternalServerErrorResponseSchema(), } -SwaggerAPI_GET_responses = { - "200": SwaggerAPI_GET_OkResponseSchema(), +NetworkRemoteUsers_POST_responses = { + "201": NetworkRemoteUsers_POST_CreatedResponseSchema(), + "400": BadRequestResponseSchema(), + "409": NetworkRemoteUsers_POST_ConflictResponseSchema(), + "500": InternalServerErrorResponseSchema(), +} +NetworkRemoteUser_PATCH_responses = { + "200": NetworkRemoteUser_PATCH_OkResponseSchema(), + "400": BadRequestResponseSchema(), + "409": NetworkRemoteUser_PATCH_ConflictResponseSchema(), + "500": InternalServerErrorResponseSchema(), +} +NetworkRemoteUser_DELETE_responses = { + "200": NetworkRemoteUser_DELETE_OkResponseSchema(), + "400": BadRequestResponseSchema(), "500": InternalServerErrorResponseSchema(), } diff --git a/magpie/cli/purge_expired_network_tokens.py b/magpie/cli/purge_expired_network_tokens.py new file mode 100644 index 000000000..f6db721ba --- /dev/null +++ b/magpie/cli/purge_expired_network_tokens.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 + +""" +Removes expired network tokens from the database. + +This ensures that the network_tokens table doesn't fill up with expired tokens. +Both an expired token and a non-existent token behave the same from an access perspective (user is denied) so +it is safe to automatically remove all expired tokens. +""" + +import argparse +from typing import TYPE_CHECKING + +import transaction + +from magpie import models +from magpie.cli.utils import make_logging_options, setup_logger_from_options +from magpie.constants import get_constant +from magpie.db import get_db_session_from_config_ini +from magpie.utils import get_logger, raise_log, print_log + +if TYPE_CHECKING: + from typing import Optional, Sequence + from magpie.typedefs import Str + +LOGGER = get_logger(__name__, + message_format="%(asctime)s - %(levelname)s - %(message)s", + datetime_format="%d-%b-%y %H:%M:%S", force_stdout=False) + + +def make_parser(): + # type: () -> argparse.ArgumentParser + parser = argparse.ArgumentParser(description="Delete all expired network tokens.") + parser.add_argument("--config", "--ini", metavar="CONFIG", dest="ini_config", + default=get_constant("MAGPIE_INI_FILE_PATH"), + help="Configuration INI file to retrieve database connection settings (default: %(default)s).") + make_logging_options(parser) + return parser + + +def main(args=None, parser=None, namespace=None): + # type: (Optional[Sequence[Str]], Optional[argparse.ArgumentParser], Optional[argparse.Namespace]) -> None + if not parser: + parser = make_parser() + args = parser.parse_args(args=args, namespace=namespace) + setup_logger_from_options(LOGGER, args) + db_session = get_db_session_from_config_ini(args.ini_config) + deleted = models.NetworkToken.get_expired(db_session).delete() + try: + transaction.commit() + db_session.close() + except Exception as exc: # noqa: W0703 # nosec: B110 # pragma: no cover + db_session.rollback() + raise_log("Failed to delete expired network tokens", exception=type(exc), logger=LOGGER) + else: + if deleted: + print_log("{} expired network tokens deleted".format(deleted), logger=LOGGER) + else: + print_log("No expired network tokens found", logger=LOGGER) + + +if __name__ == "__main__": + main() diff --git a/magpie/cli/register_defaults.py b/magpie/cli/register_defaults.py index 574d6183c..c017b6fcc 100644 --- a/magpie/cli/register_defaults.py +++ b/magpie/cli/register_defaults.py @@ -56,7 +56,8 @@ def register_user_with_group(user_name, group_name, email, password, db_session) if password is None: LOGGER.debug("No password provided for user [%s], auto-generating one.", user_name) password = pseudo_random_string(length=get_constant("MAGPIE_PASSWORD_MIN_LENGTH")) - uu.check_user_info(user_name=user_name, password=password, group_name=group_name, check_email=False) + uu.check_user_info(user_name=user_name, password=password, group_name=group_name, check_email=False, + check_anonymous=False) new_user = models.User(user_name=user_name, email=email) # noqa UserService.set_password(new_user, password) db_session.add(new_user) @@ -126,7 +127,8 @@ def init_admin(db_session, settings=None): # admin user already exist, update modified password LOGGER.warning("Detected password change for 'MAGPIE_ADMIN_USER'. Attempting to update...") try: - uu.check_user_info(password=admin_password, check_name=False, check_email=False, check_group=False) + uu.check_user_info(password=admin_password, check_name=False, check_email=False, check_group=False, + check_anonymous=False) UserService.set_password(admin_usr, admin_password) UserService.regenerate_security_code(admin_usr) except Exception as http_exc: # noqa # re-raised as value error # pragma: no cover diff --git a/magpie/constants.py b/magpie/constants.py index cc4b86486..7133e31a6 100644 --- a/magpie/constants.py +++ b/magpie/constants.py @@ -108,6 +108,9 @@ def _get_default_log_level(): MAGPIE_NETWORK_ENABLED = asbool(os.getenv("MAGPIE_NETWORK_ENABLED", False)) MAGPIE_NETWORK_INSTANCE_NAME = os.getenv("MAGPIE_NETWORK_INSTANCE_NAME") MAGPIE_NETWORK_DEFAULT_TOKEN_EXPIRY = int(os.getenv("MAGPIE_NETWORK_DEFAULT_TOKEN_EXPIRY", 86400)) +MAGPIE_NETWORK_INTERNAL_TOKEN_EXPIRY = int(os.getenv("MAGPIE_NETWORK_INTERNAL_TOKEN_EXPIRY", 30)) +MAGPIE_NETWORK_PEM_FILES = os.getenv("MAGPIE_NETWORK_PEM_FILES", os.path.join(MAGPIE_ROOT, "key.pem")) +MAGPIE_NETWORK_PEM_PASSWORDS = os.getenv("MAGPIE_NETWORK_PEM_PASSWORDS") MAGPIE_LOG_LEVEL = os.getenv("MAGPIE_LOG_LEVEL", _get_default_log_level()) # log level to apply to the loggers MAGPIE_LOG_PRINT = asbool(os.getenv("MAGPIE_LOG_PRINT", False)) # log also forces print to the console MAGPIE_LOG_REQUEST = asbool(os.getenv("MAGPIE_LOG_REQUEST", True)) # log detail of every incoming request @@ -152,6 +155,7 @@ def _get_default_log_level(): MAGPIE_NETWORK_TOKEN_NAME = "magpie_token" MAGPIE_NETWORK_PROVIDER = "magpie_network" MAGPIE_NETWORK_NAME_PREFIX = "anonymous_network_" +MAGPIE_NETWORK_ANONYMOUS_EMAIL_FORMAT = "{}{}@mail.com".format(MAGPIE_NETWORK_NAME_PREFIX, "{}") MAGPIE_NETWORK_GROUP_NAME = "magpie_network" # above this length is considered a token, @@ -197,15 +201,34 @@ def protected_user_name_regex(include_admin=True, patterns.append(get_constant("MAGPIE_ADMIN_USER", settings_container=settings_container)) if include_anonymous: patterns.append(get_constant("MAGPIE_ANONYMOUS_USER", settings_container=settings_container)) - if include_network and get_constant("MAGPIE_NETWORK_ENABLED", - settings_name="magpie.network_enabled", - settings_container=settings_container): + if include_network and get_constant("MAGPIE_NETWORK_ENABLED", settings_container=settings_container): patterns.append( "{}.*".format(get_constant("MAGPIE_NETWORK_NAME_PREFIX", settings_container=settings_container)) ) return "^({})$".format("|".join(patterns)) +def protected_user_email_regex(include_admin=True, + include_anonymous=True, + include_network=True, + additional_patterns=None, + settings_container=None): + # type: (bool, bool, bool, Optional[List[Str]], Optional[AnySettingsContainer]) -> Str + """ + Return a regular expression that matches all user emails that are protected, meaning that they are generated + by Magpie itself and no regular user account should be created with these user emails. + """ + patterns = additional_patterns or [] + if include_admin: + patterns.append(get_constant("MAGPIE_ADMIN_EMAIL", settings_container=settings_container)) + if include_anonymous: + patterns.append(get_constant("MAGPIE_ANONYMOUS_EMAIL", settings_container=settings_container)) + if include_network and get_constant("MAGPIE_NETWORK_ENABLED", settings_container=settings_container): + email_form = get_constant("MAGPIE_NETWORK_ANONYMOUS_EMAIL_FORMAT", settings_container=settings_container) + patterns.append(email_form.format('.*')) + return "^({})$".format("|".join(patterns)) + + def protected_group_name_regex(include_admin=True, include_anonymous=True, include_network=True, @@ -220,9 +243,7 @@ def protected_group_name_regex(include_admin=True, patterns.append(get_constant("MAGPIE_ADMIN_GROUP", settings_container=settings_container)) if include_anonymous: patterns.append(get_constant("MAGPIE_ANONYMOUS_GROUP", settings_container=settings_container)) - if include_network and get_constant("MAGPIE_NETWORK_ENABLED", - settings_name="magpie.network_enabled", - settings_container=settings_container): + if include_network and get_constant("MAGPIE_NETWORK_ENABLED", settings_container=settings_container): patterns.append( "{}.*".format(get_constant("MAGPIE_NETWORK_NAME_PREFIX", settings_container=settings_container)) ) diff --git a/magpie/models.py b/magpie/models.py index 1f94929e9..0987516f8 100644 --- a/magpie/models.py +++ b/magpie/models.py @@ -30,6 +30,7 @@ from ziggurat_foundations.models.user_permission import UserPermissionMixin from ziggurat_foundations.models.user_resource_permission import UserResourcePermissionMixin from ziggurat_foundations.permissions import permission_to_pyramid_acls +from cryptography.fernet import Fernet from magpie.api import exception as ax from magpie.constants import get_constant @@ -998,11 +999,11 @@ def json(self): return {"token": str(self.token), "operation": str(self.operation.value)} -class NetworkUser(BaseModel, Base): +class NetworkRemoteUser(BaseModel, Base): """ Model that defines a relationship between a User and a User that is authenticated on a different NetworkNode. """ - __tablename__ = "network_users" + __tablename__ = "network_remote_users" id = sa.Column(sa.Integer(), primary_key=True, nullable=False, autoincrement=True) user_id = sa.Column(sa.Integer, @@ -1012,11 +1013,22 @@ class NetworkUser(BaseModel, Base): network_node_id = sa.Column(sa.Integer, sa.ForeignKey("network_nodes.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=False) + network_token_id = sa.Column(sa.Integer, + sa.ForeignKey("network_tokens.id", onupdate="CASCADE", ondelete="CASCADE")) + name = sa.Column(sa.Unicode(128)) network_node = relationship("NetworkNode", foreign_keys=[network_node_id]) - network_user_name = sa.Column(sa.Unicode(128)) + network_token = relationship("NetworkToken", foreign_keys=[network_token_id]) __table_args__ = (UniqueConstraint('user_id', 'network_node_id'), - UniqueConstraint('network_user_name', 'network_node_id')) + UniqueConstraint('name', 'network_node_id')) + + def as_dict(self): + # type: () -> Dict[Str, Str] + return { + "user_name": self.user.user_name, + "remote_user_name": self.name, + "node_name": self.network_node.name + } class NetworkToken(BaseModel, Base): @@ -1028,11 +1040,47 @@ class NetworkToken(BaseModel, Base): def __init__(self, *_, **__): super(NetworkToken, self).__init__(*_, **__) if not self.token: - self.token = uuid.uuid4() + self.refresh_token() - token = sa.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4, unique=True) + @staticmethod + def _encrypt(token): + # type: (Union[Str, UUID]) -> bytes + if isinstance(token, str): + token = uuid.UUID(token) + return Fernet(get_constant("MAGPIE_SECRET")).encrypt(token.bytes) + + def refresh_token(self): + # type: () -> None + self.token = self._encrypt(uuid.uuid4()) + self.created = datetime.datetime.utcnow() + + def decrypted_token(self): + # type: () -> UUID + return uuid.UUID(bytes=Fernet(get_constant("MAGPIE_SECRET")).decrypt(self.token)) + + def expired(self): + # type: () -> bool + expiry = int(get_constant("MAGPIE_NETWORK_DEFAULT_TOKEN_EXPIRY")) + return (datetime.datetime.utcnow() - self.created) > expiry + + @classmethod + def get_expired(cls, db_session): + # type: (Session) -> Query + token_expiry = int(get_constant("MAGPIE_NETWORK_DEFAULT_TOKEN_EXPIRY")) + expiry_date_time = datetime.datetime.utcnow() - datetime.timedelta(seconds=token_expiry) + return db_session.query(cls).filter(cls.created < expiry_date_time) + + @classmethod + def by_decrypted_token(cls, token, db_session=None): + # type: (Union[Str, UUID], Optional[Session]) -> Optional[NetworkToken] + db_session = get_db_session(db_session) + return db_session.query(cls).filter(cls.token == cls._encrypt(token)).first() + + id = sa.Column(sa.Integer(), primary_key=True, nullable=False, autoincrement=True) + token = sa.Column(sa.LargeBinary, nullable=False, unique=True) + created = sa.Column(sa.DateTime, default=datetime.datetime.utcnow) user_id = sa.Column(sa.Integer, - sa.ForeignKey("users.id", onupdate="CASCADE", ondelete="CASCADE"), unique=True, nullable=False) + sa.ForeignKey("users.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=False) user = relationship("User", foreign_keys=[user_id]) @@ -1044,7 +1092,31 @@ class NetworkNode(BaseModel, Base): id = sa.Column(sa.Integer(), primary_key=True, nullable=False, autoincrement=True) name = sa.Column(sa.Unicode(128), nullable=False, unique=True) - url = sa.Column(URLType(), nullable=False) + jwks_url = sa.Column(URLType(), nullable=False) + token_url = sa.Column(URLType(), nullable=False) + authorization_url = sa.Column(URLType(), nullable=False) + redirect_uris = sa.Column(sa.String) + + def anonymous_user_name(self): + # type: () -> Str + return self.anonymous_user_name_formatter(self.name) + + @staticmethod + def anonymous_user_name_formatter(name): + # type: (Str) -> Str + return "{}{}".format(get_constant("MAGPIE_NETWORK_NAME_PREFIX"), name) + + def anonymous_user(self, db_session): + # type: (Optional[Session]) -> User + return db_session.query(User).filter(User.user_name == self.anonymous_user_name()).one() + + def anonymous_group(self, db_session): + # type: (Optional[Session]) -> User + return db_session.query(Group).filter(Group.name == self.anonymous_user_name()).one() + + def as_dict(self): + # type: () -> Dict[Str, Any] + return {attr.key: getattr(self, attr.key) for attr in sa.inspect(NetworkNode).columns if attr.key != "id"} ziggurat_model_init(User, Group, UserGroup, GroupPermission, UserPermission, diff --git a/magpie/ui/__init__.py b/magpie/ui/__init__.py index 58c39b439..a9b9ffc5f 100644 --- a/magpie/ui/__init__.py +++ b/magpie/ui/__init__.py @@ -11,3 +11,4 @@ def includeme(config): config.include("magpie.ui.login") config.include("magpie.ui.management") config.include("magpie.ui.user") + config.include("magpie.ui.network") diff --git a/magpie/ui/home/static/style.css b/magpie/ui/home/static/style.css index b8523d9ff..28e1427d3 100644 --- a/magpie/ui/home/static/style.css +++ b/magpie/ui/home/static/style.css @@ -1436,3 +1436,19 @@ div.input-container { height: 100%; -webkit-box-sizing: border-box; } + +/* authorization form */ + +table.authorization-table { + width: auto; + border: 1px solid; + border-radius: 4px; + margin-top: 10px; + padding: 1px; +} + +table.authorization-table td { + border: 0; + text-align: center; +} + diff --git a/magpie/ui/management/templates/edit_user.mako b/magpie/ui/management/templates/edit_user.mako index 399919b77..fd848b3f2 100644 --- a/magpie/ui/management/templates/edit_user.mako +++ b/magpie/ui/management/templates/edit_user.mako @@ -383,3 +383,54 @@ ${membership_alerts.edit_membership_alerts()} >${perm_type.lower()} + +%if network_nodes and user_name not in MAGPIE_FIXED_USERS_REFS: +

Network Account Links

+ + + + + + + %for node_name, remote_user_name in network_nodes: + + + %if remote_user_name is None: + + + %else: + + + %endif + + %endfor +
Network Node NameUsernameAction
${node_name} + + + ${remote_user_name} + +
+%endif diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py index de9dedec8..54af1fd64 100644 --- a/magpie/ui/management/views.py +++ b/magpie/ui/management/views.py @@ -24,7 +24,14 @@ from magpie.cli.sync_services import SYNC_SERVICES_TYPES from magpie.constants import get_constant # FIXME: remove (REMOTE_RESOURCE_TREE_SERVICE, RESOURCE_TYPE_DICT), implement getters via API -from magpie.models import REMOTE_RESOURCE_TREE_SERVICE, RESOURCE_TYPE_DICT, UserGroupStatus, UserStatuses +from magpie.models import ( + REMOTE_RESOURCE_TREE_SERVICE, + RESOURCE_TYPE_DICT, + UserGroupStatus, + UserStatuses, + NetworkNode, + User, NetworkRemoteUser +) from magpie.permissions import Permission, PermissionSet # FIXME: remove (SERVICE_TYPE_DICT), implement getters via API from magpie.services import SERVICE_TYPE_DICT @@ -161,6 +168,17 @@ def edit_user(self): user_info["invalid_{}".format(field)] = False user_info["reason_{}".format(field)] = "" + # add network information + if get_constant("MAGPIE_NETWORK_ENABLED", self.request): + network_remote_users = (self.request.db.query(NetworkRemoteUser) + .join(User).filter(User.user_name == user_name) + .all()) + existing_network_remote_user_nodes = {nu.network_node_id: nu.name for nu in network_remote_users} + network_nodes = self.request.db.query(NetworkNode).order_by(NetworkNode.id).all() + user_info["network_nodes"] = [(n.name, existing_network_remote_user_nodes.get(n.id)) for n in network_nodes] + user_info["network_routes"] = {"create": schemas.NetworkRemoteUsersAPI.name, + "delete": schemas.NetworkRemoteUserAPI.name} + if self.request.method == "POST": res_id = self.request.POST.get("resource_id") is_edit_group_membership = False diff --git a/magpie/ui/network/__init__.py b/magpie/ui/network/__init__.py new file mode 100644 index 000000000..710d5096e --- /dev/null +++ b/magpie/ui/network/__init__.py @@ -0,0 +1,11 @@ +from magpie.utils import get_logger + +LOGGER = get_logger(__name__) + + +def includeme(config): + from magpie.ui.network.views import NetworkViews + LOGGER.info("Adding UI network...") + path = "/ui/network/authorize" + config.add_route(NetworkViews.authorize.__name__, path) + config.scan() diff --git a/magpie/ui/network/templates/authorize.mako b/magpie/ui/network/templates/authorize.mako new file mode 100644 index 000000000..6616eb21c --- /dev/null +++ b/magpie/ui/network/templates/authorize.mako @@ -0,0 +1,32 @@ +<%inherit file="magpie.ui.home:templates/template.mako"/> + + + + diff --git a/magpie/ui/network/views.py b/magpie/ui/network/views.py new file mode 100644 index 000000000..37a414876 --- /dev/null +++ b/magpie/ui/network/views.py @@ -0,0 +1,46 @@ +import jwt +from pyramid.authentication import Authenticated +from pyramid.httpexceptions import HTTPBadRequest +from pyramid.view import view_config + +from magpie.api.management.network.network_utils import decode_jwt, encode_jwt +from magpie.models import NetworkNode +from magpie.ui.utils import BaseViews +from magpie.utils import get_logger + + +LOGGER = get_logger(__name__) + + +class NetworkViews(BaseViews): + @view_config(route_name="authorize", renderer="templates/authorize.mako", permission=Authenticated) + def authorize(self): + token = self.request.GET.get("token") + response_type = self.request.GET.get("response_type") + redirect_uri = self.request.GET.get("redirect_uri") + + # Extend this to other response types later if needed + if response_type != "id_token": + raise HTTPBadRequest("Invalid response type") + if token is None: + raise HTTPBadRequest("Missing token") + try: + node_name = jwt.decode(token, options={"verify_signature": False}).get("iss") + except jwt.exceptions.DecodeError: + raise HTTPBadRequest("Token is improperly formatted") + node = self.request.db.query(NetworkNode).filter(NetworkNode.name == node_name).first() + if node is None: + raise HTTPBadRequest("Invalid token: invalid or missing issuer claim") + + if redirect_uri not in (node.redirect_uris or "").split(): + raise HTTPBadRequest("Invalid redirect URI") + + decoded_token = decode_jwt(token, node, self.request) + requesting_user_name = decoded_token.get("user_name") + token_claims = {"requesting_user_name": requesting_user_name, "user_name": self.request.user.user_name} + response_token = encode_jwt(token_claims, node.name, self.request) + + return self.add_template_data(data={"authorize_uri": redirect_uri, + "token": response_token, + "requesting_user_name": requesting_user_name, + "node_name": node.name}) diff --git a/magpie/ui/user/templates/edit_current_user.mako b/magpie/ui/user/templates/edit_current_user.mako index f08cad82d..11247d51f 100644 --- a/magpie/ui/user/templates/edit_current_user.mako +++ b/magpie/ui/user/templates/edit_current_user.mako @@ -272,3 +272,48 @@ ${membership_alerts.edit_membership_alerts()} %endfor + +%if network_nodes: +

Network Account Links

+ + + + + + + %for node_name, remote_user_name in network_nodes: + + + %if remote_user_name is None: + + + %else: + + + %endif + + %endfor +
Network Node NameUsernameAction
${node_name} + + ${remote_user_name} + +
+%endif diff --git a/magpie/ui/user/views.py b/magpie/ui/user/views.py index 45cb9cac1..c7c091b20 100644 --- a/magpie/ui/user/views.py +++ b/magpie/ui/user/views.py @@ -7,7 +7,7 @@ from magpie.api import schemas from magpie.constants import get_constant -from magpie.models import UserGroupStatus +from magpie.models import UserGroupStatus, NetworkNode, NetworkRemoteUser from magpie.ui.utils import BaseViews, check_response, handle_errors, request_api from magpie.utils import get_json, get_logger @@ -89,6 +89,16 @@ def edit_current_user(self): default_value=False, print_missing=True, raise_missing=False, raise_not_set=False)) user_info["user_with_error"] = schemas.UserStatuses.get(user_info["status"]) != schemas.UserStatuses.OK + # add network information + if get_constant("MAGPIE_NETWORK_ENABLED", self.request): + network_remote_users = (self.request.db.query(NetworkRemoteUser) + .filter(NetworkRemoteUser.user_id == self.request.user.id) + .all()) + existing_network_remote_user_nodes = {nu.network_node_id: nu.name for nu in network_remote_users} + network_nodes = self.request.db.query(NetworkNode).order_by(NetworkNode.id).all() + user_info["network_nodes"] = [(n.name, existing_network_remote_user_nodes.get(n.id)) for n in network_nodes] + user_info["network_routes"] = {"create": schemas.NetworkNodeLinkAPI.name, + "delete": schemas.NetworkRemoteUserAPI.name} # reset error messages/flags user_info["error_message"] = "" for field in ["password", "user_email", "user_name"]: diff --git a/magpie/ui/utils.py b/magpie/ui/utils.py index 10e4e1721..bbb546a03 100644 --- a/magpie/ui/utils.py +++ b/magpie/ui/utils.py @@ -592,7 +592,7 @@ def create_user(self, data): if (user_email or "").lower() in [usr["email"].lower() for usr in user_details]: data["invalid_user_email"] = True data["reason_user_email"] = "Conflict" - if get_constant("MAGPIE_NETWORK_ENABLED", self.request, settings_name="magpie.network_enabled"): + if get_constant("MAGPIE_NETWORK_ENABLED", self.request): anonymous_regex = protected_user_name_regex(include_admin=False, settings_container=self.request) if re.match(anonymous_regex, user_name): data["invalid_user_name"] = True diff --git a/requirements.txt b/requirements.txt index c3a3c6d3f..ba5bd02a7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,13 +26,14 @@ gunicorn>=20; python_version >= "3" humanize jsonschema<4; python_version < "3.6" jsonschema>=4; python_version >= "3.6" +jwcrypto==1.5.0 lxml>=3.7 mako # controlled by pyramid_mako paste pastedeploy pluggy psycopg2-binary>=2.7.1 -PyJWT==2.8.0 +PyJWT[crypto]==2.8.0 pyramid>=1.10.2,<2 pyramid_beaker==0.8 pyramid_chameleon>=0.3 diff --git a/setup.py b/setup.py index ec8fdd753..cc2d3eb04 100644 --- a/setup.py +++ b/setup.py @@ -255,6 +255,7 @@ def _extra_requirements(base_requirements, other_requirements): "magpie_run_db_migration = magpie.cli.run_db_migration:main", "magpie_send_email = magpie.cli.send_email:main", "magpie_sync_resources = magpie.cli.sync_resources:main", + "magpie_purge_expired_network_tokens = magpie.cli.purge_expired_network_tokens:main" ], } ) diff --git a/tests/test_webhooks.py b/tests/test_webhooks.py index 3b31d7324..fb595967f 100644 --- a/tests/test_webhooks.py +++ b/tests/test_webhooks.py @@ -31,7 +31,7 @@ from magpie.constants import get_constant from magpie.permissions import Access, Permission, PermissionSet, Scope from magpie.services import ServiceAPI -from magpie.utils import CONTENT_TYPE_HTML +from magpie.utils import CONTENT_TYPE_HTML, get_twitcher_protected_service_url from tests import interfaces as ti from tests import runner, utils @@ -548,7 +548,7 @@ def _test(t_name, r_id, perm, action): else: expected["data"]["group_name"] = self.test_group_name expected["data"]["group_id"] = grp_id - url = "https://localhost/twitcher/ows/proxy/{}".format(self.test_service_name) + url = get_twitcher_protected_service_url(self.test_service_name) if r_id == res_id: expected["data"]["resource_name"] = self.test_resource_name expected["data"]["resource_type"] = self.test_resource_type From 4c79d8ad8074575e7e7ff889dc694c01157ca575 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Thu, 2 Nov 2023 15:31:03 -0400 Subject: [PATCH 04/55] add cron job --- magpie-cron | 1 + 1 file changed, 1 insertion(+) diff --git a/magpie-cron b/magpie-cron index e769dba72..93b91f547 100644 --- a/magpie-cron +++ b/magpie-cron @@ -1 +1,2 @@ 0 * * * * /bin/bash -c "set -a ; source <($MAGPIE_ENV_DIR/*.env) ; set +a ; magpie_sync_resources" +1 0 * * * /bin/bash -c "set -a ; source <($MAGPIE_ENV_DIR/*.env) ; set +a ; magpie_purge_expired_network_tokens" From 2c2fd34369b6f7e36a8c34f6dd115184ed2cda5c Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Thu, 2 Nov 2023 15:31:19 -0400 Subject: [PATCH 05/55] documentation updates --- CHANGES.rst | 6 ++-- docs/authentication.rst | 78 +++++++++++++++++++++++++++++------------ docs/configuration.rst | 55 ++++++++++++++++++++++++----- magpie/api/schemas.py | 8 ++--- 4 files changed, 110 insertions(+), 37 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index ab75cfc4c..ac26fefe9 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -9,9 +9,9 @@ Changes `Unreleased `_ (latest) ------------------------------------------------------------------------------------ -* Introduce "Network Mode" which allows other Magpie instances to act as external authentication providers using JSON - Web Tokens (JWT). This allows users registered across multiple Magpie instances in a network to more easily gain - access to the resources within the network, without requiring the duplication of user credentials across the network. +* Introduce "Network Mode" which allows other Magpie instances to act as external authentication providers using access + tokens. This allows users registered across multiple Magpie instances in a network to more easily gain access to the + resources within the network, without requiring the duplication of user credentials across the network. .. _changes_3.37.1: diff --git a/docs/authentication.rst b/docs/authentication.rst index 34a1173cf..4775b7593 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -428,41 +428,75 @@ their services. Network Mode ------------ -If the :envvar:`MAGPIE_NETWORK_ENABLED` is enabled, an additional external authentication provider is added to `Magpie` -which allows networked instances of `Magpie` to authenticate users for each other. +If the :envvar:`MAGPIE_NETWORK_ENABLED` is enabled, `Magpie` instances can be linked in a network which allows them to +associate user accounts across the network and provide limited resource access to users who have accounts on other +`Magpie` instances in the network. Each `Magpie` instance is considered a node in the network. -Users can then log in to any `Magpie` instance where they have an account and request a personal network token in the -form of a `JSON Web Token `_ which can be used to authenticate this user -on any `Magpie` instance in the network. +Users who have an account on one `Magpie` instance can request an access token from another instance in the network +which the user can use to access resources protected by the other `Magpie` instance. + +Users with accounts on multiple instances in the network can also choose to link their accounts. This allows users who +use access tokens to ensure that they have the same access to resources that they would have if they logged in to +`Magpie` using any other method. Managing the Network ~~~~~~~~~~~~~~~~~~~~ -In order for `Magpie` instances to authenticate each other's users, each instance must be made aware of the existence of -the others so that it knows where to send authentication verification requests. +Each `Magpie` instance must be made aware of the existence of the other instances in the network so that they know where +to send token requests and account linking requests. In order to register another `Magpie` instance as part of the same network, an admin user can create a -:term:`Network Node` with a request to ``POST /network-nodes``. The parameters given to that request include a ``name`` -and a ``url``. The ``name`` is a the name of that other `Magpie` instance in the network and should correspond to the -same value as the :envvar:`MAGPIE_NETWORK_INSTANCE_NAME` value set by the other `Magpie` instance. The ``url`` is a the root URL -of the other `Magpie` instance. - -Once a :term:`Network Node` is registered, `Magpie` can use that other instance to authenticate users as long as the -other instance also has :envvar:`MAGPIE_NETWORK_ENABLED` enabled. +:term:`Network Node` with a request to ``POST /network/nodes``. The parameters given to that request includes + +* ``name``: + * the name of that other `Magpie` instance in the network and should correspond to the same value as the + :envvar:`MAGPIE_NETWORK_INSTANCE_NAME` value set by the other `Magpie` instance. +* ``jwks_url``: + * URL that provides the instance's public key in the form of a JSON Web Key Set. + * This is usually ``https://{hostname}/network/jwks`` where ``{hostname}`` is the hostname of the other instance +* ``authorization_url`` + * URL that provides the instance's Oauth authorize endpoint. + * This is usually ``https://{hostname}/ui/network/authorize`` where ``{hostname}`` is the hostname of the other + instance +* ``token_url`` + * URL that provides the instances Oauth token endpoint. + * This is usually ``https://{hostname}/network/token`` where ``{hostname}`` is the hostname of the other instance +* ``redirect_uris`` + * Space delimited list of valid redirect URIs for the instance. These are used by the instance's Oauth authorize + endpoint to safely redirect the user back once they have authorized `Magpie` to link their accounts on two + different instances. + * This is usually ``https://{hostname}/network/nodes/link`` where ``{hostname}`` is the hostname of the other + instance + + +Once a :term:`Network Node` is registered, `Magpie` can treat the other instance as if they are in the same network as +long as: + +* Both instances have :envvar:`MAGPIE_NETWORK_ENABLED` enabled +* Both instances have :envvar:`MAGPIE_NETWORK_INSTANCE_NAME` set +* Both instances have :envvar:`MAGPIE_NETWORK_PEM_FILES` set in order to verify communication between nodes using an + asymmetric public/private key-pair. + + +Managing Personal Access Tokens +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Managing Personal JWTs -~~~~~~~~~~~~~~~~~~~~~~ +A :term:`User` can request a new access token from another node with a request to the +``GET /network/nodes/{node_name}/token`` route. -A :term:`User` can request a new network token with a request to the ``PATCH /token`` route. +Every time a :term:`User` makes a request to this route, `Magpie` send a request to the other instance, and provides it +to the user. A new token is generated every time. This effectively cancels all previously created tokens for that user. -Every time a :term:`User` makes a request to the ``PATCH /token`` route a new token is generated for them. This -effectively cancels all previously created tokens for that user. +To cancel an existing token without generating a new one. A :term:`User` can make a request to the +``DELETE /network/nodes/{node_name}/token`` route. Authentication ~~~~~~~~~~~~~~ -Once a :term:`User` gets a personal network token, they can use that token to authenticate with any `Magpie` instance in -the same network. When a user makes a request, they should set the ``provider_name`` parameter to the value of +Once a :term:`User` gets an access token, they can use that token to authenticate with the instance that issued that +token. + +When a user makes a request, they should set the ``provider_name`` parameter to the value of :envvar:`MAGPIE_NETWORK_PROVIDER` and provide the network token in the Authorization header in the following format: .. code-block:: http @@ -475,7 +509,7 @@ where the parameter name set by :envvar:`MAGPIE_NETWORK_TOKEN_NAME` and the valu Authorization ~~~~~~~~~~~~~ -Managing authorization for :term:`Users` who authenticate using personal network tokens is complicated by the fact that +Managing authorization for :term:`Users` who authenticate using access tokens is complicated by the fact that a :term:`User` is not required to have a full account on both `Magpie` instances in order to using this authentication mechanism. This means that a :term:`User` may be logged in as a node-specific "anonymous" user. diff --git a/docs/configuration.rst b/docs/configuration.rst index 6f5a28661..35b6d9788 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -998,7 +998,7 @@ to authenticate users for each other. All variables defined in this section are [:class:`bool`] (Default: ``False``) - .. versionadded:: 3.37 + .. versionadded:: 3.38 Enable "Network Mode" which enables all functionality to authenticate users using other Magpie instances as external authentication providers. @@ -1007,7 +1007,7 @@ to authenticate users for each other. All variables defined in this section are [:class:`str`] - .. versionadded:: 3.37 + .. versionadded:: 3.38 The name of this Magpie instance in the network. This variable is used to determine if an authentication token was issued by this instance of Magpie, or another instance in the network. @@ -1019,16 +1019,26 @@ to authenticate users for each other. All variables defined in this section are [:class:`int`] (Default: ``86400``) - .. versionadded:: 3.37 + .. versionadded:: 3.38 + + The default expiry time (in seconds) for an access token issued for the purpose of network authentication. + +.. envvar:: MAGPIE_NETWORK_INTERNAL_TOKEN_EXPIRY + + [:class:`int`] + (Default: ``30``) - The default expiry time (in seconds) for an authentication token issued for the purpose of network authentication. + .. versionadded:: 3.38 + + The default expiry time (in seconds) for an JSON Web Token issued for the purpose of communication between nodes in + the network. .. envvar:: MAGPIE_NETWORK_TOKEN_NAME [|constant|_] (Value: ``"magpie_token"``) - .. versionadded:: 3.37 + .. versionadded:: 3.38 The name of the request parameter key whose value is the authentication token issued for the purpose of network authentication. @@ -1038,7 +1048,7 @@ to authenticate users for each other. All variables defined in this section are [|constant|_] (Value: ``"magpie_network"``) - .. versionadded:: 3.37 + .. versionadded:: 3.38 The name of the external provider that authenticates users using other Magpie instances as external authentication providers. @@ -1048,7 +1058,7 @@ to authenticate users for each other. All variables defined in this section are [|constant|_] (Value: ``"anonymous_network_"``) - .. versionadded:: 3.37 + .. versionadded:: 3.38 A prefix added to the anonymous network user and network group names. These names are constructed by prepending the remote Magpie instance name with this prefix. For example, a Magpie instance named ``"example123"`` will have a @@ -1059,11 +1069,40 @@ to authenticate users for each other. All variables defined in this section are [|constant|_] (Value: ``"magpie_network"``) - .. versionadded:: 3.37 + .. versionadded:: 3.38 The name of the group created to manage permissions for all users authenticated using Magpie instances as external authentication providers. +.. envvar:: MAGPIE_NETWORK_PEM_FILES + + [:class:`str`] + (Default: ``${MAGPIE_ROOT}/key.pem``) + + .. versionadded:: 3.38 + + Path to a PEM file containing a public/private key-pair. This is used to sign and verify communication sent between + nodes in the network. + + Multiple PEM files can be specified if key rotation is desired. To specify multiple PEM files, separate each file + path with a ``:`` character. The first file in the list will contain the primary key and will be used to sign all + outgoing communication. + +.. envvar:: MAGPIE_NETWORK_PEM_PASSWORDS + + [:class:`str`] + (Default: ``None``) + + .. versionadded:: 3.38 + + Password used to encrypt the PEM files in :envvar:`MAGPIE_NETWORK_PEM_FILES`. + + If multiple files require passwords, they can be listed with a ``:`` character separator (Note that this means that + passwords cannot contain a ``:``. An empty string will be treated the same as no password. + + For example, if you have four files specified in :envvar:`MAGPIE_NETWORK_PEM_FILES` and only the first and third + file require a password, set this variable to ``pass1::pass2:`` where ``pass1`` and ``pass2`` are the passwords. + .. _config_phoenix: Phoenix Settings diff --git a/magpie/api/schemas.py b/magpie/api/schemas.py index aa054fc5e..4d86ae1b0 100644 --- a/magpie/api/schemas.py +++ b/magpie/api/schemas.py @@ -3557,14 +3557,14 @@ class NetworkNode_PATCH_RequestBodySchema(colander.MappingSchema): validator=colander.url, missing=colander.drop ) - token_url = colander.SchemaNode( + authorization_url = colander.SchemaNode( colander.String(), description="URL that provides the Oauth authorize endpoint for another Magpie node (instance) in the network.", example="https://nodea.example.com/ui/network/authorize", validator=colander.url, missing=colander.drop ) - authorization_url = colander.SchemaNode( + token_url = colander.SchemaNode( colander.String(), description="URL that provides the Oauth token endpoint for another Magpie node (instance) in the network.", example="https://nodea.example.com/network/token", @@ -3606,13 +3606,13 @@ class NetworkNode_BodySchema(colander.MappingSchema): example="https://nodea.example.com/jwks.json", validator=colander.url ) - token_url = colander.SchemaNode( + authorization_url = colander.SchemaNode( colander.String(), description="URL that provides the Oauth authorize endpoint for another Magpie node (instance) in the network.", example="https://nodea.example.com/ui/network/authorize", validator=colander.url, ) - authorization_url = colander.SchemaNode( + token_url = colander.SchemaNode( colander.String(), description="URL that provides the Oauth token endpoint for another Magpie node (instance) in the network.", example="https://nodea.example.com/network/token", From 35090dae5e530bdd1d6e028d3db4ab6bc5358bc0 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Thu, 2 Nov 2023 16:03:26 -0400 Subject: [PATCH 06/55] docstring updates --- .../network/node/network_node_utils.py | 10 +++++++++- .../network/remote_user/remote_user_utils.py | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/magpie/api/management/network/node/network_node_utils.py b/magpie/api/management/network/node/network_node_utils.py index ef27a332c..f590629a4 100644 --- a/magpie/api/management/network/node/network_node_utils.py +++ b/magpie/api/management/network/node/network_node_utils.py @@ -18,7 +18,9 @@ def create_associated_user_groups(new_node, request): # type: (models.NetworkNode, Request) -> None """ - Create a NetworkNode with the given name and url. + Creates an associated anonymous user and group for the newly created ``new_node``. + + This will also create the network group (named ``MAGPIE_NETWORK_GROUP_NAME``) if it does not yet exist. """ name = new_node.anonymous_user_name() @@ -48,6 +50,9 @@ def create_associated_user_groups(new_node, request): def update_associated_user_groups(node, old_node_name, request): # type: (models.NetworkNode, Str, Request) -> None + """ + If the ``NetworkNode`` name has changed, update the names of the associated anonymous user and group to match. + """ if node.name != old_node_name: old_anonymous_name = models.NetworkNode.anonymous_user_name_formatter(old_node_name) anonymous_user = request.db.query(models.User).filter(models.User.user_name == old_anonymous_name).one() @@ -69,6 +74,9 @@ def delete_network_node(request, node): def check_network_node_info(db_session=None, name=None, jwks_url=None, token_url=None, authorization_url=None, redirect_uris=None): # type: (Optional[Session], Optional[Str], Optional[Str], Optional[Str], Optional[Str], Optional[Str]) -> None + """ + Check that the parameters used to create a new ``NetworkNode`` or update an existing one are well-formed. + """ if name is not None: ax.verify_param(name, matches=True, param_name="name", param_compare=NAME_REGEX, http_error=HTTPBadRequest, diff --git a/magpie/api/management/network/remote_user/remote_user_utils.py b/magpie/api/management/network/remote_user/remote_user_utils.py index c47b71c1f..0ba456722 100644 --- a/magpie/api/management/network/remote_user/remote_user_utils.py +++ b/magpie/api/management/network/remote_user/remote_user_utils.py @@ -16,6 +16,10 @@ def _remote_user_from_names(node_name, remote_user_name, db_session): # type: (Str, Str, Session) -> models.NetworkRemoteUser + """ + Return the `NetworkRemoteUser` with the same name as ``remote_user_name`` associated + with the ``NetworkNode`` named ``node_name``. + """ return (db_session.query(models.NetworkRemoteUser) .join(models.NetworkNode) .filter(models.NetworkRemoteUser.name == remote_user_name) @@ -25,6 +29,13 @@ def _remote_user_from_names(node_name, remote_user_name, db_session): def requested_remote_user(request): # type: (Request) -> models.NetworkRemoteUser + """ + Return the ``NetworkRemoteUser`` identified by the request path. + + For example: if the current request contains the path ``/nodes/nodeA/remote_users/userB`` + this will return the ``NetworkRemoteUser`` with the name userB that is associated + with the ``NetworkNode`` with the name nodeA. + """ node_name = ar.get_value_matchdict_checked(request, "node_name") remote_user_name = ar.get_value_matchdict_checked(request, "remote_user_name") remote_user = ax.evaluate_call( @@ -36,6 +47,13 @@ def requested_remote_user(request): def check_remote_user_access_permissions(request, remote_user=None): # type: (Request, Optional[models.NetworkRemoteUser]) -> None + """ + Raises an error if the currently logged-in user has permission to view/modify the ``remote_user`` model. + If ``remote_user`` is None, the requested remote user will be extracted from the request path. + + Admins are allowed to access any model. Other users are only allowed to access those that they are associated + with. + """ if remote_user is None: remote_user = requested_remote_user(request) admin_group = get_constant("MAGPIE_ADMIN_GROUP", settings_container=request) From 79714b2854dca2799de17482bbf5289c6822ee3d Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Mon, 6 Nov 2023 12:10:07 -0500 Subject: [PATCH 07/55] general cleanup and schema updates --- ...3-08-25_2cfe144538e8_add_network_tables.py | 8 +- magpie/api/login/login.py | 4 +- .../api/management/network/network_utils.py | 2 +- .../api/management/network/network_views.py | 12 ++- .../network/node/network_node_views.py | 24 +++--- .../network/remote_user/remote_user_views.py | 2 +- magpie/api/schemas.py | 83 ++++++++++++------- magpie/cli/purge_expired_network_tokens.py | 6 ++ magpie/models.py | 45 +++++----- 9 files changed, 109 insertions(+), 77 deletions(-) diff --git a/magpie/alembic/versions/2023-08-25_2cfe144538e8_add_network_tables.py b/magpie/alembic/versions/2023-08-25_2cfe144538e8_add_network_tables.py index 7812423d8..25810487e 100644 --- a/magpie/alembic/versions/2023-08-25_2cfe144538e8_add_network_tables.py +++ b/magpie/alembic/versions/2023-08-25_2cfe144538e8_add_network_tables.py @@ -25,10 +25,8 @@ def upgrade(): op.create_table("network_tokens", sa.Column("id", sa.Integer, primary_key=True, nullable=False, autoincrement=True), - sa.Column("token", sa.LargeBinary, nullable=False, unique=True), - sa.Column("created", sa.DateTime, default=datetime.datetime.utcnow), - sa.Column("user_id", sa.Integer, - sa.ForeignKey("users.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=False) + sa.Column("token", sa.String, nullable=False, unique=True), + sa.Column("created", sa.DateTime, default=datetime.datetime.utcnow) ) op.create_table("network_nodes", sa.Column("id", sa.Integer, primary_key=True, nullable=False, autoincrement=True), @@ -47,7 +45,7 @@ def upgrade(): nullable=False), sa.Column("name", sa.Unicode(128)), sa.Column("network_token_id", sa.Integer, - sa.ForeignKey("network_tokens.id", onupdate="CASCADE", ondelete="CASCADE")) + sa.ForeignKey("network_tokens.id", onupdate="CASCADE", ondelete="CASCADE"), unique=True) ) op.create_unique_constraint("uq_network_remote_users_user_id_network_node_id", "network_remote_users", ["user_id", "network_node_id"]) diff --git a/magpie/api/login/login.py b/magpie/api/login/login.py index 8faa051ef..5bff52a4e 100644 --- a/magpie/api/login/login.py +++ b/magpie/api/login/login.py @@ -279,10 +279,10 @@ def network_login(request): if token_type != "Bearer": ax.raise_http(http_error=HTTPBadRequest, detail=s.ProviderSignin_GET_BadRequestResponseSchema.description) - network_token = models.NetworkToken.by_decrypted_token(token) + network_token = models.NetworkToken.by_token(token) if network_token is None or network_token.expired(): return login_failure_view(request, reason=s.Signin_POST_UnauthorizedResponseSchema.description) - authenticated_user = network_token.user + authenticated_user = network_token.network_remote_user.user # We should never create a token for protected users but just in case anonymous_regex = protected_user_name_regex(include_admin=False, settings_container=request) ax.verify_param(authenticated_user.name, not_matches=True, param_compare=anonymous_regex, diff --git a/magpie/api/management/network/network_utils.py b/magpie/api/management/network/network_utils.py index b9799c22f..3d908fe72 100644 --- a/magpie/api/management/network/network_utils.py +++ b/magpie/api/management/network/network_utils.py @@ -155,7 +155,7 @@ def get_network_models_from_request_token(request, create_network_remote_user=Fa node = ax.evaluate_call( lambda: request.db.query(models.NetworkNode).filter(models.NetworkNode.name == node_name).one(), http_error=HTTPNotFound, - msg_on_fail=s.NetworkNode_GET_NotFoundResponseSchema.description) + msg_on_fail=s.NetworkNode_NotFoundResponseSchema.description) decoded_token = decode_jwt(token, node, request) user_name = decoded_token.get("user_name") network_remote_user = (request.db.query(models.NetworkRemoteUser) diff --git a/magpie/api/management/network/network_views.py b/magpie/api/management/network/network_views.py index 44e69b685..91a8a3ed5 100644 --- a/magpie/api/management/network/network_views.py +++ b/magpie/api/management/network/network_views.py @@ -20,11 +20,13 @@ def post_network_token_view(request): node, network_remote_user = get_network_models_from_request_token(request, create_network_remote_user=True) network_token = network_remote_user.network_token if network_token: - network_token.refresh_token() + token = network_token.refresh_token() else: - network_token = models.NetworkToken(user_id=network_remote_user.user_id) + network_token = models.NetworkToken() + token = network_token.refresh_token() + request.db.add(network_token) network_remote_user.network_token = network_token - return ax.valid_http(http_success=HTTPCreated, content={"token": network_token.decrypted_token()}, + return ax.valid_http(http_success=HTTPCreated, content={"token": token}, detail=s.NetworkToken_POST_CreatedResponseSchema.description) @@ -46,4 +48,6 @@ def delete_network_token_view(request): @s.NetworkJSONWebKeySetAPI.get(tags=[s.NetworkTag], response_schemas=s.NetworkJSONWebKeySet_GET_responses) @view_config(route_name=s.NetworkJSONWebKeySetAPI.name, request_method="GET", permission=NO_PERMISSION_REQUIRED) def get_network_jwks_view(_request): - return ax.valid_http(http_success=HTTPOk, content=jwks().export(private_keys=False, as_dict=True)) + return ax.valid_http(http_success=HTTPOk, + detail=s.NetworkJSONWebKeySet_GET_OkResponseSchema.description, + content=jwks().export(private_keys=False, as_dict=True)) diff --git a/magpie/api/management/network/node/network_node_views.py b/magpie/api/management/network/node/network_node_views.py index 54ce461ad..edfd7ca3a 100644 --- a/magpie/api/management/network/node/network_node_views.py +++ b/magpie/api/management/network/node/network_node_views.py @@ -6,7 +6,7 @@ HTTPCreated, HTTPInternalServerError, HTTPForbidden, - HTTPTemporaryRedirect + HTTPFound ) from pyramid.security import Authenticated from pyramid.view import view_config @@ -37,7 +37,7 @@ def get_network_node_view(request): node = ax.evaluate_call( lambda: request.db.query(models.NetworkNode).filter(models.NetworkNode.name == node_name).one(), http_error=HTTPNotFound, - msg_on_fail=s.NetworkNode_GET_NotFoundResponseSchema.description) + msg_on_fail=s.NetworkNode_NotFoundResponseSchema.description) return ax.valid_http(http_success=HTTPOk, detail=s.NetworkNode_GET_OkResponseSchema.description, content=node.as_dict()) @@ -52,7 +52,7 @@ def post_network_nodes_view(request): if param in request.POST: kwargs[param] = request.POST[param] else: - ax.raise_http(http_error=HTTPBadRequest, detail=s.NetworkNodes_POST_BadRequestResponseSchema.description) + ax.raise_http(http_error=HTTPBadRequest, detail=s.BadRequestResponseSchema.description) if "redirect_uris" in request.POST: kwargs["redirect_uris"] = request.POST.get("redirect_uris") check_network_node_info(request.db, **kwargs) @@ -71,7 +71,7 @@ def patch_network_node_view(request): node = ax.evaluate_call( lambda: request.db.query(models.NetworkNode).filter(models.NetworkNode.name == node_name).one(), http_error=HTTPNotFound, - msg_on_fail=s.NetworkNode_GET_NotFoundResponseSchema.description) + msg_on_fail=s.NetworkNode_NotFoundResponseSchema.description) params = ("name", "jwks_url", "token_url", "authorization_url", "redirect_uris") kwargs = {} for param in params: @@ -96,7 +96,7 @@ def delete_network_node_view(request): node = ax.evaluate_call( lambda: request.db.query(models.NetworkNode).filter(models.NetworkNode.name == node_name).one(), http_error=HTTPNotFound, - msg_on_fail=s.NetworkNode_GET_NotFoundResponseSchema.description) + msg_on_fail=s.NetworkNode_NotFoundResponseSchema.description) ax.evaluate_call(lambda: delete_network_node(request, node), http_error=HTTPInternalServerError, fallback=lambda: request.db.rollback(), @@ -111,7 +111,7 @@ def get_network_node_token_view(request): node = ax.evaluate_call( lambda: request.db.query(models.NetworkNode).filter(models.NetworkNode.name == node_name).one(), http_error=HTTPNotFound, - msg_on_fail=s.NetworkNode_GET_NotFoundResponseSchema.description) + msg_on_fail=s.NetworkNode_NotFoundResponseSchema.description) token = encode_jwt({"user_name": request.user.user_name}, node_name, request) access_token = ax.evaluate_call(lambda: requests.post(node.token_url, json={"token": token}).json()["token"], http_error=HTTPInternalServerError, @@ -127,7 +127,7 @@ def delete_network_node_token_view(request): node = ax.evaluate_call( lambda: request.db.query(models.NetworkNode).filter(models.NetworkNode.name == node_name).one(), http_error=HTTPNotFound, - msg_on_fail=s.NetworkNode_GET_NotFoundResponseSchema.description) + msg_on_fail=s.NetworkNode_NotFoundResponseSchema.description) token = encode_jwt({"user_name": request.user.user_name}, node_name, request) ax.evaluate_call(lambda: requests.delete(node.token_url, json={"token": token}).raise_for_status(), http_error=HTTPInternalServerError, @@ -144,7 +144,7 @@ def get_network_node_link_view(request): node = ax.evaluate_call( lambda: request.db.query(models.NetworkNode).filter(models.NetworkNode.name == node_name).one(), http_error=HTTPNotFound, - msg_on_fail=s.NetworkNode_GET_NotFoundResponseSchema.description) + msg_on_fail=s.NetworkNode_NotFoundResponseSchema.description) decoded_token = decode_jwt(token, node, request) remote_user_name = ax.evaluate_call(lambda: decoded_token["user_name"], http_error=HTTPBadRequest, @@ -157,7 +157,7 @@ def get_network_node_link_view(request): new_remote_user = models.NetworkRemoteUser(user_id=request.user.id, network_node_id=node.id, name=remote_user_name) request.db.add(new_remote_user) - return ax.valid_http(http_success=HTTPOk, detail=s.NetworkNodeLink_GET_OkResponseSchema) + return ax.valid_http(http_success=HTTPOk, detail=s.NetworkNodeLink_GET_OkResponseSchema.description) @s.NetworkNodeLinkAPI.post(tags=[s.NetworkTag], response_schemas=s.NetworkNodeLink_POST_responses) @@ -167,7 +167,7 @@ def post_network_node_link_view(request): node = ax.evaluate_call( lambda: request.db.query(models.NetworkNode).filter(models.NetworkNode.name == node_name).one(), http_error=HTTPNotFound, - msg_on_fail=s.NetworkNode_GET_NotFoundResponseSchema.description + msg_on_fail=s.NetworkNode_NotFoundResponseSchema.description ) location_tuple = up.urlparse(node.authorization_url) location_query_list = up.parse_qsl(location_tuple.query) @@ -177,4 +177,6 @@ def post_network_node_link_view(request): ("redirect_uri", request.route_url(s.NetworkNodesLinkAPI.name)) )) location = up.urlunparse(location_tuple._replace(query=up.urlencode(location_query_list, doseq=True))) - return ax.valid_http(http_success=HTTPTemporaryRedirect, http_kwargs={"location": location}) + return ax.valid_http(http_success=HTTPFound, + detail=s.NetworkNodeLink_POST_FoundResponseSchema.description, + http_kwargs={"location": location}) diff --git a/magpie/api/management/network/remote_user/remote_user_views.py b/magpie/api/management/network/remote_user/remote_user_views.py index 9111c9c8c..793639a2e 100644 --- a/magpie/api/management/network/remote_user/remote_user_views.py +++ b/magpie/api/management/network/remote_user/remote_user_views.py @@ -108,7 +108,7 @@ def delete_network_remote_user_view(request): remote_user = requested_remote_user(request) check_remote_user_access_permissions(request, remote_user) request.db.delete(remote_user) - return ax.valid_http(http_success=HTTPOk, detail=s.NetworkRemoteUsers_DELETE_OkResponseSchema.description) + return ax.valid_http(http_success=HTTPOk, detail=s.NetworkRemoteUser_DELETE_OkResponseSchema.description) @s.NetworkRemoteUsersCurrentAPI.get(tags=[s.NetworkTag], response_schemas=s.NetworkRemoteUsersCurrent_GET_responses) diff --git a/magpie/api/schemas.py b/magpie/api/schemas.py index 4d86ae1b0..0db9ea866 100644 --- a/magpie/api/schemas.py +++ b/magpie/api/schemas.py @@ -3584,16 +3584,11 @@ class NetworkNode_PATCH_RequestSchema(BaseRequestSchemaAPI): body = NetworkNode_PATCH_RequestBodySchema() -class NetworkNode_GET_NotFoundResponseSchema(BaseResponseSchemaAPI): +class NetworkNode_NotFoundResponseSchema(BaseResponseSchemaAPI): description = "Network Node could not be found." body = ErrorResponseBodySchema(code=HTTPNotFound.code, description=description) -class NetworkNodes_GET_NotFoundResponseSchema(BaseResponseSchemaAPI): - description = "Network Nodes could not be found." - body = ErrorResponseBodySchema(code=HTTPNotFound.code, description=description) - - class NetworkNode_BodySchema(colander.MappingSchema): name = colander.SchemaNode( colander.String(), @@ -3655,11 +3650,6 @@ class NetworkNodes_POST_CreatedResponseSchema(BaseResponseSchemaAPI): body = BaseResponseBodySchema(code=HTTPCreated.code, description=description) -class NetworkNodes_POST_BadRequestResponseSchema(BaseResponseSchemaAPI): - description = "Missing required parameter." - body = BaseResponseBodySchema(code=HTTPBadRequest.code, description=description) - - class NetworkNodes_PATCH_BadRequestResponseSchema(BaseResponseSchemaAPI): description = "Missing parameters to update network node." body = BaseResponseBodySchema(code=HTTPBadRequest.code, description=description) @@ -3786,15 +3776,15 @@ class NetworkRemoteUsers_POST_BadRequestResponseSchema(BaseResponseSchemaAPI): class NetworkRemoteUsersSequence(colander.SequenceSchema): - node = NetworkRemoteUser_BodySchema() + remote_user = NetworkRemoteUser_BodySchema() class NetworkRemoteUsers_GET_OkResponseBodySchema(BaseResponseBodySchema): - nodes = NetworkRemoteUsersSequence() + remote_users = NetworkRemoteUsersSequence() class NetworkRemoteUsers_GET_OkResponseSchema(BaseResponseSchemaAPI): - description = "Network Nodes found." + description = "Remote Users found." body = NetworkRemoteUsers_GET_OkResponseBodySchema(code=HTTPOk.code, description=description) @@ -3812,7 +3802,7 @@ class NetworkRemoteUsers_PATCH_OkResponseSchema(BaseResponseSchemaAPI): body = BaseResponseBodySchema(code=HTTPOk.code, description=description) -class NetworkRemoteUsers_DELETE_OkResponseSchema(BaseResponseSchemaAPI): +class NetworkRemoteUser_DELETE_OkResponseSchema(BaseResponseSchemaAPI): description = "Remote user deleted." body = BaseResponseBodySchema(code=HTTPOk.code, description=description) @@ -3830,11 +3820,6 @@ class NetworkRemoteUser_PATCH_OkResponseSchema(BaseResponseSchemaAPI): body = BaseResponseBodySchema(code=HTTPOk.code, description=description) -class NetworkRemoteUser_DELETE_OkResponseSchema(BaseResponseSchemaAPI): - description = "Network Node deleted." - body = BaseResponseBodySchema(code=HTTPOk.code, description=description) - - class NetworkNodeToken_GET_OkResponseSchema(BaseResponseSchemaAPI): description = "Successfully obtained access token." body = BaseResponseBodySchema(code=HTTPOk.code, description=description) @@ -3860,6 +3845,11 @@ class NetworkNodeLink_GET_BadRequestResponseSchema(BaseResponseSchemaAPI): body = InternalServerErrorResponseBodySchema(code=HTTPBadRequest.code, description=description) +class NetworkNodeLink_POST_FoundResponseSchema(BaseResponseSchemaAPI): + description = "Redirecting to authorize endpoint on other network node." + body = BaseResponseBodySchema(code=HTTPFound.code, description=description) + + class NetworkToken_POST_CreatedResponseSchema(BaseResponseSchemaAPI): description = "Access token created or refreshed." body = BaseResponseBodySchema(code=HTTPCreated.code, description=description) @@ -3873,6 +3863,26 @@ class NetworkNodeToken_DELETE_NotFoundResponseSchema(BaseResponseSchemaAPI): body = BaseResponseBodySchema(code=HTTPNotFound.code, description=description) +class NetworkJSONWebKeySet_KeySchema(colander.MappingSchema): + kid = colander.SchemaNode(colander.String(), description="key id", example="keyidtext") + kty = colander.SchemaNode(colander.String(), description="key type", example="RSA") + e = colander.SchemaNode(colander.String(), description="public key exponent part", example="publickeyexponentpart") + n = colander.SchemaNode(colander.String(), description="public key modulus part", example="publickeymoduluspart") + + +class NetworkJSONWebKeySet_KeysSchema(colander.SequenceSchema): + key = NetworkJSONWebKeySet_KeySchema() + + +class NetworkJSONWebKeySet_GET_OkBodyResponseSchema(BaseResponseBodySchema): + keys = NetworkJSONWebKeySet_KeysSchema() + + +class NetworkJSONWebKeySet_GET_OkResponseSchema(BaseResponseSchemaAPI): + description = "JSON Web Key Set found" + body = NetworkJSONWebKeySet_GET_OkBodyResponseSchema(code=HTTPOk.code, description=description) + + class NetworkRemoteUsers_POST_ForbiddenResponseSchema(BaseResponseSchemaAPI): description = "Targeted user update not allowed by requesting user." body = ErrorResponseBodySchema(code=HTTPForbidden.code, description=description) @@ -4626,23 +4636,25 @@ class NetworkRemoteUsers_POST_ForbiddenResponseSchema(BaseResponseSchemaAPI): } NetworkToken_POST_responses = { "201": NetworkToken_POST_CreatedResponseSchema(), - "404": NetworkNode_GET_NotFoundResponseSchema(), + "404": NetworkNode_NotFoundResponseSchema(), "500": InternalServerErrorResponseSchema(), } NetworkToken_DELETE_responses = { - + "201": NetworkToken_DELETE_OkResponseSchema(), + "404": NetworkNodeToken_DELETE_NotFoundResponseSchema(), + "500": InternalServerErrorResponseSchema(), } NetworkJSONWebKeySet_GET_responses = { - + "200": NetworkJSONWebKeySet_GET_OkResponseSchema(), + "500": InternalServerErrorResponseSchema(), } NetworkNode_GET_responses = { "200": NetworkNode_GET_OkResponseSchema(), - "404": NetworkNode_GET_NotFoundResponseSchema(), + "404": NetworkNode_NotFoundResponseSchema(), "500": InternalServerErrorResponseSchema(), } NetworkNodes_GET_responses = { "200": NetworkNodes_GET_OkResponseSchema(), - "404": NetworkNodes_GET_NotFoundResponseSchema(), "500": InternalServerErrorResponseSchema(), } NetworkNodes_POST_responses = { @@ -4654,41 +4666,46 @@ class NetworkRemoteUsers_POST_ForbiddenResponseSchema(BaseResponseSchemaAPI): NetworkNode_PATCH_responses = { "200": NetworkNode_PATCH_OkResponseSchema(), "400": BadRequestResponseSchema(), + "404": NetworkNode_NotFoundResponseSchema(), "409": NetworkNodes_CheckInfo_NameValue_ConflictResponseSchema(), "500": InternalServerErrorResponseSchema(), } NetworkNode_DELETE_responses = { "200": NetworkNode_DELETE_OkResponseSchema(), - "400": BadRequestResponseSchema(), + "404": NetworkNode_NotFoundResponseSchema(), "500": InternalServerErrorResponseSchema(), } NetworkNodeToken_GET_responses = { "200": NetworkNodeToken_GET_OkResponseSchema(), - "404": NetworkNode_GET_NotFoundResponseSchema(), + "404": NetworkNode_NotFoundResponseSchema(), "500": NetworkNodeToken_GET_InternalServerErrorResponseSchema() } NetworkNodeToken_DELETE_responses = { "200": NetworkNodeToken_DELETE_OkResponseSchema(), - "404": NetworkNode_GET_NotFoundResponseSchema(), + "404": NetworkNode_NotFoundResponseSchema(), "500": NetworkNodeToken_DELETE_InternalServerErrorResponseSchema() } NetworkNodesLink_GET_responses = { - + "200": NetworkNodeLink_GET_OkResponseSchema(), + "400": NetworkNodeLink_GET_BadRequestResponseSchema(), + "404": NetworkNode_NotFoundResponseSchema(), + "500": InternalServerErrorResponseSchema(), } NetworkNodeLink_POST_responses = { - + "302": NetworkNodeLink_POST_FoundResponseSchema(), + "404": NetworkNode_NotFoundResponseSchema(), + "500": InternalServerErrorResponseSchema(), } NetworkRemoteUser_GET_responses = { "200": NetworkRemoteUser_GET_OkResponseSchema(), "404": NetworkRemoteUser_GET_NotFoundResponseSchema(), "500": InternalServerErrorResponseSchema(), } -NetworkRemoteUsersCurrent_GET_responses = NetworkRemoteUser_GET_responses NetworkRemoteUsers_GET_responses = { "200": NetworkRemoteUsers_GET_OkResponseSchema(), - "404": NetworkRemoteUsers_GET_NotFoundResponseSchema(), "500": InternalServerErrorResponseSchema(), } +NetworkRemoteUsersCurrent_GET_responses = NetworkRemoteUsers_GET_responses NetworkRemoteUsers_POST_responses = { "201": NetworkRemoteUsers_POST_CreatedResponseSchema(), "400": BadRequestResponseSchema(), @@ -4698,12 +4715,14 @@ class NetworkRemoteUsers_POST_ForbiddenResponseSchema(BaseResponseSchemaAPI): NetworkRemoteUser_PATCH_responses = { "200": NetworkRemoteUser_PATCH_OkResponseSchema(), "400": BadRequestResponseSchema(), + "404": NetworkRemoteUser_GET_NotFoundResponseSchema(), "409": NetworkRemoteUser_PATCH_ConflictResponseSchema(), "500": InternalServerErrorResponseSchema(), } NetworkRemoteUser_DELETE_responses = { "200": NetworkRemoteUser_DELETE_OkResponseSchema(), "400": BadRequestResponseSchema(), + "404": NetworkRemoteUser_GET_NotFoundResponseSchema(), "500": InternalServerErrorResponseSchema(), } diff --git a/magpie/cli/purge_expired_network_tokens.py b/magpie/cli/purge_expired_network_tokens.py index f6db721ba..dce43d2ed 100644 --- a/magpie/cli/purge_expired_network_tokens.py +++ b/magpie/cli/purge_expired_network_tokens.py @@ -46,6 +46,12 @@ def main(args=None, parser=None, namespace=None): setup_logger_from_options(LOGGER, args) db_session = get_db_session_from_config_ini(args.ini_config) deleted = models.NetworkToken.get_expired(db_session).delete() + anonymous_network_user_ids = [n.anonymous_user().id for n in db_session.query(models.NetworkNode).all()] + # clean up unused records in the database (no need to keep records associated with anonymous network users) + (db_session.query(models.NetworkRemoteUser) + .filter(models.NetworkRemoteUser.user_id.in_(anonymous_network_user_ids)) + .filter(models.NetworkRemoteUser.network_token_id == None) # noqa: E711 + .delete()) try: transaction.commit() db_session.close() diff --git a/magpie/models.py b/magpie/models.py index 0987516f8..6ca54630c 100644 --- a/magpie/models.py +++ b/magpie/models.py @@ -1,4 +1,5 @@ import datetime +import hashlib import math import uuid from typing import TYPE_CHECKING @@ -30,7 +31,6 @@ from ziggurat_foundations.models.user_permission import UserPermissionMixin from ziggurat_foundations.models.user_resource_permission import UserResourcePermissionMixin from ziggurat_foundations.permissions import permission_to_pyramid_acls -from cryptography.fernet import Fernet from magpie.api import exception as ax from magpie.constants import get_constant @@ -1014,10 +1014,11 @@ class NetworkRemoteUser(BaseModel, Base): sa.ForeignKey("network_nodes.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=False) network_token_id = sa.Column(sa.Integer, - sa.ForeignKey("network_tokens.id", onupdate="CASCADE", ondelete="CASCADE")) + sa.ForeignKey("network_tokens.id", onupdate="CASCADE", ondelete="CASCADE"), + unique=True) name = sa.Column(sa.Unicode(128)) network_node = relationship("NetworkNode", foreign_keys=[network_node_id]) - network_token = relationship("NetworkToken", foreign_keys=[network_token_id]) + network_token = relationship("NetworkToken", foreign_keys=[network_token_id], back_populates="network_remote_user") __table_args__ = (UniqueConstraint('user_id', 'network_node_id'), UniqueConstraint('name', 'network_node_id')) @@ -1039,24 +1040,26 @@ class NetworkToken(BaseModel, Base): def __init__(self, *_, **__): super(NetworkToken, self).__init__(*_, **__) - if not self.token: - self.refresh_token() + if self.token: + # The token should not be created with the object, instead call refresh_token after the object is created + # Otherwise, the unhashed token will never be available to the user + self.token = None @staticmethod - def _encrypt(token): - # type: (Union[Str, UUID]) -> bytes + def _hash_token(token): + # type: (Str) -> Str if isinstance(token, str): token = uuid.UUID(token) - return Fernet(get_constant("MAGPIE_SECRET")).encrypt(token.bytes) + h = hashlib.sha256() + h.update(token.bytes) + return h.hexdigest() def refresh_token(self): - # type: () -> None - self.token = self._encrypt(uuid.uuid4()) + # type: () -> str + unhashed_token = str(uuid.uuid4()) + self.token = self._hash_token(unhashed_token) self.created = datetime.datetime.utcnow() - - def decrypted_token(self): - # type: () -> UUID - return uuid.UUID(bytes=Fernet(get_constant("MAGPIE_SECRET")).decrypt(self.token)) + return unhashed_token def expired(self): # type: () -> bool @@ -1071,17 +1074,17 @@ def get_expired(cls, db_session): return db_session.query(cls).filter(cls.created < expiry_date_time) @classmethod - def by_decrypted_token(cls, token, db_session=None): - # type: (Union[Str, UUID], Optional[Session]) -> Optional[NetworkToken] + def by_token(cls, token, db_session=None): + # type: (Str, Optional[Session]) -> Optional[NetworkToken] db_session = get_db_session(db_session) - return db_session.query(cls).filter(cls.token == cls._encrypt(token)).first() + return db_session.query(cls).filter(cls.token == cls._hash_token(token)).first() id = sa.Column(sa.Integer(), primary_key=True, nullable=False, autoincrement=True) - token = sa.Column(sa.LargeBinary, nullable=False, unique=True) + token = sa.Column(sa.String, nullable=False, unique=True) created = sa.Column(sa.DateTime, default=datetime.datetime.utcnow) - user_id = sa.Column(sa.Integer, - sa.ForeignKey("users.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=False) - user = relationship("User", foreign_keys=[user_id]) + network_remote_user = relationship("NetworkRemoteUser", back_populates="network_token", + single_parent=True, + uselist=False) class NetworkNode(BaseModel, Base): From 18ccc4c7d005a8445ab78abfaef84382d4face26 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Mon, 6 Nov 2023 15:47:38 -0500 Subject: [PATCH 08/55] deal with pyjwt's python versions support --- docs/authentication.rst | 4 ++++ magpie/adapter/magpieowssecurity.py | 4 ++-- magpie/api/management/__init__.py | 4 ++-- magpie/api/management/group/group_utils.py | 4 ++-- magpie/api/management/group/group_views.py | 4 ++-- magpie/api/schemas.py | 3 +-- magpie/constants.py | 14 +++++++++++--- magpie/ui/management/views.py | 4 ++-- magpie/ui/user/views.py | 4 ++-- magpie/ui/utils.py | 4 ++-- requirements.txt | 4 +++- 11 files changed, 33 insertions(+), 20 deletions(-) diff --git a/docs/authentication.rst b/docs/authentication.rst index 4775b7593..c6d9efc23 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -439,6 +439,10 @@ Users with accounts on multiple instances in the network can also choose to link use access tokens to ensure that they have the same access to resources that they would have if they logged in to `Magpie` using any other method. +.. warning:: + Network Mode is only supported when the python version is at least 3.6. If the python version is less than 3.6, + setting the :envvar:`MAGPIE_NETWORK_ENABLED` will have no effect. + Managing the Network ~~~~~~~~~~~~~~~~~~~~ diff --git a/magpie/adapter/magpieowssecurity.py b/magpie/adapter/magpieowssecurity.py index a34688412..3d9a54e1e 100644 --- a/magpie/adapter/magpieowssecurity.py +++ b/magpie/adapter/magpieowssecurity.py @@ -16,7 +16,7 @@ from magpie.api.exception import evaluate_call, verify_param from magpie.api.schemas import ProviderSigninAPI from magpie.compat import LooseVersion -from magpie.constants import get_constant +from magpie.constants import get_constant, network_enabled from magpie.db import get_connected_session from magpie.models import Service from magpie.permissions import Permission @@ -269,7 +269,7 @@ def update_request_cookies(self, request): """ settings = get_settings(request) token_name = get_constant("MAGPIE_COOKIE_NAME", settings_container=settings) - network_mode = get_constant("MAGPIE_NETWORK_ENABLED", settings_container=settings) + network_mode = network_enabled(settings_container=settings) headers = dict(request.headers) network_token_name = get_constant("MAGPIE_NETWORK_TOKEN_NAME", settings_container=settings) if network_mode and "Authorization" not in headers and network_token_name in request.params: diff --git a/magpie/api/management/__init__.py b/magpie/api/management/__init__.py index fb63ad8f7..a1132b099 100644 --- a/magpie/api/management/__init__.py +++ b/magpie/api/management/__init__.py @@ -1,4 +1,4 @@ -from magpie.constants import get_constant +from magpie.constants import network_enabled from magpie.utils import get_logger LOGGER = get_logger(__name__) @@ -11,6 +11,6 @@ def includeme(config): config.include("magpie.api.management.service") config.include("magpie.api.management.resource") config.include("magpie.api.management.register") - if get_constant("MAGPIE_NETWORK_ENABLED", config): + if network_enabled(config): config.include("magpie.api.management.network") config.scan() diff --git a/magpie/api/management/group/group_utils.py b/magpie/api/management/group/group_utils.py index e7812bfad..1ab57d9eb 100644 --- a/magpie/api/management/group/group_utils.py +++ b/magpie/api/management/group/group_utils.py @@ -22,7 +22,7 @@ from magpie.api.management.resource.resource_utils import check_valid_service_or_resource_permission from magpie.api.management.service import service_formats as sf from magpie.api.webhooks import WebhookAction, get_permission_update_params, process_webhook_requests -from magpie.constants import protected_group_name_regex, get_constant +from magpie.constants import protected_group_name_regex, network_enabled from magpie.permissions import PermissionSet, PermissionType, format_permissions from magpie.services import SERVICE_TYPE_DICT @@ -95,7 +95,7 @@ def create_group(group_name, description, discoverable, terms, db_session): ax.verify_param(group_name, matches=True, param_compare=ax.PARAM_REGEX, param_name="group_name", http_error=HTTPBadRequest, content=group_content_error, msg_on_fail=s.Groups_POST_BadRequestResponseSchema.description) - if get_constant("MAGPIE_NETWORK_ENABLED"): + if network_enabled(): anonymous_regex = protected_group_name_regex(include_admin=False) ax.verify_param(group_name, not_matches=True, param_compare=anonymous_regex, param_name="group_name", http_error=HTTPBadRequest, content=group_content_error, diff --git a/magpie/api/management/group/group_views.py b/magpie/api/management/group/group_views.py index 8a3bf63a6..600b92aec 100644 --- a/magpie/api/management/group/group_views.py +++ b/magpie/api/management/group/group_views.py @@ -9,7 +9,7 @@ from magpie.api.management.group import group_formats as gf from magpie.api.management.group import group_utils as gu from magpie.api.management.service import service_utils as su -from magpie.constants import get_constant, protected_group_name_regex +from magpie.constants import get_constant, protected_group_name_regex, network_enabled from magpie.models import TemporaryToken, TokenOperation, UserGroupStatus @@ -85,7 +85,7 @@ def edit_group_view(request): ax.verify_param(GroupService.by_group_name(new_group_name, db_session=request.db), is_none=True, http_error=HTTPConflict, with_param=False, # don't return group as value msg_on_fail=s.Group_PATCH_ConflictResponseSchema.description) - if get_constant("MAGPIE_NETWORK_ENABLED"): + if network_enabled(): anonymous_regex = protected_group_name_regex(include_admin=False) ax.verify_param(new_group_name, not_matches=True, param_compare=anonymous_regex, http_error=HTTPBadRequest, diff --git a/magpie/api/schemas.py b/magpie/api/schemas.py index 0db9ea866..336ca5da6 100644 --- a/magpie/api/schemas.py +++ b/magpie/api/schemas.py @@ -2,7 +2,6 @@ from typing import TYPE_CHECKING import colander -import jwt import six from cornice import Service from cornice.service import get_services @@ -3531,7 +3530,7 @@ class JWTRequestBodySchema(colander.MappingSchema): token = colander.SchemaNode( colander.String(), description="JSON Web Token.", - example=jwt.encode({"example": "content"}, "example_secret", algorithm="HS256") + example="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleGFtcGxlIjoiIn0.3DUg2Qivw_NF8v_LArZFFpsf1-Evv19ewhCVXbh6G2U" ) diff --git a/magpie/constants.py b/magpie/constants.py index 7133e31a6..d75f43f5f 100644 --- a/magpie/constants.py +++ b/magpie/constants.py @@ -15,6 +15,7 @@ import os import re import shutil +import sys import warnings from typing import TYPE_CHECKING @@ -201,7 +202,7 @@ def protected_user_name_regex(include_admin=True, patterns.append(get_constant("MAGPIE_ADMIN_USER", settings_container=settings_container)) if include_anonymous: patterns.append(get_constant("MAGPIE_ANONYMOUS_USER", settings_container=settings_container)) - if include_network and get_constant("MAGPIE_NETWORK_ENABLED", settings_container=settings_container): + if include_network and network_enabled(settings_container=settings_container): patterns.append( "{}.*".format(get_constant("MAGPIE_NETWORK_NAME_PREFIX", settings_container=settings_container)) ) @@ -223,7 +224,7 @@ def protected_user_email_regex(include_admin=True, patterns.append(get_constant("MAGPIE_ADMIN_EMAIL", settings_container=settings_container)) if include_anonymous: patterns.append(get_constant("MAGPIE_ANONYMOUS_EMAIL", settings_container=settings_container)) - if include_network and get_constant("MAGPIE_NETWORK_ENABLED", settings_container=settings_container): + if include_network and network_enabled(settings_container=settings_container): email_form = get_constant("MAGPIE_NETWORK_ANONYMOUS_EMAIL_FORMAT", settings_container=settings_container) patterns.append(email_form.format('.*')) return "^({})$".format("|".join(patterns)) @@ -243,13 +244,20 @@ def protected_group_name_regex(include_admin=True, patterns.append(get_constant("MAGPIE_ADMIN_GROUP", settings_container=settings_container)) if include_anonymous: patterns.append(get_constant("MAGPIE_ANONYMOUS_GROUP", settings_container=settings_container)) - if include_network and get_constant("MAGPIE_NETWORK_ENABLED", settings_container=settings_container): + if include_network and network_enabled(settings_container=settings_container): patterns.append( "{}.*".format(get_constant("MAGPIE_NETWORK_NAME_PREFIX", settings_container=settings_container)) ) return "^({})$".format("|".join(patterns)) +def network_enabled(settings_container=None): + # type: (Optional[AnySettingsContainer]) -> bool + if sys.version_info.major < 3 or sys.version_info.minor < 6: + return False + return bool(network_enabled(settings_container=settings_container)) + + def get_constant_setting_name(name): # type: (Str) -> Str """ diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py index 54af1fd64..f42aaeed0 100644 --- a/magpie/ui/management/views.py +++ b/magpie/ui/management/views.py @@ -22,7 +22,7 @@ from magpie.api import schemas from magpie.cli.sync_resources import OUT_OF_SYNC, fetch_single_service, get_last_sync, merge_local_and_remote_resources from magpie.cli.sync_services import SYNC_SERVICES_TYPES -from magpie.constants import get_constant +from magpie.constants import get_constant, network_enabled # FIXME: remove (REMOTE_RESOURCE_TREE_SERVICE, RESOURCE_TYPE_DICT), implement getters via API from magpie.models import ( REMOTE_RESOURCE_TREE_SERVICE, @@ -169,7 +169,7 @@ def edit_user(self): user_info["reason_{}".format(field)] = "" # add network information - if get_constant("MAGPIE_NETWORK_ENABLED", self.request): + if network_enabled(self.request): network_remote_users = (self.request.db.query(NetworkRemoteUser) .join(User).filter(User.user_name == user_name) .all()) diff --git a/magpie/ui/user/views.py b/magpie/ui/user/views.py index c7c091b20..b372075f8 100644 --- a/magpie/ui/user/views.py +++ b/magpie/ui/user/views.py @@ -6,7 +6,7 @@ from pyramid.view import view_config from magpie.api import schemas -from magpie.constants import get_constant +from magpie.constants import get_constant, network_enabled from magpie.models import UserGroupStatus, NetworkNode, NetworkRemoteUser from magpie.ui.utils import BaseViews, check_response, handle_errors, request_api from magpie.utils import get_json, get_logger @@ -90,7 +90,7 @@ def edit_current_user(self): raise_missing=False, raise_not_set=False)) user_info["user_with_error"] = schemas.UserStatuses.get(user_info["status"]) != schemas.UserStatuses.OK # add network information - if get_constant("MAGPIE_NETWORK_ENABLED", self.request): + if network_enabled(self.request): network_remote_users = (self.request.db.query(NetworkRemoteUser) .filter(NetworkRemoteUser.user_id == self.request.user.id) .all()) diff --git a/magpie/ui/utils.py b/magpie/ui/utils.py index bbb546a03..964670eaf 100644 --- a/magpie/ui/utils.py +++ b/magpie/ui/utils.py @@ -21,7 +21,7 @@ from magpie.api import schemas from magpie.api.generic import get_exception_info, get_request_info from magpie.api.requests import get_logged_user -from magpie.constants import get_constant, protected_user_name_regex, protected_group_name_regex +from magpie.constants import get_constant, protected_user_name_regex, protected_group_name_regex, network_enabled from magpie.models import UserGroupStatus from magpie.security import mask_credentials from magpie.utils import CONTENT_TYPE_JSON, get_header, get_json, get_logger, get_magpie_url @@ -592,7 +592,7 @@ def create_user(self, data): if (user_email or "").lower() in [usr["email"].lower() for usr in user_details]: data["invalid_user_email"] = True data["reason_user_email"] = "Conflict" - if get_constant("MAGPIE_NETWORK_ENABLED", self.request): + if network_enabled(self.request): anonymous_regex = protected_user_name_regex(include_admin=False, settings_container=self.request) if re.match(anonymous_regex, user_name): data["invalid_user_name"] = True diff --git a/requirements.txt b/requirements.txt index ba5bd02a7..ace07d762 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,7 +33,9 @@ paste pastedeploy pluggy psycopg2-binary>=2.7.1 -PyJWT[crypto]==2.8.0 +PyJWT[crypto]==2.8.0; python_version >= "3.8" +PyJWT[crypto]==2.7.0; python_version == "3.7" +PyJWT[crypto]==2.4.0; python_version == "3.6" pyramid>=1.10.2,<2 pyramid_beaker==0.8 pyramid_chameleon>=0.3 From 7ec1c9b5de65b21b5d1b4cc08dd29a4754856ae3 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Mon, 6 Nov 2023 15:53:25 -0500 Subject: [PATCH 09/55] deal with jwcrypto's python versions support --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ace07d762..2e03f2d3b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,7 +26,7 @@ gunicorn>=20; python_version >= "3" humanize jsonschema<4; python_version < "3.6" jsonschema>=4; python_version >= "3.6" -jwcrypto==1.5.0 +jwcrypto==1.5.0; python_version >= "3.6" lxml>=3.7 mako # controlled by pyramid_mako paste From 50aa4e5bf43551a2257e79c84cecde71c1a28d00 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Mon, 6 Nov 2023 16:00:54 -0500 Subject: [PATCH 10/55] fix accidental infinite recursion --- magpie/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/magpie/constants.py b/magpie/constants.py index d75f43f5f..f08d0a40e 100644 --- a/magpie/constants.py +++ b/magpie/constants.py @@ -255,7 +255,7 @@ def network_enabled(settings_container=None): # type: (Optional[AnySettingsContainer]) -> bool if sys.version_info.major < 3 or sys.version_info.minor < 6: return False - return bool(network_enabled(settings_container=settings_container)) + return bool(get_constant("MAGPIE_NETWORK_ENABLED", settings_container=settings_container)) def get_constant_setting_name(name): From 3e266c70cb1c47f45bbfb6e3e4161a675230ebe5 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Mon, 6 Nov 2023 16:13:47 -0500 Subject: [PATCH 11/55] clean up planning comments and fix route loading with network mode is off --- magpie/api/management/__init__.py | 3 +-- magpie/api/management/network/__init__.py | 4 ++-- magpie/api/management/network/node/__init__.py | 10 +++++----- magpie/api/management/network/remote_user/__init__.py | 6 +++--- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/magpie/api/management/__init__.py b/magpie/api/management/__init__.py index a1132b099..fb08b1482 100644 --- a/magpie/api/management/__init__.py +++ b/magpie/api/management/__init__.py @@ -11,6 +11,5 @@ def includeme(config): config.include("magpie.api.management.service") config.include("magpie.api.management.resource") config.include("magpie.api.management.register") - if network_enabled(config): - config.include("magpie.api.management.network") + config.include("magpie.api.management.network") config.scan() diff --git a/magpie/api/management/network/__init__.py b/magpie/api/management/network/__init__.py index 94ea63cb3..96b0df9c1 100644 --- a/magpie/api/management/network/__init__.py +++ b/magpie/api/management/network/__init__.py @@ -6,8 +6,8 @@ def includeme(config): from magpie.api import schemas as s LOGGER.info("Adding API network ...") - config.add_route(**s.service_api_route_info(s.NetworkTokenAPI)) # /network/token POST and DELETE - config.add_route(**s.service_api_route_info(s.NetworkJSONWebKeySetAPI)) # /network/jwks GET + config.add_route(**s.service_api_route_info(s.NetworkTokenAPI)) + config.add_route(**s.service_api_route_info(s.NetworkJSONWebKeySetAPI)) config.include("magpie.api.management.network.node") config.include("magpie.api.management.network.remote_user") config.scan() diff --git a/magpie/api/management/network/node/__init__.py b/magpie/api/management/network/node/__init__.py index e611a0fe5..4ce70c6fc 100644 --- a/magpie/api/management/network/node/__init__.py +++ b/magpie/api/management/network/node/__init__.py @@ -6,10 +6,10 @@ def includeme(config): from magpie.api import schemas as s LOGGER.info("Adding API network node...") - config.add_route(**s.service_api_route_info(s.NetworkNodesAPI)) # /network/nodes GET and POST - config.add_route(**s.service_api_route_info(s.NetworkNodeAPI)) # /network/nodes/{node_name} GET, DELETE, PATCH - config.add_route(**s.service_api_route_info(s.NetworkNodeTokenAPI)) # /network/nodes/{node_name}/token GET and DELETE - config.add_route(**s.service_api_route_info(s.NetworkNodesLinkAPI)) # /network/nodes/link GET - config.add_route(**s.service_api_route_info(s.NetworkNodeLinkAPI)) # /network/nodes/link POST + config.add_route(**s.service_api_route_info(s.NetworkNodesAPI)) + config.add_route(**s.service_api_route_info(s.NetworkNodeAPI)) + config.add_route(**s.service_api_route_info(s.NetworkNodeTokenAPI)) + config.add_route(**s.service_api_route_info(s.NetworkNodesLinkAPI)) + config.add_route(**s.service_api_route_info(s.NetworkNodeLinkAPI)) config.scan() diff --git a/magpie/api/management/network/remote_user/__init__.py b/magpie/api/management/network/remote_user/__init__.py index 5b1a5ba98..084964248 100644 --- a/magpie/api/management/network/remote_user/__init__.py +++ b/magpie/api/management/network/remote_user/__init__.py @@ -6,8 +6,8 @@ def includeme(config): from magpie.api import schemas as s LOGGER.info("Adding API network remote users ...") - config.add_route(**s.service_api_route_info(s.NetworkRemoteUsersAPI)) # /network/remote_users GET and POST - config.add_route(**s.service_api_route_info(s.NetworkRemoteUserAPI)) # /network/remote_users/{remote_user_name} GET, DELETE, PATCH - config.add_route(**s.service_api_route_info(s.NetworkRemoteUsersCurrentAPI)) # /network/remote_users/current GET + config.add_route(**s.service_api_route_info(s.NetworkRemoteUsersAPI)) + config.add_route(**s.service_api_route_info(s.NetworkRemoteUserAPI)) + config.add_route(**s.service_api_route_info(s.NetworkRemoteUsersCurrentAPI)) config.scan() From 38410deddfffb2515553d028e78fde311cdf50b5 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Tue, 7 Nov 2023 13:00:42 -0500 Subject: [PATCH 12/55] remove unused imports --- magpie/api/management/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/magpie/api/management/__init__.py b/magpie/api/management/__init__.py index fb08b1482..39a799d3c 100644 --- a/magpie/api/management/__init__.py +++ b/magpie/api/management/__init__.py @@ -1,4 +1,3 @@ -from magpie.constants import network_enabled from magpie.utils import get_logger LOGGER = get_logger(__name__) From 4a6ed0d73c2fc735196e4149758ebc9c6ba7defc Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Tue, 7 Nov 2023 13:18:15 -0500 Subject: [PATCH 13/55] ok fine, we'll try to support end-of-life pythons --- docs/authentication.rst | 3 --- requirements.txt | 2 ++ 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/authentication.rst b/docs/authentication.rst index c6d9efc23..6bd6c96ff 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -439,9 +439,6 @@ Users with accounts on multiple instances in the network can also choose to link use access tokens to ensure that they have the same access to resources that they would have if they logged in to `Magpie` using any other method. -.. warning:: - Network Mode is only supported when the python version is at least 3.6. If the python version is less than 3.6, - setting the :envvar:`MAGPIE_NETWORK_ENABLED` will have no effect. Managing the Network ~~~~~~~~~~~~~~~~~~~~ diff --git a/requirements.txt b/requirements.txt index 2e03f2d3b..0075f6425 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,6 +27,7 @@ humanize jsonschema<4; python_version < "3.6" jsonschema>=4; python_version >= "3.6" jwcrypto==1.5.0; python_version >= "3.6" +jwcrypto==0.8.8; python_version < "3.6" lxml>=3.7 mako # controlled by pyramid_mako paste @@ -36,6 +37,7 @@ psycopg2-binary>=2.7.1 PyJWT[crypto]==2.8.0; python_version >= "3.8" PyJWT[crypto]==2.7.0; python_version == "3.7" PyJWT[crypto]==2.4.0; python_version == "3.6" +PyJWT[crypto]==2.0.0a1; python_version < "3.6" pyramid>=1.10.2,<2 pyramid_beaker==0.8 pyramid_chameleon>=0.3 From d690c418735e781b7421f4343b27060b1e637672 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Tue, 7 Nov 2023 13:27:33 -0500 Subject: [PATCH 14/55] ok let's try this version again --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0075f6425..6666f96b4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,7 +27,7 @@ humanize jsonschema<4; python_version < "3.6" jsonschema>=4; python_version >= "3.6" jwcrypto==1.5.0; python_version >= "3.6" -jwcrypto==0.8.8; python_version < "3.6" +jwcrypto==0.8; python_version < "3.6" lxml>=3.7 mako # controlled by pyramid_mako paste From 5f4087d1bbda82e3d19b810cab48fdc0ef4d6629 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Tue, 7 Nov 2023 15:41:52 -0500 Subject: [PATCH 15/55] checks updates --- Makefile | 1 + magpie/api/login/login.py | 123 +++++++++--------- magpie/api/management/group/group_utils.py | 2 +- magpie/api/management/group/group_views.py | 2 +- .../api/management/network/network_utils.py | 16 ++- .../api/management/network/network_views.py | 13 +- .../network/node/network_node_utils.py | 6 +- .../network/node/network_node_views.py | 20 +-- .../network/remote_user/remote_user_utils.py | 4 +- .../network/remote_user/remote_user_views.py | 15 +-- magpie/api/management/user/user_utils.py | 7 +- magpie/cli/purge_expired_network_tokens.py | 5 +- magpie/constants.py | 6 +- magpie/models.py | 4 +- magpie/ui/management/views.py | 7 +- magpie/ui/network/views.py | 1 - magpie/ui/user/views.py | 2 +- magpie/ui/utils.py | 2 +- requirements.txt | 8 +- 19 files changed, 123 insertions(+), 121 deletions(-) diff --git a/Makefile b/Makefile index 579d53fd0..18d023a5b 100644 --- a/Makefile +++ b/Makefile @@ -551,6 +551,7 @@ check-security-deps-only: mkdir-reports ## run security checks on package depen -r "$(APP_ROOT)/requirements-sys.txt" \ -i 42194 \ -i 51668 \ + -i 51021 \ 1> >(tee "$(REPORTS_DIR)/check-security-deps.txt")' .PHONY: check-security-code-only diff --git a/magpie/api/login/login.py b/magpie/api/login/login.py index 5bff52a4e..2a86aaf6a 100644 --- a/magpie/api/login/login.py +++ b/magpie/api/login/login.py @@ -34,7 +34,7 @@ from magpie.api import schemas as s from magpie.api.management.user.user_formats import format_user from magpie.api.management.user.user_utils import create_user -from magpie.constants import get_constant, protected_user_name_regex, protected_user_email_regex +from magpie.constants import get_constant, protected_user_email_regex, protected_user_name_regex from magpie.security import authomatic_setup, get_providers from magpie.utils import ( CONTENT_TYPE_JSON, @@ -276,7 +276,7 @@ def network_login(request): """ if "Authorization" in request.headers: token_type, token = request.headers.get("Authorization").split() - if token_type != "Bearer": + if token_type != "Bearer": # nosec: B105 ax.raise_http(http_error=HTTPBadRequest, detail=s.ProviderSignin_GET_BadRequestResponseSchema.description) network_token = models.NetworkToken.by_token(token) @@ -288,9 +288,8 @@ def network_login(request): ax.verify_param(authenticated_user.name, not_matches=True, param_compare=anonymous_regex, http_error=HTTPForbidden, msg_on_fail=s.Signin_POST_ForbiddenResponseSchema.description) return login_success_external(request, authenticated_user) - else: - ax.raise_http(http_error=HTTPBadRequest, - detail=s.ProviderSignin_GET_BadRequestResponseSchema.description) + ax.raise_http(http_error=HTTPBadRequest, + detail=s.ProviderSignin_GET_BadRequestResponseSchema.description) @s.ProviderSigninAPI.get(schema=s.ProviderSignin_GET_RequestSchema, tags=[s.SessionTag], @@ -306,65 +305,63 @@ def external_login_view(request): try: if provider_name == get_constant("MAGPIE_NETWORK_PROVIDER", request): return network_login(request) + authomatic_handler = authomatic_setup(request) + # if we directly have the Authorization header, bypass authomatic login and retrieve 'userinfo' to signin + if "Authorization" in request.headers and "authomatic" not in request.cookies: + provider_config = authomatic_handler.config.get(provider_name, {}) + provider_class = resolve_provider_class(provider_config.get("class_")) + provider = provider_class(authomatic_handler, adapter=None, provider_name=provider_name) + # provide the token user data, let the external provider update it on login afterwards + token_type, access_token = request.headers.get("Authorization").split() + data = {"access_token": access_token, "token_type": token_type} + cred = Credentials(authomatic_handler.config, token=access_token, token_type=token_type, + provider=provider) + provider.credentials = cred + result = LoginResult(provider) + # pylint: disable=W0212 + result.provider.user = result.provider._update_or_create_user(data, credentials=cred) # noqa: W0212 + + # otherwise, use the standard login procedure else: - authomatic_handler = authomatic_setup(request) - - # if we directly have the Authorization header, bypass authomatic login and retrieve 'userinfo' to signin - if "Authorization" in request.headers and "authomatic" not in request.cookies: - provider_config = authomatic_handler.config.get(provider_name, {}) - provider_class = resolve_provider_class(provider_config.get("class_")) - provider = provider_class(authomatic_handler, adapter=None, provider_name=provider_name) - # provide the token user data, let the external provider update it on login afterwards - token_type, access_token = request.headers.get("Authorization").split() - data = {"access_token": access_token, "token_type": token_type} - cred = Credentials(authomatic_handler.config, token=access_token, token_type=token_type, - provider=provider) - provider.credentials = cred - result = LoginResult(provider) - # pylint: disable=W0212 - result.provider.user = result.provider._update_or_create_user(data, credentials=cred) # noqa: W0212 - - # otherwise, use the standard login procedure - else: - result = authomatic_handler.login(WebObAdapter(request, response), provider_name) - if result is None: - if response.location is not None: - return HTTPTemporaryRedirect(location=response.location, headers=response.headers) - return response - - if result: - if result.error: - # Login procedure finished with an error. - error = result.error.to_dict() if hasattr(result.error, "to_dict") else result.error - LOGGER.debug("Login failure with error. [%r]", error) - return login_failure_view(request, reason=result.error.message) - if result.user: - # OAuth 2.0 and OAuth 1.0a provide only limited user data on login, - # update the user to get more info. - if not (result.user.name and result.user.id): - try: - response = result.user.update() - # this error can happen if providing incorrectly formed authorization header - except OAuth2Error as exc: - LOGGER.debug("Login failure with Authorization header.") - ax.raise_http(http_error=HTTPBadRequest, content={"reason": str(exc.message)}, - detail=s.ProviderSignin_GET_BadRequestResponseSchema.description) - # verify that the update procedure succeeded with provided token - if 400 <= response.status < 500: - LOGGER.debug("Login failure with invalid token.") - ax.raise_http(http_error=HTTPUnauthorized, - detail=s.ProviderSignin_GET_UnauthorizedResponseSchema.description) - # create/retrieve the user using found details from login provider - # find possibly already registered user by external_id/provider - external_id = result.user.username or result.user.id - user = ExternalIdentityService.user_by_external_id_and_provider(external_id, provider_name, - request.db) - if user is None: - # create new user with an External Identity - user = new_user_external(external_user_name=result.user.name, external_id=external_id, - email=result.user.email, provider_name=result.provider.name, - db_session=request.db) - return login_success_external(request, user) + result = authomatic_handler.login(WebObAdapter(request, response), provider_name) + if result is None: + if response.location is not None: + return HTTPTemporaryRedirect(location=response.location, headers=response.headers) + return response + + if result: + if result.error: + # Login procedure finished with an error. + error = result.error.to_dict() if hasattr(result.error, "to_dict") else result.error + LOGGER.debug("Login failure with error. [%r]", error) + return login_failure_view(request, reason=result.error.message) + if result.user: + # OAuth 2.0 and OAuth 1.0a provide only limited user data on login, + # update the user to get more info. + if not (result.user.name and result.user.id): + try: + response = result.user.update() + # this error can happen if providing incorrectly formed authorization header + except OAuth2Error as exc: + LOGGER.debug("Login failure with Authorization header.") + ax.raise_http(http_error=HTTPBadRequest, content={"reason": str(exc.message)}, + detail=s.ProviderSignin_GET_BadRequestResponseSchema.description) + # verify that the update procedure succeeded with provided token + if 400 <= response.status < 500: + LOGGER.debug("Login failure with invalid token.") + ax.raise_http(http_error=HTTPUnauthorized, + detail=s.ProviderSignin_GET_UnauthorizedResponseSchema.description) + # create/retrieve the user using found details from login provider + # find possibly already registered user by external_id/provider + external_id = result.user.username or result.user.id + user = ExternalIdentityService.user_by_external_id_and_provider(external_id, provider_name, + request.db) + if user is None: + # create new user with an External Identity + user = new_user_external(external_user_name=result.user.name, external_id=external_id, + email=result.user.email, provider_name=result.provider.name, + db_session=request.db) + return login_success_external(request, user) except Exception as exc: exc_msg = "Unhandled error during external provider '{}' login. [{!s}]".format(provider_name, exc) LOGGER.exception(exc_msg, exc_info=True) diff --git a/magpie/api/management/group/group_utils.py b/magpie/api/management/group/group_utils.py index 1ab57d9eb..89d4aea4d 100644 --- a/magpie/api/management/group/group_utils.py +++ b/magpie/api/management/group/group_utils.py @@ -22,7 +22,7 @@ from magpie.api.management.resource.resource_utils import check_valid_service_or_resource_permission from magpie.api.management.service import service_formats as sf from magpie.api.webhooks import WebhookAction, get_permission_update_params, process_webhook_requests -from magpie.constants import protected_group_name_regex, network_enabled +from magpie.constants import network_enabled, protected_group_name_regex from magpie.permissions import PermissionSet, PermissionType, format_permissions from magpie.services import SERVICE_TYPE_DICT diff --git a/magpie/api/management/group/group_views.py b/magpie/api/management/group/group_views.py index 600b92aec..b0bb5fe2e 100644 --- a/magpie/api/management/group/group_views.py +++ b/magpie/api/management/group/group_views.py @@ -9,7 +9,7 @@ from magpie.api.management.group import group_formats as gf from magpie.api.management.group import group_utils as gu from magpie.api.management.service import service_utils as su -from magpie.constants import get_constant, protected_group_name_regex, network_enabled +from magpie.constants import get_constant, network_enabled, protected_group_name_regex from magpie.models import TemporaryToken, TokenOperation, UserGroupStatus diff --git a/magpie/api/management/network/network_utils.py b/magpie/api/management/network/network_utils.py index 3d908fe72..ec1cc71a2 100644 --- a/magpie/api/management/network/network_utils.py +++ b/magpie/api/management/network/network_utils.py @@ -5,8 +5,8 @@ import jwt from cryptography.hazmat.primitives import serialization from jwcrypto import jwk - from pyramid.httpexceptions import HTTPInternalServerError, HTTPNotFound + from magpie import models from magpie.api import exception as ax from magpie.api import schemas as s @@ -14,15 +14,17 @@ from magpie.utils import get_logger if TYPE_CHECKING: - from typing import List, Optional, Dict, Tuple - from magpie.typedefs import JSON, Str, AnySettingsContainer + from typing import Dict, List, Optional, Tuple + from cryptography.hazmat.primitives.asymmetric.types import PrivateKeyTypes from pyramid.request import Request + from magpie.typedefs import JSON, AnySettingsContainer, Str + LOGGER = get_logger(__name__) PEM_FILE_DELIMITER = ":" -PEM_PASSWORD_DELIMITER = ":" +PEM_PASSWORD_DELIMITER = ":" # nosec: B105 def _pem_file_content(primary=False): @@ -33,7 +35,7 @@ def _pem_file_content(primary=False): pem_files = get_constant("MAGPIE_NETWORK_PEM_FILES").split(PEM_FILE_DELIMITER) content = [] for pem_file in pem_files: - with open(pem_file, 'rb') as f: + with open(pem_file, "rb") as f: content.append(f.read()) if primary: break @@ -143,8 +145,8 @@ def decode_jwt(token, node, settings_container=None): def get_network_models_from_request_token(request, create_network_remote_user=False): # type: (Request, bool) -> Tuple[models.NetworkNode, Optional[models.NetworkRemoteUser]] """ - Return a ``NetworkNode`` and associated ``NetworkRemoteUser`` determined by parsing the claims in the JWT included in the - ``request`` argument. + Return a ``NetworkNode`` and associated ``NetworkRemoteUser`` determined by parsing the claims in the JWT included + in the ``request`` argument. If the ``NetworkRemoteUser`` does not exist and ``create_network_remote_user`` is ``True``, this creates a new ``NetworkRemoteUser`` associated with the anonymous user for the given ``NetworkNode`` and adds it to the current diff --git a/magpie/api/management/network/network_views.py b/magpie/api/management/network/network_views.py index 91a8a3ed5..4c12e394f 100644 --- a/magpie/api/management/network/network_views.py +++ b/magpie/api/management/network/network_views.py @@ -1,23 +1,19 @@ import sqlalchemy -from pyramid.httpexceptions import ( - HTTPNotFound, - HTTPOk, - HTTPCreated, -) +from pyramid.httpexceptions import HTTPCreated, HTTPNotFound, HTTPOk from pyramid.security import NO_PERMISSION_REQUIRED from pyramid.view import view_config from magpie import models from magpie.api import exception as ax from magpie.api import schemas as s -from magpie.api.management.network.network_utils import jwks, get_network_models_from_request_token +from magpie.api.management.network.network_utils import get_network_models_from_request_token, jwks @s.NetworkTokenAPI.post(schema=s.NetworkToken_POST_RequestSchema, tags=[s.NetworkTag], response_schemas=s.NetworkToken_POST_responses) @view_config(route_name=s.NetworkTokenAPI.name, request_method="POST") def post_network_token_view(request): - node, network_remote_user = get_network_models_from_request_token(request, create_network_remote_user=True) + _, network_remote_user = get_network_models_from_request_token(request, create_network_remote_user=True) network_token = network_remote_user.network_token if network_token: token = network_token.refresh_token() @@ -41,8 +37,7 @@ def delete_network_token_view(request): sqlalchemy.inspect(network_remote_user).persisted): request.db.delete(network_remote_user) # clean up unused record in the database return ax.valid_http(http_success=HTTPOk, detail=s.NetworkToken_DELETE_OkResponseSchema.description) - else: - ax.raise_http(http_error=HTTPNotFound, detail=s.NetworkNodeToken_DELETE_NotFoundResponseSchema.description) + ax.raise_http(http_error=HTTPNotFound, detail=s.NetworkNodeToken_DELETE_NotFoundResponseSchema.description) @s.NetworkJSONWebKeySetAPI.get(tags=[s.NetworkTag], response_schemas=s.NetworkJSONWebKeySet_GET_responses) diff --git a/magpie/api/management/network/node/network_node_utils.py b/magpie/api/management/network/node/network_node_utils.py index f590629a4..d44ec62b7 100644 --- a/magpie/api/management/network/node/network_node_utils.py +++ b/magpie/api/management/network/node/network_node_utils.py @@ -1,17 +1,19 @@ from typing import TYPE_CHECKING +from pyramid.httpexceptions import HTTPBadRequest, HTTPConflict + from magpie import models from magpie.api import exception as ax from magpie.api import schemas as s from magpie.api.exception import URL_REGEX from magpie.cli.register_defaults import register_user_with_group from magpie.constants import get_constant -from pyramid.httpexceptions import HTTPBadRequest, HTTPConflict if TYPE_CHECKING: - from magpie.typedefs import Str, Optional, Session from pyramid.request import Request + from magpie.typedefs import Optional, Session, Str + NAME_REGEX = r"^[\w-]+$" diff --git a/magpie/api/management/network/node/network_node_views.py b/magpie/api/management/network/node/network_node_views.py index edfd7ca3a..b18eeddf2 100644 --- a/magpie/api/management/network/node/network_node_views.py +++ b/magpie/api/management/network/node/network_node_views.py @@ -1,25 +1,29 @@ import jwt +import requests from pyramid.httpexceptions import ( HTTPBadRequest, - HTTPNotFound, - HTTPOk, HTTPCreated, - HTTPInternalServerError, HTTPForbidden, - HTTPFound + HTTPFound, + HTTPInternalServerError, + HTTPNotFound, + HTTPOk ) from pyramid.security import Authenticated from pyramid.view import view_config from six.moves.urllib import parse as up -import requests from magpie import models from magpie.api import exception as ax from magpie.api import requests as ar from magpie.api import schemas as s -from magpie.api.management.network.network_utils import encode_jwt, decode_jwt -from magpie.api.management.network.node.network_node_utils import delete_network_node, \ - check_network_node_info, create_associated_user_groups, update_associated_user_groups +from magpie.api.management.network.network_utils import decode_jwt, encode_jwt +from magpie.api.management.network.node.network_node_utils import ( + check_network_node_info, + create_associated_user_groups, + delete_network_node, + update_associated_user_groups +) @s.NetworkNodesAPI.get(tags=[s.NetworkTag], response_schemas=s.NetworkNodes_GET_responses) diff --git a/magpie/api/management/network/remote_user/remote_user_utils.py b/magpie/api/management/network/remote_user/remote_user_utils.py index 0ba456722..1871830f3 100644 --- a/magpie/api/management/network/remote_user/remote_user_utils.py +++ b/magpie/api/management/network/remote_user/remote_user_utils.py @@ -1,6 +1,6 @@ from typing import TYPE_CHECKING -from pyramid.httpexceptions import HTTPNotFound, HTTPForbidden +from pyramid.httpexceptions import HTTPForbidden, HTTPNotFound from magpie import models from magpie.api import exception as ax @@ -10,7 +10,9 @@ if TYPE_CHECKING: from typing import Optional + from pyramid.request import Request + from magpie.typedefs import Session, Str diff --git a/magpie/api/management/network/remote_user/remote_user_views.py b/magpie/api/management/network/remote_user/remote_user_views.py index 793639a2e..6b2f42011 100644 --- a/magpie/api/management/network/remote_user/remote_user_views.py +++ b/magpie/api/management/network/remote_user/remote_user_views.py @@ -1,19 +1,14 @@ -from pyramid.httpexceptions import ( - HTTPBadRequest, - HTTPNotFound, - HTTPOk, - HTTPCreated, - HTTPForbidden -) +from pyramid.httpexceptions import HTTPBadRequest, HTTPCreated, HTTPForbidden, HTTPNotFound, HTTPOk from pyramid.security import Authenticated - from pyramid.view import view_config from magpie import models from magpie.api import exception as ax from magpie.api import schemas as s -from magpie.api.management.network.remote_user.remote_user_utils import requested_remote_user, \ - check_remote_user_access_permissions +from magpie.api.management.network.remote_user.remote_user_utils import ( + check_remote_user_access_permissions, + requested_remote_user +) from magpie.constants import protected_user_name_regex diff --git a/magpie/api/management/user/user_utils.py b/magpie/api/management/user/user_utils.py index 39f46a5b1..8b1c9ed12 100644 --- a/magpie/api/management/user/user_utils.py +++ b/magpie/api/management/user/user_utils.py @@ -33,7 +33,7 @@ get_permission_update_params, process_webhook_requests ) -from magpie.constants import get_constant, protected_user_name_regex, protected_user_email_regex +from magpie.constants import get_constant, protected_user_email_regex, protected_user_name_regex from magpie.models import TemporaryToken, TokenOperation from magpie.permissions import PermissionSet, PermissionType, format_permissions from magpie.services import SERVICE_TYPE_DICT, service_factory @@ -886,7 +886,10 @@ def check_user_info(user_name=None, email=None, password=None, group_name=None, msg_on_fail=s.Users_CheckInfo_ReservedKeyword_BadRequestResponseSchema.description) if check_anonymous: anonymous_user_name_regex = protected_user_name_regex() - ax.verify_param(user_name, not_matches=True, param_compare=anonymous_user_name_regex, param_name="user_name", + ax.verify_param(user_name, + not_matches=True, + param_compare=anonymous_user_name_regex, + param_name="user_name", http_error=HTTPBadRequest, msg_on_fail=s.Users_CheckInfo_Email_BadRequestResponseSchema.description) if check_email: diff --git a/magpie/cli/purge_expired_network_tokens.py b/magpie/cli/purge_expired_network_tokens.py index dce43d2ed..50f8db700 100644 --- a/magpie/cli/purge_expired_network_tokens.py +++ b/magpie/cli/purge_expired_network_tokens.py @@ -17,10 +17,11 @@ from magpie.cli.utils import make_logging_options, setup_logger_from_options from magpie.constants import get_constant from magpie.db import get_db_session_from_config_ini -from magpie.utils import get_logger, raise_log, print_log +from magpie.utils import get_logger, print_log, raise_log if TYPE_CHECKING: from typing import Optional, Sequence + from magpie.typedefs import Str LOGGER = get_logger(__name__, @@ -50,7 +51,7 @@ def main(args=None, parser=None, namespace=None): # clean up unused records in the database (no need to keep records associated with anonymous network users) (db_session.query(models.NetworkRemoteUser) .filter(models.NetworkRemoteUser.user_id.in_(anonymous_network_user_ids)) - .filter(models.NetworkRemoteUser.network_token_id == None) # noqa: E711 + .filter(models.NetworkRemoteUser.network_token_id == None) # noqa: E711 # pylint: disable=singleton-comparison .delete()) try: transaction.commit() diff --git a/magpie/constants.py b/magpie/constants.py index f08d0a40e..da690d22f 100644 --- a/magpie/constants.py +++ b/magpie/constants.py @@ -24,7 +24,7 @@ if TYPE_CHECKING: # pylint: disable=W0611,unused-import - from typing import Optional, List + from typing import List, Optional from magpie.typedefs import AnySettingsContainer, SettingValue, Str @@ -153,7 +153,7 @@ def _get_default_log_level(): MAGPIE_CONTEXT_PERMISSION = "MAGPIE_CONTEXT_USER" # path user must be itself, MAGPIE_LOGGED_USER or unauthenticated MAGPIE_LOGGED_USER = "current" MAGPIE_DEFAULT_PROVIDER = "ziggurat" -MAGPIE_NETWORK_TOKEN_NAME = "magpie_token" +MAGPIE_NETWORK_TOKEN_NAME = "magpie_token" # nosec: B105 MAGPIE_NETWORK_PROVIDER = "magpie_network" MAGPIE_NETWORK_NAME_PREFIX = "anonymous_network_" MAGPIE_NETWORK_ANONYMOUS_EMAIL_FORMAT = "{}{}@mail.com".format(MAGPIE_NETWORK_NAME_PREFIX, "{}") @@ -226,7 +226,7 @@ def protected_user_email_regex(include_admin=True, patterns.append(get_constant("MAGPIE_ANONYMOUS_EMAIL", settings_container=settings_container)) if include_network and network_enabled(settings_container=settings_container): email_form = get_constant("MAGPIE_NETWORK_ANONYMOUS_EMAIL_FORMAT", settings_container=settings_container) - patterns.append(email_form.format('.*')) + patterns.append(email_form.format(".*")) return "^({})$".format("|".join(patterns)) diff --git a/magpie/models.py b/magpie/models.py index 6ca54630c..2dfb83410 100644 --- a/magpie/models.py +++ b/magpie/models.py @@ -1020,8 +1020,8 @@ class NetworkRemoteUser(BaseModel, Base): network_node = relationship("NetworkNode", foreign_keys=[network_node_id]) network_token = relationship("NetworkToken", foreign_keys=[network_token_id], back_populates="network_remote_user") - __table_args__ = (UniqueConstraint('user_id', 'network_node_id'), - UniqueConstraint('name', 'network_node_id')) + __table_args__ = (UniqueConstraint("user_id", "network_node_id"), + UniqueConstraint("name", "network_node_id")) def as_dict(self): # type: () -> Dict[Str, Str] diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py index f42aaeed0..20a299058 100644 --- a/magpie/ui/management/views.py +++ b/magpie/ui/management/views.py @@ -27,10 +27,11 @@ from magpie.models import ( REMOTE_RESOURCE_TREE_SERVICE, RESOURCE_TYPE_DICT, - UserGroupStatus, - UserStatuses, NetworkNode, - User, NetworkRemoteUser + NetworkRemoteUser, + User, + UserGroupStatus, + UserStatuses ) from magpie.permissions import Permission, PermissionSet # FIXME: remove (SERVICE_TYPE_DICT), implement getters via API diff --git a/magpie/ui/network/views.py b/magpie/ui/network/views.py index 37a414876..c6abceb5b 100644 --- a/magpie/ui/network/views.py +++ b/magpie/ui/network/views.py @@ -8,7 +8,6 @@ from magpie.ui.utils import BaseViews from magpie.utils import get_logger - LOGGER = get_logger(__name__) diff --git a/magpie/ui/user/views.py b/magpie/ui/user/views.py index b372075f8..5f2cb0b33 100644 --- a/magpie/ui/user/views.py +++ b/magpie/ui/user/views.py @@ -7,7 +7,7 @@ from magpie.api import schemas from magpie.constants import get_constant, network_enabled -from magpie.models import UserGroupStatus, NetworkNode, NetworkRemoteUser +from magpie.models import NetworkNode, NetworkRemoteUser, UserGroupStatus from magpie.ui.utils import BaseViews, check_response, handle_errors, request_api from magpie.utils import get_json, get_logger diff --git a/magpie/ui/utils.py b/magpie/ui/utils.py index 964670eaf..0f8d32d3b 100644 --- a/magpie/ui/utils.py +++ b/magpie/ui/utils.py @@ -21,7 +21,7 @@ from magpie.api import schemas from magpie.api.generic import get_exception_info, get_request_info from magpie.api.requests import get_logged_user -from magpie.constants import get_constant, protected_user_name_regex, protected_group_name_regex, network_enabled +from magpie.constants import get_constant, network_enabled, protected_group_name_regex, protected_user_name_regex from magpie.models import UserGroupStatus from magpie.security import mask_credentials from magpie.utils import CONTENT_TYPE_JSON, get_header, get_json, get_logger, get_magpie_url diff --git a/requirements.txt b/requirements.txt index 6666f96b4..3456f867b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,7 +27,7 @@ humanize jsonschema<4; python_version < "3.6" jsonschema>=4; python_version >= "3.6" jwcrypto==1.5.0; python_version >= "3.6" -jwcrypto==0.8; python_version < "3.6" +jwcrypto==0.8; python_version < "3.6" # pyup: ignore lxml>=3.7 mako # controlled by pyramid_mako paste @@ -35,9 +35,9 @@ pastedeploy pluggy psycopg2-binary>=2.7.1 PyJWT[crypto]==2.8.0; python_version >= "3.8" -PyJWT[crypto]==2.7.0; python_version == "3.7" -PyJWT[crypto]==2.4.0; python_version == "3.6" -PyJWT[crypto]==2.0.0a1; python_version < "3.6" +PyJWT[crypto]==2.7.0; python_version == "3.7" # pyup: ignore +PyJWT[crypto]==2.4.0; python_version == "3.6" # pyup: ignore +PyJWT[crypto]==2.0.0a1; python_version < "3.6" # pyup: ignore pyramid>=1.10.2,<2 pyramid_beaker==0.8 pyramid_chameleon>=0.3 From 18ee9dda7a18a9396ac1410a2bbdd9b7cbf31661 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Tue, 7 Nov 2023 15:48:42 -0500 Subject: [PATCH 16/55] fix css --- magpie/ui/home/static/style.css | 1 - 1 file changed, 1 deletion(-) diff --git a/magpie/ui/home/static/style.css b/magpie/ui/home/static/style.css index 28e1427d3..d1a56a1ca 100644 --- a/magpie/ui/home/static/style.css +++ b/magpie/ui/home/static/style.css @@ -1451,4 +1451,3 @@ table.authorization-table td { border: 0; text-align: center; } - From a3aa357c5417947c2c1a451bf19244af761ab37e Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Mon, 18 Dec 2023 14:41:51 -0500 Subject: [PATCH 17/55] review suggestions --- .gitignore | 3 +- docs/authentication.rst | 10 +-- docs/configuration.rst | 7 +- ...3-08-25_2cfe144538e8_add_network_tables.py | 2 +- magpie/api/management/network/__init__.py | 1 + .../api/management/network/network_utils.py | 17 +++-- .../api/management/network/network_views.py | 49 ++++++++++++-- .../api/management/network/node/__init__.py | 2 +- .../network/node/network_node_utils.py | 2 +- .../network/node/network_node_views.py | 19 ++++-- .../network/remote_user/remote_user_views.py | 5 +- magpie/api/schemas.py | 66 +++++++++++++++---- magpie/cli/purge_expired_network_tokens.py | 66 ++++++++++++++----- magpie/constants.py | 12 ++-- magpie/models.py | 11 ++-- magpie/ui/management/views.py | 10 ++- magpie/ui/network/__init__.py | 4 +- magpie/ui/network/templates/authorize.mako | 9 ++- magpie/ui/network/views.py | 42 +++++++----- magpie/ui/user/views.py | 16 +++-- setup.cfg | 4 +- 21 files changed, 248 insertions(+), 109 deletions(-) diff --git a/.gitignore b/.gitignore index 79605d7f2..46fd41765 100644 --- a/.gitignore +++ b/.gitignore @@ -58,4 +58,5 @@ gunicorn.app.wsgiapp error_log.txt # Secrets -key.pem +*.pem +*.key diff --git a/docs/authentication.rst b/docs/authentication.rst index 6bd6c96ff..2c50c043c 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -458,16 +458,16 @@ In order to register another `Magpie` instance as part of the same network, an a * ``authorization_url`` * URL that provides the instance's Oauth authorize endpoint. * This is usually ``https://{hostname}/ui/network/authorize`` where ``{hostname}`` is the hostname of the other - instance + instance. * ``token_url`` * URL that provides the instances Oauth token endpoint. - * This is usually ``https://{hostname}/network/token`` where ``{hostname}`` is the hostname of the other instance + * This is usually ``https://{hostname}/network/token`` where ``{hostname}`` is the hostname of the other instance. * ``redirect_uris`` - * Space delimited list of valid redirect URIs for the instance. These are used by the instance's Oauth authorize + * JSON array of valid redirect URIs for the instance. These are used by the instance's Oauth authorize endpoint to safely redirect the user back once they have authorized `Magpie` to link their accounts on two different instances. - * This is usually ``https://{hostname}/network/nodes/link`` where ``{hostname}`` is the hostname of the other - instance + * This is usually ``https://{hostname}/network/link`` where ``{hostname}`` is the hostname of the other + instance. Once a :term:`Network Node` is registered, `Magpie` can treat the other instance as if they are in the same network as diff --git a/docs/configuration.rst b/docs/configuration.rst index 35b6d9788..8cbd96b6c 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1097,11 +1097,12 @@ to authenticate users for each other. All variables defined in this section are Password used to encrypt the PEM files in :envvar:`MAGPIE_NETWORK_PEM_FILES`. - If multiple files require passwords, they can be listed with a ``:`` character separator (Note that this means that - passwords cannot contain a ``:``. An empty string will be treated the same as no password. + If multiple files require passwords, they can be listed as a JSON array. An empty string will be treated the same as + no password. For example, if you have four files specified in :envvar:`MAGPIE_NETWORK_PEM_FILES` and only the first and third - file require a password, set this variable to ``pass1::pass2:`` where ``pass1`` and ``pass2`` are the passwords. + file require a password, set this variable to ``["pass1", "" ,"pass2", ""]`` where ``pass1`` and ``pass2`` are the + passwords. .. _config_phoenix: diff --git a/magpie/alembic/versions/2023-08-25_2cfe144538e8_add_network_tables.py b/magpie/alembic/versions/2023-08-25_2cfe144538e8_add_network_tables.py index 25810487e..54fcdba07 100644 --- a/magpie/alembic/versions/2023-08-25_2cfe144538e8_add_network_tables.py +++ b/magpie/alembic/versions/2023-08-25_2cfe144538e8_add_network_tables.py @@ -34,7 +34,7 @@ def upgrade(): sa.Column("jwks_url", URLType(), nullable=False), sa.Column("token_url", URLType(), nullable=False), sa.Column("authorization_url", URLType(), nullable=False), - sa.Column("redirect_uris", sa.String) + sa.Column("redirect_uris", sa.JSON, nullable=False, server_default='[]') ) op.create_table("network_remote_users", sa.Column("id", sa.Integer, primary_key=True, autoincrement=True), diff --git a/magpie/api/management/network/__init__.py b/magpie/api/management/network/__init__.py index 96b0df9c1..4ed465338 100644 --- a/magpie/api/management/network/__init__.py +++ b/magpie/api/management/network/__init__.py @@ -8,6 +8,7 @@ def includeme(config): LOGGER.info("Adding API network ...") config.add_route(**s.service_api_route_info(s.NetworkTokenAPI)) config.add_route(**s.service_api_route_info(s.NetworkJSONWebKeySetAPI)) + config.add_route(**s.service_api_route_info(s.NetworkDecodeJWTAPI)) config.include("magpie.api.management.network.node") config.include("magpie.api.management.network.remote_user") config.scan() diff --git a/magpie/api/management/network/network_utils.py b/magpie/api/management/network/network_utils.py index ec1cc71a2..3f3750591 100644 --- a/magpie/api/management/network/network_utils.py +++ b/magpie/api/management/network/network_utils.py @@ -1,3 +1,4 @@ +import json from datetime import datetime, timedelta from itertools import zip_longest from typing import TYPE_CHECKING @@ -54,15 +55,13 @@ def _pem_file_passwords(primary=False): ``["password1", None, "password2"]`` """ pem_passwords = get_constant("MAGPIE_NETWORK_PEM_PASSWORDS", raise_missing=False, raise_not_set=False) - passwords = [] - if pem_passwords: - for password in pem_passwords.split(PEM_PASSWORD_DELIMITER): - if password: - passwords.append(password.encode()) - else: - passwords.append(None) - if primary: - break + try: + passwords = json.loads(pem_passwords) + except json.decoder.JSONDecodeError: + passwords = [pem_passwords] + passwords = [p.encode() if p else None for p in passwords] + if primary: + return passwords[:1] return passwords diff --git a/magpie/api/management/network/network_views.py b/magpie/api/management/network/network_views.py index 4c12e394f..83b1dfd83 100644 --- a/magpie/api/management/network/network_views.py +++ b/magpie/api/management/network/network_views.py @@ -1,17 +1,20 @@ +import jwt import sqlalchemy -from pyramid.httpexceptions import HTTPCreated, HTTPNotFound, HTTPOk +from pyramid.httpexceptions import HTTPCreated, HTTPNotFound, HTTPOk, HTTPBadRequest from pyramid.security import NO_PERMISSION_REQUIRED +from pyramid.settings import asbool from pyramid.view import view_config from magpie import models from magpie.api import exception as ax from magpie.api import schemas as s -from magpie.api.management.network.network_utils import get_network_models_from_request_token, jwks +from magpie.api.management.network.network_utils import get_network_models_from_request_token, jwks, decode_jwt +from magpie.models import NetworkNode, NetworkToken @s.NetworkTokenAPI.post(schema=s.NetworkToken_POST_RequestSchema, tags=[s.NetworkTag], response_schemas=s.NetworkToken_POST_responses) -@view_config(route_name=s.NetworkTokenAPI.name, request_method="POST") +@view_config(route_name=s.NetworkTokenAPI.name, request_method="POST", permission=NO_PERMISSION_REQUIRED) def post_network_token_view(request): _, network_remote_user = get_network_models_from_request_token(request, create_network_remote_user=True) network_token = network_remote_user.network_token @@ -28,7 +31,7 @@ def post_network_token_view(request): @s.NetworkTokenAPI.delete(schema=s.NetworkToken_DELETE_RequestSchema, tags=[s.NetworkTag], response_schemas=s.NetworkToken_DELETE_responses) -@view_config(route_name=s.NetworkTokenAPI.name, request_method="DELETE") +@view_config(route_name=s.NetworkTokenAPI.name, request_method="DELETE", permission=NO_PERMISSION_REQUIRED) def delete_network_token_view(request): node, network_remote_user = get_network_models_from_request_token(request) if network_remote_user.network_token: @@ -40,9 +43,47 @@ def delete_network_token_view(request): ax.raise_http(http_error=HTTPNotFound, detail=s.NetworkNodeToken_DELETE_NotFoundResponseSchema.description) +@s.NetworkTokensAPI.delete(schema=s.NetworkTokens_DELETE_RequestSchema, tags=[s.NetworkTag], + response_schemas=s.NetworkTokens_DELETE_responses) +@view_config(route_name=s.NetworkTokenAPI.name, request_method="DELETE") +def delete_network_tokens_view(request): + if asbool(request.GET.get("expired_only")): + deleted = models.NetworkToken.delete_expired(request.db) + else: + deleted = request.db.query(NetworkToken).delete() + anonymous_network_user_ids = [n.anonymous_user().id for n in request.db.query(models.NetworkNode).all()] + # clean up unused records in the database (no need to keep records associated with anonymous network users) + (request.db.query(models.NetworkRemoteUser) + .filter(models.NetworkRemoteUser.user_id.in_(anonymous_network_user_ids)) + .filter(models.NetworkRemoteUser.network_token_id == None) # noqa: E711 # pylint: disable=singleton-comparison + .delete()) + return ax.valid_http(http_success=HTTPOk, + content={"deleted": deleted}, + detail=s.NetworkTokens_DELETE_OkResponseSchema.description) + + @s.NetworkJSONWebKeySetAPI.get(tags=[s.NetworkTag], response_schemas=s.NetworkJSONWebKeySet_GET_responses) @view_config(route_name=s.NetworkJSONWebKeySetAPI.name, request_method="GET", permission=NO_PERMISSION_REQUIRED) def get_network_jwks_view(_request): return ax.valid_http(http_success=HTTPOk, detail=s.NetworkJSONWebKeySet_GET_OkResponseSchema.description, content=jwks().export(private_keys=False, as_dict=True)) + + +@s.NetworkDecodeJWTAPI.get(tags=[s.NetworkTag], response_schemas=s.NetworkDecodeJWT_GET_Responses) +@view_config(route_name=s.NetworkDecodeJWTAPI.name, request_method="GET") +def get_decode_jwt(request): + token = request.GET.get("token") + if token is None: + raise HTTPBadRequest("Missing token") + try: + node_name = jwt.decode(token, options={"verify_signature": False}).get("iss") + except jwt.exceptions.DecodeError: + raise HTTPBadRequest("Token is improperly formatted") + node = request.db.query(NetworkNode).filter(NetworkNode.name == node_name).first() + if node is None: + raise HTTPBadRequest("Invalid token: invalid or missing issuer claim") + jwt_content = decode_jwt(token, node, request) + return ax.valid_http(http_success=HTTPOk, + content={"jwt_content": jwt_content}, + detail=s.NetworkDecodeJWT_GET_OkResponseSchema.description) diff --git a/magpie/api/management/network/node/__init__.py b/magpie/api/management/network/node/__init__.py index 4ce70c6fc..d4dcac7a1 100644 --- a/magpie/api/management/network/node/__init__.py +++ b/magpie/api/management/network/node/__init__.py @@ -9,7 +9,7 @@ def includeme(config): config.add_route(**s.service_api_route_info(s.NetworkNodesAPI)) config.add_route(**s.service_api_route_info(s.NetworkNodeAPI)) config.add_route(**s.service_api_route_info(s.NetworkNodeTokenAPI)) - config.add_route(**s.service_api_route_info(s.NetworkNodesLinkAPI)) + config.add_route(**s.service_api_route_info(s.NetworkLinkAPI)) config.add_route(**s.service_api_route_info(s.NetworkNodeLinkAPI)) config.scan() diff --git a/magpie/api/management/network/node/network_node_utils.py b/magpie/api/management/network/node/network_node_utils.py index d44ec62b7..03ea9cecf 100644 --- a/magpie/api/management/network/node/network_node_utils.py +++ b/magpie/api/management/network/node/network_node_utils.py @@ -100,7 +100,7 @@ def check_network_node_info(db_session=None, name=None, jwks_url=None, token_url http_error=HTTPBadRequest, msg_on_fail=s.NetworkNodes_CheckInfo_AuthorizationURLValue_BadRequestResponseSchema.description) if redirect_uris is not None: - for uri in redirect_uris.split(): + for uri in redirect_uris: ax.verify_param(uri, matches=True, param_name="redirect_uris", param_compare=URL_REGEX, http_error=HTTPBadRequest, msg_on_fail=s.NetworkNodes_CheckInfo_RedirectURIsValue_BadRequestResponseSchema.description) diff --git a/magpie/api/management/network/node/network_node_views.py b/magpie/api/management/network/node/network_node_views.py index b18eeddf2..761d2aa4c 100644 --- a/magpie/api/management/network/node/network_node_views.py +++ b/magpie/api/management/network/node/network_node_views.py @@ -24,12 +24,19 @@ delete_network_node, update_associated_user_groups ) +from magpie.constants import get_constant @s.NetworkNodesAPI.get(tags=[s.NetworkTag], response_schemas=s.NetworkNodes_GET_responses) -@view_config(route_name=s.NetworkNodesAPI.name, request_method="GET") +@view_config(route_name=s.NetworkNodesAPI.name, request_method="GET", permission=Authenticated) def get_network_nodes_view(request): + is_admin = False + if request.user is not None: + admin_group = get_constant("MAGPIE_ADMIN_GROUP", settings_container=request) + is_admin = admin_group in [group.group_name for group in request.user.groups] nodes = [n.as_dict() for n in request.db.query(models.NetworkNode).all()] + if not is_admin: + nodes = [{"name": n["name"]} for n in nodes] return ax.valid_http(http_success=HTTPOk, detail=s.NetworkNodes_GET_OkResponseSchema.description, content={"nodes": nodes}) @@ -139,11 +146,11 @@ def delete_network_node_token_view(request): return ax.valid_http(http_success=HTTPOk, detail=s.NetworkNodeToken_DELETE_OkResponseSchema) -@s.NetworkNodesLinkAPI.get(schema=s.NetworkNodesLink_GET_RequestSchema, tags=[s.NetworkTag], - response_schemas=s.NetworkNodesLink_GET_responses) -@view_config(route_name=s.NetworkNodesLinkAPI.name, request_method="GET", permission=Authenticated) +@s.NetworkLinkAPI.get(schema=s.NetworkLink_GET_RequestSchema, tags=[s.NetworkTag], + response_schemas=s.NetworkLink_GET_responses) +@view_config(route_name=s.NetworkLinkAPI.name, request_method="GET", permission=Authenticated) def get_network_node_link_view(request): - token = request.POST.get("token") + token = request.GET.get("token") node_name = jwt.decode(token, options={"verify_signature": False}).get("iss") node = ax.evaluate_call( lambda: request.db.query(models.NetworkNode).filter(models.NetworkNode.name == node_name).one(), @@ -178,7 +185,7 @@ def post_network_node_link_view(request): location_query_list.extend(( ("token", encode_jwt({"user_name": request.user.user_name}, node.name, request)), ("response_type", "id_token"), - ("redirect_uri", request.route_url(s.NetworkNodesLinkAPI.name)) + ("redirect_uri", request.route_url(s.NetworkLinkAPI.name)) )) location = up.urlunparse(location_tuple._replace(query=up.urlencode(location_query_list, doseq=True))) return ax.valid_http(http_success=HTTPFound, diff --git a/magpie/api/management/network/remote_user/remote_user_views.py b/magpie/api/management/network/remote_user/remote_user_views.py index 6b2f42011..b737e69b8 100644 --- a/magpie/api/management/network/remote_user/remote_user_views.py +++ b/magpie/api/management/network/remote_user/remote_user_views.py @@ -15,7 +15,10 @@ @s.NetworkRemoteUsersAPI.get(tags=[s.NetworkTag], response_schemas=s.NetworkRemoteUsers_GET_responses) @view_config(route_name=s.NetworkRemoteUsersAPI.name, request_method="GET") def get_network_remote_users_view(request): - nodes = [n.as_dict() for n in request.db.query(models.NetworkRemoteUser).all()] + query = request.db.query(models.NetworkRemoteUser) + if request.GET.get("user_name"): + query = query.join(models.User).filter(models.User.user_name == request.GET["user_name"]) + nodes = [n.as_dict() for n in query.all()] return ax.valid_http(http_success=HTTPOk, detail=s.NetworkRemoteUsers_GET_OkResponseSchema.description, content={"nodes": nodes}) diff --git a/magpie/api/schemas.py b/magpie/api/schemas.py index 336ca5da6..ed9d38ef6 100644 --- a/magpie/api/schemas.py +++ b/magpie/api/schemas.py @@ -285,9 +285,9 @@ def service_api_route_info(service_api, **kwargs): NetworkNodeTokenAPI = Service( path="/network/nodes/{node_name}/token", name="NetworkNodeToken") -NetworkNodesLinkAPI = Service( - path="/network/nodes/link", - name="NetworkNodesLink") +NetworkLinkAPI = Service( + path="/network/link", + name="NetworkLink") NetworkNodeLinkAPI = Service( path="/network/nodes/{node_name}/link", name="NetworkNodeLink") @@ -303,10 +303,15 @@ def service_api_route_info(service_api, **kwargs): NetworkTokenAPI = Service( path="/network/token", name="NetworkToken") +NetworkTokensAPI = Service( + path="/network/tokens", + name="NetworkTokens") NetworkJSONWebKeySetAPI = Service( path="/network/jwks", - name="NetworkJSONWebKeySet" -) + name="NetworkJSONWebKeySet") +NetworkDecodeJWTAPI = Service( + path="/network/decode_jwt", + name="NetworkDecodeJWT") # Path parameters GroupNameParameter = colander.SchemaNode( @@ -3542,6 +3547,19 @@ class NetworkToken_DELETE_RequestSchema(BaseRequestSchemaAPI): body = JWTRequestBodySchema() +class NetworkTokens_DELETE_RequestBodySchema(colander.MappingSchema): + expired_only = colander.SchemaNode( + colander.Boolean(), + description="Boolean indicating whether to only delete expired tokens.", + missing=colander.drop, + example=True + ) + + +class NetworkTokens_DELETE_RequestSchema(BaseRequestSchemaAPI): + body = NetworkTokens_DELETE_RequestBodySchema() + + class NetworkNode_PATCH_RequestBodySchema(colander.MappingSchema): name = colander.SchemaNode( colander.String(), @@ -3572,9 +3590,8 @@ class NetworkNode_PATCH_RequestBodySchema(colander.MappingSchema): ) redirect_uris = colander.SchemaNode( colander.String(), - description="Space delimited list of valid redirect URIs for another Magpie node (instance) in the " - "network.", - example="https://node.example.com/network/nodes/link https://node.example.com/some/other/uri", + description="JSON array of valid redirect URIs for another Magpie node (instance) in the network.", + example="https://node.example.com/network/link https://node.example.com/some/other/uri", missing=colander.drop ) @@ -3614,8 +3631,8 @@ class NetworkNode_BodySchema(colander.MappingSchema): ) redirect_uris = colander.SchemaNode( colander.String(), - description="Space delimited list of valid redirect URIs for another Magpie node (instance) in the network", - example="https://node.example.com/network/nodes/link https://node.example.com/some/other/uri", + description="JSON array of valid redirect URIs for another Magpie node (instance) in the network", + example="https://node.example.com/network/link https://node.example.com/some/other/uri", ) @@ -3694,7 +3711,7 @@ class NetworkNode_DELETE_OkResponseSchema(BaseResponseSchemaAPI): body = BaseResponseBodySchema(code=HTTPOk.code, description=description) -class NetworkNodesLink_GET_RequestSchema(BaseRequestSchemaAPI): +class NetworkLink_GET_RequestSchema(BaseRequestSchemaAPI): body = JWTRequestBodySchema() @@ -3834,6 +3851,11 @@ class NetworkNodeToken_DELETE_OkResponseSchema(BaseResponseSchemaAPI): body = BaseResponseBodySchema(code=HTTPOk.code, description=description) +class NetworkTokens_DELETE_OkResponseSchema(BaseResponseSchemaAPI): + description = "Successfully deleted access tokens." + body = BaseResponseBodySchema(code=HTTPOk.code, description=description) + + class NetworkNodeToken_DELETE_InternalServerErrorResponseSchema(BaseResponseSchemaAPI): description = "Unable to delete an access token." body = InternalServerErrorResponseBodySchema(code=HTTPInternalServerError.code, description=description) @@ -3882,6 +3904,15 @@ class NetworkJSONWebKeySet_GET_OkResponseSchema(BaseResponseSchemaAPI): body = NetworkJSONWebKeySet_GET_OkBodyResponseSchema(code=HTTPOk.code, description=description) +class NetworkDecodeJWT_GET_OkBodyResponseSchema(BaseResponseBodySchema): + jwt_content = colander.MappingSchema() + + +class NetworkDecodeJWT_GET_OkResponseSchema(BaseRequestSchemaAPI): + description = "JSON Web Token is Valid" + body = NetworkDecodeJWT_GET_OkBodyResponseSchema(code=HTTPOk.code, description=description) + + class NetworkRemoteUsers_POST_ForbiddenResponseSchema(BaseResponseSchemaAPI): description = "Targeted user update not allowed by requesting user." body = ErrorResponseBodySchema(code=HTTPForbidden.code, description=description) @@ -4639,14 +4670,23 @@ class NetworkRemoteUsers_POST_ForbiddenResponseSchema(BaseResponseSchemaAPI): "500": InternalServerErrorResponseSchema(), } NetworkToken_DELETE_responses = { - "201": NetworkToken_DELETE_OkResponseSchema(), + "200": NetworkToken_DELETE_OkResponseSchema(), "404": NetworkNodeToken_DELETE_NotFoundResponseSchema(), "500": InternalServerErrorResponseSchema(), } +NetworkTokens_DELETE_responses = { + "200": NetworkTokens_DELETE_OkResponseSchema(), + "500": InternalServerErrorResponseSchema(), +} NetworkJSONWebKeySet_GET_responses = { "200": NetworkJSONWebKeySet_GET_OkResponseSchema(), "500": InternalServerErrorResponseSchema(), } +NetworkDecodeJWT_GET_Responses = { + "200": NetworkDecodeJWT_GET_OkResponseSchema(), + "400": BadRequestResponseSchema(), + "500": InternalServerErrorResponseSchema() +} NetworkNode_GET_responses = { "200": NetworkNode_GET_OkResponseSchema(), "404": NetworkNode_NotFoundResponseSchema(), @@ -4684,7 +4724,7 @@ class NetworkRemoteUsers_POST_ForbiddenResponseSchema(BaseResponseSchemaAPI): "404": NetworkNode_NotFoundResponseSchema(), "500": NetworkNodeToken_DELETE_InternalServerErrorResponseSchema() } -NetworkNodesLink_GET_responses = { +NetworkLink_GET_responses = { "200": NetworkNodeLink_GET_OkResponseSchema(), "400": NetworkNodeLink_GET_BadRequestResponseSchema(), "404": NetworkNode_NotFoundResponseSchema(), diff --git a/magpie/cli/purge_expired_network_tokens.py b/magpie/cli/purge_expired_network_tokens.py index 50f8db700..fa717f3ee 100644 --- a/magpie/cli/purge_expired_network_tokens.py +++ b/magpie/cli/purge_expired_network_tokens.py @@ -11,6 +11,7 @@ import argparse from typing import TYPE_CHECKING +import requests import transaction from magpie import models @@ -35,35 +36,64 @@ def make_parser(): parser.add_argument("--config", "--ini", metavar="CONFIG", dest="ini_config", default=get_constant("MAGPIE_INI_FILE_PATH"), help="Configuration INI file to retrieve database connection settings (default: %(default)s).") + subparsers = parser.add_subparsers(help="run with API or directly access the database", dest="api_or_db") + api_parser = subparsers.add_parser("api") + _db_parser = subparsers.add_parser("db") + + api_parser.add_argument("url", help="URL used to access the magpie service.") + api_parser.add_argument("username", help="Admin username for magpie login.") + api_parser.add_argument("password", help="Admin password for magpie login.") make_logging_options(parser) return parser +def get_login_session(magpie_url, username, password): + session = requests.Session() + data = {"user_name": username, "password": password} + response = session.post(magpie_url + "/signin", json=data) + if response.status_code != 200: + LOGGER.error(response.content) + return None + return session + + def main(args=None, parser=None, namespace=None): # type: (Optional[Sequence[Str]], Optional[argparse.ArgumentParser], Optional[argparse.Namespace]) -> None if not parser: parser = make_parser() args = parser.parse_args(args=args, namespace=namespace) setup_logger_from_options(LOGGER, args) - db_session = get_db_session_from_config_ini(args.ini_config) - deleted = models.NetworkToken.get_expired(db_session).delete() - anonymous_network_user_ids = [n.anonymous_user().id for n in db_session.query(models.NetworkNode).all()] - # clean up unused records in the database (no need to keep records associated with anonymous network users) - (db_session.query(models.NetworkRemoteUser) - .filter(models.NetworkRemoteUser.user_id.in_(anonymous_network_user_ids)) - .filter(models.NetworkRemoteUser.network_token_id == None) # noqa: E711 # pylint: disable=singleton-comparison - .delete()) - try: - transaction.commit() - db_session.close() - except Exception as exc: # noqa: W0703 # nosec: B110 # pragma: no cover - db_session.rollback() - raise_log("Failed to delete expired network tokens", exception=type(exc), logger=LOGGER) + if args.api_or_db == "api": + session = get_login_session(args.url, args.username, args.password) + if session is None: + raise_log("Failed to login, invalid username or password", logger=LOGGER) + response = session.delete("{}/network/tokens?expired_only=true".format(args.url)) + try: + response.raise_for_status() + except requests.HTTPError as exc: + raise_log("Failed to delete expired network tokens: {}".format(exc), exception=type(exc), logger=LOGGER) + data = response.json() + deleted = int(data["deleted"]) + else: + db_session = get_db_session_from_config_ini(args.ini_config) + deleted = models.NetworkToken.delete_expired(db_session) + anonymous_network_user_ids = [n.anonymous_user(db_session).id for n in + db_session.query(models.NetworkNode).all()] + # clean up unused records in the database (no need to keep records associated with anonymous network users) + (db_session.query(models.NetworkRemoteUser) + .filter(models.NetworkRemoteUser.user_id.in_(anonymous_network_user_ids)) + .filter(models.NetworkRemoteUser.network_token_id == None) # noqa: E711 # pylint: disable=singleton-comparison + .delete()) + try: + transaction.commit() + db_session.close() + except Exception as exc: # noqa: W0703 # nosec: B110 # pragma: no cover + db_session.rollback() + raise_log("Failed to delete expired network tokens", exception=type(exc), logger=LOGGER) + if deleted: + print_log("{} expired network tokens deleted".format(deleted), logger=LOGGER) else: - if deleted: - print_log("{} expired network tokens deleted".format(deleted), logger=LOGGER) - else: - print_log("No expired network tokens found", logger=LOGGER) + print_log("No expired network tokens found", logger=LOGGER) if __name__ == "__main__": diff --git a/magpie/constants.py b/magpie/constants.py index da690d22f..101bc7294 100644 --- a/magpie/constants.py +++ b/magpie/constants.py @@ -192,7 +192,7 @@ def protected_user_name_regex(include_admin=True, include_network=True, additional_patterns=None, settings_container=None): - # type: (bool, bool, bool, Optional[List[Str]], Optional[AnySettingsContainer]) -> Str + # type: (bool, bool, bool, Optional[List[Str]], Optional[AnySettingsContainer]) -> re.Pattern """ Return a regular expression that matches all user names that are protected, meaning that they are generated by Magpie itself and no regular user account should be created with these user names. @@ -206,7 +206,7 @@ def protected_user_name_regex(include_admin=True, patterns.append( "{}.*".format(get_constant("MAGPIE_NETWORK_NAME_PREFIX", settings_container=settings_container)) ) - return "^({})$".format("|".join(patterns)) + return re.compile("^({})$".format("|".join(patterns))) def protected_user_email_regex(include_admin=True, @@ -214,7 +214,7 @@ def protected_user_email_regex(include_admin=True, include_network=True, additional_patterns=None, settings_container=None): - # type: (bool, bool, bool, Optional[List[Str]], Optional[AnySettingsContainer]) -> Str + # type: (bool, bool, bool, Optional[List[Str]], Optional[AnySettingsContainer]) -> re.Pattern """ Return a regular expression that matches all user emails that are protected, meaning that they are generated by Magpie itself and no regular user account should be created with these user emails. @@ -227,14 +227,14 @@ def protected_user_email_regex(include_admin=True, if include_network and network_enabled(settings_container=settings_container): email_form = get_constant("MAGPIE_NETWORK_ANONYMOUS_EMAIL_FORMAT", settings_container=settings_container) patterns.append(email_form.format(".*")) - return "^({})$".format("|".join(patterns)) + return re.compile("^({})$".format("|".join(patterns))) def protected_group_name_regex(include_admin=True, include_anonymous=True, include_network=True, settings_container=None): - # type: (bool, bool, bool, Optional[AnySettingsContainer]) -> Str + # type: (bool, bool, bool, Optional[AnySettingsContainer]) -> re.Pattern """ Return a regular expression that matches all group names that are protected, meaning that they are generated by Magpie itself and no regular user account should be created with these group names. @@ -248,7 +248,7 @@ def protected_group_name_regex(include_admin=True, patterns.append( "{}.*".format(get_constant("MAGPIE_NETWORK_NAME_PREFIX", settings_container=settings_container)) ) - return "^({})$".format("|".join(patterns)) + return re.compile("^({})$".format("|".join(patterns))) def network_enabled(settings_container=None): diff --git a/magpie/models.py b/magpie/models.py index 2dfb83410..150203980 100644 --- a/magpie/models.py +++ b/magpie/models.py @@ -1001,7 +1001,8 @@ def json(self): class NetworkRemoteUser(BaseModel, Base): """ - Model that defines a relationship between a User and a User that is authenticated on a different NetworkNode. + Model that defines a relationship between a :class:`User` and a :class:`User` that is authenticated on a different + :class:`NetworkNode`. """ __tablename__ = "network_remote_users" @@ -1067,11 +1068,11 @@ def expired(self): return (datetime.datetime.utcnow() - self.created) > expiry @classmethod - def get_expired(cls, db_session): - # type: (Session) -> Query + def delete_expired(cls, db_session): + # type: (Session) -> int token_expiry = int(get_constant("MAGPIE_NETWORK_DEFAULT_TOKEN_EXPIRY")) expiry_date_time = datetime.datetime.utcnow() - datetime.timedelta(seconds=token_expiry) - return db_session.query(cls).filter(cls.created < expiry_date_time) + return db_session.query(cls).filter(cls.created < expiry_date_time).delete() @classmethod def by_token(cls, token, db_session=None): @@ -1098,7 +1099,7 @@ class NetworkNode(BaseModel, Base): jwks_url = sa.Column(URLType(), nullable=False) token_url = sa.Column(URLType(), nullable=False) authorization_url = sa.Column(URLType(), nullable=False) - redirect_uris = sa.Column(sa.String) + redirect_uris = sa.Column(sa.JSON, nullable=False, server_default='[]') def anonymous_user_name(self): # type: () -> Str diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py index 20a299058..b44441ae8 100644 --- a/magpie/ui/management/views.py +++ b/magpie/ui/management/views.py @@ -171,12 +171,10 @@ def edit_user(self): # add network information if network_enabled(self.request): - network_remote_users = (self.request.db.query(NetworkRemoteUser) - .join(User).filter(User.user_name == user_name) - .all()) - existing_network_remote_user_nodes = {nu.network_node_id: nu.name for nu in network_remote_users} - network_nodes = self.request.db.query(NetworkNode).order_by(NetworkNode.id).all() - user_info["network_nodes"] = [(n.name, existing_network_remote_user_nodes.get(n.id)) for n in network_nodes] + request_uri = "{}?user_name={}".format(schemas.NetworkRemoteUsersAPI.path, user_name) + resp = request_api(self.request, request_uri, "GET") + check_response(resp) + user_info["network_nodes"] = [(n["node_name"], n["remote_user_name"]) for n in get_json(resp)["nodes"]] user_info["network_routes"] = {"create": schemas.NetworkRemoteUsersAPI.name, "delete": schemas.NetworkRemoteUserAPI.name} diff --git a/magpie/ui/network/__init__.py b/magpie/ui/network/__init__.py index 710d5096e..062e47764 100644 --- a/magpie/ui/network/__init__.py +++ b/magpie/ui/network/__init__.py @@ -1,4 +1,4 @@ -from magpie.utils import get_logger +from magpie.utils import get_logger, fully_qualified_name LOGGER = get_logger(__name__) @@ -7,5 +7,5 @@ def includeme(config): from magpie.ui.network.views import NetworkViews LOGGER.info("Adding UI network...") path = "/ui/network/authorize" - config.add_route(NetworkViews.authorize.__name__, path) + config.add_route(fully_qualified_name(NetworkViews.authorize), path) config.scan() diff --git a/magpie/ui/network/templates/authorize.mako b/magpie/ui/network/templates/authorize.mako index 6616eb21c..548487164 100644 --- a/magpie/ui/network/templates/authorize.mako +++ b/magpie/ui/network/templates/authorize.mako @@ -13,10 +13,15 @@ - + + + + WARNING This will give this user full access to your account. - WARNING diff --git a/magpie/ui/network/views.py b/magpie/ui/network/views.py index c6abceb5b..229fd30fa 100644 --- a/magpie/ui/network/views.py +++ b/magpie/ui/network/views.py @@ -1,18 +1,22 @@ +from urllib.parse import urlparse + import jwt from pyramid.authentication import Authenticated from pyramid.httpexceptions import HTTPBadRequest from pyramid.view import view_config +from magpie.api import schemas from magpie.api.management.network.network_utils import decode_jwt, encode_jwt from magpie.models import NetworkNode -from magpie.ui.utils import BaseViews -from magpie.utils import get_logger +from magpie.ui.utils import BaseViews, request_api, check_response, AdminRequests +from magpie.utils import get_logger, get_json LOGGER = get_logger(__name__) -class NetworkViews(BaseViews): - @view_config(route_name="authorize", renderer="templates/authorize.mako", permission=Authenticated) +class NetworkViews(AdminRequests): + @view_config(route_name="magpie.ui.network.views.NetworkViews.authorize", + renderer="templates/authorize.mako", permission=Authenticated) def authorize(self): token = self.request.GET.get("token") response_type = self.request.GET.get("response_type") @@ -23,23 +27,27 @@ def authorize(self): raise HTTPBadRequest("Invalid response type") if token is None: raise HTTPBadRequest("Missing token") - try: - node_name = jwt.decode(token, options={"verify_signature": False}).get("iss") - except jwt.exceptions.DecodeError: - raise HTTPBadRequest("Token is improperly formatted") - node = self.request.db.query(NetworkNode).filter(NetworkNode.name == node_name).first() - if node is None: - raise HTTPBadRequest("Invalid token: invalid or missing issuer claim") - - if redirect_uri not in (node.redirect_uris or "").split(): + admin_cookies = self.get_admin_session() + jwt_path = "{}?token={}".format(schemas.NetworkDecodeJWTAPI.path, token) + jwt_resp = request_api(self.request, jwt_path, "GET", cookies=admin_cookies) + check_response(jwt_resp) + token_content = get_json(jwt_resp)["jwt_content"] + + node_name = token_content["iss"] + node_path = schemas.NetworkNodeAPI.path.format(node_name=node_name) + node_resp = request_api(self.request, node_path, "GET", cookies=admin_cookies) + check_response(node_resp) + node_details = get_json(node_resp) + + if redirect_uri not in node_details["redirect_uris"]: raise HTTPBadRequest("Invalid redirect URI") - decoded_token = decode_jwt(token, node, self.request) - requesting_user_name = decoded_token.get("user_name") + requesting_user_name = token_content.get("user_name") token_claims = {"requesting_user_name": requesting_user_name, "user_name": self.request.user.user_name} - response_token = encode_jwt(token_claims, node.name, self.request) + response_token = encode_jwt(token_claims, node_name, self.request) return self.add_template_data(data={"authorize_uri": redirect_uri, "token": response_token, "requesting_user_name": requesting_user_name, - "node_name": node.name}) + "node_name": node_name, + "referrer": urlparse(self.request.referrer).hostname}) diff --git a/magpie/ui/user/views.py b/magpie/ui/user/views.py index 5f2cb0b33..ef04d4524 100644 --- a/magpie/ui/user/views.py +++ b/magpie/ui/user/views.py @@ -7,7 +7,7 @@ from magpie.api import schemas from magpie.constants import get_constant, network_enabled -from magpie.models import NetworkNode, NetworkRemoteUser, UserGroupStatus +from magpie.models import UserGroupStatus from magpie.ui.utils import BaseViews, check_response, handle_errors, request_api from magpie.utils import get_json, get_logger @@ -91,12 +91,14 @@ def edit_current_user(self): user_info["user_with_error"] = schemas.UserStatuses.get(user_info["status"]) != schemas.UserStatuses.OK # add network information if network_enabled(self.request): - network_remote_users = (self.request.db.query(NetworkRemoteUser) - .filter(NetworkRemoteUser.user_id == self.request.user.id) - .all()) - existing_network_remote_user_nodes = {nu.network_node_id: nu.name for nu in network_remote_users} - network_nodes = self.request.db.query(NetworkNode).order_by(NetworkNode.id).all() - user_info["network_nodes"] = [(n.name, existing_network_remote_user_nodes.get(n.id)) for n in network_nodes] + node_resp = request_api(self.request, schemas.NetworkNodesAPI.path, "GET") + check_response(node_resp) + nodes = {node["name"]: None for node in get_json(node_resp)["nodes"]} + user_resp = request_api(self.request, schemas.NetworkRemoteUsersCurrentAPI.path, "GET") + check_response(user_resp) + for info in get_json(user_resp)["nodes"]: + nodes[info["node_name"]] = info["remote_user_name"] + user_info["network_nodes"] = list(nodes.items()) user_info["network_routes"] = {"create": schemas.NetworkNodeLinkAPI.name, "delete": schemas.NetworkRemoteUserAPI.name} # reset error messages/flags diff --git a/setup.cfg b/setup.cfg index 8878edf28..cb9c18039 100644 --- a/setup.cfg +++ b/setup.cfg @@ -51,11 +51,12 @@ exclude = eggs, parts, share, + magpieenv, [pylint] [bandit] -exclude = *.egg-info,build,dist,env,./tests,test_* +exclude = *.egg-info,build,dist,env,./tests,test_*,./magpieenv targets = . [tool:isort] @@ -69,6 +70,7 @@ known_first_party = magpie known_third_party = mock forced_separate = twitcher combine_as_imports = false +skip=magpieenv [coverage:run] branch = true From 1bf55843aba6b7743e4b1ea64994cc04f108c1e7 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Mon, 18 Dec 2023 15:36:02 -0500 Subject: [PATCH 18/55] style fixes --- .../2023-08-25_2cfe144538e8_add_network_tables.py | 2 +- magpie/api/management/network/network_views.py | 4 ++-- magpie/cli/purge_expired_network_tokens.py | 2 +- magpie/models.py | 2 +- magpie/ui/management/views.py | 10 +--------- magpie/ui/network/__init__.py | 2 +- magpie/ui/network/views.py | 8 +++----- 7 files changed, 10 insertions(+), 20 deletions(-) diff --git a/magpie/alembic/versions/2023-08-25_2cfe144538e8_add_network_tables.py b/magpie/alembic/versions/2023-08-25_2cfe144538e8_add_network_tables.py index 54fcdba07..395d361c9 100644 --- a/magpie/alembic/versions/2023-08-25_2cfe144538e8_add_network_tables.py +++ b/magpie/alembic/versions/2023-08-25_2cfe144538e8_add_network_tables.py @@ -34,7 +34,7 @@ def upgrade(): sa.Column("jwks_url", URLType(), nullable=False), sa.Column("token_url", URLType(), nullable=False), sa.Column("authorization_url", URLType(), nullable=False), - sa.Column("redirect_uris", sa.JSON, nullable=False, server_default='[]') + sa.Column("redirect_uris", sa.JSON, nullable=False, server_default="[]") ) op.create_table("network_remote_users", sa.Column("id", sa.Integer, primary_key=True, autoincrement=True), diff --git a/magpie/api/management/network/network_views.py b/magpie/api/management/network/network_views.py index 83b1dfd83..803609c0b 100644 --- a/magpie/api/management/network/network_views.py +++ b/magpie/api/management/network/network_views.py @@ -1,6 +1,6 @@ import jwt import sqlalchemy -from pyramid.httpexceptions import HTTPCreated, HTTPNotFound, HTTPOk, HTTPBadRequest +from pyramid.httpexceptions import HTTPBadRequest, HTTPCreated, HTTPNotFound, HTTPOk from pyramid.security import NO_PERMISSION_REQUIRED from pyramid.settings import asbool from pyramid.view import view_config @@ -8,7 +8,7 @@ from magpie import models from magpie.api import exception as ax from magpie.api import schemas as s -from magpie.api.management.network.network_utils import get_network_models_from_request_token, jwks, decode_jwt +from magpie.api.management.network.network_utils import decode_jwt, get_network_models_from_request_token, jwks from magpie.models import NetworkNode, NetworkToken diff --git a/magpie/cli/purge_expired_network_tokens.py b/magpie/cli/purge_expired_network_tokens.py index fa717f3ee..fd4517cd3 100644 --- a/magpie/cli/purge_expired_network_tokens.py +++ b/magpie/cli/purge_expired_network_tokens.py @@ -38,7 +38,7 @@ def make_parser(): help="Configuration INI file to retrieve database connection settings (default: %(default)s).") subparsers = parser.add_subparsers(help="run with API or directly access the database", dest="api_or_db") api_parser = subparsers.add_parser("api") - _db_parser = subparsers.add_parser("db") + subparsers.add_parser("db") api_parser.add_argument("url", help="URL used to access the magpie service.") api_parser.add_argument("username", help="Admin username for magpie login.") diff --git a/magpie/models.py b/magpie/models.py index 150203980..5f9dbee98 100644 --- a/magpie/models.py +++ b/magpie/models.py @@ -1099,7 +1099,7 @@ class NetworkNode(BaseModel, Base): jwks_url = sa.Column(URLType(), nullable=False) token_url = sa.Column(URLType(), nullable=False) authorization_url = sa.Column(URLType(), nullable=False) - redirect_uris = sa.Column(sa.JSON, nullable=False, server_default='[]') + redirect_uris = sa.Column(sa.JSON, nullable=False, server_default="[]") def anonymous_user_name(self): # type: () -> Str diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py index b44441ae8..7ee81d99f 100644 --- a/magpie/ui/management/views.py +++ b/magpie/ui/management/views.py @@ -24,15 +24,7 @@ from magpie.cli.sync_services import SYNC_SERVICES_TYPES from magpie.constants import get_constant, network_enabled # FIXME: remove (REMOTE_RESOURCE_TREE_SERVICE, RESOURCE_TYPE_DICT), implement getters via API -from magpie.models import ( - REMOTE_RESOURCE_TREE_SERVICE, - RESOURCE_TYPE_DICT, - NetworkNode, - NetworkRemoteUser, - User, - UserGroupStatus, - UserStatuses -) +from magpie.models import REMOTE_RESOURCE_TREE_SERVICE, RESOURCE_TYPE_DICT, UserGroupStatus, UserStatuses from magpie.permissions import Permission, PermissionSet # FIXME: remove (SERVICE_TYPE_DICT), implement getters via API from magpie.services import SERVICE_TYPE_DICT diff --git a/magpie/ui/network/__init__.py b/magpie/ui/network/__init__.py index 062e47764..6908ea479 100644 --- a/magpie/ui/network/__init__.py +++ b/magpie/ui/network/__init__.py @@ -1,4 +1,4 @@ -from magpie.utils import get_logger, fully_qualified_name +from magpie.utils import fully_qualified_name, get_logger LOGGER = get_logger(__name__) diff --git a/magpie/ui/network/views.py b/magpie/ui/network/views.py index 229fd30fa..501e5979e 100644 --- a/magpie/ui/network/views.py +++ b/magpie/ui/network/views.py @@ -1,15 +1,13 @@ from urllib.parse import urlparse -import jwt from pyramid.authentication import Authenticated from pyramid.httpexceptions import HTTPBadRequest from pyramid.view import view_config from magpie.api import schemas -from magpie.api.management.network.network_utils import decode_jwt, encode_jwt -from magpie.models import NetworkNode -from magpie.ui.utils import BaseViews, request_api, check_response, AdminRequests -from magpie.utils import get_logger, get_json +from magpie.api.management.network.network_utils import encode_jwt +from magpie.ui.utils import AdminRequests, check_response, request_api +from magpie.utils import get_json, get_logger LOGGER = get_logger(__name__) From 3e941612f01e9cdf9c067f29b6e2680623a47ec2 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Mon, 18 Dec 2023 15:46:41 -0500 Subject: [PATCH 19/55] route name fix --- magpie/api/management/network/__init__.py | 1 + magpie/api/management/network/network_views.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/magpie/api/management/network/__init__.py b/magpie/api/management/network/__init__.py index 4ed465338..2b8ea4d85 100644 --- a/magpie/api/management/network/__init__.py +++ b/magpie/api/management/network/__init__.py @@ -9,6 +9,7 @@ def includeme(config): config.add_route(**s.service_api_route_info(s.NetworkTokenAPI)) config.add_route(**s.service_api_route_info(s.NetworkJSONWebKeySetAPI)) config.add_route(**s.service_api_route_info(s.NetworkDecodeJWTAPI)) + config.add_route(**s.service_api_route_info(s.NetworkTokensAPI)) config.include("magpie.api.management.network.node") config.include("magpie.api.management.network.remote_user") config.scan() diff --git a/magpie/api/management/network/network_views.py b/magpie/api/management/network/network_views.py index 803609c0b..b00825c17 100644 --- a/magpie/api/management/network/network_views.py +++ b/magpie/api/management/network/network_views.py @@ -45,7 +45,7 @@ def delete_network_token_view(request): @s.NetworkTokensAPI.delete(schema=s.NetworkTokens_DELETE_RequestSchema, tags=[s.NetworkTag], response_schemas=s.NetworkTokens_DELETE_responses) -@view_config(route_name=s.NetworkTokenAPI.name, request_method="DELETE") +@view_config(route_name=s.NetworkTokensAPI.name, request_method="DELETE") def delete_network_tokens_view(request): if asbool(request.GET.get("expired_only")): deleted = models.NetworkToken.delete_expired(request.db) From c632d08a516b92ec678fd2fd9779553b7ebc142f Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Fri, 19 Jan 2024 15:08:04 -0500 Subject: [PATCH 20/55] added documentation about skipping a check and added a cache to regex functions --- Makefile | 1 + magpie/constants.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/Makefile b/Makefile index bc8a50804..101bb7b2f 100644 --- a/Makefile +++ b/Makefile @@ -539,6 +539,7 @@ check-security-only: check-security-code-only check-security-deps-only ## run s # ignored codes: # 42194: https://github.com/kvesteri/sqlalchemy-utils/issues/166 # not fixed since 2015 # 51668: https://github.com/sqlalchemy/sqlalchemy/pull/8563 # still in beta + major version change sqlalchemy 2.0.0b1 +# 51021: This is patched in jwcrypto>=1.4.0 but that version is not available for python version < 3.6 .PHONY: check-security-deps-only check-security-deps-only: mkdir-reports ## run security checks on package dependencies @echo "Running security checks of dependencies..." diff --git a/magpie/constants.py b/magpie/constants.py index 101bc7294..aff5f75c0 100644 --- a/magpie/constants.py +++ b/magpie/constants.py @@ -11,6 +11,7 @@ constant ``MAGPIE_INI_FILE_PATH`` (or any other `path variable` defined before it - see below) has to be defined by environment variable if the default location is not desired (ie: if you want to provide your own configuration). """ +import functools import logging import os import re @@ -187,6 +188,7 @@ def _get_default_log_level(): _REGEX_ASCII_ONLY = re.compile(r"\W|^(?=\d)") +@functools.lru_cache def protected_user_name_regex(include_admin=True, include_anonymous=True, include_network=True, @@ -209,6 +211,7 @@ def protected_user_name_regex(include_admin=True, return re.compile("^({})$".format("|".join(patterns))) +@functools.lru_cache def protected_user_email_regex(include_admin=True, include_anonymous=True, include_network=True, @@ -230,6 +233,7 @@ def protected_user_email_regex(include_admin=True, return re.compile("^({})$".format("|".join(patterns))) +@functools.lru_cache def protected_group_name_regex(include_admin=True, include_anonymous=True, include_network=True, From 50a86b7f70d28cada30a5e133e0481b0b268e841 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Fri, 19 Jan 2024 15:16:59 -0500 Subject: [PATCH 21/55] make lru_cache compatible with older pythons --- magpie/constants.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/magpie/constants.py b/magpie/constants.py index aff5f75c0..e1834b02c 100644 --- a/magpie/constants.py +++ b/magpie/constants.py @@ -188,7 +188,7 @@ def _get_default_log_level(): _REGEX_ASCII_ONLY = re.compile(r"\W|^(?=\d)") -@functools.lru_cache +@functools.lru_cache(maxsize=128) def protected_user_name_regex(include_admin=True, include_anonymous=True, include_network=True, @@ -211,7 +211,7 @@ def protected_user_name_regex(include_admin=True, return re.compile("^({})$".format("|".join(patterns))) -@functools.lru_cache +@functools.lru_cache(maxsize=128) def protected_user_email_regex(include_admin=True, include_anonymous=True, include_network=True, @@ -233,7 +233,7 @@ def protected_user_email_regex(include_admin=True, return re.compile("^({})$".format("|".join(patterns))) -@functools.lru_cache +@functools.lru_cache(maxsize=128) def protected_group_name_regex(include_admin=True, include_anonymous=True, include_network=True, From c32501b4a8ebd586b46199f7aa010117fabd5ca2 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Fri, 19 Jan 2024 15:44:44 -0500 Subject: [PATCH 22/55] make all arguments hashable for lru_cache functions --- magpie/api/management/user/user_utils.py | 2 +- magpie/constants.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/magpie/api/management/user/user_utils.py b/magpie/api/management/user/user_utils.py index d0cc35a30..048a4c934 100644 --- a/magpie/api/management/user/user_utils.py +++ b/magpie/api/management/user/user_utils.py @@ -228,7 +228,7 @@ def update_user(user, request, new_user_name=None, new_password=None, new_email= # logged user updating itself is forbidden if it corresponds to special users # cannot edit reserved keywords nor apply them to another user forbidden_user_names_regex = protected_user_name_regex( - additional_patterns=[get_constant("MAGPIE_LOGGED_USER", request)], settings_container=request + additional_patterns=(get_constant("MAGPIE_LOGGED_USER", request),), settings_container=request ) check_user_name_cases = [user.user_name, new_user_name] if update_username else [user.user_name] for check_user_name in check_user_name_cases: diff --git a/magpie/constants.py b/magpie/constants.py index e1834b02c..37c4ae979 100644 --- a/magpie/constants.py +++ b/magpie/constants.py @@ -25,7 +25,7 @@ if TYPE_CHECKING: # pylint: disable=W0611,unused-import - from typing import List, Optional + from typing import Optional, Tuple from magpie.typedefs import AnySettingsContainer, SettingValue, Str @@ -194,12 +194,12 @@ def protected_user_name_regex(include_admin=True, include_network=True, additional_patterns=None, settings_container=None): - # type: (bool, bool, bool, Optional[List[Str]], Optional[AnySettingsContainer]) -> re.Pattern + # type: (bool, bool, bool, Optional[Tuple[Str]], Optional[AnySettingsContainer]) -> re.Pattern """ Return a regular expression that matches all user names that are protected, meaning that they are generated by Magpie itself and no regular user account should be created with these user names. """ - patterns = additional_patterns or [] + patterns = list(additional_patterns or []) if include_admin: patterns.append(get_constant("MAGPIE_ADMIN_USER", settings_container=settings_container)) if include_anonymous: @@ -217,12 +217,12 @@ def protected_user_email_regex(include_admin=True, include_network=True, additional_patterns=None, settings_container=None): - # type: (bool, bool, bool, Optional[List[Str]], Optional[AnySettingsContainer]) -> re.Pattern + # type: (bool, bool, bool, Optional[Tuple[Str]], Optional[AnySettingsContainer]) -> re.Pattern """ Return a regular expression that matches all user emails that are protected, meaning that they are generated by Magpie itself and no regular user account should be created with these user emails. """ - patterns = additional_patterns or [] + patterns = list(additional_patterns or []) if include_admin: patterns.append(get_constant("MAGPIE_ADMIN_EMAIL", settings_container=settings_container)) if include_anonymous: From 521af0957c2f9059f7020dd144ba052b73162031 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Fri, 26 Jan 2024 13:37:32 -0500 Subject: [PATCH 23/55] spacing fix --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 101bb7b2f..49f136a39 100644 --- a/Makefile +++ b/Makefile @@ -539,7 +539,7 @@ check-security-only: check-security-code-only check-security-deps-only ## run s # ignored codes: # 42194: https://github.com/kvesteri/sqlalchemy-utils/issues/166 # not fixed since 2015 # 51668: https://github.com/sqlalchemy/sqlalchemy/pull/8563 # still in beta + major version change sqlalchemy 2.0.0b1 -# 51021: This is patched in jwcrypto>=1.4.0 but that version is not available for python version < 3.6 +# 51021: This is patched in jwcrypto>=1.4.0 but that version is not available for python version < 3.6 .PHONY: check-security-deps-only check-security-deps-only: mkdir-reports ## run security checks on package dependencies @echo "Running security checks of dependencies..." From 9be6121d9438a0084eebe2fd00deaf16042390b2 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Fri, 16 Feb 2024 11:21:04 -0500 Subject: [PATCH 24/55] add timeout to requests (CWE-400) --- magpie/api/management/network/node/network_node_views.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/magpie/api/management/network/node/network_node_views.py b/magpie/api/management/network/node/network_node_views.py index 761d2aa4c..7a833aec8 100644 --- a/magpie/api/management/network/node/network_node_views.py +++ b/magpie/api/management/network/node/network_node_views.py @@ -124,7 +124,9 @@ def get_network_node_token_view(request): http_error=HTTPNotFound, msg_on_fail=s.NetworkNode_NotFoundResponseSchema.description) token = encode_jwt({"user_name": request.user.user_name}, node_name, request) - access_token = ax.evaluate_call(lambda: requests.post(node.token_url, json={"token": token}).json()["token"], + access_token = ax.evaluate_call(lambda: requests.post(node.token_url, + json={"token": token}, + timeout=5).json()["token"], http_error=HTTPInternalServerError, msg_on_fail=s.NetworkNodeToken_GET_InternalServerErrorResponseSchema.description) return ax.valid_http(http_success=HTTPOk, content={"token": access_token}, @@ -140,7 +142,7 @@ def delete_network_node_token_view(request): http_error=HTTPNotFound, msg_on_fail=s.NetworkNode_NotFoundResponseSchema.description) token = encode_jwt({"user_name": request.user.user_name}, node_name, request) - ax.evaluate_call(lambda: requests.delete(node.token_url, json={"token": token}).raise_for_status(), + ax.evaluate_call(lambda: requests.delete(node.token_url, json={"token": token}, timeout=5).raise_for_status(), http_error=HTTPInternalServerError, msg_on_fail=s.NetworkNodeToken_DELETE_InternalServerErrorResponseSchema.description) return ax.valid_http(http_success=HTTPOk, detail=s.NetworkNodeToken_DELETE_OkResponseSchema) From aa1e9e40d98c7ec83ac4bb9b12ab59290410ef7f Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Sat, 24 Feb 2024 21:58:57 -0500 Subject: [PATCH 25/55] initial tests and test setup --- .github/workflows/tests.yml | 7 + .../api/management/network/network_views.py | 14 +- .../network/node/network_node_views.py | 24 ++-- .../network/remote_user/remote_user_views.py | 16 ++- magpie/api/requests.py | 30 +++- magpie/api/schemas.py | 6 + magpie/ui/network/views.py | 2 + setup.cfg | 1 + tests/interfaces.py | 131 ++++++++++++++++++ tests/runner.py | 1 + tests/test_utils.py | 20 ++- tests/utils.py | 47 +++++++ 12 files changed, 276 insertions(+), 23 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 124e948f6..0c593cca5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -67,6 +67,13 @@ jobs: python-version: "3.11" allow-failure: true test-case: start test-remote + # remote network tests using a remote server with network mode enabled + - os: ubuntu-latest + python-version: "3.11" + allow-failure: true + test-case: start test-remote + test-option: >- + PYTEST_ADDOPTS='-m "remote and network"' MAGPIE_NETWORK_ENABLED=on MAGPIE_NETWORK_INSTANCE_NAME=example # coverage test - os: ubuntu-latest python-version: "3.11" diff --git a/magpie/api/management/network/network_views.py b/magpie/api/management/network/network_views.py index b00825c17..5477a9f9c 100644 --- a/magpie/api/management/network/network_views.py +++ b/magpie/api/management/network/network_views.py @@ -9,12 +9,14 @@ from magpie.api import exception as ax from magpie.api import schemas as s from magpie.api.management.network.network_utils import decode_jwt, get_network_models_from_request_token, jwks +from magpie.api.requests import check_network_mode_enabled from magpie.models import NetworkNode, NetworkToken @s.NetworkTokenAPI.post(schema=s.NetworkToken_POST_RequestSchema, tags=[s.NetworkTag], response_schemas=s.NetworkToken_POST_responses) -@view_config(route_name=s.NetworkTokenAPI.name, request_method="POST", permission=NO_PERMISSION_REQUIRED) +@view_config(route_name=s.NetworkTokenAPI.name, request_method="POST", + decorator=check_network_mode_enabled, permission=NO_PERMISSION_REQUIRED) def post_network_token_view(request): _, network_remote_user = get_network_models_from_request_token(request, create_network_remote_user=True) network_token = network_remote_user.network_token @@ -31,7 +33,8 @@ def post_network_token_view(request): @s.NetworkTokenAPI.delete(schema=s.NetworkToken_DELETE_RequestSchema, tags=[s.NetworkTag], response_schemas=s.NetworkToken_DELETE_responses) -@view_config(route_name=s.NetworkTokenAPI.name, request_method="DELETE", permission=NO_PERMISSION_REQUIRED) +@view_config(route_name=s.NetworkTokenAPI.name, request_method="DELETE", + decorator=check_network_mode_enabled, permission=NO_PERMISSION_REQUIRED) def delete_network_token_view(request): node, network_remote_user = get_network_models_from_request_token(request) if network_remote_user.network_token: @@ -45,7 +48,7 @@ def delete_network_token_view(request): @s.NetworkTokensAPI.delete(schema=s.NetworkTokens_DELETE_RequestSchema, tags=[s.NetworkTag], response_schemas=s.NetworkTokens_DELETE_responses) -@view_config(route_name=s.NetworkTokensAPI.name, request_method="DELETE") +@view_config(route_name=s.NetworkTokensAPI.name, request_method="DELETE", decorator=check_network_mode_enabled) def delete_network_tokens_view(request): if asbool(request.GET.get("expired_only")): deleted = models.NetworkToken.delete_expired(request.db) @@ -63,7 +66,8 @@ def delete_network_tokens_view(request): @s.NetworkJSONWebKeySetAPI.get(tags=[s.NetworkTag], response_schemas=s.NetworkJSONWebKeySet_GET_responses) -@view_config(route_name=s.NetworkJSONWebKeySetAPI.name, request_method="GET", permission=NO_PERMISSION_REQUIRED) +@view_config(route_name=s.NetworkJSONWebKeySetAPI.name, request_method="GET", + decorator=check_network_mode_enabled, permission=NO_PERMISSION_REQUIRED) def get_network_jwks_view(_request): return ax.valid_http(http_success=HTTPOk, detail=s.NetworkJSONWebKeySet_GET_OkResponseSchema.description, @@ -71,7 +75,7 @@ def get_network_jwks_view(_request): @s.NetworkDecodeJWTAPI.get(tags=[s.NetworkTag], response_schemas=s.NetworkDecodeJWT_GET_Responses) -@view_config(route_name=s.NetworkDecodeJWTAPI.name, request_method="GET") +@view_config(route_name=s.NetworkDecodeJWTAPI.name, request_method="GET", decorator=check_network_mode_enabled) def get_decode_jwt(request): token = request.GET.get("token") if token is None: diff --git a/magpie/api/management/network/node/network_node_views.py b/magpie/api/management/network/node/network_node_views.py index 7a833aec8..e182705e7 100644 --- a/magpie/api/management/network/node/network_node_views.py +++ b/magpie/api/management/network/node/network_node_views.py @@ -24,11 +24,13 @@ delete_network_node, update_associated_user_groups ) +from magpie.api.requests import check_network_mode_enabled from magpie.constants import get_constant @s.NetworkNodesAPI.get(tags=[s.NetworkTag], response_schemas=s.NetworkNodes_GET_responses) -@view_config(route_name=s.NetworkNodesAPI.name, request_method="GET", permission=Authenticated) +@view_config(route_name=s.NetworkNodesAPI.name, request_method="GET", + decorator=check_network_mode_enabled, permission=Authenticated) def get_network_nodes_view(request): is_admin = False if request.user is not None: @@ -42,7 +44,7 @@ def get_network_nodes_view(request): @s.NetworkNodeAPI.get(tags=[s.NetworkTag], response_schemas=s.NetworkNode_GET_responses) -@view_config(route_name=s.NetworkNodeAPI.name, request_method="GET") +@view_config(route_name=s.NetworkNodeAPI.name, request_method="GET", decorator=check_network_mode_enabled) def get_network_node_view(request): node_name = ar.get_value_matchdict_checked(request, "node_name") node = ax.evaluate_call( @@ -55,7 +57,7 @@ def get_network_node_view(request): @s.NetworkNodesAPI.post(schema=s.NetworkNode_POST_RequestSchema, tags=[s.NetworkTag], response_schemas=s.NetworkNodes_POST_responses) -@view_config(route_name=s.NetworkNodesAPI.name, request_method="POST") +@view_config(route_name=s.NetworkNodesAPI.name, request_method="POST", decorator=check_network_mode_enabled) def post_network_nodes_view(request): required_params = ("name", "jwks_url", "token_url", "authorization_url") kwargs = {} @@ -76,7 +78,7 @@ def post_network_nodes_view(request): @s.NetworkNodeAPI.patch(schema=s.NetworkNode_PATCH_RequestSchema, tags=[s.NetworkTag], response_schemas=s.NetworkNode_PATCH_responses) -@view_config(route_name=s.NetworkNodeAPI.name, request_method="PATCH") +@view_config(route_name=s.NetworkNodeAPI.name, request_method="PATCH", decorator=check_network_mode_enabled) def patch_network_node_view(request): node_name = ar.get_value_matchdict_checked(request, "node_name") node = ax.evaluate_call( @@ -101,7 +103,7 @@ def patch_network_node_view(request): @s.NetworkNodeAPI.delete(tags=[s.NetworkTag], response_schemas=s.NetworkNode_DELETE_responses) -@view_config(route_name=s.NetworkNodeAPI.name, request_method="DELETE") +@view_config(route_name=s.NetworkNodeAPI.name, request_method="DELETE", decorator=check_network_mode_enabled) def delete_network_node_view(request): node_name = ar.get_value_matchdict_checked(request, "node_name") node = ax.evaluate_call( @@ -116,7 +118,8 @@ def delete_network_node_view(request): @s.NetworkNodeTokenAPI.get(tags=[s.NetworkTag], response_schemas=s.NetworkNodeToken_GET_responses) -@view_config(route_name=s.NetworkNodeTokenAPI.name, request_method="GET", permission=Authenticated) +@view_config(route_name=s.NetworkNodeTokenAPI.name, request_method="GET", + decorator=check_network_mode_enabled, permission=Authenticated) def get_network_node_token_view(request): node_name = ar.get_value_matchdict_checked(request, "node_name") node = ax.evaluate_call( @@ -134,7 +137,8 @@ def get_network_node_token_view(request): @s.NetworkNodeTokenAPI.delete(tags=[s.NetworkTag], response_schemas=s.NetworkNodeToken_DELETE_responses) -@view_config(route_name=s.NetworkNodeTokenAPI.name, request_method="DELETE", permission=Authenticated) +@view_config(route_name=s.NetworkNodeTokenAPI.name, request_method="DELETE", + decorator=check_network_mode_enabled, permission=Authenticated) def delete_network_node_token_view(request): node_name = ar.get_value_matchdict_checked(request, "node_name") node = ax.evaluate_call( @@ -150,7 +154,8 @@ def delete_network_node_token_view(request): @s.NetworkLinkAPI.get(schema=s.NetworkLink_GET_RequestSchema, tags=[s.NetworkTag], response_schemas=s.NetworkLink_GET_responses) -@view_config(route_name=s.NetworkLinkAPI.name, request_method="GET", permission=Authenticated) +@view_config(route_name=s.NetworkLinkAPI.name, request_method="GET", + decorator=check_network_mode_enabled, permission=Authenticated) def get_network_node_link_view(request): token = request.GET.get("token") node_name = jwt.decode(token, options={"verify_signature": False}).get("iss") @@ -174,7 +179,8 @@ def get_network_node_link_view(request): @s.NetworkNodeLinkAPI.post(tags=[s.NetworkTag], response_schemas=s.NetworkNodeLink_POST_responses) -@view_config(route_name=s.NetworkNodeLinkAPI.name, request_method="POST", permission=Authenticated) +@view_config(route_name=s.NetworkNodeLinkAPI.name, request_method="POST", + decorator=check_network_mode_enabled, permission=Authenticated) def post_network_node_link_view(request): node_name = ar.get_value_matchdict_checked(request, "node_name") node = ax.evaluate_call( diff --git a/magpie/api/management/network/remote_user/remote_user_views.py b/magpie/api/management/network/remote_user/remote_user_views.py index b737e69b8..a7e5f8d64 100644 --- a/magpie/api/management/network/remote_user/remote_user_views.py +++ b/magpie/api/management/network/remote_user/remote_user_views.py @@ -9,11 +9,12 @@ check_remote_user_access_permissions, requested_remote_user ) +from magpie.api.requests import check_network_mode_enabled from magpie.constants import protected_user_name_regex @s.NetworkRemoteUsersAPI.get(tags=[s.NetworkTag], response_schemas=s.NetworkRemoteUsers_GET_responses) -@view_config(route_name=s.NetworkRemoteUsersAPI.name, request_method="GET") +@view_config(route_name=s.NetworkRemoteUsersAPI.name, request_method="GET", decorator=check_network_mode_enabled) def get_network_remote_users_view(request): query = request.db.query(models.NetworkRemoteUser) if request.GET.get("user_name"): @@ -24,7 +25,8 @@ def get_network_remote_users_view(request): @s.NetworkRemoteUserAPI.get(tags=[s.NetworkTag], response_schemas=s.NetworkRemoteUser_GET_responses) -@view_config(route_name=s.NetworkRemoteUserAPI.name, request_method="GET", permission=Authenticated) +@view_config(route_name=s.NetworkRemoteUserAPI.name, request_method="GET", + decorator=check_network_mode_enabled, permission=Authenticated) def get_network_remote_user_view(request): remote_user = requested_remote_user(request) check_remote_user_access_permissions(request, remote_user) @@ -34,7 +36,7 @@ def get_network_remote_user_view(request): @s.NetworkRemoteUsersAPI.post(schema=s.NetworkRemoteUsers_POST_RequestSchema, tags=[s.NetworkTag], response_schemas=s.NetworkRemoteUsers_POST_responses) -@view_config(route_name=s.NetworkRemoteUsersAPI.name, request_method="POST") +@view_config(route_name=s.NetworkRemoteUsersAPI.name, request_method="POST", decorator=check_network_mode_enabled) def post_network_remote_users_view(request): required_params = ("remote_user_name", "user_name", "node_name") for param in required_params: @@ -69,7 +71,7 @@ def post_network_remote_users_view(request): @s.NetworkRemoteUserAPI.patch(schema=s.NetworkRemoteUser_PATCH_RequestSchema, tags=[s.NetworkTag], response_schemas=s.NetworkRemoteUser_PATCH_responses) -@view_config(route_name=s.NetworkRemoteUserAPI.name, request_method="PATCH") +@view_config(route_name=s.NetworkRemoteUserAPI.name, request_method="PATCH", decorator=check_network_mode_enabled) def patch_network_remote_user_view(request): update_params = [p for p in request.POST if p in ("remote_user_name", "user_name", "node_name")] if not update_params: @@ -101,7 +103,8 @@ def patch_network_remote_user_view(request): @s.NetworkRemoteUserAPI.delete(tags=[s.NetworkTag], response_schemas=s.NetworkRemoteUser_DELETE_responses) -@view_config(route_name=s.NetworkRemoteUserAPI.name, request_method="DELETE", permission=Authenticated) +@view_config(route_name=s.NetworkRemoteUserAPI.name, request_method="DELETE", + decorator=check_network_mode_enabled, permission=Authenticated) def delete_network_remote_user_view(request): remote_user = requested_remote_user(request) check_remote_user_access_permissions(request, remote_user) @@ -110,7 +113,8 @@ def delete_network_remote_user_view(request): @s.NetworkRemoteUsersCurrentAPI.get(tags=[s.NetworkTag], response_schemas=s.NetworkRemoteUsersCurrent_GET_responses) -@view_config(route_name=s.NetworkRemoteUsersCurrentAPI.name, request_method="GET", permission=Authenticated) +@view_config(route_name=s.NetworkRemoteUsersCurrentAPI.name, request_method="GET", + decorator=check_network_mode_enabled, permission=Authenticated) def get_network_remote_users_current_view(request): nodes = [n.as_dict() for n in request.db.query(models.NetworkRemoteUser).filter(models.NetworkRemoteUser.user_id == request.user.id)] diff --git a/magpie/api/requests.py b/magpie/api/requests.py index 8815dbc35..201cbdee0 100644 --- a/magpie/api/requests.py +++ b/magpie/api/requests.py @@ -1,3 +1,4 @@ +import functools from typing import TYPE_CHECKING import six @@ -8,6 +9,7 @@ HTTPForbidden, HTTPInternalServerError, HTTPNotFound, + HTTPNotImplemented, HTTPUnprocessableEntity ) from ziggurat_foundations.models.services.group import GroupService @@ -17,13 +19,13 @@ from magpie import models from magpie.api import exception as ax from magpie.api import schemas as s -from magpie.constants import get_constant +from magpie.constants import get_constant, network_enabled from magpie.permissions import PermissionSet from magpie.utils import CONTENT_TYPE_JSON, get_logger if TYPE_CHECKING: # pylint: disable=W0611,unused-import - from typing import Any, Dict, Iterable, List, Optional, Union + from typing import Any, Callable, Dict, Iterable, List, Optional, Union from pyramid.request import Request @@ -379,3 +381,27 @@ def get_query_param(request, case_insensitive_key, default=None): if param.lower() == key.lower(): return request.params.get(param) return default + + +def check_network_mode_enabled(view_func): + # type: (Callable) -> Callable + """ + Decorator for views that returns a :class:`HTTPNotImplemented` response if network mode is not enabled. + This is intended to be used for all views that should only be accessed if network mode is enabled. + + Instead of decorating a view function directly, pass this function to the ``decorator`` argument of the + ``view_config`` decorator. For example: + + .. code-block:: python + + @view_config(..., decorator=check_network_mode_enabled) + def get_some_view(request): + ... + """ + @functools.wraps(view_func) + def wrapper(context, request): + if not network_enabled(request): + return ax.raise_http(http_error=HTTPNotImplemented, + detail=s.NetworkMode_NotEnabledResponseSchema.description) + return view_func(context, request) + return wrapper diff --git a/magpie/api/schemas.py b/magpie/api/schemas.py index ed9d38ef6..da47396d0 100644 --- a/magpie/api/schemas.py +++ b/magpie/api/schemas.py @@ -17,6 +17,7 @@ HTTPInternalServerError, HTTPMethodNotAllowed, HTTPNotAcceptable, + HTTPNotImplemented, HTTPNotFound, HTTPOk, HTTPUnauthorized, @@ -3918,6 +3919,11 @@ class NetworkRemoteUsers_POST_ForbiddenResponseSchema(BaseResponseSchemaAPI): body = ErrorResponseBodySchema(code=HTTPForbidden.code, description=description) +class NetworkMode_NotEnabledResponseSchema(BaseRequestSchemaAPI): + description = "Network Mode is not enabled." + body = ErrorResponseBodySchema(code=HTTPNotImplemented.code, description=description) + + # Responses for specific views Resource_GET_responses = { "200": Resource_GET_OkResponseSchema(), diff --git a/magpie/ui/network/views.py b/magpie/ui/network/views.py index 501e5979e..200c2d1ec 100644 --- a/magpie/ui/network/views.py +++ b/magpie/ui/network/views.py @@ -6,6 +6,7 @@ from magpie.api import schemas from magpie.api.management.network.network_utils import encode_jwt +from magpie.api.requests import check_network_mode_enabled from magpie.ui.utils import AdminRequests, check_response, request_api from magpie.utils import get_json, get_logger @@ -14,6 +15,7 @@ class NetworkViews(AdminRequests): @view_config(route_name="magpie.ui.network.views.NetworkViews.authorize", + decorator=check_network_mode_enabled, renderer="templates/authorize.mako", permission=Authenticated) def authorize(self): token = self.request.GET.get("token") diff --git a/setup.cfg b/setup.cfg index cc86f7e61..4393e060a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -130,3 +130,4 @@ markers = auth_admin: magpie operations that require admin-level access auth_users: magpie operations that require user-level access (non admin) auth_public: magpie operations that are publicly accessible (no auth) + network: magpie operations that are enabled in network mode \ No newline at end of file diff --git a/tests/interfaces.py b/tests/interfaces.py index 04e5b0547..8606f7e1b 100644 --- a/tests/interfaces.py +++ b/tests/interfaces.py @@ -474,6 +474,137 @@ def test_LoginAnonymous_Forbidden(self): resp = utils.test_request(self, "POST", s.SigninAPI.path, json=data, expect_errors=True) utils.check_response_basic_info(resp, 403, expected_method="POST") + @runner.MAGPIE_TEST_NETWORK + @runner.MAGPIE_TEST_LOGIN + @utils.check_network_mode + def test_LoginProtectedUsername_Forbidden(self): + """ + Test different login variations and ensure that users with usernames that start with + ``MAGPIE_NETWORK_NAME_PREFIX`` are blocked in network mode. + + .. versionadded:: 3.38 + """ + + utils.warn_version(self, "Login protected usernames explicitly blocked.", "3.38.0", skip=True) + protected_username = "{}{!s}".format(get_constant("MAGPIE_NETWORK_NAME_PREFIX"), uuid.uuid4()) + data = { + "user_name": protected_username, + "password": get_constant("MAGPIE_ANONYMOUS_PASSWORD") + } + headers = {"Accept": CONTENT_TYPE_JSON} + + resp = utils.test_request(self, "GET", s.SigninAPI.path, params=data, headers=headers, expect_errors=True) + utils.check_response_basic_info(resp, 403, expected_method="GET") + + form = "user_name={user_name}&password={password}".format(**data) + headers["Content-Type"] = CONTENT_TYPE_FORM + resp = utils.test_request(self, "POST", s.SigninAPI.path, data=form, expect_errors=True) + utils.check_response_basic_info(resp, 403, expected_method="POST") + + headers["Content-Type"] = CONTENT_TYPE_JSON + resp = utils.test_request(self, "POST", s.SigninAPI.path, json=data, expect_errors=True) + utils.check_response_basic_info(resp, 403, expected_method="POST") + + @runner.MAGPIE_TEST_NETWORK + @runner.MAGPIE_TEST_LOGIN + @utils.check_network_mode + def test_LoginProtectedEmail_Forbidden(self): + """ + Test different login variations and ensure that users with emails that start with + ``MAGPIE_NETWORK_NAME_PREFIX`` are blocked in network mode. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Login protected emails explicitly blocked.", "3.38.0", skip=True) + protected_email = get_constant("MAGPIE_NETWORK_ANONYMOUS_EMAIL_FORMAT").format(uuid.uuid4()) + data = { + "user_name": protected_email, + "password": get_constant("MAGPIE_ANONYMOUS_PASSWORD") + } + headers = {"Accept": CONTENT_TYPE_JSON} + + resp = utils.test_request(self, "GET", s.SigninAPI.path, params=data, headers=headers, expect_errors=True) + utils.check_response_basic_info(resp, 403, expected_method="GET") + + form = "user_name={user_name}&password={password}".format(**data) + headers["Content-Type"] = CONTENT_TYPE_FORM + resp = utils.test_request(self, "POST", s.SigninAPI.path, data=form, expect_errors=True) + utils.check_response_basic_info(resp, 403, expected_method="POST") + + headers["Content-Type"] = CONTENT_TYPE_JSON + resp = utils.test_request(self, "POST", s.SigninAPI.path, json=data, expect_errors=True) + utils.check_response_basic_info(resp, 403, expected_method="POST") + + @runner.MAGPIE_TEST_NETWORK + @runner.MAGPIE_TEST_LOGIN + @utils.check_network_mode(enable=False) + def test_LoginProtectedUsername_Allowed(self): + """ + Test different login variations and ensure that users with usernames that start with + ``MAGPIE_NETWORK_NAME_PREFIX`` are allowed if network mode is not enabled. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Login protected usernames explicitly blocked.", "3.38.0", skip=True) + protected_username = "{}{!s}".format(get_constant("MAGPIE_NETWORK_NAME_PREFIX"), uuid.uuid4()) + self.login_admin() + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_user_name=protected_username, override_cookies=self.cookies) + + data = { + "user_name": protected_username, + "password": protected_username + } + headers = {"Accept": CONTENT_TYPE_JSON} + + resp = utils.test_request(self, "GET", s.SigninAPI.path, params=data, headers=headers, expect_errors=True) + utils.check_response_basic_info(resp, 200, expected_method="GET") + + form = "user_name={user_name}&password={password}".format(**data) + headers["Content-Type"] = CONTENT_TYPE_FORM + resp = utils.test_request(self, "POST", s.SigninAPI.path, data=form, expect_errors=True) + utils.check_response_basic_info(resp, 200, expected_method="POST") + + headers["Content-Type"] = CONTENT_TYPE_JSON + resp = utils.test_request(self, "POST", s.SigninAPI.path, json=data, expect_errors=True) + utils.check_response_basic_info(resp, 200, expected_method="POST") + + @runner.MAGPIE_TEST_NETWORK + @runner.MAGPIE_TEST_LOGIN + @utils.check_network_mode(enable=False) + def test_LoginProtectedEmail_Allowed(self): + """ + Test different login variations and ensure that users with emails that start with + ``MAGPIE_NETWORK_NAME_PREFIX`` are allowed if network mode is not enabled. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Login protected emails explicitly blocked.", "3.38.0", skip=True) + protected_email = get_constant("MAGPIE_NETWORK_ANONYMOUS_EMAIL_FORMAT").format(uuid.uuid4()) + user_name = "{!s}".format(uuid.uuid4()) + self.login_admin() + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_user_name=user_name, + override_email=protected_email, + override_cookies=self.cookies) + data = { + "user_name": protected_email, + "password": user_name + } + headers = {"Accept": CONTENT_TYPE_JSON} + + resp = utils.test_request(self, "GET", s.SigninAPI.path, params=data, headers=headers, expect_errors=True) + utils.check_response_basic_info(resp, 200, expected_method="GET") + + form = "user_name={user_name}&password={password}".format(**data) + headers["Content-Type"] = CONTENT_TYPE_FORM + resp = utils.test_request(self, "POST", s.SigninAPI.path, data=form, expect_errors=True) + utils.check_response_basic_info(resp, 200, expected_method="POST") + + headers["Content-Type"] = CONTENT_TYPE_JSON + resp = utils.test_request(self, "POST", s.SigninAPI.path, json=data, expect_errors=True) + utils.check_response_basic_info(resp, 200, expected_method="POST") + @runner.MAGPIE_TEST_LOGIN def test_Login_GetRequestFormat(self): """ diff --git a/tests/runner.py b/tests/runner.py index fcd81ec79..965867808 100644 --- a/tests/runner.py +++ b/tests/runner.py @@ -51,6 +51,7 @@ def filter_test_files(root, filename): MAGPIE_TEST_AUTH_ADMIN = RunOptionDecorator("MAGPIE_TEST_AUTH_ADMIN", "operations that require admin-level access") MAGPIE_TEST_AUTH_USERS = RunOptionDecorator("MAGPIE_TEST_AUTH_USERS", "operations that require user-level access") MAGPIE_TEST_AUTH_PUBLIC = RunOptionDecorator("MAGPIE_TEST_AUTH_PUBLIC", "operations that are publicly accessible") +MAGPIE_TEST_NETWORK = RunOptionDecorator("MAGPIE_TEST_NETWORK", "operations that are enabled in network mode") def test_suite(): diff --git a/tests/test_utils.py b/tests/test_utils.py index 910d43963..211d6ffeb 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -191,6 +191,10 @@ def test_verify_param_proper_verifications_raised(self): utils.check_raises(lambda: ax.verify_param("abc", matches=True, param_compare=r"[0-9]+"), HTTPBadRequest) utils.check_raises(lambda: ax.verify_param("abc", matches=True, param_compare=re.compile(r"[0-9]+")), HTTPBadRequest) + utils.check_raises(lambda: ax.verify_param("abc", not_matches=True, param_compare=r"[a-z]+"), + HTTPBadRequest) + utils.check_raises(lambda: ax.verify_param("abc", not_matches=True, param_compare=re.compile(r"[a-z]+")), + HTTPBadRequest) # with requested error utils.check_raises(lambda: @@ -218,6 +222,16 @@ def test_verify_param_proper_verifications_raised(self): param_compare=re.compile(r"[0-9]+"), http_error=HTTPForbidden), HTTPForbidden) + utils.check_raises(lambda: + ax.verify_param("abc", not_matches=True, + param_compare=r"[a-z]+", + http_error=HTTPForbidden), + HTTPForbidden) + utils.check_raises(lambda: + ax.verify_param("abc", not_matches=True, + param_compare=re.compile(r"[a-z]+"), + http_error=HTTPForbidden), + HTTPForbidden) def test_verify_param_proper_verifications_passed(self): ax.verify_param("x", param_compare=["a", "b"], not_in=True) @@ -235,6 +249,8 @@ def test_verify_param_proper_verifications_passed(self): ax.verify_param("", is_empty=True) ax.verify_param("abc", matches=True, param_compare=r"[a-z]+") ax.verify_param("abc", matches=True, param_compare=re.compile(r"[a-z]+")) + ax.verify_param("abc", not_matches=True, param_compare=r"[0-9]+") + ax.verify_param("abc", not_matches=True, param_compare=re.compile(r"[0-9]+")) def test_verify_param_args_incorrect_usage(self): """ @@ -252,8 +268,10 @@ def test_verify_param_args_incorrect_usage(self): HTTPInternalServerError, msg="incorrect non-iterable compare should raise invalid type") utils.check_raises(lambda: ax.verify_param("a", matches=True, param_compare=1), HTTPInternalServerError, msg="incorrect matching pattern not a string or compiled pattern") + utils.check_raises(lambda: ax.verify_param("a", not_matches=True, param_compare=1), + HTTPInternalServerError, msg="incorrect matching pattern not a string or compiled pattern") for flag in ["not_none", "not_empty", "not_in", "not_equal", "is_none", "is_empty", "is_in", "is_equal", - "is_true", "is_false", "is_type", "matches"]: + "is_true", "is_false", "is_type", "matches", "not_matches"]: utils.check_raises(lambda: ax.verify_param("x", **{flag: 1}), HTTPInternalServerError, msg="invalid flag '{}' type should be caught".format(flag)) diff --git a/tests/utils.py b/tests/utils.py index e8cd192dc..021f6a9c9 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -34,6 +34,7 @@ from webtest.response import TestResponse from magpie import __meta__, app, services +from magpie.api import schemas from magpie.compat import LooseVersion from magpie.constants import get_constant from magpie.permissions import Access, PermissionSet, Scope @@ -731,6 +732,52 @@ def wrapped(*_, **__): # type: (*Any, **Any) -> Any return mocked_get_settings_decorator +def check_network_mode(_test_func=None, enable=True): + # type: (Optional[Callable[[Callable], Any]], bool) -> Callable + """ + Returns a decorator that checks whether a test function is marked as local or remote according to the + :class:`RunOptionDecorator` on the current test. + If local, then ensure that the test application has network mode enabled (if :paramref:`enable` is ``True``) or + disabled (if :paramref:`enable` is ``False``). + If remote, then check whether the network mode is enabled or not for the remote application. If the network mode + status does not match the :paramref:`enable` value, then skip the test. For example, if the remote application has + network mode enabled but :paramref:`enable` is ``False`` then the test is skipped. + + .. note:: + + A test cannot affect the settings of a remote application. For this reason, we skip tests that would fail only + because a remote application is not configured in a way that would allow the test to pass. + + :param _test_func: Test function being decorated. + :param enable: Boolean value indicating whether ``_test_func`` expects network mode to be enabled in order to pass. + """ + if enable: + settings = {"magpie.network_enabled": True, "magpie.network_instance_name": "node1"} + else: + settings = {"magpie.network_enabled": False} + + def decorator_func(test_func): + @functools.wraps(test_func) + def wrapper(*args, **kwargs): + test = args[0] + if hasattr(test, "pytestmark") and any(mark.name == "remote" for mark in test.pytestmark): + resp = test_request(test, "GET", schemas.NetworkJSONWebKeySetAPI.path) + remote_enabled = get_json_body(resp).get("code") != 501 + if enable and not remote_enabled: + test.skipTest("Test requires remote to be running with network mode enabled.") + elif not enable and remote_enabled: + test.skipTest("Test requires remote to be running without network mode enabled.") + return test_func(*args, **kwargs) + else: + # assume local test + return mocked_get_settings(settings=settings)(test_func)(*args, **kwargs) + return wrapper + if _test_func is None: + return decorator_func + else: + return decorator_func(_test_func) + + def mock_request(request_path_query="", # type: Str method="GET", # type: Str params=None, # type: Optional[Dict[Str, Str]] From b337464e37b2d14134bfe4dbf92c8f1267d3a701 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Tue, 2 Apr 2024 16:14:07 -0400 Subject: [PATCH 26/55] bug fixes and testing helper methods --- ci/magpie.env | 3 + env/magpie.env.example | 3 + .../api/management/network/network_utils.py | 9 +- .../network/node/network_node_views.py | 22 +- .../network/remote_user/remote_user_views.py | 49 ++-- magpie/models.py | 2 +- requirements-dev.txt | 4 + tests/interfaces.py | 30 +- tests/test_magpie_api.py | 20 ++ tests/utils.py | 258 +++++++++++++++++- 10 files changed, 367 insertions(+), 33 deletions(-) diff --git a/ci/magpie.env b/ci/magpie.env index 44d514426..4c07a05d5 100644 --- a/ci/magpie.env +++ b/ci/magpie.env @@ -12,8 +12,11 @@ MAGPIE_ADMIN_PASSWORD=qwerty-ci-tests MAGPIE_LOG_LEVEL=INFO MAGPIE_LOG_REQUEST=false MAGPIE_LOG_EXCEPTION=false +MAGPIE_NETWORK_INSTANCE_NAME=node1 MAGPIE_TEST_VERSION=latest MAGPIE_TEST_REMOTE_SERVER_URL=http://localhost:2001 +MAGPIE_TEST_REMOTE_NODE_SERVER_HOST=localhost +MAGPIE_TEST_REMOTE_NODE_SERVER_PORT=2002 MAGPIE_TEST_ADMIN_USERNAME=unittest-admin # auto-generate password MAGPIE_TEST_ADMIN_PASSWORD= diff --git a/env/magpie.env.example b/env/magpie.env.example index 70a428072..0b3f7521e 100644 --- a/env/magpie.env.example +++ b/env/magpie.env.example @@ -32,6 +32,9 @@ MAGPIE_TEST_VERSION=latest # this means you must have a separately running Magpie instance reachable at that endpoint to run 'remote' tests # to ignore this, consider setting an empty value or running 'make test-local' instead MAGPIE_TEST_REMOTE_SERVER_URL=http://localhost:2001 +# below URL specifies a host and port for a fake network node (used for network tests) +MAGPIE_TEST_REMOTE_NODE_SERVER_HOST=localhost +MAGPIE_TEST_REMOTE_NODE_SERVER_PORT=2002 # below are the credentials employed to run tests (especially if running on non-default remote server) MAGPIE_TEST_ADMIN_USERNAME=admin MAGPIE_TEST_ADMIN_PASSWORD=qwerty diff --git a/magpie/api/management/network/network_utils.py b/magpie/api/management/network/network_utils.py index 3f3750591..ad4fe5814 100644 --- a/magpie/api/management/network/network_utils.py +++ b/magpie/api/management/network/network_utils.py @@ -6,10 +6,11 @@ import jwt from cryptography.hazmat.primitives import serialization from jwcrypto import jwk -from pyramid.httpexceptions import HTTPInternalServerError, HTTPNotFound +from pyramid.httpexceptions import HTTPBadRequest, HTTPInternalServerError, HTTPNotFound from magpie import models from magpie.api import exception as ax +from magpie.api import requests as ar from magpie.api import schemas as s from magpie.constants import get_constant from magpie.utils import get_logger @@ -56,7 +57,7 @@ def _pem_file_passwords(primary=False): """ pem_passwords = get_constant("MAGPIE_NETWORK_PEM_PASSWORDS", raise_missing=False, raise_not_set=False) try: - passwords = json.loads(pem_passwords) + passwords = json.loads(pem_passwords or "") except json.decoder.JSONDecodeError: passwords = [pem_passwords] passwords = [p.encode() if p else None for p in passwords] @@ -151,7 +152,9 @@ def get_network_models_from_request_token(request, create_network_remote_user=Fa ``NetworkRemoteUser`` associated with the anonymous user for the given ``NetworkNode`` and adds it to the current database transaction. """ - token = request.POST.get("token") + token = ar.get_multiformat_body(request, "token", default=None) + if token is None: + ax.raise_http(http_error=HTTPBadRequest, detail=s.BadRequestResponseSchema.description) node_name = jwt.decode(token, options={"verify_signature": False}).get("iss") node = ax.evaluate_call( lambda: request.db.query(models.NetworkNode).filter(models.NetworkNode.name == node_name).one(), diff --git a/magpie/api/management/network/node/network_node_views.py b/magpie/api/management/network/node/network_node_views.py index e182705e7..4c6078fa1 100644 --- a/magpie/api/management/network/node/network_node_views.py +++ b/magpie/api/management/network/node/network_node_views.py @@ -1,5 +1,8 @@ +import json + import jwt import requests +import six from pyramid.httpexceptions import ( HTTPBadRequest, HTTPCreated, @@ -62,12 +65,21 @@ def post_network_nodes_view(request): required_params = ("name", "jwks_url", "token_url", "authorization_url") kwargs = {} for param in required_params: - if param in request.POST: - kwargs[param] = request.POST[param] - else: + value = ar.get_multiformat_body(request, param, default=None) + if value is None: ax.raise_http(http_error=HTTPBadRequest, detail=s.BadRequestResponseSchema.description) - if "redirect_uris" in request.POST: - kwargs["redirect_uris"] = request.POST.get("redirect_uris") + kwargs[param] = value + redirect_uris = ar.get_multiformat_body(request, "redirect_uris", default=None) + if redirect_uris is not None: + if isinstance(redirect_uris, six.string_types): + kwargs["redirect_uris"] = ax.evaluate_call( + lambda: json.loads(redirect_uris), + http_error=HTTPBadRequest, + fallback=lambda: request.db.rollback(), + msg_on_fail=s.NetworkNodes_CheckInfo_RedirectURIsValue_BadRequestResponseSchema.description + ) + else: + kwargs["redirect_uris"] = redirect_uris check_network_node_info(request.db, **kwargs) node = models.NetworkNode(**kwargs) diff --git a/magpie/api/management/network/remote_user/remote_user_views.py b/magpie/api/management/network/remote_user/remote_user_views.py index a7e5f8d64..be8f08142 100644 --- a/magpie/api/management/network/remote_user/remote_user_views.py +++ b/magpie/api/management/network/remote_user/remote_user_views.py @@ -4,6 +4,7 @@ from magpie import models from magpie.api import exception as ax +from magpie.api import requests as ar from magpie.api import schemas as s from magpie.api.management.network.remote_user.remote_user_utils import ( check_remote_user_access_permissions, @@ -17,11 +18,12 @@ @view_config(route_name=s.NetworkRemoteUsersAPI.name, request_method="GET", decorator=check_network_mode_enabled) def get_network_remote_users_view(request): query = request.db.query(models.NetworkRemoteUser) - if request.GET.get("user_name"): - query = query.join(models.User).filter(models.User.user_name == request.GET["user_name"]) + user_name = ar.get_multiformat_body(request, "user_name", default=None) + if user_name is not None: + query = query.join(models.User).filter(models.User.user_name == user_name) nodes = [n.as_dict() for n in query.all()] return ax.valid_http(http_success=HTTPOk, detail=s.NetworkRemoteUsers_GET_OkResponseSchema.description, - content={"nodes": nodes}) + content={"remote_users": nodes}) @s.NetworkRemoteUserAPI.get(tags=[s.NetworkTag], response_schemas=s.NetworkRemoteUser_GET_responses) @@ -39,27 +41,29 @@ def get_network_remote_user_view(request): @view_config(route_name=s.NetworkRemoteUsersAPI.name, request_method="POST", decorator=check_network_mode_enabled) def post_network_remote_users_view(request): required_params = ("remote_user_name", "user_name", "node_name") + kwargs = {} for param in required_params: - if param not in request.POST: + value = ar.get_multiformat_body(request, param, default=None) + if value is None: ax.raise_http(http_error=HTTPBadRequest, detail=s.NetworkRemoteUsers_POST_BadRequestResponseSchema.description) + kwargs[param] = value node = ax.evaluate_call( - lambda: request.db.query(models.NetworkNode).filter(models.NetworkNode.name == request.POST["node_name"]).one(), + lambda: request.db.query(models.NetworkNode).filter(models.NetworkNode.name == kwargs["node_name"]).one(), http_error=HTTPNotFound, - msg_on_fail="No network node with name '{}' found".format(request.POST["node_name"]) + msg_on_fail="No network node with name '{}' found".format(kwargs["node_name"]) ) forbidden_user_names_regex = protected_user_name_regex(include_admin=False, settings_container=request) - user_name = request.POST["user_name"] - ax.verify_param(user_name, not_matches=True, param_compare=forbidden_user_names_regex, + ax.verify_param(kwargs["user_name"], not_matches=True, param_compare=forbidden_user_names_regex, param_name="user_name", - http_error=HTTPForbidden, content={"user_name": user_name}, + http_error=HTTPForbidden, content={"user_name": kwargs["user_name"]}, msg_on_fail=s.NetworkRemoteUsers_POST_ForbiddenResponseSchema.description) user = ax.evaluate_call( - lambda: request.db.query(models.User).filter(models.User.user_name == request.POST["user_name"]).one(), + lambda: request.db.query(models.User).filter(models.User.user_name == kwargs["user_name"]).one(), http_error=HTTPNotFound, - msg_on_fail="No user with user_name '{}' found".format(request.POST["user_name"]) + msg_on_fail="No user with user_name '{}' found".format(kwargs["user_name"]) ) - remote_user_name = request.POST["remote_user_name"] + remote_user_name = kwargs["remote_user_name"] ax.verify_param(remote_user_name, not_empty=True, param_name="remote_user_name", http_error=HTTPForbidden, @@ -73,30 +77,31 @@ def post_network_remote_users_view(request): tags=[s.NetworkTag], response_schemas=s.NetworkRemoteUser_PATCH_responses) @view_config(route_name=s.NetworkRemoteUserAPI.name, request_method="PATCH", decorator=check_network_mode_enabled) def patch_network_remote_user_view(request): - update_params = [p for p in request.POST if p in ("remote_user_name", "user_name", "node_name")] - if not update_params: + kwargs = {p: ar.get_multiformat_body(request, p, default=None) for p in + ("remote_user_name", "user_name", "node_name")} + if not any(kwargs.values()): ax.raise_http(http_error=HTTPBadRequest, detail=s.NetworkRemoteUser_PATCH_BadRequestResponseSchema.description) remote_user = requested_remote_user(request) - if "remote_user_name" in request.POST: - remote_user_name = request.POST["remote_user_name"] + if "remote_user_name" in kwargs: + remote_user_name = kwargs["remote_user_name"] ax.verify_param(remote_user_name, not_empty=True, param_name="remote_user_name", http_error=HTTPForbidden, msg_on_fail="remote_user_name is empty") remote_user.name = remote_user_name - if "user_name" in request.POST: + if "user_name" in kwargs: user = ax.evaluate_call( - lambda: request.db.query(models.User).filter(models.User.user_name == request.POST["user_name"]).one(), + lambda: request.db.query(models.User).filter(models.User.user_name == kwargs["user_name"]).one(), http_error=HTTPNotFound, - msg_on_fail="No user with user_name '{}' found".format(request.POST["user_name"]) + msg_on_fail="No user with user_name '{}' found".format(kwargs["user_name"]) ) remote_user.user_id = user.id - if "node_name" in request.POST: + if "node_name" in kwargs: node = ax.evaluate_call( lambda: request.db.query(models.NetworkNode).filter( - models.NetworkNode.name == request.POST["node_name"]).one(), + models.NetworkNode.name == kwargs["node_name"]).one(), http_error=HTTPNotFound, - msg_on_fail="No network node with name '{}' found".format(request.POST["node_name"]) + msg_on_fail="No network node with name '{}' found".format(kwargs["node_name"]) ) remote_user.network_node_id = node.id return ax.valid_http(http_success=HTTPOk, detail=s.NetworkRemoteUsers_PATCH_OkResponseSchema.description) diff --git a/magpie/models.py b/magpie/models.py index 5f9dbee98..b0ba4b3c0 100644 --- a/magpie/models.py +++ b/magpie/models.py @@ -1116,7 +1116,7 @@ def anonymous_user(self, db_session): def anonymous_group(self, db_session): # type: (Optional[Session]) -> User - return db_session.query(Group).filter(Group.name == self.anonymous_user_name()).one() + return db_session.query(Group).filter(Group.group_name == self.anonymous_user_name()).one() def as_dict(self): # type: () -> Dict[Str, Any] diff --git a/requirements-dev.txt b/requirements-dev.txt index 9c9672878..8ee56618f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -29,6 +29,10 @@ pylint-quotes pyramid-twitcher>=0.5.3; python_version < "3.6" # pyup: ignore pyramid-twitcher>=0.9.0; python_version >= "3.6" pytest +pytest-httpserver==1.0.1; python_version < "3.6" # pyup: ignore +pytest-httpserver==1.0.5; python_version == "3.6" # pyup: ignore +pytest-httpserver==1.0.6; python_version == "3.7" # pyup: ignore +pytest-httpserver==1.0.10; python_version > "3.7" python2-secrets; python_version <= "3.5" safety tox>=3.0 diff --git a/tests/interfaces.py b/tests/interfaces.py index 8606f7e1b..1670ece0f 100644 --- a/tests/interfaces.py +++ b/tests/interfaces.py @@ -48,7 +48,7 @@ CONTENT_TYPE_TXT_XML ) from tests import runner, utils -from tests.utils import TestVersion +from tests.utils import TestVersion, check_network_mode if six.PY3: # WARNING: Twitcher does not support Python 2 since 0.4.0, adapter cannot work without it @@ -99,6 +99,15 @@ class ConfigTestCase(object): test_admin = None # type: Optional[Str] test_user_name = None # type: Optional[Str] # also used as password when creating test user test_group_name = None # type: Optional[Str] + test_node_host = None # type: Optional[Str] + test_node_port = None # type: Optional[int] + test_node_name = None # type: Optional[Str] + test_node_jwks_url = None # type: Optional[Str] + test_node_token_url = None # type: Optional[Str] + test_authorization_url = None # type: Optional[Str] + test_redirect_uris = None # type: Optional[JSON] + test_remote_user_name = None # type: Optional[Str] + # extra parameters to indicate cleanup on final tear down # add new test values on test case startup before they *potentially* get interrupted because of error reserved_users = None # type: Optional[List[Str]] @@ -107,6 +116,9 @@ class ConfigTestCase(object): extra_group_names = set() # type: Set[Str] extra_resource_ids = set() # type: Set[int] extra_service_names = set() # type: Set[Str] + extra_node_names = set() # type: Set[Str] + extra_remote_user_names = set() # type: Set[Tuple[Str, Str]] + extra_network_tokens = set() # type: Set[Tuple[Str, Str]] @six.add_metaclass(ABCMeta) @@ -171,6 +183,18 @@ def cleanup(cls): cls.extra_user_names.add(cls.test_admin) cls.extra_user_names.add(cls.test_user_name) cls.extra_group_names.add(cls.test_group_name) + for remote_user_name, node_name in list(cls.extra_network_tokens): + check_network_mode(utils.TestSetup.delete_TestNetworkToken, + enable=True)(cls, + override_remote_user_name=remote_user_name, + override_node_name=node_name) + + cls.extra_network_tokens.discard((remote_user_name, node_name)) + for remote_user_name, node_name in list(cls.extra_remote_user_names): + check_network_mode(utils.TestSetup.delete_TestNetworkRemoteUser, + enable=True)(cls, override_remote_user_name=remote_user_name, + override_node_name=node_name) + cls.extra_remote_user_names.discard((remote_user_name, node_name)) for usr in list(cls.extra_user_names): # copy to update removed ones if usr not in cls.reserved_users: utils.TestSetup.delete_TestUser(cls, override_user_name=usr) @@ -187,6 +211,10 @@ def cleanup(cls): for res in list(cls.extra_resource_ids): # copy to update removed ones utils.TestSetup.delete_TestResource(cls, res) cls.extra_resource_ids.discard(res) + for node in list(cls.extra_node_names): + check_network_mode(utils.TestSetup.delete_TestNetworkNode, + enable=True)(cls, override_name=node) + cls.extra_node_names.discard(node) @property def update_method(self): diff --git a/tests/test_magpie_api.py b/tests/test_magpie_api.py index 81341eace..68d374b3b 100644 --- a/tests/test_magpie_api.py +++ b/tests/test_magpie_api.py @@ -46,6 +46,16 @@ def setUpClass(cls): raise_missing=False, raise_not_set=False) cls.test_group_name = get_constant("MAGPIE_TEST_GROUP", default_value="unittest-no-auth_api-group-local", raise_missing=False, raise_not_set=False) + cls.test_node_name = "node2" + cls.test_remote_user_name = "remote_user_1" + cls.test_node_host = get_constant("MAGPIE_TEST_REMOTE_NODE_SERVER_HOST", default_value="localhost", + raise_missing=False, raise_not_set=False) + cls.test_node_port = int(get_constant("MAGPIE_TEST_REMOTE_NODE_SERVER_PORT", default_value=2002, + raise_missing=False, raise_not_set=False)) + cls.test_node_jwks_url = "http://{}:{}/network/jwks".format(cls.test_node_host, cls.test_node_port) + cls.test_node_token_url = "http://{}:{}/network/token".format(cls.test_node_host, cls.test_node_port) + cls.test_authorization_url = "http://{}:{}/ui/network/authorize".format(cls.test_node_host, cls.test_node_port) + cls.test_redirect_uris = '["http://{}:{}/network/link"]'.format(cls.test_node_host, cls.test_node_port) @runner.MAGPIE_TEST_API @@ -662,6 +672,16 @@ def setUpClass(cls): raise_missing=False, raise_not_set=False) cls.test_group_name = get_constant("MAGPIE_TEST_GROUP", default_value="unittest-no-auth_api-group-remote", raise_missing=False, raise_not_set=False) + cls.test_node_name = "node2" + cls.test_remote_user_name = "remote_user_1" + cls.test_node_host = get_constant("MAGPIE_TEST_REMOTE_NODE_SERVER_HOST", default_value="localhost", + raise_missing=False, raise_not_set=False) + cls.test_node_port = int(get_constant("MAGPIE_TEST_REMOTE_NODE_SERVER_PORT", default_value=2002, + raise_missing=False, raise_not_set=False)) + cls.test_node_jwks_url = "http://{}:{}/network/jwks".format(cls.test_node_host, cls.test_node_port) + cls.test_node_token_url = "http://{}:{}/network/token".format(cls.test_node_host, cls.test_node_port) + cls.test_authorization_url = "http://{}:{}/ui/network/authorize".format(cls.test_node_host, cls.test_node_port) + cls.test_redirect_uris = '["http://{}:{}/network/link"]'.format(cls.test_node_host, cls.test_node_port) @runner.MAGPIE_TEST_API diff --git a/tests/utils.py b/tests/utils.py index 021f6a9c9..a2f441c4e 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -10,21 +10,27 @@ import uuid import warnings from copy import deepcopy +from datetime import datetime, timedelta from errno import EADDRINUSE from typing import TYPE_CHECKING +import jwt import mock import pytest import requests import requests.exceptions import six from beaker.cache import cache_managers, cache_regions +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives.serialization import Encoding, PrivateFormat, NoEncryption +from jwt.utils import to_base64url_uint from pyramid.config import Configurator from pyramid.httpexceptions import HTTPException from pyramid.response import Response as PyramidResponse from pyramid.settings import asbool from pyramid.testing import DummyRequest from pyramid.testing import setUp as PyramidSetUp +from pytest_httpserver import HTTPServer from requests.cookies import RequestsCookieJar, create_cookie from requests.models import Response as RequestsResponse from six.moves.urllib.parse import urlparse @@ -191,6 +197,7 @@ def test_func(): All ```` definitions should be added to ``setup.cfg`` to allow :mod:`pytest` to reference them. """ + @functools.wraps(run_option) def wrap(test_func, *_, **__): # type: (Callable, *Any, **Any) -> Callable @@ -213,6 +220,7 @@ class RunOptionDecorator(object): RunOptionDecorator("MAGPIE_TEST_CUSTOM_MARKER") """ + def __new__(cls, name, description=None): # type: (Type[RunOptionDecorator], Str, Optional[Str]) -> RunOptionDecorator return make_run_option_decorator(RunOption(name, description=description)) @@ -752,7 +760,7 @@ def check_network_mode(_test_func=None, enable=True): :param enable: Boolean value indicating whether ``_test_func`` expects network mode to be enabled in order to pass. """ if enable: - settings = {"magpie.network_enabled": True, "magpie.network_instance_name": "node1"} + settings = {"magpie.network_enabled": True} else: settings = {"magpie.network_enabled": False} @@ -2974,6 +2982,254 @@ def create_TestUser(test_case, # type: AnyMagpieTestCaseType accept_terms=accept_terms) return check_response_basic_info(create_user_resp, 201, expected_method="POST") + @staticmethod + def create_TestNetworkNode(test_case, # type: AnyMagpieTestCaseType + override_name=null, # type: Optional[Str] + override_jwks_url=null, # type: Optional[Str] + override_token_url=null, # type: Optional[Str] + override_authorization_url=null, # type: Optional[Str] + override_redirect_uris=null, # type: Optional[JSON] + override_headers=null, # type: Optional[HeadersType] + override_cookies=null, # type: Optional[CookiesType] + override_exist=False, # type: bool + override_data=null # type: Optional[JSON] + ): # type: (...) -> JSON + """ + Creates a Network Node + """ + app_or_url = get_app_or_url(test_case) + if override_data is not null: + data = override_data + else: + data = { + "name": override_name if override_name is not null else test_case.test_node_name, + "jwks_url": override_jwks_url if override_jwks_url is not null else test_case.test_node_jwks_url, + "token_url": override_token_url if override_token_url is not null else test_case.test_node_token_url, + "authorization_url": (override_authorization_url if override_authorization_url is not null + else test_case.test_authorization_url), + "redirect_uris": (override_redirect_uris if override_redirect_uris is not null + else test_case.test_redirect_uris) + } + headers = override_headers if override_headers is not null else test_case.json_headers + cookies = override_cookies if override_cookies is not null else test_case.cookies + + resp = test_request(app_or_url, "POST", "/network/nodes", json=data, expect_errors=override_exist, + headers=headers, cookies=cookies) + + if data.get("name"): + test_case.extra_node_names.add(data["name"]) # indicate potential removal at a later point + + if resp.status_code == 409 and override_exist and data.get("name"): + TestSetup.delete_TestNetworkNode(test_case, + override_name=data["name"], + override_headers=headers, + override_cookies=cookies) + return TestSetup.create_TestNetworkNode(test_case, + override_data=data, + override_headers=headers, + override_cookies=cookies, + override_exist=False) + + return check_response_basic_info(resp, 201, expected_method="POST") + + @staticmethod + def delete_TestNetworkNode(test_case, # type: AnyMagpieTestCaseType + override_name=null, # type: Optional[Str] + override_headers=null, # type: Optional[HeadersType] + override_cookies=null, # type: Optional[CookiesType] + ): # type: (...) -> None + """ + Deletes a Network Node. + """ + app_or_url = get_app_or_url(test_case) + headers = override_headers if override_headers is not null else test_case.json_headers + cookies = override_cookies if override_cookies is not null else test_case.cookies + name = override_name if override_name is not null else test_case.test_user_name + path = "/network/nodes/{}".format(name) + resp = test_request(app_or_url, "DELETE", path, headers=headers, cookies=cookies) + check_response_basic_info(resp, 200, expected_method="DELETE") + + @staticmethod + def create_TestNetworkRemoteUser(test_case, # type: AnyMagpieTestCaseType + override_remote_user_name=null, # type: Optional[Str] + override_user_name=null, # type: Optional[Str] + override_node_name=null, # type: Optional[Str] + override_node_host=null, # type: Optional[Str] + override_node_port=null, # type: Optional[int] + override_headers=null, # type: Optional[HeadersType] + override_cookies=null, # type: Optional[CookiesType] + override_exist=False, # type: bool + ): # type: (...) -> JSON + """ + Creates a Network Remote User. + """ + app_or_url = get_app_or_url(test_case) + data = { + "remote_user_name": (override_remote_user_name if override_remote_user_name is not null + else test_case.test_remote_user_name), + "user_name": override_user_name if override_user_name is not null else test_case.test_user_name, + "node_name": override_node_name if override_node_name is not null else test_case.test_node_name + } + headers = override_headers if override_headers is not null else test_case.json_headers + cookies = override_cookies if override_cookies is not null else test_case.cookies + + resp = test_request(app_or_url, "POST", "/network/remote_users", json=data, + expect_errors=override_exist, headers=headers, cookies=cookies) + + if data.get("remote_user_name") and data.get("node_name"): + # indicate potential removal at a later point + test_case.extra_remote_user_names.add((data["remote_user_name"], data["node_name"])) + + if resp.status_code == 409 and override_exist and data.get("remote_user_name") and data.get("node_name"): + TestSetup.delete_TestNetworkRemoteUser(test_case, + override_remote_user_name=data["remote_user_name"], + override_node_name=data["node_name"], + override_headers=headers, + override_cookies=cookies) + return TestSetup.create_TestNetworkRemoteUser(test_case, + override_remote_user_name=data["remote_user_name"], + override_node_name=data["node_name"], + override_user_name=override_user_name, + override_node_host=override_node_host, + override_node_port=override_node_port, + override_headers=headers, + override_cookies=cookies, + override_exist=False) + + return check_response_basic_info(resp, 201, expected_method="POST") + + @staticmethod + def delete_TestNetworkRemoteUser(test_case, # type: AnyMagpieTestCaseType + override_remote_user_name=null, # type: Optional[Str] + override_node_name=null, # type: Optional[Str] + override_headers=null, # type: Optional[HeadersType] + override_cookies=null, # type: Optional[CookiesType] + ): # type: (...) -> None + """ + Deletes a Network Remote User. + """ + app_or_url = get_app_or_url(test_case) + headers = override_headers if override_headers is not null else test_case.json_headers + cookies = override_cookies if override_cookies is not null else test_case.cookies + name = override_remote_user_name if override_remote_user_name is not null else test_case.test_user_name + node_name = override_node_name if override_node_name is not null else test_case.test_node_name + + path = "/network/nodes/{}/remote_users/{}".format(node_name, name) + resp = test_request(app_or_url, "DELETE", path, headers=headers, cookies=cookies) + check_response_basic_info(resp, 200, expected_method="DELETE") + + @staticmethod + @contextlib.contextmanager + def remote_node(test_case, override_node_host=null, override_node_port=null): + # type: (AnyMagpieTestCaseType, Optional[Str], Optional[int]) -> Any + """ + Starts a :class:`pytest_httpserver.HTTPServer` instance which can be used to generate fake responses + from a fake network node. + """ + node_host = override_node_host if override_node_host is not null else test_case.test_node_host + node_port = override_node_port if override_node_port is not null else test_case.test_node_port + server = HTTPServer(host=node_host, port=node_port) + server.start() + try: + yield server + finally: + server.clear() + if server.is_running(): + server.stop() + + @staticmethod + @contextlib.contextmanager + def valid_jwt(test_case, # type: AnyMagpieTestCaseType + override_jwt_claims=null, # type: Optional[Dict[Str, Str]] + override_jwt_headers=null, # type: Optional[Dict[Str, Str]] + override_jwt_issuer=null, # type: Optional[Str] + override_jwks_url=null, # type: Optional[Str] + override_node_host=null, # type: Optional[Str] + override_node_port=null # type: Optional[int] + ): # type: (...) -> Any + jwks_url = override_jwks_url if override_jwks_url is not null else test_case.test_node_jwks_url + jwks_request_path = urlparse(jwks_url).path + + private_key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + public_key = private_key.public_key() + private_bytes = private_key.private_bytes(Encoding.PEM, PrivateFormat.TraditionalOpenSSL, NoEncryption()) + public_numbers = public_key.public_numbers() + kid = str(uuid.uuid4()) + jwk = { + "kid": kid, + "alg": "RS256", + "kty": "RSA", + "use": "sig", + "n": to_base64url_uint(public_numbers.n).decode("ascii"), + "e": to_base64url_uint(public_numbers.e).decode("ascii") + } + jwt_headers = {} if override_jwt_headers is null else override_jwt_headers + issuer = test_case.test_node_name if override_jwt_issuer is null else override_jwt_issuer + jwt_claims = {} if override_jwt_claims is null else override_jwt_claims + expiry = int(get_constant("MAGPIE_NETWORK_INTERNAL_TOKEN_EXPIRY")) + expiry_time = datetime.utcnow() + timedelta(seconds=expiry) + audience = get_constant("MAGPIE_NETWORK_INSTANCE_NAME") + claims = {"iss": issuer, "aud": audience, "exp": expiry_time, **jwt_claims} + token = jwt.encode(claims, private_bytes, headers={"kid": jwk["kid"], **jwt_headers}, algorithm="RS256") + with TestSetup.remote_node(test_case, override_node_host=override_node_host, + override_node_port=override_node_port) as node_server: + node_server.expect_request(jwks_request_path, method="GET").respond_with_json({"keys": [jwk]}) + yield token + + @staticmethod + def create_TestNetworkToken(test_case, # type: AnyMagpieTestCaseType + override_jwks_url=null, # type: Optional[Str] + override_node_host=null, # type: Optional[Str] + override_node_port=null, # type: Optional[int] + override_node_name=null, # type: Optional[Str] + override_remote_user_name=null, # type: Optional[Str] + override_headers=null, # type: Optional[HeadersType] + override_cookies=null, # type: Optional[CookiesType] + ): # type: (...) -> JSON + app_or_url = get_app_or_url(test_case) + + node_name = override_node_name if override_node_name is not null else test_case.test_node_name + headers = override_headers if override_headers is not null else test_case.json_headers + cookies = override_cookies if override_cookies is not null else test_case.cookies + if override_remote_user_name is null: + remote_user_name = test_case.test_remote_user_name + else: + remote_user_name = override_remote_user_name + with TestSetup.valid_jwt(test_case, override_jwt_claims={"user_name": remote_user_name}, + override_jwt_issuer=node_name, override_jwks_url=override_jwks_url, + override_node_host=override_node_host, override_node_port=override_node_port) as token: + resp = test_request(app_or_url, "POST", "/network/token", json={"token": token}, headers=headers, + cookies=cookies) + json_body = check_response_basic_info(resp, 201, expected_method="POST") + test_case.extra_network_tokens.add((remote_user_name, node_name)) + return json_body + + @staticmethod + def delete_TestNetworkToken(test_case, # type: AnyMagpieTestCaseType + override_jwks_url=null, # type: Optional[Str] + override_node_host=null, # type: Optional[Str] + override_node_port=null, # type: Optional[int] + override_node_name=null, # type: Optional[Str] + override_remote_user_name=null, # type: Optional[Str] + override_headers=null, # type: Optional[HeadersType] + override_cookies=null, # type: Optional[CookiesType] + ): # type: (...) -> None + app_or_url = get_app_or_url(test_case) + + node_name = override_node_name if override_node_name is not null else test_case.test_node_name + headers = override_headers if override_headers is not null else test_case.json_headers + cookies = override_cookies if override_cookies is not null else test_case.cookies + if override_remote_user_name is null: + remote_user_name = test_case.test_remote_user_name + else: + remote_user_name = override_remote_user_name + with TestSetup.valid_jwt(test_case, override_jwt_claims={"user_name": remote_user_name}, + override_jwt_issuer=node_name, override_jwks_url=override_jwks_url, + override_node_host=override_node_host, override_node_port=override_node_port) as token: + resp = test_request(app_or_url, "DELETE", "/network/token", json={"token": token}, headers=headers, + cookies=cookies) + check_response_basic_info(resp, 200, expected_method="DELETE") + @staticmethod def delete_TestUser(test_case, # type: AnyMagpieTestCaseType override_user_name=null, # type: Optional[Str] From f5de4625179c8579a1046eb98abb56afa2e8bd17 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Fri, 5 Apr 2024 16:05:18 -0400 Subject: [PATCH 27/55] some more tests and bugfixes --- ...3-08-25_2cfe144538e8_add_network_tables.py | 2 +- magpie/api/login/login.py | 32 +- .../api/management/network/network_views.py | 6 +- magpie/models.py | 12 +- tests/interfaces.py | 505 +++++++++++++++++- tests/utils.py | 51 +- 6 files changed, 556 insertions(+), 52 deletions(-) diff --git a/magpie/alembic/versions/2023-08-25_2cfe144538e8_add_network_tables.py b/magpie/alembic/versions/2023-08-25_2cfe144538e8_add_network_tables.py index 395d361c9..1d536b795 100644 --- a/magpie/alembic/versions/2023-08-25_2cfe144538e8_add_network_tables.py +++ b/magpie/alembic/versions/2023-08-25_2cfe144538e8_add_network_tables.py @@ -45,7 +45,7 @@ def upgrade(): nullable=False), sa.Column("name", sa.Unicode(128)), sa.Column("network_token_id", sa.Integer, - sa.ForeignKey("network_tokens.id", onupdate="CASCADE", ondelete="CASCADE"), unique=True) + sa.ForeignKey("network_tokens.id", onupdate="CASCADE", ondelete="SET NULL"), unique=True) ) op.create_unique_constraint("uq_network_remote_users_user_id_network_node_id", "network_remote_users", ["user_id", "network_node_id"]) diff --git a/magpie/api/login/login.py b/magpie/api/login/login.py index 2a86aaf6a..e81829592 100644 --- a/magpie/api/login/login.py +++ b/magpie/api/login/login.py @@ -34,7 +34,7 @@ from magpie.api import schemas as s from magpie.api.management.user.user_formats import format_user from magpie.api.management.user.user_utils import create_user -from magpie.constants import get_constant, protected_user_email_regex, protected_user_name_regex +from magpie.constants import get_constant, protected_user_email_regex, protected_user_name_regex, network_enabled from magpie.security import authomatic_setup, get_providers from magpie.utils import ( CONTENT_TYPE_JSON, @@ -46,7 +46,7 @@ ) if TYPE_CHECKING: - from magpie.typedefs import Session, Str + from magpie.typedefs import AnySettingsContainer, Session, Str LOGGER = get_logger(__name__) @@ -77,14 +77,17 @@ def process_sign_in_external(request, username, provider): return HTTPTemporaryRedirect(location=external_login_route, headers=request.response.headers) -def verify_provider(provider_name): - # type: (Str) -> None +def verify_provider(provider_name, settings_container=None): + # type: (Str, AnySettingsContainer) -> None """ Verifies that the specified name is a valid external provider against the login configuration providers. :raises HTTPNotFound: if provider name is not one of known providers. """ - ax.verify_param(provider_name, param_name="provider_name", param_compare=list(MAGPIE_PROVIDER_KEYS), is_in=True, + provider_keys = list(MAGPIE_PROVIDER_KEYS) + if network_enabled(settings_container): + provider_keys.append(get_constant("MAGPIE_NETWORK_PROVIDER", settings_container)) + ax.verify_param(provider_name, param_name="provider_name", param_compare=provider_keys, is_in=True, http_error=HTTPNotFound, msg_on_fail=s.ProviderSignin_GET_NotFoundResponseSchema.description) @@ -138,7 +141,7 @@ def sign_in_view(request): http_error=HTTPForbidden, msg_on_fail=s.Signin_POST_ForbiddenResponseSchema.description) except HTTPForbidden as http_error: return http_error - verify_provider(provider_name) + verify_provider(provider_name, request) if provider_name in MAGPIE_INTERNAL_PROVIDERS: # password can be None for external login, validate only here as it is required for internal login @@ -275,17 +278,22 @@ def network_login(request): Sign in a user authenticating using a `Magpie` network access token. """ if "Authorization" in request.headers: - token_type, token = request.headers.get("Authorization").split() + token_list = request.headers.get("Authorization").split(maxsplit=1) + if len(token_list) != 2: + return ax.raise_http(http_error=HTTPUnauthorized, + detail=s.Signin_POST_UnauthorizedResponseSchema.description, nothrow=True) + token_type, token = token_list if token_type != "Bearer": # nosec: B105 - ax.raise_http(http_error=HTTPBadRequest, - detail=s.ProviderSignin_GET_BadRequestResponseSchema.description) - network_token = models.NetworkToken.by_token(token) + return ax.raise_http(http_error=HTTPBadRequest, + detail=s.ProviderSignin_GET_BadRequestResponseSchema.description, nothrow=True) + network_token = models.NetworkToken.by_token(token, db_session=request.db) if network_token is None or network_token.expired(): - return login_failure_view(request, reason=s.Signin_POST_UnauthorizedResponseSchema.description) + return ax.raise_http(http_error=HTTPUnauthorized, + detail=s.Signin_POST_UnauthorizedResponseSchema.description, nothrow=True) authenticated_user = network_token.network_remote_user.user # We should never create a token for protected users but just in case anonymous_regex = protected_user_name_regex(include_admin=False, settings_container=request) - ax.verify_param(authenticated_user.name, not_matches=True, param_compare=anonymous_regex, + ax.verify_param(authenticated_user.user_name, not_matches=True, param_compare=anonymous_regex, http_error=HTTPForbidden, msg_on_fail=s.Signin_POST_ForbiddenResponseSchema.description) return login_success_external(request, authenticated_user) ax.raise_http(http_error=HTTPBadRequest, diff --git a/magpie/api/management/network/network_views.py b/magpie/api/management/network/network_views.py index 5477a9f9c..50fb66d48 100644 --- a/magpie/api/management/network/network_views.py +++ b/magpie/api/management/network/network_views.py @@ -37,10 +37,10 @@ def post_network_token_view(request): decorator=check_network_mode_enabled, permission=NO_PERMISSION_REQUIRED) def delete_network_token_view(request): node, network_remote_user = get_network_models_from_request_token(request) - if network_remote_user.network_token: + if network_remote_user and network_remote_user.network_token: request.db.delete(network_remote_user.network_token) if (network_remote_user.user.id == node.anonymous_user(request.db).id and - sqlalchemy.inspect(network_remote_user).persisted): + sqlalchemy.inspect(network_remote_user).persistent): request.db.delete(network_remote_user) # clean up unused record in the database return ax.valid_http(http_success=HTTPOk, detail=s.NetworkToken_DELETE_OkResponseSchema.description) ax.raise_http(http_error=HTTPNotFound, detail=s.NetworkNodeToken_DELETE_NotFoundResponseSchema.description) @@ -54,7 +54,7 @@ def delete_network_tokens_view(request): deleted = models.NetworkToken.delete_expired(request.db) else: deleted = request.db.query(NetworkToken).delete() - anonymous_network_user_ids = [n.anonymous_user().id for n in request.db.query(models.NetworkNode).all()] + anonymous_network_user_ids = [n.anonymous_user(request.db).id for n in request.db.query(models.NetworkNode).all()] # clean up unused records in the database (no need to keep records associated with anonymous network users) (request.db.query(models.NetworkRemoteUser) .filter(models.NetworkRemoteUser.user_id.in_(anonymous_network_user_ids)) diff --git a/magpie/models.py b/magpie/models.py index b0ba4b3c0..b62c61fba 100644 --- a/magpie/models.py +++ b/magpie/models.py @@ -1015,7 +1015,7 @@ class NetworkRemoteUser(BaseModel, Base): sa.ForeignKey("network_nodes.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=False) network_token_id = sa.Column(sa.Integer, - sa.ForeignKey("network_tokens.id", onupdate="CASCADE", ondelete="CASCADE"), + sa.ForeignKey("network_tokens.id", onupdate="CASCADE", ondelete="SET NULL"), unique=True) name = sa.Column(sa.Unicode(128)) network_node = relationship("NetworkNode", foreign_keys=[network_node_id]) @@ -1064,8 +1064,8 @@ def refresh_token(self): def expired(self): # type: () -> bool - expiry = int(get_constant("MAGPIE_NETWORK_DEFAULT_TOKEN_EXPIRY")) - return (datetime.datetime.utcnow() - self.created) > expiry + expire = int(get_constant("MAGPIE_NETWORK_DEFAULT_TOKEN_EXPIRY")) + return (datetime.datetime.utcnow() - self.created) > datetime.timedelta(seconds=expire) @classmethod def delete_expired(cls, db_session): @@ -1078,7 +1078,11 @@ def delete_expired(cls, db_session): def by_token(cls, token, db_session=None): # type: (Str, Optional[Session]) -> Optional[NetworkToken] db_session = get_db_session(db_session) - return db_session.query(cls).filter(cls.token == cls._hash_token(token)).first() + try: + hashed_token = cls._hash_token(token) + except ValueError: + return + return db_session.query(cls).filter(cls.token == hashed_token).first() id = sa.Column(sa.Integer(), primary_key=True, nullable=False, autoincrement=True) token = sa.Column(sa.String, nullable=False, unique=True) diff --git a/tests/interfaces.py b/tests/interfaces.py index 1670ece0f..15301a3f7 100644 --- a/tests/interfaces.py +++ b/tests/interfaces.py @@ -61,6 +61,7 @@ from sqlalchemy.orm.session import Session from magpie.typedefs import JSON, CookiesType, HeadersType, PermissionDict, SettingsType, Str + from pytest_httpserver import HTTPServer @six.add_metaclass(ABCMeta) @@ -119,6 +120,7 @@ class ConfigTestCase(object): extra_node_names = set() # type: Set[Str] extra_remote_user_names = set() # type: Set[Tuple[Str, Str]] extra_network_tokens = set() # type: Set[Tuple[Str, Str]] + extra_remote_servers = {} # type: Dict[Tuple[Str, int], HTTPServer] @six.add_metaclass(ABCMeta) @@ -183,18 +185,6 @@ def cleanup(cls): cls.extra_user_names.add(cls.test_admin) cls.extra_user_names.add(cls.test_user_name) cls.extra_group_names.add(cls.test_group_name) - for remote_user_name, node_name in list(cls.extra_network_tokens): - check_network_mode(utils.TestSetup.delete_TestNetworkToken, - enable=True)(cls, - override_remote_user_name=remote_user_name, - override_node_name=node_name) - - cls.extra_network_tokens.discard((remote_user_name, node_name)) - for remote_user_name, node_name in list(cls.extra_remote_user_names): - check_network_mode(utils.TestSetup.delete_TestNetworkRemoteUser, - enable=True)(cls, override_remote_user_name=remote_user_name, - override_node_name=node_name) - cls.extra_remote_user_names.discard((remote_user_name, node_name)) for usr in list(cls.extra_user_names): # copy to update removed ones if usr not in cls.reserved_users: utils.TestSetup.delete_TestUser(cls, override_user_name=usr) @@ -215,6 +205,23 @@ def cleanup(cls): check_network_mode(utils.TestSetup.delete_TestNetworkNode, enable=True)(cls, override_name=node) cls.extra_node_names.discard(node) + for remote_user_name, node_name in list(cls.extra_network_tokens): + check_network_mode(utils.TestSetup.delete_TestNetworkToken, + enable=True)(cls, + override_remote_user_name=remote_user_name, + override_node_name=node_name, + allow_missing=True) # should already be deleted with associated models + cls.extra_network_tokens.discard((remote_user_name, node_name)) + for remote_user_name, node_name in list(cls.extra_remote_user_names): + check_network_mode(utils.TestSetup.delete_TestNetworkRemoteUser, + enable=True)(cls, override_remote_user_name=remote_user_name, + override_node_name=node_name, + allow_missing=True) # should already be deleted with associated models + cls.extra_remote_user_names.discard((remote_user_name, node_name)) + for host_port, server in list(cls.extra_remote_servers.items()): + if server.is_running(): + server.stop() + cls.extra_remote_servers.pop(host_port) @property def update_method(self): @@ -633,6 +640,210 @@ def test_LoginProtectedEmail_Allowed(self): resp = utils.test_request(self, "POST", s.SigninAPI.path, json=data, expect_errors=True) utils.check_response_basic_info(resp, 200, expected_method="POST") + @runner.MAGPIE_TEST_NETWORK + @runner.MAGPIE_TEST_LOGIN + @utils.check_network_mode + def test_Login_NetworkToken_Authorized(self): + """ + Test logging in with a network token. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Login with a network token", "3.38.0", skip=True) + self.login_admin() + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True) + utils.check_or_try_logout_user(self) + self.cookies = self.headers = None + token = utils.TestSetup.create_TestNetworkToken(self) + + headers = {"Accept": CONTENT_TYPE_JSON, "Authorization": "Bearer {}".format(token["token"])} + path = s.ProviderSigninAPI.path.format(provider_name=get_constant("MAGPIE_NETWORK_PROVIDER")) + + resp = utils.test_request(self, "GET", path, headers=headers, expect_errors=True, allow_redirects=False) + utils.check_response_basic_info(resp, 302, expected_method="GET") + + app_or_url = utils.get_app_or_url(self) + if isinstance(app_or_url, TestApp): + resp_cookies = app_or_url.cookies + else: + resp_cookies = resp.cookies + resp = utils.test_request(self, "GET", "/session", headers=self.json_headers, cookies=resp_cookies) + session_json = utils.get_json_body(resp) + assert session_json.get("authenticated") + assert session_json.get("user", {}).get("user_name") == self.test_user_name + + @runner.MAGPIE_TEST_NETWORK + @runner.MAGPIE_TEST_LOGIN + @utils.check_network_mode + def test_Login_NetworkToken_Authorized_Refreshed(self): + """ + Test logging in with a refreshed network token. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Login with a network token", "3.38.0", skip=True) + self.login_admin() + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True) + utils.check_or_try_logout_user(self) + self.cookies = self.headers = None + initial_token = utils.TestSetup.create_TestNetworkToken(self) + token = utils.TestSetup.create_TestNetworkToken(self) + + assert token != initial_token + + headers = {"Accept": CONTENT_TYPE_JSON, "Authorization": "Bearer {}".format(token["token"])} + path = s.ProviderSigninAPI.path.format(provider_name=get_constant("MAGPIE_NETWORK_PROVIDER")) + + resp = utils.test_request(self, "GET", path, headers=headers, expect_errors=True, allow_redirects=False) + utils.check_response_basic_info(resp, 302, expected_method="GET") + + app_or_url = utils.get_app_or_url(self) + if isinstance(app_or_url, TestApp): + resp_cookies = app_or_url.cookies + else: + resp_cookies = resp.cookies + resp = utils.test_request(self, "GET", "/session", headers=self.json_headers, cookies=resp_cookies) + session_json = utils.get_json_body(resp) + assert session_json.get("authenticated") + assert session_json.get("user", {}).get("user_name") == self.test_user_name + + @runner.MAGPIE_TEST_NETWORK + @runner.MAGPIE_TEST_LOGIN + @utils.check_network_mode + def test_Login_NetworkToken_Unauthorized_BadFormat(self): + """ + Test logging in with an incorrectly formatted network token. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Login with a network token", "3.38.0", skip=True) + self.login_admin() + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True) + utils.check_or_try_logout_user(self) + self.cookies = self.headers = None + + headers = {"Accept": CONTENT_TYPE_JSON, "Authorization": "Bearer {}".format("abc123")} + path = s.ProviderSigninAPI.path.format(provider_name=get_constant("MAGPIE_NETWORK_PROVIDER")) + + resp = utils.test_request(self, "GET", path, headers=headers, expect_errors=True, allow_redirects=False) + utils.check_response_basic_info(resp, 401, expected_method="GET") + + app_or_url = utils.get_app_or_url(self) + if isinstance(app_or_url, TestApp): + resp_cookies = app_or_url.cookies + else: + resp_cookies = resp.cookies + resp = utils.test_request(self, "GET", "/session", headers=self.json_headers, cookies=resp_cookies) + session_json = utils.get_json_body(resp) + assert session_json.get("authenticated") is False + assert session_json.get("user", {}).get("user_name") is None + + @runner.MAGPIE_TEST_NETWORK + @runner.MAGPIE_TEST_LOGIN + @utils.check_network_mode + def test_Login_NetworkToken_Unauthorized_Deleted(self): + """ + Test logging in with a deleted network token. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Login with a network token", "3.38.0", skip=True) + self.login_admin() + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True) + utils.check_or_try_logout_user(self) + self.cookies = self.headers = None + token = utils.TestSetup.create_TestNetworkToken(self) + utils.TestSetup.delete_TestNetworkToken(self) + + headers = {"Accept": CONTENT_TYPE_JSON, "Authorization": "Bearer {}".format(token["token"])} + path = s.ProviderSigninAPI.path.format(provider_name=get_constant("MAGPIE_NETWORK_PROVIDER")) + + resp = utils.test_request(self, "GET", path, headers=headers, expect_errors=True, allow_redirects=False) + utils.check_response_basic_info(resp, 401, expected_method="GET") + + app_or_url = utils.get_app_or_url(self) + if isinstance(app_or_url, TestApp): + resp_cookies = app_or_url.cookies + else: + resp_cookies = resp.cookies + resp = utils.test_request(self, "GET", "/session", headers=self.json_headers, cookies=resp_cookies) + session_json = utils.get_json_body(resp) + assert session_json.get("authenticated") is False + assert session_json.get("user", {}).get("user_name") is None + + @runner.MAGPIE_TEST_NETWORK + @runner.MAGPIE_TEST_LOGIN + @utils.check_network_mode(enable=False) + def test_Login_NetworkToken_Unauthorized_NetworkNotEnabled(self): + """ + Test logging in with a network token when network mode is not enabled. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Login with a network token", "3.38.0", skip=True) + + headers = {"Accept": CONTENT_TYPE_JSON, "Authorization": "Bearer {}".format(str(uuid.uuid4()))} + path = s.ProviderSigninAPI.path.format(provider_name=get_constant("MAGPIE_NETWORK_PROVIDER")) + + resp = utils.test_request(self, "GET", path, headers=headers, expect_errors=True, allow_redirects=False) + utils.check_response_basic_info(resp, 404, expected_method="GET") # provider is not found + + app_or_url = utils.get_app_or_url(self) + if isinstance(app_or_url, TestApp): + resp_cookies = app_or_url.cookies + else: + resp_cookies = resp.cookies + resp = utils.test_request(self, "GET", "/session", headers=self.json_headers, cookies=resp_cookies) + session_json = utils.get_json_body(resp) + assert session_json.get("authenticated") is False + assert session_json.get("user", {}).get("user_name") is None + + @runner.MAGPIE_TEST_NETWORK + @runner.MAGPIE_TEST_LOGIN + @utils.check_network_mode + def test_Login_NetworkToken_Unauthorized_BadToken(self): + """ + Test logging in with a correctly formatted but invalid network token. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Login with a network token", "3.38.0", skip=True) + self.login_admin() + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True) + utils.check_or_try_logout_user(self) + self.cookies = self.headers = None + + headers = {"Accept": CONTENT_TYPE_JSON, "Authorization": "Bearer {}".format(str(uuid.uuid4()))} + path = s.ProviderSigninAPI.path.format(provider_name=get_constant("MAGPIE_NETWORK_PROVIDER")) + + resp = utils.test_request(self, "GET", path, headers=headers, expect_errors=True, allow_redirects=False) + utils.check_response_basic_info(resp, 401, expected_method="GET") + + app_or_url = utils.get_app_or_url(self) + if isinstance(app_or_url, TestApp): + resp_cookies = app_or_url.cookies + else: + resp_cookies = resp.cookies + resp = utils.test_request(self, "GET", "/session", headers=self.json_headers, cookies=resp_cookies) + session_json = utils.get_json_body(resp) + assert session_json.get("authenticated") is False + assert session_json.get("user", {}).get("user_name") is None + @runner.MAGPIE_TEST_LOGIN def test_Login_GetRequestFormat(self): """ @@ -987,6 +1198,194 @@ def test_DeleteUser_OtherUser_Unauthorized(self): self.headers, self.cookies = utils.check_or_try_login_user(self, username=self.usr, password=self.pwd) utils.TestSetup.get_UserInfo(self, override_username=self.test_user_name) # if found, no deletion was applied + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_PostNetworkToken(self): + """ + Test get a network token. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Acquire a network token", "3.38.0", skip=True) + self.login_admin() + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True) + utils.check_or_try_logout_user(self) + self.cookies = self.headers = None + token = utils.TestSetup.create_TestNetworkToken(self) + assert token.get("token") + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_PostNetworkToken_Create_AnonymousUser(self): + """ + Test creating network token for an anonymous user and check that the anonymous remote user is created as well. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Delete a network token", "3.38.0", skip=True) + self.login_admin() + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + utils.check_or_try_logout_user(self) + self.cookies = self.headers = None + utils.TestSetup.create_TestNetworkToken(self) + + self.login_admin() + resp = utils.test_request(self, "GET", "/network/nodes/{}/remote_users/{}".format(self.test_node_name, + self.test_remote_user_name), + headers=self.headers, cookies=type(self).cookies) + json_body = utils.get_json_body(resp) + anon_user_name = "{}{}".format(get_constant("MAGPIE_NETWORK_NAME_PREFIX"), self.test_node_name) + assert json_body.get("user_name") == anon_user_name + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_PostNetworkToken_InvalidNode(self): + """ + Test do not get a network token if the associated node doesn't exist. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Acquire a network token", "3.38.0", skip=True) + self.login_admin() + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True) + utils.check_or_try_logout_user(self) + self.cookies = self.headers = None + token = utils.TestSetup.create_TestNetworkToken(self, override_node_name="some_other_node", expect_errors=True) + assert token.get("token") is None + assert token.get("code") == 404 + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_PostNetworkToken_InvalidNodeCredentials(self): + """ + Test do not get a network token if the associated node exists but has invalid/missing + JSON web keys. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Acquire a network token", "3.38.0", skip=True) + self.login_admin() + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True, override_jwks_url="http://example.com/jwks") + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True) + utils.check_or_try_logout_user(self) + self.cookies = self.headers = None + token = utils.TestSetup.create_TestNetworkToken(self, expect_errors=True) + assert token.get("token") is None + assert token.get("code") == 500 + assert token.get("call", {}).get("exception") == "PyJWKClientConnectionError" + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_DeleteNetworkToken(self): + """ + Test delete a network token. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Delete a network token", "3.38.0", skip=True) + self.login_admin() + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True) + utils.check_or_try_logout_user(self) + self.cookies = self.headers = None + utils.TestSetup.create_TestNetworkToken(self) + utils.TestSetup.delete_TestNetworkToken(self) + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_DeleteNetworkToken_NotRemoteUser(self): + """ + Test delete a network token but not the associated NetworkRemoteUser. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Delete a network token", "3.38.0", skip=True) + self.login_admin() + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True) + utils.check_or_try_logout_user(self) + utils.TestSetup.create_TestNetworkToken(self) + utils.TestSetup.delete_TestNetworkToken(self) + + resp = utils.test_request(self, "GET", "/network/remote_users", cookies=self.cookies, headers=self.headers) + json_body = utils.get_json_body(resp) + assert json_body.get("remote_users", [{}])[0].get("remote_user_name") == self.test_remote_user_name + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_DeleteNetworkToken_And_AnonymousUser(self): + """ + Test delete a network token for an anonymous user and check that the anonymous remote user is deleted as well. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Delete a network token", "3.38.0", skip=True) + self.login_admin() + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + utils.TestSetup.create_TestNetworkToken(self, override_cookies={}, override_headers={}) + utils.TestSetup.delete_TestNetworkToken(self, override_cookies={}, override_headers={}) + + resp = utils.test_request(self, "GET", "/network/remote_users", cookies=self.cookies, headers=self.headers) + json_body = utils.get_json_body(resp) + assert not json_body["remote_users"] + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_DeleteNetworkToken_InvalidNode(self): + """ + Test do not delete a network token if the node doesn't match. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Delete a network token", "3.38.0", skip=True) + self.login_admin() + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True) + utils.check_or_try_logout_user(self) + self.cookies = self.headers = None + utils.TestSetup.create_TestNetworkToken(self) + resp = utils.TestSetup.delete_TestNetworkToken(self, override_node_name="some_other_node", allow_missing=True) + utils.check_response_basic_info(resp, expected_code=404, expected_method="DELETE") + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_DeleteNetworkToken_InvalidRemoteUser(self): + """ + Test do not delete a network token if the node doesn't match. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Delete a network token", "3.38.0", skip=True) + self.login_admin() + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True) + utils.check_or_try_logout_user(self) + self.cookies = self.headers = None + utils.TestSetup.create_TestNetworkToken(self) + resp = utils.TestSetup.delete_TestNetworkToken(self, override_remote_user_name="some_other_user", + allow_missing=True) + utils.check_response_basic_info(resp, expected_code=404, expected_method="DELETE") + @runner.MAGPIE_TEST_API @six.add_metaclass(ABCMeta) @@ -1948,6 +2347,17 @@ def setup_test_values(cls): cls.test_group_name = "magpie-unittest-dummy-group" cls.test_user_name = "magpie-unittest-toto" + cls.test_node_name = "node2" + cls.test_remote_user_name = "remote_user_1" + cls.test_node_host = get_constant("MAGPIE_TEST_REMOTE_NODE_SERVER_HOST", default_value="localhost", + raise_missing=False, raise_not_set=False) + cls.test_node_port = int(get_constant("MAGPIE_TEST_REMOTE_NODE_SERVER_PORT", default_value=2002, + raise_missing=False, raise_not_set=False)) + cls.test_node_jwks_url = "http://{}:{}/network/jwks".format(cls.test_node_host, cls.test_node_port) + cls.test_node_token_url = "http://{}:{}/network/token".format(cls.test_node_host, cls.test_node_port) + cls.test_authorization_url = "http://{}:{}/ui/network/authorize".format(cls.test_node_host, cls.test_node_port) + cls.test_redirect_uris = '["http://{}:{}/network/link"]'.format(cls.test_node_host, cls.test_node_port) + @runner.MAGPIE_TEST_STATUS def test_unauthorized_forbidden_responses(self): """ @@ -7262,6 +7672,77 @@ def test_GetResourceTypes_ServiceGeoserver(self): utils.check_val_equal(body["root_service_name"], self.test_service_name) utils.check_val_equal(body["root_service_type"], ServiceGeoserver.service_type) + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_DeleteNetworkTokens(self): + """ + Test delete all network tokens. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Delete all network tokens", "3.38.0", skip=True) + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True, override_name="test1") + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True, override_remote_user_name="test1", + override_node_name="test1") + utils.TestSetup.create_TestNetworkNode(self, override_exist=True, override_name="test2") + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True, override_remote_user_name="test2", + override_node_name="test2") + utils.TestSetup.create_TestNetworkToken(self, override_remote_user_name="test1", override_node_name="test1") + utils.TestSetup.create_TestNetworkToken(self, override_remote_user_name="test2", override_node_name="test2") + + resp = utils.test_request(self, "DELETE", "/network/tokens", cookies=self.cookies, headers=self.headers) + utils.check_response_basic_info(resp, 200, expected_method="DELETE") + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_DeleteNetworkTokens_NotRemoteUser(self): + """ + Test deleting network tokens does not delete associated non-anonymous network users. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Delete all network tokens", "3.38.0", skip=True) + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True, override_name="test1") + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True, override_remote_user_name="test1", + override_node_name="test1") + utils.TestSetup.create_TestNetworkNode(self, override_exist=True, override_name="test2") + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True, override_remote_user_name="test2", + override_node_name="test2") + utils.TestSetup.create_TestNetworkToken(self, override_remote_user_name="test1", override_node_name="test1") + utils.TestSetup.create_TestNetworkToken(self, override_remote_user_name="test2", override_node_name="test2") + + utils.test_request(self, "DELETE", "/network/tokens", cookies=self.cookies, headers=self.headers) + + resp = utils.test_request(self, "GET", "/network/remote_users", cookies=self.cookies, headers=self.headers) + json_body = utils.get_json_body(resp) + assert {u.get("remote_user_name") for u in json_body.get("remote_users", [{}])} == {"test1", "test2"} + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_DeleteNetworkTokens_AndAnonymousNetworkUsers(self): + """ + Test deleting network tokens does delete associated anonymous network users. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Delete all network tokens", "3.38.0", skip=True) + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True, override_name="test1") + utils.TestSetup.create_TestNetworkNode(self, override_exist=True, override_name="test2") + utils.TestSetup.create_TestNetworkToken(self, override_remote_user_name="test1", override_node_name="test1") + utils.TestSetup.create_TestNetworkToken(self, override_remote_user_name="test2", override_node_name="test2") + + utils.test_request(self, "DELETE", "/network/tokens", cookies=self.cookies, headers=self.headers) + + resp = utils.test_request(self, "GET", "/network/remote_users", cookies=self.cookies, headers=self.headers) + json_body = utils.get_json_body(resp) + assert not json_body["remote_users"] + @runner.MAGPIE_TEST_UI @six.add_metaclass(ABCMeta) diff --git a/tests/utils.py b/tests/utils.py index a2f441c4e..5feebc3c6 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1201,6 +1201,8 @@ def test_request(test_item, # type: AnyMagpieTestItemType # automatically follow the redirect if any and evaluate its response max_redirect = kwargs.get("max_redirects", 5) + if not allow_redirects: + max_redirect = 0 while 300 <= resp.status_code < 400 and max_redirect > 0: # noqa resp = resp.follow() max_redirect -= 1 @@ -3104,7 +3106,8 @@ def delete_TestNetworkRemoteUser(test_case, # type: AnyMag override_node_name=null, # type: Optional[Str] override_headers=null, # type: Optional[HeadersType] override_cookies=null, # type: Optional[CookiesType] - ): # type: (...) -> None + allow_missing=False, # type: bool + ): # type: (...) -> Optional[AnyResponseType] """ Deletes a Network Remote User. """ @@ -3115,27 +3118,29 @@ def delete_TestNetworkRemoteUser(test_case, # type: AnyMag node_name = override_node_name if override_node_name is not null else test_case.test_node_name path = "/network/nodes/{}/remote_users/{}".format(node_name, name) - resp = test_request(app_or_url, "DELETE", path, headers=headers, cookies=cookies) + resp = test_request(app_or_url, "DELETE", path, headers=headers, cookies=cookies, expect_errors=allow_missing) + if resp.status_code == 404 and allow_missing: + return resp check_response_basic_info(resp, 200, expected_method="DELETE") @staticmethod - @contextlib.contextmanager - def remote_node(test_case, override_node_host=null, override_node_port=null): - # type: (AnyMagpieTestCaseType, Optional[Str], Optional[int]) -> Any + def remote_node(test_case, override_node_host=null, override_node_port=null, clear=True): + # type: (AnyMagpieTestCaseType, Optional[Str], Optional[int], bool) -> Any """ Starts a :class:`pytest_httpserver.HTTPServer` instance which can be used to generate fake responses from a fake network node. """ node_host = override_node_host if override_node_host is not null else test_case.test_node_host node_port = override_node_port if override_node_port is not null else test_case.test_node_port - server = HTTPServer(host=node_host, port=node_port) - server.start() - try: - yield server - finally: + server = test_case.extra_remote_servers.get((node_host, node_port)) + if server is None: + server = HTTPServer(host=node_host, port=node_port) + test_case.extra_remote_servers[(node_host, node_port)] = server + if not server.is_running(): + server.start() + if clear: server.clear() - if server.is_running(): - server.stop() + return server @staticmethod @contextlib.contextmanager @@ -3171,10 +3176,10 @@ def valid_jwt(test_case, # type: AnyMagpieTestCaseType audience = get_constant("MAGPIE_NETWORK_INSTANCE_NAME") claims = {"iss": issuer, "aud": audience, "exp": expiry_time, **jwt_claims} token = jwt.encode(claims, private_bytes, headers={"kid": jwk["kid"], **jwt_headers}, algorithm="RS256") - with TestSetup.remote_node(test_case, override_node_host=override_node_host, - override_node_port=override_node_port) as node_server: - node_server.expect_request(jwks_request_path, method="GET").respond_with_json({"keys": [jwk]}) - yield token + node_server = TestSetup.remote_node(test_case, override_node_host=override_node_host, + override_node_port=override_node_port) + node_server.expect_request(jwks_request_path, method="GET").respond_with_json({"keys": [jwk]}) + yield token @staticmethod def create_TestNetworkToken(test_case, # type: AnyMagpieTestCaseType @@ -3185,6 +3190,7 @@ def create_TestNetworkToken(test_case, # type: AnyMagpieTe override_remote_user_name=null, # type: Optional[Str] override_headers=null, # type: Optional[HeadersType] override_cookies=null, # type: Optional[CookiesType] + expect_errors=False, # type: bool ): # type: (...) -> JSON app_or_url = get_app_or_url(test_case) @@ -3199,9 +3205,11 @@ def create_TestNetworkToken(test_case, # type: AnyMagpieTe override_jwt_issuer=node_name, override_jwks_url=override_jwks_url, override_node_host=override_node_host, override_node_port=override_node_port) as token: resp = test_request(app_or_url, "POST", "/network/token", json={"token": token}, headers=headers, - cookies=cookies) - json_body = check_response_basic_info(resp, 201, expected_method="POST") + cookies=cookies, expect_errors=expect_errors) test_case.extra_network_tokens.add((remote_user_name, node_name)) + if expect_errors: + return get_json_body(resp) + json_body = check_response_basic_info(resp, 201, expected_method="POST") return json_body @staticmethod @@ -3213,7 +3221,8 @@ def delete_TestNetworkToken(test_case, # type: AnyMagpieTe override_remote_user_name=null, # type: Optional[Str] override_headers=null, # type: Optional[HeadersType] override_cookies=null, # type: Optional[CookiesType] - ): # type: (...) -> None + allow_missing=False, # type: bool + ): # type: (...) -> Optional[AnyResponseType] app_or_url = get_app_or_url(test_case) node_name = override_node_name if override_node_name is not null else test_case.test_node_name @@ -3227,7 +3236,9 @@ def delete_TestNetworkToken(test_case, # type: AnyMagpieTe override_jwt_issuer=node_name, override_jwks_url=override_jwks_url, override_node_host=override_node_host, override_node_port=override_node_port) as token: resp = test_request(app_or_url, "DELETE", "/network/token", json={"token": token}, headers=headers, - cookies=cookies) + cookies=cookies, expect_errors=allow_missing) + if resp.status_code == 404 and allow_missing: + return resp check_response_basic_info(resp, 200, expected_method="DELETE") @staticmethod From 72d2bb32785ee88a8f31cf0360c36fb3348a247f Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Tue, 9 Apr 2024 11:23:09 -0400 Subject: [PATCH 28/55] ensure network mode is configured properly at startup --- magpie/api/management/network/__init__.py | 8 +++++++ magpie/utils.py | 26 ++++++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/magpie/api/management/network/__init__.py b/magpie/api/management/network/__init__.py index 2b8ea4d85..2565b9165 100644 --- a/magpie/api/management/network/__init__.py +++ b/magpie/api/management/network/__init__.py @@ -5,7 +5,15 @@ def includeme(config): from magpie.api import schemas as s + from magpie import utils + from pyramid.exceptions import ConfigurationError + LOGGER.info("Adding API network ...") + try: + utils.check_network_configured(config) + except ConfigurationError as exc: + LOGGER.error("API network failed with following configuration error: {}".format(exc)) + raise config.add_route(**s.service_api_route_info(s.NetworkTokenAPI)) config.add_route(**s.service_api_route_info(s.NetworkJSONWebKeySetAPI)) config.add_route(**s.service_api_route_info(s.NetworkDecodeJWTAPI)) diff --git a/magpie/utils.py b/magpie/utils.py index bf36e0fe9..ac31a7ece 100644 --- a/magpie/utils.py +++ b/magpie/utils.py @@ -35,7 +35,7 @@ from zope.interface import implementer from magpie import __meta__ -from magpie.constants import get_constant +from magpie.constants import get_constant, network_enabled if sys.version_info >= (3, 6): from enum import Enum @@ -1176,3 +1176,27 @@ class classproperty(property): # pylint: disable=C0103,invalid-name """ def __get__(self, cls, owner): # noqa return classmethod(self.fget).__get__(None, owner)() + + +def check_network_configured(settings_container=None): + # type: (Optional[AnySettingsContainer]) -> None + """ + Check that the required variables are set and configured properly to support network mode if network mode is + enabled. If not, raises a :class:`ConfigurationError`. + + .. note:: + This should be called when the application starts up to detect a misconfigured application right away. + """ + if network_enabled(settings_container): + instance_name = get_constant("MAGPIE_NETWORK_INSTANCE_NAME", settings_container=settings_container) + if not instance_name: + raise ConfigurationError("MAGPIE_NETWORK_INSTANCE_NAME is required when network mode is enabled.") + # import here to avoid a potential cyclical import + from magpie.api.management.network.network_utils import jwks + try: + jwks() + except Exception as exc: + msg = ("Error occurred when loading PEM keys which are required when network mode is enabled. " + "Check that the MAGPIE_NETWORK_PEM_FILES and MAGPIE_NETWORK_PEM_PASSWORDS are set properly. " + "Original error message: '{}'".format(exc)) + raise ConfigurationError(msg) from exc From 93d177f33a36f6904fd5f38a2f1554068c5c617a Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Wed, 10 Apr 2024 12:00:44 -0400 Subject: [PATCH 29/55] add option to create private key at startup --- .github/workflows/tests.yml | 2 +- docs/configuration.rst | 11 ++++ .../api/management/network/network_utils.py | 63 +++++++++++++++---- .../api/management/network/network_views.py | 8 +-- magpie/constants.py | 3 +- magpie/utils.py | 27 +++++--- 6 files changed, 87 insertions(+), 27 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0c593cca5..ea5e7b1cf 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -73,7 +73,7 @@ jobs: allow-failure: true test-case: start test-remote test-option: >- - PYTEST_ADDOPTS='-m "remote and network"' MAGPIE_NETWORK_ENABLED=on MAGPIE_NETWORK_INSTANCE_NAME=example + PYTEST_ADDOPTS='-m "remote and network"' MAGPIE_NETWORK_CREATE_MISSING_PEM_FILE=True MAGPIE_NETWORK_ENABLED=on MAGPIE_NETWORK_INSTANCE_NAME=example # coverage test - os: ubuntu-latest python-version: "3.11" diff --git a/docs/configuration.rst b/docs/configuration.rst index 8cbd96b6c..c971bfc5b 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1104,6 +1104,17 @@ to authenticate users for each other. All variables defined in this section are file require a password, set this variable to ``["pass1", "" ,"pass2", ""]`` where ``pass1`` and ``pass2`` are the passwords. +.. envvar:: MAGPIE_NETWORK_CREATE_MISSING_PEM_FILE + + [:class:`bool`] + (Default: ``False``) + + .. versionadded:: 3.38 + + If enabled *and* there is a single file specified in :envvar:`MAGPIE_NETWORK_PEM_FILES` *and* that file is missing, + `Magpie` will generate a new private key file when starting up. If a password is specified for that file in + :envvar:`MAGPIE_NETWORK_PEM_PASSWORDS` then the private key file will be encrypted with that password as well. + .. _config_phoenix: Phoenix Settings diff --git a/magpie/api/management/network/network_utils.py b/magpie/api/management/network/network_utils.py index ad4fe5814..7973547c7 100644 --- a/magpie/api/management/network/network_utils.py +++ b/magpie/api/management/network/network_utils.py @@ -1,10 +1,12 @@ import json +import os from datetime import datetime, timedelta from itertools import zip_longest from typing import TYPE_CHECKING import jwt from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa from jwcrypto import jwk from pyramid.httpexceptions import HTTPBadRequest, HTTPInternalServerError, HTTPNotFound @@ -25,18 +27,24 @@ LOGGER = get_logger(__name__) -PEM_FILE_DELIMITER = ":" -PEM_PASSWORD_DELIMITER = ":" # nosec: B105 +def pem_files(settings_container=None): + # type: (Optional[AnySettingsContainer]) -> List[Str] + pem_files_ = get_constant("MAGPIE_NETWORK_PEM_FILES", settings_container=settings_container) + try: + return json.loads(pem_files_) + except json.decoder.JSONDecodeError: + return [pem_files_] -def _pem_file_content(primary=False): - # type: (bool) -> List[bytes] + +def _pem_file_content(primary=False, settings_container=None): + # type: (bool, Optional[AnySettingsContainer]) -> List[bytes] """ Return the content of all PEM files """ - pem_files = get_constant("MAGPIE_NETWORK_PEM_FILES").split(PEM_FILE_DELIMITER) + content = [] - for pem_file in pem_files: + for pem_file in pem_files(settings_container=settings_container): with open(pem_file, "rb") as f: content.append(f.read()) if primary: @@ -44,8 +52,8 @@ def _pem_file_content(primary=False): return content -def _pem_file_passwords(primary=False): - # type: (bool) -> List[Optional[bytes]] +def _pem_file_passwords(primary=False, settings_container=None): + # type: (bool, Optional[AnySettingsContainer]) -> List[Optional[bytes]] """ Return the passwords used to encrypt the PEM files. The passwords will be returned in the same order as the file content from `_pem_file_content`. @@ -55,7 +63,8 @@ def _pem_file_passwords(primary=False): For example: if there are 4 PEM files and the second and fourth are not encrypted, this will return ``["password1", None, "password2"]`` """ - pem_passwords = get_constant("MAGPIE_NETWORK_PEM_PASSWORDS", raise_missing=False, raise_not_set=False) + pem_passwords = get_constant("MAGPIE_NETWORK_PEM_PASSWORDS", settings_container=settings_container, + raise_missing=False, raise_not_set=False) try: passwords = json.loads(pem_passwords or "") except json.decoder.JSONDecodeError: @@ -66,14 +75,44 @@ def _pem_file_passwords(primary=False): return passwords -def jwks(primary=False): - # type: (bool) -> jwk.JWKSet +def create_private_key(filename, password=None, settings_container=None): + # type: (Str, Optional[bytes], Optional[AnySettingsContainer]) -> None + """ + Create a private key file at the specified filename. Encrypt it using the password if specified. + If password is None and the filename matches a file in MAGPIE_NETWORK_PEM_FILES, the associated + password specified in MAGPIE_NETWORK_PEM_PASSWORDS will be used instead. + + .. warning:: + This function should only be used to create a file if MAGPIE_NETWORK_CREATE_MISSING_PEM_FILE is + truthy. This is not enforced in this function. + """ + if password is None: + for pem_file, pem_password in zip_longest(pem_files(settings_container), + _pem_file_passwords(False, settings_container)): + if os.path.realpath(pem_file) == os.path.realpath(filename): + password = pem_password + + LOGGER.info("Creating a valid PEM file at '{}'.".format(filename)) + private_key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + if password: + encryption_algorithm = serialization.BestAvailableEncryption(password) + else: + encryption_algorithm = serialization.NoEncryption() + private_bytes = private_key.private_bytes(serialization.Encoding.PEM, + serialization.PrivateFormat.TraditionalOpenSSL, encryption_algorithm) + with open(filename, mode='wb') as f: + f.write(private_bytes) + + +def jwks(primary=False, settings_container=None): + # type: (bool, Optional[AnySettingsContainer]) -> jwk.JWKSet """ Return a JSON Web Key Set containing all JSON Web Keys loaded from the PEM files listed in ``MAGPIE_NETWORK_PEM_FILES``. """ jwks_ = jwk.JWKSet() - for pem_content, pem_password in zip_longest(_pem_file_content(primary), _pem_file_passwords(primary)): + for pem_content, pem_password in zip_longest(_pem_file_content(primary, settings_container), + _pem_file_passwords(primary, settings_container)): jwks_["keys"].add(jwk.JWK.from_pem(pem_content, password=pem_password)) return jwks_ diff --git a/magpie/api/management/network/network_views.py b/magpie/api/management/network/network_views.py index 50fb66d48..9153be64f 100644 --- a/magpie/api/management/network/network_views.py +++ b/magpie/api/management/network/network_views.py @@ -9,7 +9,7 @@ from magpie.api import exception as ax from magpie.api import schemas as s from magpie.api.management.network.network_utils import decode_jwt, get_network_models_from_request_token, jwks -from magpie.api.requests import check_network_mode_enabled +from magpie.api.requests import check_network_mode_enabled, get_multiformat_body from magpie.models import NetworkNode, NetworkToken @@ -50,7 +50,7 @@ def delete_network_token_view(request): response_schemas=s.NetworkTokens_DELETE_responses) @view_config(route_name=s.NetworkTokensAPI.name, request_method="DELETE", decorator=check_network_mode_enabled) def delete_network_tokens_view(request): - if asbool(request.GET.get("expired_only")): + if asbool(get_multiformat_body(request, "expired_only", default=False)): deleted = models.NetworkToken.delete_expired(request.db) else: deleted = request.db.query(NetworkToken).delete() @@ -68,10 +68,10 @@ def delete_network_tokens_view(request): @s.NetworkJSONWebKeySetAPI.get(tags=[s.NetworkTag], response_schemas=s.NetworkJSONWebKeySet_GET_responses) @view_config(route_name=s.NetworkJSONWebKeySetAPI.name, request_method="GET", decorator=check_network_mode_enabled, permission=NO_PERMISSION_REQUIRED) -def get_network_jwks_view(_request): +def get_network_jwks_view(request): return ax.valid_http(http_success=HTTPOk, detail=s.NetworkJSONWebKeySet_GET_OkResponseSchema.description, - content=jwks().export(private_keys=False, as_dict=True)) + content=jwks(settings_container=request).export(private_keys=False, as_dict=True)) @s.NetworkDecodeJWTAPI.get(tags=[s.NetworkTag], response_schemas=s.NetworkDecodeJWT_GET_Responses) diff --git a/magpie/constants.py b/magpie/constants.py index 37c4ae979..710871bff 100644 --- a/magpie/constants.py +++ b/magpie/constants.py @@ -113,6 +113,7 @@ def _get_default_log_level(): MAGPIE_NETWORK_INTERNAL_TOKEN_EXPIRY = int(os.getenv("MAGPIE_NETWORK_INTERNAL_TOKEN_EXPIRY", 30)) MAGPIE_NETWORK_PEM_FILES = os.getenv("MAGPIE_NETWORK_PEM_FILES", os.path.join(MAGPIE_ROOT, "key.pem")) MAGPIE_NETWORK_PEM_PASSWORDS = os.getenv("MAGPIE_NETWORK_PEM_PASSWORDS") +MAGPIE_NETWORK_CREATE_MISSING_PEM_FILE = asbool(os.getenv("MAGPIE_NETWORK_CREATE_MISSING_PEM_FILE", False)) MAGPIE_LOG_LEVEL = os.getenv("MAGPIE_LOG_LEVEL", _get_default_log_level()) # log level to apply to the loggers MAGPIE_LOG_PRINT = asbool(os.getenv("MAGPIE_LOG_PRINT", False)) # log also forces print to the console MAGPIE_LOG_REQUEST = asbool(os.getenv("MAGPIE_LOG_REQUEST", True)) # log detail of every incoming request @@ -259,7 +260,7 @@ def network_enabled(settings_container=None): # type: (Optional[AnySettingsContainer]) -> bool if sys.version_info.major < 3 or sys.version_info.minor < 6: return False - return bool(get_constant("MAGPIE_NETWORK_ENABLED", settings_container=settings_container)) + return asbool(get_constant("MAGPIE_NETWORK_ENABLED", settings_container=settings_container)) def get_constant_setting_name(name): diff --git a/magpie/utils.py b/magpie/utils.py index ac31a7ece..dcaf7f427 100644 --- a/magpie/utils.py +++ b/magpie/utils.py @@ -1188,15 +1188,24 @@ def check_network_configured(settings_container=None): This should be called when the application starts up to detect a misconfigured application right away. """ if network_enabled(settings_container): - instance_name = get_constant("MAGPIE_NETWORK_INSTANCE_NAME", settings_container=settings_container) - if not instance_name: - raise ConfigurationError("MAGPIE_NETWORK_INSTANCE_NAME is required when network mode is enabled.") + try: + get_constant("MAGPIE_NETWORK_INSTANCE_NAME", settings_container=settings_container, empty_missing=True) + except (ValueError, LookupError) as exc: + raise ConfigurationError("MAGPIE_NETWORK_INSTANCE_NAME is required when network mode is enabled.") from exc + # import here to avoid a potential cyclical import - from magpie.api.management.network.network_utils import jwks + from magpie.api.management.network.network_utils import jwks, pem_files, create_private_key try: - jwks() + jwks(settings_container=settings_container) except Exception as exc: - msg = ("Error occurred when loading PEM keys which are required when network mode is enabled. " - "Check that the MAGPIE_NETWORK_PEM_FILES and MAGPIE_NETWORK_PEM_PASSWORDS are set properly. " - "Original error message: '{}'".format(exc)) - raise ConfigurationError(msg) from exc + create_missing = asbool( + get_constant("MAGPIE_NETWORK_CREATE_MISSING_PEM_FILE", settings_container=settings_container)) + pem_files_ = pem_files(settings_container) + if isinstance(exc, FileNotFoundError) and create_missing and len(pem_files_) == 1: + LOGGER.warning("No network PEM files found") + create_private_key(pem_files_[0], settings_container=settings_container) + else: + msg = ("Error occurred when loading PEM keys which are required when network mode is enabled. " + "Check that the MAGPIE_NETWORK_PEM_FILES and MAGPIE_NETWORK_PEM_PASSWORDS are set properly. " + "Original error message: '{}'".format(exc)) + raise ConfigurationError(msg) from exc From fedc5eda13a067d7c84deeb5a65fbe2623a2b24f Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Tue, 16 Apr 2024 11:26:30 -0400 Subject: [PATCH 30/55] finish up tests for network views --- .../api/management/network/network_utils.py | 2 +- .../api/management/network/network_views.py | 6 +- tests/interfaces.py | 120 ++++++++++++++++++ tests/test_magpie_api.py | 50 +++++++- tests/utils.py | 30 ++++- 5 files changed, 201 insertions(+), 7 deletions(-) diff --git a/magpie/api/management/network/network_utils.py b/magpie/api/management/network/network_utils.py index 7973547c7..b403be26a 100644 --- a/magpie/api/management/network/network_utils.py +++ b/magpie/api/management/network/network_utils.py @@ -177,7 +177,7 @@ def decode_jwt(token, node, settings_container=None): algorithms=["RS256"], issuer=node.name, audience=instance_name), - http_error=HTTPInternalServerError, + http_error=HTTPBadRequest, msg_on_fail="Cannot verify JWT") diff --git a/magpie/api/management/network/network_views.py b/magpie/api/management/network/network_views.py index 9153be64f..3c70b929e 100644 --- a/magpie/api/management/network/network_views.py +++ b/magpie/api/management/network/network_views.py @@ -79,14 +79,14 @@ def get_network_jwks_view(request): def get_decode_jwt(request): token = request.GET.get("token") if token is None: - raise HTTPBadRequest("Missing token") + ax.raise_http(http_error=HTTPBadRequest, detail="Missing token") try: node_name = jwt.decode(token, options={"verify_signature": False}).get("iss") except jwt.exceptions.DecodeError: - raise HTTPBadRequest("Token is improperly formatted") + ax.raise_http(http_error=HTTPBadRequest, detail="Token is improperly formatted") node = request.db.query(NetworkNode).filter(NetworkNode.name == node_name).first() if node is None: - raise HTTPBadRequest("Invalid token: invalid or missing issuer claim") + ax.raise_http(http_error=HTTPBadRequest, detail="Invalid token: invalid or missing issuer claim") jwt_content = decode_jwt(token, node, request) return ax.valid_http(http_success=HTTPOk, content={"jwt_content": jwt_content}, diff --git a/tests/interfaces.py b/tests/interfaces.py index 15301a3f7..806fe3aa2 100644 --- a/tests/interfaces.py +++ b/tests/interfaces.py @@ -1,3 +1,4 @@ +import datetime import html import itertools import os @@ -1386,6 +1387,25 @@ def test_DeleteNetworkToken_InvalidRemoteUser(self): allow_missing=True) utils.check_response_basic_info(resp, expected_code=404, expected_method="DELETE") + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_GetJSONWebKeySet(self): + """ + Test get a valid JSON Web Key Set. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Get a valid JSON Web Key Set", "3.38.0", skip=True) + resp = utils.test_request(self, "GET", "/network/jwks") + utils.check_response_basic_info(resp) + json_body = utils.get_json_body(resp) + assert json_body.get("keys") + for key in json_body["keys"]: + assert key.get("kty") == "RSA" + assert key.get("kid") + assert key.get("n") + assert key.get("e") + @runner.MAGPIE_TEST_API @six.add_metaclass(ABCMeta) @@ -7743,6 +7763,106 @@ def test_DeleteNetworkTokens_AndAnonymousNetworkUsers(self): json_body = utils.get_json_body(resp) assert not json_body["remote_users"] + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_GetDecodeJWT(self): + """ + Test decode a JSON web token. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Decode a JSON web token", "3.38.0", skip=True) + + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + claims = {"test": 123, "test2": "another value"} + with utils.TestSetup.valid_jwt(self, override_jwt_claims=claims) as token: + resp = utils.test_request(self, "GET", "/network/decode_jwt?token={}".format(token), cookies=self.cookies, + headers=self.headers) + utils.check_response_basic_info(resp) + json_body = utils.get_json_body(resp) + assert json_body.get("jwt_content") + assert json_body["jwt_content"].get("iss") == self.test_node_name + assert json_body["jwt_content"].get("aud") == get_constant("MAGPIE_NETWORK_INSTANCE_NAME") + for key, val in claims.items(): + assert json_body["jwt_content"].get(key) == val + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_GetDecodeJWT_NoToken(self): + """ + Test that decoding an empty JSON web token returns an error. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Decode a JSON web token", "3.38.0", skip=True) + + resp = utils.test_request(self, "GET", "/network/decode_jwt", cookies=self.cookies, + headers=self.headers, expect_errors=True) + utils.check_response_basic_info(resp, expected_code=400) + json_body = utils.get_json_body(resp) + assert "Missing token" in json_body.get("detail", '') + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_GetDecodeJWT_BadToken(self): + """ + Test that decoding an improperly formatted JSON web token returns an error. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Decode a JSON web token", "3.38.0", skip=True) + + resp = utils.test_request(self, "GET", "/network/decode_jwt?token=abc123", cookies=self.cookies, + headers=self.headers, expect_errors=True) + utils.check_response_basic_info(resp, expected_code=400) + json_body = utils.get_json_body(resp) + assert "Token is improperly formatted" in json_body.get("detail", '') + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_GetDecodeJWT_BadIssuer(self): + """ + Test that decoding a JSON web token with an issuer that doesn't exist as a node returns an error. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Decode a JSON web token", "3.38.0", skip=True) + + with utils.TestSetup.valid_jwt(self) as token: + resp = utils.test_request(self, "GET", "/network/decode_jwt?token={}".format(token), cookies=self.cookies, + headers=self.headers, expect_errors=True) + utils.check_response_basic_info(resp, expected_code=400) + json_body = utils.get_json_body(resp) + assert "invalid or missing issuer claim" in json_body.get("detail", '') + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_GetDecodeJWT_Expired(self): + """ + Test raise error when decoding a JSON web token that has expired. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Decode a JSON web token", "3.38.0", skip=True) + + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + expiry = datetime.datetime.utcnow() - datetime.timedelta(days=365) + with utils.TestSetup.valid_jwt(self, override_jwt_expiry=expiry) as token: + resp = utils.test_request(self, "GET", "/network/decode_jwt?token={}".format(token), + cookies=self.cookies, headers=self.headers, expect_errors=True) + json_info = utils.check_response_basic_info(resp, expected_code=400) + assert json_info.get("call", {}).get("exception") == "ExpiredSignatureError" + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_GetNetworkNodes(self): + """ + Test admin can view full information of all network nodes. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "View network node information", "3.38.0", skip=True) + @runner.MAGPIE_TEST_UI @six.add_metaclass(ABCMeta) diff --git a/tests/test_magpie_api.py b/tests/test_magpie_api.py index 68d374b3b..ded9eaee6 100644 --- a/tests/test_magpie_api.py +++ b/tests/test_magpie_api.py @@ -7,6 +7,7 @@ Tests for :mod:`magpie.api` module. """ +import datetime import unittest import mock @@ -17,7 +18,7 @@ from magpie.models import UserGroupStatus, UserStatuses from magpie.utils import CONTENT_TYPE_JSON from tests import runner, utils -from tests.utils import TestVersion +from tests.utils import TestVersion, patch_datetime @runner.MAGPIE_TEST_API @@ -556,6 +557,53 @@ def test_PostUsers_WithExtraRegex_ValidBoth(self): headers=self.json_headers, cookies=self.cookies, expect_errors=True) utils.check_response_basic_info(resp, 201, expected_method="POST") + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_DeleteNetworkTokens_AndAnonymousNetworkUsers_ExpiredOnly(self): + """ + Test deleting only expired network tokens and their associated anonymous network users. + + Note that this test only runs locally because it requires mocking the expiry time. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Delete expired network tokens", "3.38.0", skip=True) + + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True, override_name="test1") + utils.TestSetup.create_TestNetworkNode(self, override_exist=True, override_name="test2") + utils.TestSetup.create_TestNetworkToken(self, override_remote_user_name="test1", override_node_name="test1") + + with patch_datetime({"utcnow": datetime.datetime.utcnow() - datetime.timedelta(days=365)}): + utils.TestSetup.create_TestNetworkToken(self, override_remote_user_name="test2", override_node_name="test2") + + + utils.test_request(self, "DELETE", "/network/tokens", data={"expired_only": True}, cookies=self.cookies, + headers=self.headers) + + resp = utils.test_request(self, "GET", "/network/remote_users", cookies=self.cookies, headers=self.headers) + json_body = utils.get_json_body(resp) + assert {u["remote_user_name"] for u in json_body["remote_users"]} == {"test1"} + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_GetDecodeJWT_DifferentAudience(self): + """ + Test raise error when decoding a JSON web token for a different audience. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Decode a JSON web token", "3.38.0", skip=True) + + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + with utils.TestSetup.valid_jwt(self) as token: + with utils.mocked_get_settings(settings={"MAGPIE_NETWORK_INSTANCE_NAME": "some_other_instance"}): + resp = utils.test_request(self, "GET", "/network/decode_jwt?token={}".format(token), + cookies=self.cookies, headers=self.headers, expect_errors=True) + json_info = utils.check_response_basic_info(resp, expected_code=400) + assert json_info.get("call", {}).get("exception") == "InvalidAudienceError" + @runner.MAGPIE_TEST_API @runner.MAGPIE_TEST_LOCAL diff --git a/tests/utils.py b/tests/utils.py index 5feebc3c6..9c4f375c6 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1522,6 +1522,28 @@ def check_val_type(val, ref, msg=None): assert isinstance(val, ref), format_test_val_ref(val, repr(ref), pre="Type Fail", msg=msg) +@contextlib.contextmanager +def patch_datetime(patches): + # type: (Dict[Str, Any]) -> Any + """ + Patch functions in the datetime.datetime module where keys in patches are names of functions and values are their + patched return values. + + Note: mocking datetime is complicated because it's implemented in C. The technique used here is inspired by: + https://williambert.online/2011/07/how-to-unit-testing-in-django-with-mocking-and-patching/ + """ + + from datetime import datetime as org_datetime + class FakeDatetime(org_datetime): + def __new__(cls, *args, **kwargs): + return org_datetime.__new__(org_datetime, *args, **kwargs) + + with mock.patch("datetime.datetime", FakeDatetime): + for method, return_value in patches.items(): + setattr(FakeDatetime, method, classmethod(lambda cls: return_value)) + yield + + def check_raises(func, exception_type, msg=None): # type: (Callable[[], Any], Type[Exception], Optional[Str]) -> Exception """ @@ -3148,6 +3170,7 @@ def valid_jwt(test_case, # type: AnyMagpieTestCaseType override_jwt_claims=null, # type: Optional[Dict[Str, Str]] override_jwt_headers=null, # type: Optional[Dict[Str, Str]] override_jwt_issuer=null, # type: Optional[Str] + override_jwt_expiry=null, # type: Optional[datetime] override_jwks_url=null, # type: Optional[Str] override_node_host=null, # type: Optional[Str] override_node_port=null # type: Optional[int] @@ -3171,8 +3194,11 @@ def valid_jwt(test_case, # type: AnyMagpieTestCaseType jwt_headers = {} if override_jwt_headers is null else override_jwt_headers issuer = test_case.test_node_name if override_jwt_issuer is null else override_jwt_issuer jwt_claims = {} if override_jwt_claims is null else override_jwt_claims - expiry = int(get_constant("MAGPIE_NETWORK_INTERNAL_TOKEN_EXPIRY")) - expiry_time = datetime.utcnow() + timedelta(seconds=expiry) + if override_jwt_expiry is null: + expiry = int(get_constant("MAGPIE_NETWORK_INTERNAL_TOKEN_EXPIRY")) + expiry_time = datetime.utcnow() + timedelta(seconds=expiry) + else: + expiry_time = override_jwt_expiry audience = get_constant("MAGPIE_NETWORK_INSTANCE_NAME") claims = {"iss": issuer, "aud": audience, "exp": expiry_time, **jwt_claims} token = jwt.encode(claims, private_bytes, headers={"kid": jwk["kid"], **jwt_headers}, algorithm="RS256") From 7f6a8a0ee1b2d0c8be022f6edbebc0b9289d28d4 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Wed, 17 Apr 2024 14:06:46 -0400 Subject: [PATCH 31/55] most of the tests for network nodes --- .../network/node/network_node_utils.py | 21 +- .../network/node/network_node_views.py | 23 +- tests/interfaces.py | 369 +++++++++++++++++- tests/test_magpie_api.py | 24 +- tests/utils.py | 23 +- 5 files changed, 403 insertions(+), 57 deletions(-) diff --git a/magpie/api/management/network/node/network_node_utils.py b/magpie/api/management/network/node/network_node_utils.py index 03ea9cecf..e01017419 100644 --- a/magpie/api/management/network/node/network_node_utils.py +++ b/magpie/api/management/network/node/network_node_utils.py @@ -1,5 +1,7 @@ +import json from typing import TYPE_CHECKING +import six from pyramid.httpexceptions import HTTPBadRequest, HTTPConflict from magpie import models @@ -12,7 +14,7 @@ if TYPE_CHECKING: from pyramid.request import Request - from magpie.typedefs import Optional, Session, Str + from magpie.typedefs import AnyRequestType, JSON, List, Optional, Session, Str NAME_REGEX = r"^[\w-]+$" @@ -60,6 +62,7 @@ def update_associated_user_groups(node, old_node_name, request): anonymous_user = request.db.query(models.User).filter(models.User.user_name == old_anonymous_name).one() anonymous_group = request.db.query(models.Group).filter(models.Group.group_name == old_anonymous_name).one() anonymous_user.user_name = node.anonymous_user_name() + anonymous_user.email = get_constant("MAGPIE_NETWORK_ANONYMOUS_EMAIL_FORMAT").format(node.name) anonymous_group.group_name = node.anonymous_user_name() @@ -104,3 +107,19 @@ def check_network_node_info(db_session=None, name=None, jwks_url=None, token_url ax.verify_param(uri, matches=True, param_name="redirect_uris", param_compare=URL_REGEX, http_error=HTTPBadRequest, msg_on_fail=s.NetworkNodes_CheckInfo_RedirectURIsValue_BadRequestResponseSchema.description) + + +def load_redirect_uris(uris, request): + # type: (JSON, AnyRequestType) -> List[Str] + """ + If the uris are a string type, load them as a JSON into a list and return the list. + """ + if isinstance(uris, six.string_types): + return ax.evaluate_call( + lambda: json.loads(uris), + http_error=HTTPBadRequest, + fallback=lambda: request.db.rollback(), + msg_on_fail=s.NetworkNodes_CheckInfo_RedirectURIsValue_BadRequestResponseSchema.description + ) + else: + return uris diff --git a/magpie/api/management/network/node/network_node_views.py b/magpie/api/management/network/node/network_node_views.py index 4c6078fa1..660c2c0fa 100644 --- a/magpie/api/management/network/node/network_node_views.py +++ b/magpie/api/management/network/node/network_node_views.py @@ -1,8 +1,5 @@ -import json - import jwt import requests -import six from pyramid.httpexceptions import ( HTTPBadRequest, HTTPCreated, @@ -25,7 +22,7 @@ check_network_node_info, create_associated_user_groups, delete_network_node, - update_associated_user_groups + update_associated_user_groups, load_redirect_uris ) from magpie.api.requests import check_network_mode_enabled from magpie.constants import get_constant @@ -71,15 +68,7 @@ def post_network_nodes_view(request): kwargs[param] = value redirect_uris = ar.get_multiformat_body(request, "redirect_uris", default=None) if redirect_uris is not None: - if isinstance(redirect_uris, six.string_types): - kwargs["redirect_uris"] = ax.evaluate_call( - lambda: json.loads(redirect_uris), - http_error=HTTPBadRequest, - fallback=lambda: request.db.rollback(), - msg_on_fail=s.NetworkNodes_CheckInfo_RedirectURIsValue_BadRequestResponseSchema.description - ) - else: - kwargs["redirect_uris"] = redirect_uris + kwargs["redirect_uris"] = load_redirect_uris(redirect_uris, request) check_network_node_info(request.db, **kwargs) node = models.NetworkNode(**kwargs) @@ -100,8 +89,12 @@ def patch_network_node_view(request): params = ("name", "jwks_url", "token_url", "authorization_url", "redirect_uris") kwargs = {} for param in params: - if param in request.POST: - kwargs[param] = request.POST[param] + param_value = ar.get_multiformat_body(request, param, default=None) + if param_value is not None: + if param == "redirect_uris": + kwargs[param] = load_redirect_uris(param_value, request) + else: + kwargs[param] = param_value if not kwargs: ax.raise_http(http_error=HTTPBadRequest, detail=s.NetworkNodes_PATCH_BadRequestResponseSchema.description) diff --git a/tests/interfaces.py b/tests/interfaces.py index 806fe3aa2..6691b86fe 100644 --- a/tests/interfaces.py +++ b/tests/interfaces.py @@ -204,7 +204,7 @@ def cleanup(cls): cls.extra_resource_ids.discard(res) for node in list(cls.extra_node_names): check_network_mode(utils.TestSetup.delete_TestNetworkNode, - enable=True)(cls, override_name=node) + enable=True)(cls, override_name=node, allow_missing=True) cls.extra_node_names.discard(node) for remote_user_name, node_name in list(cls.extra_network_tokens): check_network_mode(utils.TestSetup.delete_TestNetworkToken, @@ -274,6 +274,22 @@ def setup_admin(cls): override_headers=admin_headers, override_cookies=admin_cookies) utils.check_or_try_logout_user(cls) + @classmethod + def setup_network_attrs(cls): + """ + Sets initial default attributes and values for use in network tests + """ + cls.test_node_name = "node2" + cls.test_remote_user_name = "remote_user_1" + cls.test_node_host = get_constant("MAGPIE_TEST_REMOTE_NODE_SERVER_HOST", default_value="localhost", + raise_missing=False, raise_not_set=False) + cls.test_node_port = int(get_constant("MAGPIE_TEST_REMOTE_NODE_SERVER_PORT", default_value=2002, + raise_missing=False, raise_not_set=False)) + cls.test_node_jwks_url = "http://{}:{}/network/jwks".format(cls.test_node_host, cls.test_node_port) + cls.test_node_token_url = "http://{}:{}/network/token".format(cls.test_node_host, cls.test_node_port) + cls.test_authorization_url = "http://{}:{}/ui/network/authorize".format(cls.test_node_host, cls.test_node_port) + cls.test_redirect_uris = '["http://{}:{}/network/link"]'.format(cls.test_node_host, cls.test_node_port) + @classmethod def login_admin(cls): """ @@ -2316,6 +2332,62 @@ def test_DeleteDiscoverableGroup_Forbidden(self): resp = utils.test_request(self, "GET", path, cookies=self.cookies, headers=self.headers) utils.check_response_basic_info(resp, 200) + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_GetNetworkNodes_NameOnly(self): + """ + Test user can view network node names. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "View network node names", "3.38.0", skip=True) + + utils.TestSetup.create_TestNetworkNode(self, override_exist=True, override_name="test123") + headers, cookies = self.login_test_user() + resp = utils.test_request(self, "GET", "/network/nodes", cookies=cookies, headers=headers) + + utils.check_response_basic_info(resp) + json_body = utils.get_json_body(resp) + node_info = json_body.get("nodes", [{}])[0] + assert node_info == {"name": "test123"}, node_info + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_GetNetworkNodeToken(self): + """ + Test can get a token from another node in the network for the current user. + + .. versionadded:: 3.38 + """ + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + node_server = utils.TestSetup.remote_node(self) + token = str(uuid.uuid4()) + path = urlparse(self.test_node_token_url).path + node_server.expect_request(path, method="POST").respond_with_json({"token": token}) + + resp = utils.test_request(self, "GET", "/network/nodes/{}/token".format(self.test_node_name), + cookies=self.cookies, headers=self.headers) + utils.check_response_basic_info(resp) + json_body = utils.get_json_body(resp) + assert json_body.get("token") == token + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_DeleteNetworkNodeToken(self): + """ + Test can delete a token from another node in the network for the current user. + + .. versionadded:: 3.38 + """ + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + node_server = utils.TestSetup.remote_node(self) + path = urlparse(self.test_node_token_url).path + node_server.expect_request(path, method="DELETE").respond_with_json({}) + + resp = utils.test_request(self, "DELETE", "/network/nodes/{}/token".format(self.test_node_name), + cookies=self.cookies, headers=self.headers) + utils.check_response_basic_info(resp, expected_method="DELETE") + @runner.MAGPIE_TEST_API @six.add_metaclass(ABCMeta) @@ -2366,17 +2438,7 @@ def setup_test_values(cls): cls.test_group_name = "magpie-unittest-dummy-group" cls.test_user_name = "magpie-unittest-toto" - - cls.test_node_name = "node2" - cls.test_remote_user_name = "remote_user_1" - cls.test_node_host = get_constant("MAGPIE_TEST_REMOTE_NODE_SERVER_HOST", default_value="localhost", - raise_missing=False, raise_not_set=False) - cls.test_node_port = int(get_constant("MAGPIE_TEST_REMOTE_NODE_SERVER_PORT", default_value=2002, - raise_missing=False, raise_not_set=False)) - cls.test_node_jwks_url = "http://{}:{}/network/jwks".format(cls.test_node_host, cls.test_node_port) - cls.test_node_token_url = "http://{}:{}/network/token".format(cls.test_node_host, cls.test_node_port) - cls.test_authorization_url = "http://{}:{}/ui/network/authorize".format(cls.test_node_host, cls.test_node_port) - cls.test_redirect_uris = '["http://{}:{}/network/link"]'.format(cls.test_node_host, cls.test_node_port) + cls.setup_network_attrs() @runner.MAGPIE_TEST_STATUS def test_unauthorized_forbidden_responses(self): @@ -7855,7 +7917,7 @@ def test_GetDecodeJWT_Expired(self): @runner.MAGPIE_TEST_NETWORK @utils.check_network_mode - def test_GetNetworkNodes(self): + def test_GetNetworkNodes_AllInfo(self): """ Test admin can view full information of all network nodes. @@ -7863,6 +7925,287 @@ def test_GetNetworkNodes(self): """ utils.warn_version(self, "View network node information", "3.38.0", skip=True) + expected_node_info = { + "name": "test123", + "jwks_url": "http://test1.example.com/jwks", + "token_url": "http://test1.example.com/token", + "authorization_url": "http://test1.example.com/authorization", + "redirect_uris": ["http://uri.test.some.example.com"] + } + utils.TestSetup.create_TestNetworkNode(self, override_exist=True, override_data=expected_node_info) + resp = utils.test_request(self, "GET", "/network/nodes", cookies=self.cookies, headers=self.headers) + + utils.check_response_basic_info(resp) + json_body = utils.get_json_body(resp) + node_info = json_body.get("nodes", [{}])[0] + assert {k: v for k, v in node_info.items() if k in expected_node_info} == expected_node_info + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_GetNetworkNode_AllInfo(self): + """ + Test admin can view full information of a network node. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "View network node information", "3.38.0", skip=True) + + expected_node_info = { + "name": "test123", + "jwks_url": "http://test1.example.com/jwks", + "token_url": "http://test1.example.com/token", + "authorization_url": "http://test1.example.com/authorization", + "redirect_uris": ["http://uri.test.some.example.com"] + } + utils.TestSetup.create_TestNetworkNode(self, override_exist=True, override_data=expected_node_info) + resp = utils.test_request(self, "GET", "/network/nodes/test123", cookies=self.cookies, headers=self.headers) + + utils.check_response_basic_info(resp) + json_body = utils.get_json_body(resp) + assert {k: v for k, v in json_body.items() if k in expected_node_info} == expected_node_info + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_GetNetworkNode_NotFound(self): + """ + Test non-existant node returns 404 error. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Can't find non-existant node", "3.38.0", skip=True) + + resp = utils.test_request(self, "GET", "/network/nodes/test123", cookies=self.cookies, headers=self.headers, + expect_errors=True) + utils.check_response_basic_info(resp, expected_code=404) + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_PostNetworkNode(self): + """ + Test create a new network node. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Create a network node", "3.38.0", skip=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_PostNetworkNode_CreateAssociatedRecords(self): + """ + Test create a new network node and check that the associated Group and anonymous User are created. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Create a network node", "3.38.0", skip=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + + anonymous_name = "{}{}".format(get_constant("MAGPIE_NETWORK_NAME_PREFIX"), self.test_node_name) + + resp = utils.test_request(self, "GET", "/users/{}".format(anonymous_name), cookies=self.cookies, + headers=self.headers, expect_errors=True) + utils.check_response_basic_info(resp) + + resp = utils.test_request(self, "GET", "/groups/{}".format(anonymous_name), cookies=self.cookies, + headers=self.headers, expect_errors=True) + utils.check_response_basic_info(resp) + + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_PostNetworkNode_MissingParamName(self): + """ + Test cannot create a new network node without required parameters. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Create a network node without required parameters", "3.38.0", skip=True) + node_info = { + "name": "test1", + "jwks_url": "http://test1.example.com/jwks", + "token_url": "http://test1.example.com/token", + "authorization_url": "http://test1.example.com/authorization", + "redirect_uris": ["http://uri.test.some.example.com"] + } + + for param, value in node_info.items(): + json_body = utils.TestSetup.create_TestNetworkNode(self, override_exist=True, + override_data={**node_info, param: None}, + expect_errors=True) + assert json_body.get("code") == 201 if param == "redirect_uris" else 400 + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_PostNetworkNode_InvalidParam(self): + """ + Test cannot create a new network node with invalid params. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Create a network node with invalid parameters", "3.38.0", skip=True) + node_info = { + "name": "test1", + "jwks_url": "http://test1.example.com/jwks", + "token_url": "http://test1.example.com/token", + "authorization_url": "http://test1.example.com/authorization", + "redirect_uris": ["http://uri.test.some.example.com"] + } + + for param in node_info.keys(): + json_body = utils.TestSetup.create_TestNetworkNode(self, override_exist=True, + override_data={**node_info, param: ""}, + expect_errors=True) + assert json_body.get("code") == 400 + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_PatchNetworkNode(self): + """ + Test can update attributes of a network node. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Update a network node", "3.38.0", skip=True) + + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + node_info = { + "name": "test123", + "jwks_url": "http://test1.example.com/jwks", + "token_url": "http://test1.example.com/token", + "authorization_url": "http://test1.example.com/authorization", + "redirect_uris": ["http://uri.test.some.example.com"] + } + resp = utils.test_request(self, "PATCH", "/network/nodes/{}".format(self.test_node_name), cookies=self.cookies, + headers=self.headers, data=node_info) + utils.check_response_basic_info(resp, expected_method="PATCH") + self.extra_node_names.add("test123") # indicate potential removal at a later point + + resp = utils.test_request(self, "GET", "/network/nodes/test123", cookies=self.cookies, headers=self.headers) + json_body = utils.check_response_basic_info(resp) + + assert {k: v for k, v in json_body.items() if k in node_info} == node_info + + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_PatchNetworkNode_CreateAssociatedRecords(self): + """ + Test can update attributes of a network node. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Update a network node", "3.38.0", skip=True) + + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + resp = utils.test_request(self, "PATCH", "/network/nodes/{}".format(self.test_node_name), cookies=self.cookies, + headers=self.headers, data={"name": "test123"}) + utils.check_response_basic_info(resp, expected_method="PATCH") + self.extra_node_names.add("test123") # indicate potential removal at a later point + + anonymous_name = "{}test123".format(get_constant("MAGPIE_NETWORK_NAME_PREFIX")) + + resp = utils.test_request(self, "GET", "/users/{}".format(anonymous_name), cookies=self.cookies, + headers=self.headers, expect_errors=True) + utils.check_response_basic_info(resp) + + resp = utils.test_request(self, "GET", "/groups/{}".format(anonymous_name), cookies=self.cookies, + headers=self.headers, expect_errors=True) + utils.check_response_basic_info(resp) + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_PatchNetworkNode_DuplicateName(self): + """ + Test cannot update attributes of a network node if another node exists with the same name. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Update a network node", "3.38.0", skip=True) + + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True, override_name="test123") + node_info = { + "name": "test123", + "jwks_url": "http://test1.example.com/jwks", + "token_url": "http://test1.example.com/token", + "authorization_url": "http://test1.example.com/authorization", + "redirect_uris": ["http://uri.test.some.example.com"] + } + resp = utils.test_request(self, "PATCH", "/network/nodes/{}".format(self.test_node_name), cookies=self.cookies, + headers=self.headers, data=node_info, expect_errors=True) + utils.check_response_basic_info(resp, expected_method="PATCH", expected_code=409) + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_PatchNetworkNode_InvalidName(self): + """ + Test cannot update attributes of a network node if the new name is invalid. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Update a network node", "3.38.0", skip=True) + + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + node_info = { + "name": "test123", + "jwks_url": "http://test1.example.com/jwks", + "token_url": "http://test1.example.com/token", + "authorization_url": "http://test1.example.com/authorization", + "redirect_uris": ["http://uri.test.some.example.com"] + } + + for param in node_info.keys(): + resp = utils.test_request(self, "PATCH", "/network/nodes/{}".format(self.test_node_name), + cookies=self.cookies, headers=self.headers, + data={**node_info, param: ""}, expect_errors=True) + utils.check_response_basic_info(resp, expected_method="PATCH", expected_code=400) + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_DeleteNetworkNode(self): + """ + Test can delete a network node. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Delete a network node", "3.38.0", skip=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + utils.TestSetup.delete_TestNetworkNode(self) + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_DeleteNetworkNode_BadName(self): + """ + Test cannot delete a network node if the node doesn't exist. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Delete a network node", "3.38.0", skip=True) + resp = utils.TestSetup.delete_TestNetworkNode(self, allow_missing=True) + utils.check_response_basic_info(resp, expected_method="DELETE", expected_code=404) + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_DeleteNetworkNode_RemoveAssociatedRecords(self): + """ + Test can delete a network node and the associated User and Group records are removed as well. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Delete a network node", "3.38.0", skip=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + utils.TestSetup.delete_TestNetworkNode(self) + + anonymous_name = "{}{}".format(get_constant("MAGPIE_NETWORK_NAME_PREFIX"), self.test_node_name) + + resp = utils.test_request(self, "GET", "/users/{}".format(anonymous_name), cookies=self.cookies, + headers=self.headers, expect_errors=True) + utils.check_response_basic_info(resp, expected_code=404) + + resp = utils.test_request(self, "GET", "/groups/{}".format(anonymous_name), cookies=self.cookies, + headers=self.headers, expect_errors=True) + utils.check_response_basic_info(resp, expected_code=404) + @runner.MAGPIE_TEST_UI @six.add_metaclass(ABCMeta) diff --git a/tests/test_magpie_api.py b/tests/test_magpie_api.py index ded9eaee6..3075ab5f2 100644 --- a/tests/test_magpie_api.py +++ b/tests/test_magpie_api.py @@ -47,16 +47,7 @@ def setUpClass(cls): raise_missing=False, raise_not_set=False) cls.test_group_name = get_constant("MAGPIE_TEST_GROUP", default_value="unittest-no-auth_api-group-local", raise_missing=False, raise_not_set=False) - cls.test_node_name = "node2" - cls.test_remote_user_name = "remote_user_1" - cls.test_node_host = get_constant("MAGPIE_TEST_REMOTE_NODE_SERVER_HOST", default_value="localhost", - raise_missing=False, raise_not_set=False) - cls.test_node_port = int(get_constant("MAGPIE_TEST_REMOTE_NODE_SERVER_PORT", default_value=2002, - raise_missing=False, raise_not_set=False)) - cls.test_node_jwks_url = "http://{}:{}/network/jwks".format(cls.test_node_host, cls.test_node_port) - cls.test_node_token_url = "http://{}:{}/network/token".format(cls.test_node_host, cls.test_node_port) - cls.test_authorization_url = "http://{}:{}/ui/network/authorize".format(cls.test_node_host, cls.test_node_port) - cls.test_redirect_uris = '["http://{}:{}/network/link"]'.format(cls.test_node_host, cls.test_node_port) + cls.setup_network_attrs() @runner.MAGPIE_TEST_API @@ -91,6 +82,7 @@ def setUpClass(cls): cls.test_resource_type = "route" cls.test_group_name = "unittest-user-auth-local_test-group" cls.test_user_name = "unittest-user-auth-local_test-user-username" + cls.setup_network_attrs() @runner.MAGPIE_TEST_USERS @runner.MAGPIE_TEST_GROUPS @@ -720,16 +712,7 @@ def setUpClass(cls): raise_missing=False, raise_not_set=False) cls.test_group_name = get_constant("MAGPIE_TEST_GROUP", default_value="unittest-no-auth_api-group-remote", raise_missing=False, raise_not_set=False) - cls.test_node_name = "node2" - cls.test_remote_user_name = "remote_user_1" - cls.test_node_host = get_constant("MAGPIE_TEST_REMOTE_NODE_SERVER_HOST", default_value="localhost", - raise_missing=False, raise_not_set=False) - cls.test_node_port = int(get_constant("MAGPIE_TEST_REMOTE_NODE_SERVER_PORT", default_value=2002, - raise_missing=False, raise_not_set=False)) - cls.test_node_jwks_url = "http://{}:{}/network/jwks".format(cls.test_node_host, cls.test_node_port) - cls.test_node_token_url = "http://{}:{}/network/token".format(cls.test_node_host, cls.test_node_port) - cls.test_authorization_url = "http://{}:{}/ui/network/authorize".format(cls.test_node_host, cls.test_node_port) - cls.test_redirect_uris = '["http://{}:{}/network/link"]'.format(cls.test_node_host, cls.test_node_port) + cls.setup_network_attrs() @runner.MAGPIE_TEST_API @@ -763,6 +746,7 @@ def setUpClass(cls): cls.test_resource_type = "route" cls.test_group_name = "unittest-user-auth-remote_test-group" cls.test_user_name = "unittest-user-auth-remote_test-user-username" + cls.setup_network_attrs() @runner.MAGPIE_TEST_API diff --git a/tests/utils.py b/tests/utils.py index 9c4f375c6..56c4dfc29 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -3016,7 +3016,8 @@ def create_TestNetworkNode(test_case, # type: AnyMagpieTe override_headers=null, # type: Optional[HeadersType] override_cookies=null, # type: Optional[CookiesType] override_exist=False, # type: bool - override_data=null # type: Optional[JSON] + override_data=null, # type: Optional[JSON] + expect_errors=False, # type: bool ): # type: (...) -> JSON """ Creates a Network Node @@ -3037,8 +3038,8 @@ def create_TestNetworkNode(test_case, # type: AnyMagpieTe headers = override_headers if override_headers is not null else test_case.json_headers cookies = override_cookies if override_cookies is not null else test_case.cookies - resp = test_request(app_or_url, "POST", "/network/nodes", json=data, expect_errors=override_exist, - headers=headers, cookies=cookies) + resp = test_request(app_or_url, "POST", "/network/nodes", json=data, + expect_errors=(override_exist or expect_errors), headers=headers, cookies=cookies) if data.get("name"): test_case.extra_node_names.add(data["name"]) # indicate potential removal at a later point @@ -3052,8 +3053,11 @@ def create_TestNetworkNode(test_case, # type: AnyMagpieTe override_data=data, override_headers=headers, override_cookies=cookies, - override_exist=False) + override_exist=False, + expect_errors=expect_errors) + if expect_errors: + return get_json_body(resp) return check_response_basic_info(resp, 201, expected_method="POST") @staticmethod @@ -3061,16 +3065,19 @@ def delete_TestNetworkNode(test_case, # type: AnyMagpieTestCaseType override_name=null, # type: Optional[Str] override_headers=null, # type: Optional[HeadersType] override_cookies=null, # type: Optional[CookiesType] - ): # type: (...) -> None + allow_missing=False, # type: bool + ): # type: (...) -> Optional[AnyResponseType] """ Deletes a Network Node. """ app_or_url = get_app_or_url(test_case) headers = override_headers if override_headers is not null else test_case.json_headers cookies = override_cookies if override_cookies is not null else test_case.cookies - name = override_name if override_name is not null else test_case.test_user_name + name = override_name if override_name is not null else test_case.test_node_name path = "/network/nodes/{}".format(name) - resp = test_request(app_or_url, "DELETE", path, headers=headers, cookies=cookies) + resp = test_request(app_or_url, "DELETE", path, headers=headers, cookies=cookies, expect_errors=allow_missing) + if resp.status_code == 404 and allow_missing: + return resp check_response_basic_info(resp, 200, expected_method="DELETE") @staticmethod @@ -3147,7 +3154,7 @@ def delete_TestNetworkRemoteUser(test_case, # type: AnyMag @staticmethod def remote_node(test_case, override_node_host=null, override_node_port=null, clear=True): - # type: (AnyMagpieTestCaseType, Optional[Str], Optional[int], bool) -> Any + # type: (AnyMagpieTestCaseType, Optional[Str], Optional[int], bool) -> HTTPServer """ Starts a :class:`pytest_httpserver.HTTPServer` instance which can be used to generate fake responses from a fake network node. From 249f4a1130b0f73ada789c6fe5888ee09397d1f2 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Fri, 19 Apr 2024 09:40:17 -0400 Subject: [PATCH 32/55] use check functions instead of plain assert --- tests/interfaces.py | 285 ++++++++++++++++++++++++++++++++------- tests/test_magpie_api.py | 4 +- 2 files changed, 237 insertions(+), 52 deletions(-) diff --git a/tests/interfaces.py b/tests/interfaces.py index 6691b86fe..e56e9a769 100644 --- a/tests/interfaces.py +++ b/tests/interfaces.py @@ -1,6 +1,7 @@ import datetime import html import itertools +import json import os import secrets import string @@ -10,12 +11,13 @@ from copy import deepcopy from typing import TYPE_CHECKING +import jwt import pyramid.testing import pytest import six import yaml from pyramid.interfaces import IRequestExtensions -from six.moves.urllib.parse import urlparse +from six.moves.urllib.parse import parse_qs, urlparse from webtest.app import TestApp from magpie import __meta__ @@ -689,8 +691,8 @@ def test_Login_NetworkToken_Authorized(self): resp_cookies = resp.cookies resp = utils.test_request(self, "GET", "/session", headers=self.json_headers, cookies=resp_cookies) session_json = utils.get_json_body(resp) - assert session_json.get("authenticated") - assert session_json.get("user", {}).get("user_name") == self.test_user_name + utils.check_val_true(session_json.get("authenticated")) + utils.check_val_equal(session_json.get("user", {}).get("user_name"), self.test_user_name) @runner.MAGPIE_TEST_NETWORK @runner.MAGPIE_TEST_LOGIN @@ -712,7 +714,7 @@ def test_Login_NetworkToken_Authorized_Refreshed(self): initial_token = utils.TestSetup.create_TestNetworkToken(self) token = utils.TestSetup.create_TestNetworkToken(self) - assert token != initial_token + utils.check_val_not_equal(token, initial_token) headers = {"Accept": CONTENT_TYPE_JSON, "Authorization": "Bearer {}".format(token["token"])} path = s.ProviderSigninAPI.path.format(provider_name=get_constant("MAGPIE_NETWORK_PROVIDER")) @@ -727,8 +729,8 @@ def test_Login_NetworkToken_Authorized_Refreshed(self): resp_cookies = resp.cookies resp = utils.test_request(self, "GET", "/session", headers=self.json_headers, cookies=resp_cookies) session_json = utils.get_json_body(resp) - assert session_json.get("authenticated") - assert session_json.get("user", {}).get("user_name") == self.test_user_name + utils.check_val_true(bool(session_json.get("authenticated"))) + utils.check_val_equal(session_json.get("user", {}).get("user_name"), self.test_user_name) @runner.MAGPIE_TEST_NETWORK @runner.MAGPIE_TEST_LOGIN @@ -761,8 +763,8 @@ def test_Login_NetworkToken_Unauthorized_BadFormat(self): resp_cookies = resp.cookies resp = utils.test_request(self, "GET", "/session", headers=self.json_headers, cookies=resp_cookies) session_json = utils.get_json_body(resp) - assert session_json.get("authenticated") is False - assert session_json.get("user", {}).get("user_name") is None + utils.check_val_false(session_json.get("authenticated")) + utils.check_val_equal(session_json.get("user", {}).get("user_name"), None) @runner.MAGPIE_TEST_NETWORK @runner.MAGPIE_TEST_LOGIN @@ -797,8 +799,8 @@ def test_Login_NetworkToken_Unauthorized_Deleted(self): resp_cookies = resp.cookies resp = utils.test_request(self, "GET", "/session", headers=self.json_headers, cookies=resp_cookies) session_json = utils.get_json_body(resp) - assert session_json.get("authenticated") is False - assert session_json.get("user", {}).get("user_name") is None + utils.check_val_false(session_json.get("authenticated")) + utils.check_val_equal(session_json.get("user", {}).get("user_name"), None) @runner.MAGPIE_TEST_NETWORK @runner.MAGPIE_TEST_LOGIN @@ -824,8 +826,8 @@ def test_Login_NetworkToken_Unauthorized_NetworkNotEnabled(self): resp_cookies = resp.cookies resp = utils.test_request(self, "GET", "/session", headers=self.json_headers, cookies=resp_cookies) session_json = utils.get_json_body(resp) - assert session_json.get("authenticated") is False - assert session_json.get("user", {}).get("user_name") is None + utils.check_val_false(session_json.get("authenticated")) + utils.check_val_equal(session_json.get("user", {}).get("user_name"), None) @runner.MAGPIE_TEST_NETWORK @runner.MAGPIE_TEST_LOGIN @@ -858,8 +860,8 @@ def test_Login_NetworkToken_Unauthorized_BadToken(self): resp_cookies = resp.cookies resp = utils.test_request(self, "GET", "/session", headers=self.json_headers, cookies=resp_cookies) session_json = utils.get_json_body(resp) - assert session_json.get("authenticated") is False - assert session_json.get("user", {}).get("user_name") is None + utils.check_val_false(session_json.get("authenticated")) + utils.check_val_equal(session_json.get("user", {}).get("user_name"), None) @runner.MAGPIE_TEST_LOGIN def test_Login_GetRequestFormat(self): @@ -1232,7 +1234,7 @@ def test_PostNetworkToken(self): utils.check_or_try_logout_user(self) self.cookies = self.headers = None token = utils.TestSetup.create_TestNetworkToken(self) - assert token.get("token") + utils.check_val_true(bool(token.get("token"))) @runner.MAGPIE_TEST_NETWORK @utils.check_network_mode @@ -1257,7 +1259,7 @@ def test_PostNetworkToken_Create_AnonymousUser(self): headers=self.headers, cookies=type(self).cookies) json_body = utils.get_json_body(resp) anon_user_name = "{}{}".format(get_constant("MAGPIE_NETWORK_NAME_PREFIX"), self.test_node_name) - assert json_body.get("user_name") == anon_user_name + utils.check_val_equal(json_body.get("user_name"), anon_user_name) @runner.MAGPIE_TEST_NETWORK @utils.check_network_mode @@ -1276,8 +1278,8 @@ def test_PostNetworkToken_InvalidNode(self): utils.check_or_try_logout_user(self) self.cookies = self.headers = None token = utils.TestSetup.create_TestNetworkToken(self, override_node_name="some_other_node", expect_errors=True) - assert token.get("token") is None - assert token.get("code") == 404 + utils.check_val_equal(token.get("token"), None) + utils.check_val_equal(token.get("code"), 404) @runner.MAGPIE_TEST_NETWORK @utils.check_network_mode @@ -1297,9 +1299,9 @@ def test_PostNetworkToken_InvalidNodeCredentials(self): utils.check_or_try_logout_user(self) self.cookies = self.headers = None token = utils.TestSetup.create_TestNetworkToken(self, expect_errors=True) - assert token.get("token") is None - assert token.get("code") == 500 - assert token.get("call", {}).get("exception") == "PyJWKClientConnectionError" + utils.check_val_equal(token.get("token"), None) + utils.check_val_equal(token.get("code"), 500) + utils.check_val_equal(token.get("call", {}).get("exception"), "PyJWKClientConnectionError") @runner.MAGPIE_TEST_NETWORK @utils.check_network_mode @@ -1340,7 +1342,8 @@ def test_DeleteNetworkToken_NotRemoteUser(self): resp = utils.test_request(self, "GET", "/network/remote_users", cookies=self.cookies, headers=self.headers) json_body = utils.get_json_body(resp) - assert json_body.get("remote_users", [{}])[0].get("remote_user_name") == self.test_remote_user_name + utils.check_val_equal(json_body.get("remote_users", [{}])[0].get("remote_user_name"), + self.test_remote_user_name) @runner.MAGPIE_TEST_NETWORK @utils.check_network_mode @@ -1360,7 +1363,8 @@ def test_DeleteNetworkToken_And_AnonymousUser(self): resp = utils.test_request(self, "GET", "/network/remote_users", cookies=self.cookies, headers=self.headers) json_body = utils.get_json_body(resp) - assert not json_body["remote_users"] + utils.check_val_false(bool(json_body["remote_users"])) + @runner.MAGPIE_TEST_NETWORK @utils.check_network_mode @@ -1415,12 +1419,12 @@ def test_GetJSONWebKeySet(self): resp = utils.test_request(self, "GET", "/network/jwks") utils.check_response_basic_info(resp) json_body = utils.get_json_body(resp) - assert json_body.get("keys") + utils.check_val_true(bool(json_body.get("keys"))) for key in json_body["keys"]: - assert key.get("kty") == "RSA" - assert key.get("kid") - assert key.get("n") - assert key.get("e") + utils.check_val_equal(key.get("kty"), "RSA") + utils.check_val_true(bool(key.get("kid"))) + utils.check_val_true(bool(key.get("n"))) + utils.check_val_true(bool(key.get("e"))) @runner.MAGPIE_TEST_API @@ -2349,7 +2353,7 @@ def test_GetNetworkNodes_NameOnly(self): utils.check_response_basic_info(resp) json_body = utils.get_json_body(resp) node_info = json_body.get("nodes", [{}])[0] - assert node_info == {"name": "test123"}, node_info + utils.check_val_equal(node_info, {"name": "test123"}) @runner.MAGPIE_TEST_NETWORK @utils.check_network_mode @@ -2359,17 +2363,19 @@ def test_GetNetworkNodeToken(self): .. versionadded:: 3.38 """ + utils.warn_version(self, "Get token from another node", "3.38.0", skip=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) node_server = utils.TestSetup.remote_node(self) token = str(uuid.uuid4()) path = urlparse(self.test_node_token_url).path node_server.expect_request(path, method="POST").respond_with_json({"token": token}) - + headers, cookies = self.login_test_user() resp = utils.test_request(self, "GET", "/network/nodes/{}/token".format(self.test_node_name), - cookies=self.cookies, headers=self.headers) + cookies=cookies, headers=headers) utils.check_response_basic_info(resp) json_body = utils.get_json_body(resp) - assert json_body.get("token") == token + utils.check_val_equal(json_body.get("token"), token) @runner.MAGPIE_TEST_NETWORK @utils.check_network_mode @@ -2379,15 +2385,194 @@ def test_DeleteNetworkNodeToken(self): .. versionadded:: 3.38 """ + utils.warn_version(self, "Delete token on another node", "3.38.0", skip=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) node_server = utils.TestSetup.remote_node(self) path = urlparse(self.test_node_token_url).path node_server.expect_request(path, method="DELETE").respond_with_json({}) - + headers, cookies = self.login_test_user() resp = utils.test_request(self, "DELETE", "/network/nodes/{}/token".format(self.test_node_name), - cookies=self.cookies, headers=self.headers) + cookies=cookies, headers=headers) utils.check_response_basic_info(resp, expected_method="DELETE") + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_GetNetworkLink(self): + """ + Test can create a NetworkRemoteUser associated with another node for the current user. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Link current user with a user on another node", "3.38.0", skip=True) + + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + claims = {"user_name": "test123", "requesting_user_name": self.test_user_name} + headers, cookies = self.login_test_user() + with utils.TestSetup.valid_jwt(self, override_jwt_claims=claims) as token: + resp = utils.test_request(self, "GET", "/network/link?token={}".format(token), cookies=cookies, + headers=headers) + utils.check_response_basic_info(resp) + + resp = utils.test_request(self, "GET", + "/network/nodes/{}/remote_users/test123".format(self.test_node_name), + cookies=self.cookies, headers=self.headers) + utils.check_response_basic_info(resp) + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_GetNetworkLink_DifferentUser(self): + """ + Test cannot create a NetworkRemoteUser associated with another node for a user other than the current user. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Link current user with a user on another node", "3.38.0", skip=True) + + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + claims = {"user_name": "test123", "requesting_user_name": "some_other_user"} + headers, cookies = self.login_test_user() + with utils.TestSetup.valid_jwt(self, override_jwt_claims=claims) as token: + resp = utils.test_request(self, "GET", "/network/link?token={}".format(token), cookies=cookies, + headers=headers, expect_errors=True) + utils.check_response_basic_info(resp, expected_code=403) + + resp = utils.test_request(self, "GET", + "/network/nodes/{}/remote_users/test123".format(self.test_node_name), + cookies=self.cookies, headers=self.headers, expect_errors=True) + utils.check_response_basic_info(resp, expected_code=404) + + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_GetNetworkLink_MissingClaim_user_name(self): + """ + Test cannot create a NetworkRemoteUser associated with another node if the user_name claim is missing. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Link current user with a user on another node", "3.38.0", skip=True) + + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + claims = {"requesting_user_name": self.test_user_name} + headers, cookies = self.login_test_user() + with utils.TestSetup.valid_jwt(self, override_jwt_claims=claims) as token: + resp = utils.test_request(self, "GET", "/network/link?token={}".format(token), cookies=cookies, + headers=headers, expect_errors=True) + utils.check_response_basic_info(resp, expected_code=400) + + resp = utils.test_request(self, "GET", + "/network/nodes/{}/remote_users/test123".format(self.test_node_name), + cookies=self.cookies, headers=self.headers, expect_errors=True) + utils.check_response_basic_info(resp, expected_code=404) + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_GetNetworkLink_MissingClaim_requesting_user_name(self): + """ + Test cannot create a NetworkRemoteUser associated with another node if the requesting_user_name claim is + missing. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Link current user with a user on another node", "3.38.0", skip=True) + + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + claims = {"user_name": "test123"} + headers, cookies = self.login_test_user() + with utils.TestSetup.valid_jwt(self, override_jwt_claims=claims) as token: + resp = utils.test_request(self, "GET", "/network/link?token={}".format(token), cookies=cookies, + headers=headers, expect_errors=True) + utils.check_response_basic_info(resp, expected_code=400) + + resp = utils.test_request(self, "GET", + "/network/nodes/{}/remote_users/test123".format(self.test_node_name), + cookies=self.cookies, headers=self.headers, expect_errors=True) + utils.check_response_basic_info(resp, expected_code=404) + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_GetNetworkLink_InvalidIssuer(self): + """ + Test cannot create a NetworkRemoteUser associated with another node if the issuer claim is for a different node. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Link current user with a user on another node", "3.38.0", skip=True) + + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True, override_name="some_other_name") + claims = {"user_name": "test123", "requesting_user_name": "some_other_user"} + headers, cookies = self.login_test_user() + with utils.TestSetup.valid_jwt(self, override_jwt_claims=claims) as token: + resp = utils.test_request(self, "GET", "/network/link?token={}".format(token), cookies=cookies, + headers=headers, expect_errors=True) + utils.check_response_basic_info(resp, expected_code=404) + + resp = utils.test_request(self, "GET", + "/network/nodes/{}/remote_users/test123".format(self.test_node_name), + cookies=self.cookies, headers=self.headers, expect_errors=True) + utils.check_response_basic_info(resp, expected_code=404) + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_PostNetworkLink(self): + """ + Test redirect to authorization url of the requested node with an appropriate request token. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Request a link with a remote node for the current user", "3.38.0", skip=True) + + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + headers, cookies = self.login_test_user() + resp = utils.test_request(self, "POST", "/network/nodes/{}/link".format(self.test_node_name), cookies=cookies, + headers=headers, expect_errors=True, allow_redirects=False) + utils.check_response_basic_info(resp, expected_method="POST", expected_code=302) + redirect_uri = resp.headers.get("Location") + utils.check_val_true(bool(redirect_uri)) + + parsed_url = urlparse(redirect_uri) + utils.check_val_equal(parsed_url.hostname, self.test_node_host) + utils.check_val_equal(parsed_url.port, self.test_node_port) + utils.check_val_equal(parsed_url.path, "/ui/network/authorize") + + query = parse_qs(parsed_url.query) + utils.check_val_equal(query.get("response_type", [None])[0], "id_token") + utils.check_val_is_in(urlparse(query.get("redirect_uri", [None])[0]).path, + ["/network/link", "/magpie/network/link"]) + + token = query.get("token", [None])[0] + utils.check_val_true(bool(token)) + + jwt_claims = jwt.decode(token, options={"verify_signature": False}) + jwt_claims.pop("exp") + utils.check_val_equal(jwt_claims, + {"user_name": self.test_user_name, + "iss": get_constant("MAGPIE_NETWORK_INSTANCE_NAME"), + "aud": self.test_node_name}) + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_PostNetworkLink_DifferentNode(self): + """ + Test redirect to authorization url of the requested node if the node doesn't exist. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Request a link with a remote node for the current user", "3.38.0", skip=True) + + utils.TestSetup.create_TestUser(self, override_exist=True) + headers, cookies = self.login_test_user() + resp = utils.test_request(self, "POST", "/network/nodes/{}/link".format(self.test_node_name), cookies=cookies, + headers=headers, expect_errors=True, allow_redirects=False) + utils.check_response_basic_info(resp, expected_method="POST", expected_code=404) + @runner.MAGPIE_TEST_API @six.add_metaclass(ABCMeta) @@ -7801,7 +7986,8 @@ def test_DeleteNetworkTokens_NotRemoteUser(self): resp = utils.test_request(self, "GET", "/network/remote_users", cookies=self.cookies, headers=self.headers) json_body = utils.get_json_body(resp) - assert {u.get("remote_user_name") for u in json_body.get("remote_users", [{}])} == {"test1", "test2"} + utils.check_val_equal({u.get("remote_user_name") for u in json_body.get("remote_users", [{}])}, + {"test1", "test2"}) @runner.MAGPIE_TEST_NETWORK @utils.check_network_mode @@ -7823,7 +8009,7 @@ def test_DeleteNetworkTokens_AndAnonymousNetworkUsers(self): resp = utils.test_request(self, "GET", "/network/remote_users", cookies=self.cookies, headers=self.headers) json_body = utils.get_json_body(resp) - assert not json_body["remote_users"] + utils.check_val_false(bool(json_body["remote_users"])) @runner.MAGPIE_TEST_NETWORK @utils.check_network_mode @@ -7842,11 +8028,11 @@ def test_GetDecodeJWT(self): headers=self.headers) utils.check_response_basic_info(resp) json_body = utils.get_json_body(resp) - assert json_body.get("jwt_content") - assert json_body["jwt_content"].get("iss") == self.test_node_name - assert json_body["jwt_content"].get("aud") == get_constant("MAGPIE_NETWORK_INSTANCE_NAME") + utils.check_val_true(bool(json_body.get("jwt_content"))) + utils.check_val_equal(json_body["jwt_content"].get("iss"), self.test_node_name) + utils.check_val_equal(json_body["jwt_content"].get("aud"), get_constant("MAGPIE_NETWORK_INSTANCE_NAME")) for key, val in claims.items(): - assert json_body["jwt_content"].get(key) == val + utils.check_val_equal(json_body["jwt_content"].get(key), val) @runner.MAGPIE_TEST_NETWORK @utils.check_network_mode @@ -7862,7 +8048,7 @@ def test_GetDecodeJWT_NoToken(self): headers=self.headers, expect_errors=True) utils.check_response_basic_info(resp, expected_code=400) json_body = utils.get_json_body(resp) - assert "Missing token" in json_body.get("detail", '') + utils.check_val_is_in("Missing token", json_body.get("detail", '')) @runner.MAGPIE_TEST_NETWORK @utils.check_network_mode @@ -7878,7 +8064,7 @@ def test_GetDecodeJWT_BadToken(self): headers=self.headers, expect_errors=True) utils.check_response_basic_info(resp, expected_code=400) json_body = utils.get_json_body(resp) - assert "Token is improperly formatted" in json_body.get("detail", '') + utils.check_val_is_in("Token is improperly formatted", json_body.get("detail", '')) @runner.MAGPIE_TEST_NETWORK @utils.check_network_mode @@ -7895,7 +8081,7 @@ def test_GetDecodeJWT_BadIssuer(self): headers=self.headers, expect_errors=True) utils.check_response_basic_info(resp, expected_code=400) json_body = utils.get_json_body(resp) - assert "invalid or missing issuer claim" in json_body.get("detail", '') + utils.check_val_is_in("invalid or missing issuer claim", json_body.get("detail", '')) @runner.MAGPIE_TEST_NETWORK @utils.check_network_mode @@ -7913,7 +8099,7 @@ def test_GetDecodeJWT_Expired(self): resp = utils.test_request(self, "GET", "/network/decode_jwt?token={}".format(token), cookies=self.cookies, headers=self.headers, expect_errors=True) json_info = utils.check_response_basic_info(resp, expected_code=400) - assert json_info.get("call", {}).get("exception") == "ExpiredSignatureError" + utils.check_val_equal(json_info.get("call", {}).get("exception"), "ExpiredSignatureError") @runner.MAGPIE_TEST_NETWORK @utils.check_network_mode @@ -7938,7 +8124,7 @@ def test_GetNetworkNodes_AllInfo(self): utils.check_response_basic_info(resp) json_body = utils.get_json_body(resp) node_info = json_body.get("nodes", [{}])[0] - assert {k: v for k, v in node_info.items() if k in expected_node_info} == expected_node_info + utils.check_val_equal({k: v for k, v in node_info.items() if k in expected_node_info}, expected_node_info) @runner.MAGPIE_TEST_NETWORK @utils.check_network_mode @@ -7962,7 +8148,7 @@ def test_GetNetworkNode_AllInfo(self): utils.check_response_basic_info(resp) json_body = utils.get_json_body(resp) - assert {k: v for k, v in json_body.items() if k in expected_node_info} == expected_node_info + utils.check_val_equal({k: v for k, v in json_body.items() if k in expected_node_info}, expected_node_info) @runner.MAGPIE_TEST_NETWORK @utils.check_network_mode @@ -8032,7 +8218,7 @@ def test_PostNetworkNode_MissingParamName(self): json_body = utils.TestSetup.create_TestNetworkNode(self, override_exist=True, override_data={**node_info, param: None}, expect_errors=True) - assert json_body.get("code") == 201 if param == "redirect_uris" else 400 + utils.check_val_equal(json_body.get("code"), 201 if param == "redirect_uris" else 400) @runner.MAGPIE_TEST_NETWORK @utils.check_network_mode @@ -8055,7 +8241,7 @@ def test_PostNetworkNode_InvalidParam(self): json_body = utils.TestSetup.create_TestNetworkNode(self, override_exist=True, override_data={**node_info, param: ""}, expect_errors=True) - assert json_body.get("code") == 400 + utils.check_val_equal(json_body.get("code"), 400) @runner.MAGPIE_TEST_NETWORK @utils.check_network_mode @@ -8083,8 +8269,7 @@ def test_PatchNetworkNode(self): resp = utils.test_request(self, "GET", "/network/nodes/test123", cookies=self.cookies, headers=self.headers) json_body = utils.check_response_basic_info(resp) - assert {k: v for k, v in json_body.items() if k in node_info} == node_info - + utils.check_val_equal({k: v for k, v in json_body.items() if k in node_info}, node_info) @runner.MAGPIE_TEST_NETWORK @utils.check_network_mode diff --git a/tests/test_magpie_api.py b/tests/test_magpie_api.py index 3075ab5f2..01a840b35 100644 --- a/tests/test_magpie_api.py +++ b/tests/test_magpie_api.py @@ -576,7 +576,7 @@ def test_DeleteNetworkTokens_AndAnonymousNetworkUsers_ExpiredOnly(self): resp = utils.test_request(self, "GET", "/network/remote_users", cookies=self.cookies, headers=self.headers) json_body = utils.get_json_body(resp) - assert {u["remote_user_name"] for u in json_body["remote_users"]} == {"test1"} + utils.check_val_equal({u["remote_user_name"] for u in json_body["remote_users"]}, {"test1"}) @runner.MAGPIE_TEST_NETWORK @utils.check_network_mode @@ -594,7 +594,7 @@ def test_GetDecodeJWT_DifferentAudience(self): resp = utils.test_request(self, "GET", "/network/decode_jwt?token={}".format(token), cookies=self.cookies, headers=self.headers, expect_errors=True) json_info = utils.check_response_basic_info(resp, expected_code=400) - assert json_info.get("call", {}).get("exception") == "InvalidAudienceError" + utils.check_val_equal(json_info.get("call", {}).get("exception"), "InvalidAudienceError") @runner.MAGPIE_TEST_API From df90ba2bd66ba4dbb5abd500dedbeff9c7096aa2 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Mon, 22 Apr 2024 13:38:06 -0400 Subject: [PATCH 33/55] finish remote user tests and fix issue where multiple remote users couldn't be associated with an anonymous user --- ...3-08-25_2cfe144538e8_add_network_tables.py | 2 +- magpie/api/login/login.py | 6 +- .../api/management/network/network_utils.py | 3 +- .../api/management/network/network_views.py | 5 +- .../network/remote_user/remote_user_utils.py | 6 +- .../network/remote_user/remote_user_views.py | 61 ++- magpie/api/schemas.py | 21 +- magpie/cli/purge_expired_network_tokens.py | 4 +- magpie/models.py | 7 +- tests/interfaces.py | 514 ++++++++++++++++++ tests/utils.py | 18 +- 11 files changed, 599 insertions(+), 48 deletions(-) diff --git a/magpie/alembic/versions/2023-08-25_2cfe144538e8_add_network_tables.py b/magpie/alembic/versions/2023-08-25_2cfe144538e8_add_network_tables.py index 1d536b795..56d68a757 100644 --- a/magpie/alembic/versions/2023-08-25_2cfe144538e8_add_network_tables.py +++ b/magpie/alembic/versions/2023-08-25_2cfe144538e8_add_network_tables.py @@ -39,7 +39,7 @@ def upgrade(): op.create_table("network_remote_users", sa.Column("id", sa.Integer, primary_key=True, autoincrement=True), sa.Column("user_id", sa.Integer, sa.ForeignKey("users.id", onupdate="CASCADE", ondelete="CASCADE"), - nullable=False), + nullable=True), sa.Column("network_node_id", sa.Integer, sa.ForeignKey("network_nodes.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=False), diff --git a/magpie/api/login/login.py b/magpie/api/login/login.py index e81829592..18932ff70 100644 --- a/magpie/api/login/login.py +++ b/magpie/api/login/login.py @@ -291,8 +291,12 @@ def network_login(request): return ax.raise_http(http_error=HTTPUnauthorized, detail=s.Signin_POST_UnauthorizedResponseSchema.description, nothrow=True) authenticated_user = network_token.network_remote_user.user + if authenticated_user is None: + authenticated_user = network_token.network_remote_user.network_node.anonymous_user(request.db) # We should never create a token for protected users but just in case - anonymous_regex = protected_user_name_regex(include_admin=False, settings_container=request) + # Note that we *should* create tokens for anonymous network users + anonymous_regex = protected_user_name_regex(include_admin=False, include_network=False, + settings_container=request) ax.verify_param(authenticated_user.user_name, not_matches=True, param_compare=anonymous_regex, http_error=HTTPForbidden, msg_on_fail=s.Signin_POST_ForbiddenResponseSchema.description) return login_success_external(request, authenticated_user) diff --git a/magpie/api/management/network/network_utils.py b/magpie/api/management/network/network_utils.py index b403be26a..06687c067 100644 --- a/magpie/api/management/network/network_utils.py +++ b/magpie/api/management/network/network_utils.py @@ -206,7 +206,6 @@ def get_network_models_from_request_token(request, create_network_remote_user=Fa .filter(models.NetworkRemoteUser.network_node_id == node.id) .first()) if network_remote_user is None and create_network_remote_user: - anonymous_user = node.anonymous_user(request.db) - network_remote_user = models.NetworkRemoteUser(user=anonymous_user, network_node=node, name=user_name) + network_remote_user = models.NetworkRemoteUser(network_node=node, name=user_name) request.db.add(network_remote_user) return node, network_remote_user diff --git a/magpie/api/management/network/network_views.py b/magpie/api/management/network/network_views.py index 3c70b929e..bc7e5c289 100644 --- a/magpie/api/management/network/network_views.py +++ b/magpie/api/management/network/network_views.py @@ -39,8 +39,7 @@ def delete_network_token_view(request): node, network_remote_user = get_network_models_from_request_token(request) if network_remote_user and network_remote_user.network_token: request.db.delete(network_remote_user.network_token) - if (network_remote_user.user.id == node.anonymous_user(request.db).id and - sqlalchemy.inspect(network_remote_user).persistent): + if network_remote_user.user is None and sqlalchemy.inspect(network_remote_user).persistent: request.db.delete(network_remote_user) # clean up unused record in the database return ax.valid_http(http_success=HTTPOk, detail=s.NetworkToken_DELETE_OkResponseSchema.description) ax.raise_http(http_error=HTTPNotFound, detail=s.NetworkNodeToken_DELETE_NotFoundResponseSchema.description) @@ -57,7 +56,7 @@ def delete_network_tokens_view(request): anonymous_network_user_ids = [n.anonymous_user(request.db).id for n in request.db.query(models.NetworkNode).all()] # clean up unused records in the database (no need to keep records associated with anonymous network users) (request.db.query(models.NetworkRemoteUser) - .filter(models.NetworkRemoteUser.user_id.in_(anonymous_network_user_ids)) + .filter(models.NetworkRemoteUser.user_id == None) # noqa: E711 # pylint: disable=singleton-comparison .filter(models.NetworkRemoteUser.network_token_id == None) # noqa: E711 # pylint: disable=singleton-comparison .delete()) return ax.valid_http(http_success=HTTPOk, diff --git a/magpie/api/management/network/remote_user/remote_user_utils.py b/magpie/api/management/network/remote_user/remote_user_utils.py index 1871830f3..68cceb9ac 100644 --- a/magpie/api/management/network/remote_user/remote_user_utils.py +++ b/magpie/api/management/network/remote_user/remote_user_utils.py @@ -60,7 +60,11 @@ def check_remote_user_access_permissions(request, remote_user=None): remote_user = requested_remote_user(request) admin_group = get_constant("MAGPIE_ADMIN_GROUP", settings_container=request) is_admin = admin_group in [group.group_name for group in request.user.groups] - is_logged_user = request.user.user_name == remote_user.user.user_name + if remote_user.user is None: + associated_user = remote_user.network_node.anonymous_user(request.db) + else: + associated_user = remote_user.user + is_logged_user = request.user.user_name == associated_user.user_name if not (is_admin or is_logged_user): # admins can access any remote user, other users can only delete remote users associated with themselves ax.raise_http(http_error=HTTPForbidden, diff --git a/magpie/api/management/network/remote_user/remote_user_views.py b/magpie/api/management/network/remote_user/remote_user_views.py index be8f08142..9ef1d0211 100644 --- a/magpie/api/management/network/remote_user/remote_user_views.py +++ b/magpie/api/management/network/remote_user/remote_user_views.py @@ -1,5 +1,6 @@ from pyramid.httpexceptions import HTTPBadRequest, HTTPCreated, HTTPForbidden, HTTPNotFound, HTTPOk from pyramid.security import Authenticated +from pyramid.settings import asbool from pyramid.view import view_config from magpie import models @@ -14,11 +15,12 @@ from magpie.constants import protected_user_name_regex -@s.NetworkRemoteUsersAPI.get(tags=[s.NetworkTag], response_schemas=s.NetworkRemoteUsers_GET_responses) +@s.NetworkRemoteUsersAPI.get(tags=[s.NetworkTag], schema=s.NetworkRemoteUsers_GET_RequestSchema, + response_schemas=s.NetworkRemoteUsers_GET_responses) @view_config(route_name=s.NetworkRemoteUsersAPI.name, request_method="GET", decorator=check_network_mode_enabled) def get_network_remote_users_view(request): query = request.db.query(models.NetworkRemoteUser) - user_name = ar.get_multiformat_body(request, "user_name", default=None) + user_name = request.GET.get("user_name") if user_name is not None: query = query.join(models.User).filter(models.User.user_name == user_name) nodes = [n.as_dict() for n in query.all()] @@ -40,8 +42,8 @@ def get_network_remote_user_view(request): response_schemas=s.NetworkRemoteUsers_POST_responses) @view_config(route_name=s.NetworkRemoteUsersAPI.name, request_method="POST", decorator=check_network_mode_enabled) def post_network_remote_users_view(request): - required_params = ("remote_user_name", "user_name", "node_name") - kwargs = {} + required_params = ("remote_user_name", "node_name") + kwargs = {"user_name": ar.get_multiformat_body(request, "user_name", default=None)} for param in required_params: value = ar.get_multiformat_body(request, param, default=None) if value is None: @@ -53,22 +55,25 @@ def post_network_remote_users_view(request): http_error=HTTPNotFound, msg_on_fail="No network node with name '{}' found".format(kwargs["node_name"]) ) - forbidden_user_names_regex = protected_user_name_regex(include_admin=False, settings_container=request) - ax.verify_param(kwargs["user_name"], not_matches=True, param_compare=forbidden_user_names_regex, - param_name="user_name", - http_error=HTTPForbidden, content={"user_name": kwargs["user_name"]}, - msg_on_fail=s.NetworkRemoteUsers_POST_ForbiddenResponseSchema.description) - user = ax.evaluate_call( - lambda: request.db.query(models.User).filter(models.User.user_name == kwargs["user_name"]).one(), - http_error=HTTPNotFound, - msg_on_fail="No user with user_name '{}' found".format(kwargs["user_name"]) - ) + anonymous_user = node.anonymous_user(request.db) + if kwargs["user_name"] is None: + user = anonymous_user + else: + user = ax.evaluate_call( + lambda: request.db.query(models.User).filter(models.User.user_name == kwargs["user_name"]).one(), + http_error=HTTPNotFound, + msg_on_fail="No user with user_name '{}' found".format(kwargs["user_name"]) + ) remote_user_name = kwargs["remote_user_name"] ax.verify_param(remote_user_name, not_empty=True, param_name="remote_user_name", http_error=HTTPForbidden, msg_on_fail="remote_user_name is empty") - remote_user = models.NetworkRemoteUser(user_id=user.id, network_node_id=node.id, name=remote_user_name) + if user.id == anonymous_user.id: + user_id = None + else: + user_id = user.id + remote_user = models.NetworkRemoteUser(user_id=user_id, network_node_id=node.id, name=remote_user_name) request.db.add(remote_user) return ax.valid_http(http_success=HTTPCreated, detail=s.NetworkRemoteUsers_POST_CreatedResponseSchema.description) @@ -78,25 +83,18 @@ def post_network_remote_users_view(request): @view_config(route_name=s.NetworkRemoteUserAPI.name, request_method="PATCH", decorator=check_network_mode_enabled) def patch_network_remote_user_view(request): kwargs = {p: ar.get_multiformat_body(request, p, default=None) for p in - ("remote_user_name", "user_name", "node_name")} + ("remote_user_name", "user_name", "node_name", "assign_anonymous")} if not any(kwargs.values()): ax.raise_http(http_error=HTTPBadRequest, detail=s.NetworkRemoteUser_PATCH_BadRequestResponseSchema.description) remote_user = requested_remote_user(request) - if "remote_user_name" in kwargs: + if kwargs["remote_user_name"]: remote_user_name = kwargs["remote_user_name"] ax.verify_param(remote_user_name, not_empty=True, param_name="remote_user_name", - http_error=HTTPForbidden, + http_error=HTTPBadRequest, msg_on_fail="remote_user_name is empty") remote_user.name = remote_user_name - if "user_name" in kwargs: - user = ax.evaluate_call( - lambda: request.db.query(models.User).filter(models.User.user_name == kwargs["user_name"]).one(), - http_error=HTTPNotFound, - msg_on_fail="No user with user_name '{}' found".format(kwargs["user_name"]) - ) - remote_user.user_id = user.id - if "node_name" in kwargs: + if kwargs["node_name"]: node = ax.evaluate_call( lambda: request.db.query(models.NetworkNode).filter( models.NetworkNode.name == kwargs["node_name"]).one(), @@ -104,6 +102,15 @@ def patch_network_remote_user_view(request): msg_on_fail="No network node with name '{}' found".format(kwargs["node_name"]) ) remote_user.network_node_id = node.id + if kwargs["user_name"]: + user = ax.evaluate_call( + lambda: request.db.query(models.User).filter(models.User.user_name == kwargs["user_name"]).one(), + http_error=HTTPNotFound, + msg_on_fail="No user with user_name '{}' found".format(kwargs["user_name"]) + ) + remote_user.user_id = user.id + elif asbool(kwargs["assign_anonymous"]): + remote_user.user_id = None return ax.valid_http(http_success=HTTPOk, detail=s.NetworkRemoteUsers_PATCH_OkResponseSchema.description) @@ -124,4 +131,4 @@ def get_network_remote_users_current_view(request): nodes = [n.as_dict() for n in request.db.query(models.NetworkRemoteUser).filter(models.NetworkRemoteUser.user_id == request.user.id)] return ax.valid_http(http_success=HTTPOk, detail=s.NetworkRemoteUsers_GET_OkResponseSchema.description, - content={"nodes": nodes}) + content={"remote_users": nodes}) diff --git a/magpie/api/schemas.py b/magpie/api/schemas.py index da47396d0..0ec62129c 100644 --- a/magpie/api/schemas.py +++ b/magpie/api/schemas.py @@ -3724,7 +3724,7 @@ class NetworkNodeLink_GET_OkResponseSchema(BaseResponseSchemaAPI): class NetworkRemoteUser_PATCH_RequestBodySchema(colander.MappingSchema): remote_user_name = colander.SchemaNode( colander.String(), - description="Name of the associated user a remote Magpie node (instance) in the network.", + description="Name of the associated user from another remote Magpie node (instance) in the network.", example="userAremote", missing=colander.drop ) @@ -3740,6 +3740,13 @@ class NetworkRemoteUser_PATCH_RequestBodySchema(colander.MappingSchema): example="NodeA", missing=colander.drop ) + assign_anonymous = colander.SchemaNode( + colander.Boolean(), + description="Associate this network user with the anonymous user for the associated Magpie node (instance). " + "Only has an effect if the user_name parameter is not specified.", + example=True, + default=False + ) class NetworkRemoteUser_PATCH_RequestSchema(BaseRequestSchemaAPI): @@ -3761,6 +3768,14 @@ class NetworkRemoteUsers_GET_NotFoundResponseSchema(BaseResponseSchemaAPI): body = ErrorResponseBodySchema(code=HTTPNotFound.code, description=description) +class NetworkRemoteUser_GET_BodySchema(colander.MappingSchema): + user_name = colander.SchemaNode( + colander.String(), + description="Name of the associated user on this Magpie node (instance).", + example="userAlocal" + ) + + class NetworkRemoteUser_BodySchema(colander.MappingSchema): remote_user_name = colander.SchemaNode( colander.String(), @@ -3805,6 +3820,10 @@ class NetworkRemoteUsers_GET_OkResponseSchema(BaseResponseSchemaAPI): body = NetworkRemoteUsers_GET_OkResponseBodySchema(code=HTTPOk.code, description=description) +class NetworkRemoteUsers_GET_RequestSchema(BaseRequestSchemaAPI): + body = NetworkRemoteUser_GET_BodySchema() + + class NetworkRemoteUsers_POST_RequestSchema(BaseRequestSchemaAPI): body = NetworkRemoteUser_BodySchema() diff --git a/magpie/cli/purge_expired_network_tokens.py b/magpie/cli/purge_expired_network_tokens.py index fd4517cd3..b1a3d825a 100644 --- a/magpie/cli/purge_expired_network_tokens.py +++ b/magpie/cli/purge_expired_network_tokens.py @@ -77,11 +77,9 @@ def main(args=None, parser=None, namespace=None): else: db_session = get_db_session_from_config_ini(args.ini_config) deleted = models.NetworkToken.delete_expired(db_session) - anonymous_network_user_ids = [n.anonymous_user(db_session).id for n in - db_session.query(models.NetworkNode).all()] # clean up unused records in the database (no need to keep records associated with anonymous network users) (db_session.query(models.NetworkRemoteUser) - .filter(models.NetworkRemoteUser.user_id.in_(anonymous_network_user_ids)) + .filter(models.NetworkRemoteUser.user_id == None) # noqa: E711 # pylint: disable=singleton-comparison .filter(models.NetworkRemoteUser.network_token_id == None) # noqa: E711 # pylint: disable=singleton-comparison .delete()) try: diff --git a/magpie/models.py b/magpie/models.py index b62c61fba..a1c426d0e 100644 --- a/magpie/models.py +++ b/magpie/models.py @@ -1007,8 +1007,11 @@ class NetworkRemoteUser(BaseModel, Base): __tablename__ = "network_remote_users" id = sa.Column(sa.Integer(), primary_key=True, nullable=False, autoincrement=True) + # Note: a null user_id indicates that this NetworkRemoteUser is associated with the anonymous user for the + # associated network node. This is to ensure that the unique constraint defined below allows for multiple + # NetworkRemoteUsers to be associated with the anonymous users. user_id = sa.Column(sa.Integer, - sa.ForeignKey("users.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=False) + sa.ForeignKey("users.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True) user = relationship("User", foreign_keys=[user_id]) network_node_id = sa.Column(sa.Integer, @@ -1027,7 +1030,7 @@ class NetworkRemoteUser(BaseModel, Base): def as_dict(self): # type: () -> Dict[Str, Str] return { - "user_name": self.user.user_name, + "user_name": getattr(self.user, "user_name", self.network_node.anonymous_user_name()), "remote_user_name": self.name, "node_name": self.network_node.name } diff --git a/tests/interfaces.py b/tests/interfaces.py index e56e9a769..e03a87964 100644 --- a/tests/interfaces.py +++ b/tests/interfaces.py @@ -694,6 +694,42 @@ def test_Login_NetworkToken_Authorized(self): utils.check_val_true(session_json.get("authenticated")) utils.check_val_equal(session_json.get("user", {}).get("user_name"), self.test_user_name) + @runner.MAGPIE_TEST_NETWORK + @runner.MAGPIE_TEST_LOGIN + @utils.check_network_mode + def test_Login_NetworkToken_Authorized_Anonymous(self): + """ + Test logging in with a network token associated with an anonymous network user. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Login with a network token", "3.38.0", skip=True) + self.login_admin() + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True, + override_data={"node_name": self.test_node_name, + "remote_user_name": self.test_remote_user_name}) + utils.check_or_try_logout_user(self) + self.cookies = self.headers = None + token = utils.TestSetup.create_TestNetworkToken(self) + + headers = {"Accept": CONTENT_TYPE_JSON, "Authorization": "Bearer {}".format(token["token"])} + path = s.ProviderSigninAPI.path.format(provider_name=get_constant("MAGPIE_NETWORK_PROVIDER")) + + resp = utils.test_request(self, "GET", path, headers=headers, expect_errors=True, allow_redirects=False) + utils.check_response_basic_info(resp, 302, expected_method="GET") + + app_or_url = utils.get_app_or_url(self) + if isinstance(app_or_url, TestApp): + resp_cookies = app_or_url.cookies + else: + resp_cookies = resp.cookies + resp = utils.test_request(self, "GET", "/session", headers=self.json_headers, cookies=resp_cookies) + session_json = utils.get_json_body(resp) + utils.check_val_true(session_json.get("authenticated")) + anonymous_username = "{}{}".format(get_constant("MAGPIE_NETWORK_NAME_PREFIX"), self.test_node_name) + utils.check_val_equal(session_json.get("user", {}).get("user_name"), anonymous_username) + @runner.MAGPIE_TEST_NETWORK @runner.MAGPIE_TEST_LOGIN @utils.check_network_mode @@ -1061,6 +1097,7 @@ def test_GetUser_ReservedKeyword_Anonymous_Allowed(self): body = utils.check_response_basic_info(resp) info = utils.TestSetup.get_UserInfo(self, override_body=body) utils.check_val_equal(info["user_name"], anonymous) + utils.check_val_not_in("user_id", info) @runner.MAGPIE_TEST_USERS @runner.MAGPIE_TEST_LOGGED @@ -1089,6 +1126,7 @@ def test_GetUser_ReservedKeyword_LoggedUser_Allowed(self): body = utils.check_response_basic_info(resp) info = utils.TestSetup.get_UserInfo(self, override_body=body) utils.check_val_equal(info["user_name"], anonymous) + utils.check_val_not_in("user_id", info) @runner.MAGPIE_TEST_USERS @runner.MAGPIE_TEST_STATUS @@ -2573,6 +2611,111 @@ def test_PostNetworkLink_DifferentNode(self): headers=headers, expect_errors=True, allow_redirects=False) utils.check_response_basic_info(resp, expected_method="POST", expected_code=404) + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_GetNetworkRemoteUser(self): + """ + Test get remote user information for currently logged-in user. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Get remote user information", "3.38.0", skip=True) + + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True) + + headers, cookies = self.login_test_user() + resp = utils.test_request(self, "GET", + "/network/nodes/{}/remote_users/{}".format(self.test_node_name, + self.test_remote_user_name), + cookies=cookies, + headers=headers) + json_body = utils.check_response_basic_info(resp) + expected = {"user_name": self.test_user_name, + "remote_user_name": self.test_remote_user_name, + "node_name": self.test_node_name} + actual = {k: v for k, v in json_body.items() if k in expected} + utils.check_val_equal(actual, expected) + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_GetNetworkRemoteUser_DifferentUser(self): + """ + Test cannot get remote user information for a user other than the currently logged-in user. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Get remote user information", "3.38.0", skip=True) + + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True, override_user_name="test123", + override_password=self.test_user_name, override_email="test123@example.com") + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True, override_user_name="test123") + + headers, cookies = self.login_test_user() + resp = utils.test_request(self, "GET", + "/network/nodes/{}/remote_users/{}".format(self.test_node_name, + self.test_remote_user_name), + cookies=cookies, + headers=headers, + expect_errors=True) + utils.check_response_basic_info(resp, expected_code=403) + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_DeleteNetworkRemoteUser(self): + """ + Test delete a remote user. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Delete remote user", "3.38.0", skip=True) + + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True) + + headers, cookies = self.login_test_user() + utils.TestSetup.delete_TestNetworkRemoteUser(self, override_cookies=cookies, override_headers=headers) + resp = utils.test_request(self, "GET", + "/network/nodes/{}/remote_users/{}".format(self.test_node_name, + self.test_remote_user_name), + cookies=self.cookies, + headers=self.headers, + expect_errors=True) + utils.check_response_basic_info(resp, expected_code=404) + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_GetNetworkRemoteUsersCurrent(self): + """ + Test get all remote user information for currently logged-in user. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Get remote user information", "3.38.0", skip=True) + + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True) + + headers, cookies = self.login_test_user() + resp = utils.test_request(self, "GET", + "/network/remote_users/current", + cookies=cookies, + headers=headers) + json_body = utils.check_response_basic_info(resp) + expected = [{"user_name": self.test_user_name, + "remote_user_name": self.test_remote_user_name, + "node_name": self.test_node_name}] + utils.check_val_equal(json_body.get("remote_users"), expected) + @runner.MAGPIE_TEST_API @six.add_metaclass(ABCMeta) @@ -8391,6 +8534,377 @@ def test_DeleteNetworkNode_RemoveAssociatedRecords(self): headers=self.headers, expect_errors=True) utils.check_response_basic_info(resp, expected_code=404) + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_GetNetworkRemoteUsers(self): + """ + Test get all remote user information. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Get remote user information", "3.38.0", skip=True) + + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True) + + resp = utils.test_request(self, "GET", "/network/remote_users", cookies=self.cookies, + headers=self.headers) + json_body = utils.check_response_basic_info(resp) + utils.check_val_equal(json_body.get("remote_users", [{}]), [{"user_name": self.test_user_name, + "remote_user_name": self.test_remote_user_name, + "node_name": self.test_node_name}]) + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_GetNetworkRemoteUsers_OneUser(self): + """ + Test get remote user information for a single user. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Get remote user information", "3.38.0", skip=True) + + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True, override_user_name="test123", + override_password=self.test_user_name, override_email="test123@example.com") + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True, override_remote_user_name="test123", + override_user_name="test123") + + resp = utils.test_request(self, "GET", "/network/remote_users?user_name={}".format(self.test_user_name), + cookies=self.cookies, headers=self.headers) + json_body = utils.check_response_basic_info(resp) + utils.check_val_equal(json_body.get("remote_users", [{}]), [{"user_name": self.test_user_name, + "remote_user_name": self.test_remote_user_name, + "node_name": self.test_node_name}]) + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_GetNetworkRemoteUser(self): + """ + Test get remote user information for a single user. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Get remote user information", "3.38.0", skip=True) + + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True) + + resp = utils.test_request(self, "GET", + "/network/nodes/{}/remote_users/{}".format(self.test_node_name, + self.test_remote_user_name), + cookies=self.cookies, + headers=self.headers) + json_body = utils.check_response_basic_info(resp) + expected = {"user_name": self.test_user_name, + "remote_user_name": self.test_remote_user_name, + "node_name": self.test_node_name} + actual = {k: v for k, v in json_body.items() if k in expected} + utils.check_val_equal(actual, expected) + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_PostNetworkRemoteUser(self): + """ + Test create a remote user. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Create a remote user", "3.38.0", skip=True) + + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True) + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_PostNetworkRemoteUser_MissingRequiredParams(self): + """ + Test cannot create a remote user if required params are missing. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Create a remote user", "3.38.0", skip=True) + + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True) + data = { + "remote_user_name": self.test_remote_user_name, + "node_name": self.test_node_name + } + for key in data.keys(): + resp = utils.test_request(self, "POST", "/network/remote_users", json={**data, key: None}, + expect_errors=True, headers=self.headers, cookies=self.cookies) + utils.check_response_basic_info(resp, expected_method="POST", expected_code=400) + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_PostNetworkRemoteUser_MissingNode(self): + """ + Test cannot create a remote user when associated node is not found. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Create a remote user", "3.38.0", skip=True) + + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + + data = { + "remote_user_name": self.test_remote_user_name, + "user_name": self.test_user_name, + "node_name": self.test_node_name + } + resp = utils.test_request(self, "POST", "/network/remote_users", json=data, + expect_errors=True, headers=self.headers, cookies=self.cookies) + utils.check_response_basic_info(resp, expected_method="POST", expected_code=404) + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_PostNetworkRemoteUser_DuplicateRemoteUserName(self): + """ + Test cannot create a remote user with the same name for the same node. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Create a remote user", "3.38.0", skip=True) + + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True, override_user_name="test123", + override_password=self.test_user_name, override_email="test123@example.com") + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True) + + data = { + "remote_user_name": self.test_remote_user_name, + "user_name": "test123", + "node_name": self.test_node_name + } + resp = utils.test_request(self, "POST", "/network/remote_users", json=data, + expect_errors=True, headers=self.headers, cookies=self.cookies) + utils.check_response_basic_info(resp, expected_method="POST", expected_code=500) + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_PostNetworkRemoteUser_DuplicateUser(self): + """ + Test cannot create a remote user with the same associated user for the same node. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Create a remote user", "3.38.0", skip=True) + + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True) + + data = { + "remote_user_name": "some_other_name", + "user_name": self.test_user_name, + "node_name": self.test_node_name + } + resp = utils.test_request(self, "POST", "/network/remote_users", json=data, + expect_errors=True, headers=self.headers, cookies=self.cookies) + utils.check_response_basic_info(resp, expected_method="POST", expected_code=500) + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_PostNetworkRemoteUser_DifferentUserAndName(self): + """ + Test create a remote user with a different name and associated user for the same node. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Create a remote user", "3.38.0", skip=True) + + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True, override_user_name="test123", + override_password=self.test_user_name, override_email="test123@example.com") + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True) + + data = { + "remote_user_name": "test123", + "user_name": "test123", + "node_name": self.test_node_name + } + resp = utils.test_request(self, "POST", "/network/remote_users", json=data, headers=self.headers, + cookies=self.cookies) + utils.check_response_basic_info(resp, expected_code=201) + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_PostNetworkRemoteUser_DifferentNodeSameName(self): + """ + Test create a remote user with the same name and associated user for a different node. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Create a remote user", "3.38.0", skip=True) + + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True, override_name="test123") + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True, override_node_name="test123") + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_PostNetworkRemoteUser_NoUser(self): + """ + Test create a remote user with no user specified gets associated to the node's anonymous user. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Create a remote user", "3.38.0", skip=True) + + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + + data = { + "remote_user_name": self.test_remote_user_name, + "node_name": self.test_node_name + } + resp = utils.test_request(self, "POST", "/network/remote_users", json=data, headers=self.headers, + cookies=self.cookies, expect_errors=True) + utils.check_response_basic_info(resp, expected_code=201, expected_method="POST") + + self.extra_remote_user_names.add((self.test_remote_user_name, self.test_node_name)) + + resp = utils.test_request(self, "GET", + "/network/nodes/{}/remote_users/{}".format(self.test_node_name, + self.test_remote_user_name), + cookies=self.cookies, + headers=self.headers) + json_body = utils.check_response_basic_info(resp) + data["user_name"] = "{}{}".format(get_constant("MAGPIE_NETWORK_NAME_PREFIX"), self.test_node_name) + actual = {k: v for k, v in json_body.items() if k in data} + utils.check_val_equal(actual, data) + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_PostNetworkRemoteUser_MultipleAnonymous(self): + """ + Test create a remote user with no user specified gets associated to the node's anonymous user even when another + remote user is associated with the anonymous user for that node. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Create a remote user", "3.38.0", skip=True) + + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + anonymous_username = "{}{}".format(get_constant("MAGPIE_NETWORK_NAME_PREFIX"), self.test_node_name) + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True, + override_user_name=anonymous_username) + + data = { + "remote_user_name": "other_remote_user_name", + "user_name": anonymous_username, + "node_name": self.test_node_name + } + resp = utils.test_request(self, "POST", "/network/remote_users", json=data, headers=self.headers, + cookies=self.cookies, expect_errors=True) + utils.check_response_basic_info(resp, expected_code=201, expected_method="POST") + + self.extra_remote_user_names.add((self.test_remote_user_name, self.test_node_name)) + + resp = utils.test_request(self, "GET", + "/network/nodes/{}/remote_users/{}".format(self.test_node_name, + "other_remote_user_name"), + cookies=self.cookies, + headers=self.headers) + json_body = utils.check_response_basic_info(resp) + actual = {k: v for k, v in json_body.items() if k in data} + utils.check_val_equal(actual, data) + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_PatchNetworkRemoteUser(self): + """ + Test update a remote user. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Update a remote user", "3.38.0", skip=True) + + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True, override_user_name="test123_user_name", + override_password=self.test_user_name, override_email="test123@example.com") + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True, override_name="test123_node_name") + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True) + + data = { + "remote_user_name": "test123", + "user_name": "test123_user_name", + "node_name": "test123_node_name" + } + + resp = utils.test_request(self, "PATCH", + "/network/nodes/{}/remote_users/{}".format(self.test_node_name, + self.test_remote_user_name), + json=data, + headers=self.headers, + cookies=self.cookies) + utils.check_response_basic_info(resp, expected_code=200) + self.extra_remote_user_names.add(("test123_user_name", "test123_node_name")) + + resp = utils.test_request(self, "GET", + "/network/nodes/test123_node_name/remote_users/test123", + cookies=self.cookies, + headers=self.headers) + json_body = utils.check_response_basic_info(resp) + actual = {k: v for k, v in json_body.items() if k in data} + utils.check_val_equal(actual, data) + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_PatchNetworkRemoteUser_AssignAnonymous(self): + """ + Test update a remote user so that it is assigned with the anonymous user for the same node. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "Update a remote user", "3.38.0", skip=True) + + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True) + + resp = utils.test_request(self, "PATCH", + "/network/nodes/{}/remote_users/{}".format(self.test_node_name, + self.test_remote_user_name), + json={"assign_anonymous": True}, + headers=self.headers, + cookies=self.cookies) + utils.check_response_basic_info(resp, expected_code=200) + + resp = utils.test_request(self, "GET", + "/network/nodes/{}/remote_users/{}".format(self.test_node_name, + self.test_remote_user_name), + cookies=self.cookies, + headers=self.headers) + json_body = utils.check_response_basic_info(resp) + anonymous_username = "{}{}".format(get_constant("MAGPIE_NETWORK_NAME_PREFIX"), self.test_node_name) + utils.check_val_equal(json_body.get("user_name"), anonymous_username) + @runner.MAGPIE_TEST_UI @six.add_metaclass(ABCMeta) diff --git a/tests/utils.py b/tests/utils.py index 56c4dfc29..a615c2ee8 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -3089,18 +3089,22 @@ def create_TestNetworkRemoteUser(test_case, # type: AnyMag override_node_port=null, # type: Optional[int] override_headers=null, # type: Optional[HeadersType] override_cookies=null, # type: Optional[CookiesType] + override_data=null, # type: Optional[JSON] override_exist=False, # type: bool ): # type: (...) -> JSON """ Creates a Network Remote User. """ app_or_url = get_app_or_url(test_case) - data = { - "remote_user_name": (override_remote_user_name if override_remote_user_name is not null - else test_case.test_remote_user_name), - "user_name": override_user_name if override_user_name is not null else test_case.test_user_name, - "node_name": override_node_name if override_node_name is not null else test_case.test_node_name - } + if override_data is not null: + data = override_data + else: + data = { + "remote_user_name": (override_remote_user_name if override_remote_user_name is not null + else test_case.test_remote_user_name), + "user_name": override_user_name if override_user_name is not null else test_case.test_user_name, + "node_name": override_node_name if override_node_name is not null else test_case.test_node_name + } headers = override_headers if override_headers is not null else test_case.json_headers cookies = override_cookies if override_cookies is not null else test_case.cookies @@ -3143,7 +3147,7 @@ def delete_TestNetworkRemoteUser(test_case, # type: AnyMag app_or_url = get_app_or_url(test_case) headers = override_headers if override_headers is not null else test_case.json_headers cookies = override_cookies if override_cookies is not null else test_case.cookies - name = override_remote_user_name if override_remote_user_name is not null else test_case.test_user_name + name = override_remote_user_name if override_remote_user_name is not null else test_case.test_remote_user_name node_name = override_node_name if override_node_name is not null else test_case.test_node_name path = "/network/nodes/{}/remote_users/{}".format(node_name, name) From 9de7fee8a9649cee52b65a1e19e9f3a21782aa71 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Mon, 22 Apr 2024 14:10:56 -0400 Subject: [PATCH 34/55] don't allow explicit assign to anonymous users so that we don't get mismatch between node and anonymous user for another node --- .../management/network/remote_user/remote_user_views.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/magpie/api/management/network/remote_user/remote_user_views.py b/magpie/api/management/network/remote_user/remote_user_views.py index 9ef1d0211..20cc9d636 100644 --- a/magpie/api/management/network/remote_user/remote_user_views.py +++ b/magpie/api/management/network/remote_user/remote_user_views.py @@ -59,6 +59,10 @@ def post_network_remote_users_view(request): if kwargs["user_name"] is None: user = anonymous_user else: + anonymous_regex = protected_user_name_regex(include_admin=False) + ax.verify_param(kwargs["user_name"], not_matches=True, param_compare=anonymous_regex, param_name="user_name", + http_error=HTTPForbidden, + msg_on_fail="Cannot explicitly assign to an anonymous user.") user = ax.evaluate_call( lambda: request.db.query(models.User).filter(models.User.user_name == kwargs["user_name"]).one(), http_error=HTTPNotFound, @@ -103,6 +107,10 @@ def patch_network_remote_user_view(request): ) remote_user.network_node_id = node.id if kwargs["user_name"]: + anonymous_regex = protected_user_name_regex(include_admin=False) + ax.verify_param(kwargs["user_name"], not_matches=True, param_compare=anonymous_regex, param_name="user_name", + http_error=HTTPForbidden, + msg_on_fail="Cannot explicitly assign to an anonymous user.") user = ax.evaluate_call( lambda: request.db.query(models.User).filter(models.User.user_name == kwargs["user_name"]).one(), http_error=HTTPNotFound, From e56504e1e8a36aa4cd6fde2f38f6a7dbbfa46e71 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Tue, 23 Apr 2024 11:44:31 -0400 Subject: [PATCH 35/55] add cli and constants tests --- magpie/cli/purge_expired_network_tokens.py | 3 +- tests/test_constants.py | 124 ++++++++++++++++++++- tests/test_magpie_cli.py | 45 +++++++- 3 files changed, 168 insertions(+), 4 deletions(-) diff --git a/magpie/cli/purge_expired_network_tokens.py b/magpie/cli/purge_expired_network_tokens.py index b1a3d825a..63e49be9c 100644 --- a/magpie/cli/purge_expired_network_tokens.py +++ b/magpie/cli/purge_expired_network_tokens.py @@ -58,7 +58,7 @@ def get_login_session(magpie_url, username, password): def main(args=None, parser=None, namespace=None): - # type: (Optional[Sequence[Str]], Optional[argparse.ArgumentParser], Optional[argparse.Namespace]) -> None + # type: (Optional[Sequence[Str]], Optional[argparse.ArgumentParser], Optional[argparse.Namespace]) -> int if not parser: parser = make_parser() args = parser.parse_args(args=args, namespace=namespace) @@ -92,6 +92,7 @@ def main(args=None, parser=None, namespace=None): print_log("{} expired network tokens deleted".format(deleted), logger=LOGGER) else: print_log("No expired network tokens found", logger=LOGGER) + return 0 if __name__ == "__main__": diff --git a/tests/test_constants.py b/tests/test_constants.py index 97e28be08..3d581ce7a 100644 --- a/tests/test_constants.py +++ b/tests/test_constants.py @@ -1,8 +1,10 @@ +import re + import mock import pytest from magpie import constants as c -from tests import runner +from tests import runner, utils @runner.MAGPIE_TEST_UTILS @@ -130,3 +132,123 @@ def test_constant_protected_no_override(): with mock.patch.dict("os.environ", {const_name: "override-value"}): const = c.get_constant(const_name) assert const != "override-value" + + +@runner.MAGPIE_TEST_UTILS +class TestProtectedUserNameRegex: + def test_include_admin(self): + c.protected_user_name_regex.cache_clear() + assert re.search(c.protected_user_name_regex(), c.get_constant("MAGPIE_ADMIN_USER")) + + def test_include_anonymous(self): + c.protected_user_name_regex.cache_clear() + assert re.search(c.protected_user_name_regex(), c.get_constant("MAGPIE_ANONYMOUS_USER")) + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_include_network(self): + c.protected_user_name_regex.cache_clear() + assert re.search(c.protected_user_name_regex(), c.get_constant("MAGPIE_NETWORK_NAME_PREFIX") + "test") + + def test_extra_patterns(self): + assert re.search(c.protected_user_name_regex(additional_patterns=("test.*end",)), "test abc end") + + def test_no_admin(self): + assert re.search(c.protected_user_name_regex(include_admin=False), c.get_constant("MAGPIE_ADMIN_USER")) is None + + def test_no_anonymous(self): + assert re.search(c.protected_user_name_regex(include_anonymous=False), + c.get_constant("MAGPIE_ANONYMOUS_USER")) is None + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_no_network_network_mode_on(self): + assert re.search(c.protected_user_name_regex(include_network=False), + c.get_constant("MAGPIE_NETWORK_NAME_PREFIX") + "test") is None + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode(enable=False) + def test_include_network(self): + c.protected_user_name_regex.cache_clear() + assert re.search(c.protected_user_name_regex(), + c.get_constant("MAGPIE_NETWORK_NAME_PREFIX") + "test") is None + + +@runner.MAGPIE_TEST_UTILS +class TestProtectedUserEmailRegex: + def test_include_admin(self): + c.protected_user_email_regex.cache_clear() + assert re.search(c.protected_user_email_regex(), c.get_constant("MAGPIE_ADMIN_EMAIL")) + + def test_include_anonymous(self): + c.protected_user_email_regex.cache_clear() + assert re.search(c.protected_user_email_regex(), c.get_constant("MAGPIE_ANONYMOUS_EMAIL")) + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_include_network(self): + c.protected_user_email_regex.cache_clear() + assert re.search(c.protected_user_email_regex(), + c.get_constant("MAGPIE_NETWORK_ANONYMOUS_EMAIL_FORMAT").format("test")) + + def test_extra_patterns(self): + assert re.search(c.protected_user_email_regex(additional_patterns=("test.*end",)), "test abc end") + + def test_no_admin(self): + assert re.search(c.protected_user_email_regex(include_admin=False), + c.get_constant("MAGPIE_ADMIN_EMAIL")) is None + + def test_no_anonymous(self): + assert re.search(c.protected_user_email_regex(include_anonymous=False), + c.get_constant("MAGPIE_ANONYMOUS_EMAIL")) is None + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_no_network_network_mode_on(self): + assert re.search(c.protected_user_email_regex(include_network=False), + c.get_constant("MAGPIE_NETWORK_ANONYMOUS_EMAIL_FORMAT").format("test")) is None + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode(enable=False) + def test_include_network(self): + c.protected_user_email_regex.cache_clear() + assert re.search(c.protected_user_email_regex(), + c.get_constant("MAGPIE_NETWORK_ANONYMOUS_EMAIL_FORMAT").format("test")) is None + + +@runner.MAGPIE_TEST_UTILS +class TestProtectedGroupNameRegex: + def test_include_admin(self): + c.protected_group_name_regex.cache_clear() + assert re.search(c.protected_group_name_regex(), c.get_constant("MAGPIE_ADMIN_GROUP")) + + def test_include_anonymous(self): + c.protected_group_name_regex.cache_clear() + assert re.search(c.protected_group_name_regex(), c.get_constant("MAGPIE_ANONYMOUS_GROUP")) + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_include_network(self): + c.protected_group_name_regex.cache_clear() + assert re.search(c.protected_group_name_regex(), c.get_constant("MAGPIE_NETWORK_NAME_PREFIX") + "test") + + def test_no_admin(self): + assert re.search(c.protected_group_name_regex(include_admin=False), c.get_constant("MAGPIE_ADMIN_GROUP")) is None + + def test_no_anonymous(self): + assert re.search(c.protected_group_name_regex(include_anonymous=False), + c.get_constant("MAGPIE_ANONYMOUS_GROUP")) is None + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_no_network_network_mode_on(self): + assert re.search(c.protected_group_name_regex(include_network=False), + c.get_constant("MAGPIE_NETWORK_NAME_PREFIX") + "test") is None + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode(enable=False) + def test_include_network(self): + c.protected_group_name_regex.cache_clear() + assert re.search(c.protected_group_name_regex(), + c.get_constant("MAGPIE_NETWORK_NAME_PREFIX") + "test") is None + diff --git a/tests/test_magpie_cli.py b/tests/test_magpie_cli.py index 11a313cd1..8dd49fe3c 100644 --- a/tests/test_magpie_cli.py +++ b/tests/test_magpie_cli.py @@ -14,9 +14,10 @@ import tempfile import mock +import requests import six -from magpie.cli import batch_update_users, magpie_helper_cli +from magpie.cli import batch_update_users, magpie_helper_cli, purge_expired_network_tokens from magpie.constants import get_constant from tests import runner, utils @@ -31,7 +32,8 @@ "register_providers", "run_db_migration", "send_email", - "sync_resources" + "sync_resources", + "purge_expired_network_tokens" ] @@ -266,3 +268,42 @@ def test_magpie_sync_resources_help_directly(): out_lines = run_and_get_output("magpie_sync_resources --help") assert "usage: magpie_sync_resources" in out_lines[0] assert "Synchronize local and remote resources based on Magpie Service sync-type" in out_lines[1] + + +@runner.MAGPIE_TEST_CLI +@runner.MAGPIE_TEST_LOCAL +@runner.MAGPIE_TEST_NETWORK +def test_purge_expired_network_tokens_help_via_magpie_helper(): + out_lines = run_and_get_output("magpie_helper purge_expired_network_tokens --help") + assert "usage: magpie_helper purge_expired_network_tokens" in out_lines[0] + assert "Delete all expired network tokens." in out_lines[1] + + +@runner.MAGPIE_TEST_CLI +@runner.MAGPIE_TEST_LOCAL +@runner.MAGPIE_TEST_NETWORK +def test_purge_expired_network_tokens_help_directly(): + out_lines = run_and_get_output("magpie_purge_expired_network_tokens --help") + assert "usage: magpie_purge_expired_network_tokens" in out_lines[0] + assert "Delete all expired network tokens." in out_lines[1] + + +def test_purge_expired_network_tokens(): + test_url = "http://localhost" + test_username = "test_username" + test_password = "qwertyqwerty" + + def mocked_request(*args, **_kwargs): + method, url, *_ = args + response = requests.Response() + response.status_code = 200 + if method == "DELETE": + response._content = '{"deleted": 101}'.encode() + return response + + with mock.patch("requests.Session.request", side_effect=mocked_request) as session_mock: + cmd = ["api", test_url, test_username, test_password] + assert purge_expired_network_tokens.main(cmd) == 0, "failed execution due to invalid arguments" + session_mock.assert_any_call("POST", "{}/signin".format(test_url), data=None, + json={'user_name': 'test_username', 'password': 'qwertyqwerty'}) + session_mock.assert_any_call("DELETE", "{}/network/tokens?expired_only=true".format(test_url)) From aa8f43dab9e2d5079ce19304306b0541dfd2eec9 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Tue, 23 Apr 2024 22:45:58 -0400 Subject: [PATCH 36/55] add ui tests --- magpie/ui/management/templates/edit_user.mako | 2 +- magpie/ui/management/views.py | 4 +- .../ui/user/templates/edit_current_user.mako | 2 +- magpie/ui/user/views.py | 3 +- tests/interfaces.py | 250 ++++++++++++++++++ tests/test_magpie_ui.py | 4 + tests/utils.py | 12 +- 7 files changed, 267 insertions(+), 10 deletions(-) diff --git a/magpie/ui/management/templates/edit_user.mako b/magpie/ui/management/templates/edit_user.mako index fd848b3f2..2bd382159 100644 --- a/magpie/ui/management/templates/edit_user.mako +++ b/magpie/ui/management/templates/edit_user.mako @@ -384,7 +384,7 @@ ${membership_alerts.edit_membership_alerts()} -%if network_nodes and user_name not in MAGPIE_FIXED_USERS_REFS: +%if network_enabled and user_name not in MAGPIE_FIXED_USERS_REFS:

Network Account Links

diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py index 7ee81d99f..df0b80bb2 100644 --- a/magpie/ui/management/views.py +++ b/magpie/ui/management/views.py @@ -166,7 +166,9 @@ def edit_user(self): request_uri = "{}?user_name={}".format(schemas.NetworkRemoteUsersAPI.path, user_name) resp = request_api(self.request, request_uri, "GET") check_response(resp) - user_info["network_nodes"] = [(n["node_name"], n["remote_user_name"]) for n in get_json(resp)["nodes"]] + user_info["network_enabled"] = True + user_info["network_nodes"] = [(n["node_name"], n["remote_user_name"]) for n in + get_json(resp)["remote_users"]] user_info["network_routes"] = {"create": schemas.NetworkRemoteUsersAPI.name, "delete": schemas.NetworkRemoteUserAPI.name} diff --git a/magpie/ui/user/templates/edit_current_user.mako b/magpie/ui/user/templates/edit_current_user.mako index 11247d51f..9f9582cae 100644 --- a/magpie/ui/user/templates/edit_current_user.mako +++ b/magpie/ui/user/templates/edit_current_user.mako @@ -273,7 +273,7 @@ ${membership_alerts.edit_membership_alerts()}
-%if network_nodes: +%if network_enabled:

Network Account Links

diff --git a/magpie/ui/user/views.py b/magpie/ui/user/views.py index ef04d4524..0290a85a3 100644 --- a/magpie/ui/user/views.py +++ b/magpie/ui/user/views.py @@ -96,8 +96,9 @@ def edit_current_user(self): nodes = {node["name"]: None for node in get_json(node_resp)["nodes"]} user_resp = request_api(self.request, schemas.NetworkRemoteUsersCurrentAPI.path, "GET") check_response(user_resp) - for info in get_json(user_resp)["nodes"]: + for info in get_json(user_resp)["remote_users"]: nodes[info["node_name"]] = info["remote_user_name"] + user_info["network_enabled"] = True user_info["network_nodes"] = list(nodes.items()) user_info["network_routes"] = {"create": schemas.NetworkNodeLinkAPI.name, "delete": schemas.NetworkRemoteUserAPI.name} diff --git a/tests/interfaces.py b/tests/interfaces.py index e03a87964..f300d4846 100644 --- a/tests/interfaces.py +++ b/tests/interfaces.py @@ -9168,6 +9168,192 @@ def test_UserAccount_DeleteSelf(self): self.login_admin() utils.TestSetup.check_NonExistingTestUser(self, override_user_name=other_user) + @runner.MAGPIE_TEST_USERS + @runner.MAGPIE_TEST_LOGGED + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_UserAccount_NetworkInfo_WithRemoteUser(self): + """ + Test that remote user information is visible in a table + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "View network info.", "3.38.0", skip=True) + + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True) + + self.login_test_user() + path = "/ui/users/{}".format(get_constant("MAGPIE_LOGGED_USER")) + resp = utils.TestSetup.check_UpStatus(self, method="GET", path=path) + + nodes_table = utils.find_html_body_contents(resp, [{"class": ["content"]}, {"id": "network_node_list"}]) + utils.check_val_true(bool(nodes_table)) + + node_info = nodes_table.find_all("tr", recursive=True)[-1].find_all("td") + utils.check_val_equal(node_info[0].text, self.test_node_name) + utils.check_val_equal(node_info[1].text, self.test_remote_user_name) + utils.check_val_equal(node_info[2].find("input").attrs.get("name"), + "delete_node_link_{}".format(self.test_node_name)) + + @runner.MAGPIE_TEST_USERS + @runner.MAGPIE_TEST_LOGGED + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_UserAccount_NetworkInfo_WithoutRemoteUser(self): + """ + Test that node information is visible in a table even if there is no remote user for that node + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "View network info.", "3.38.0", skip=True) + + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + + self.login_test_user() + path = "/ui/users/{}".format(get_constant("MAGPIE_LOGGED_USER")) + resp = utils.TestSetup.check_UpStatus(self, method="GET", path=path) + + nodes_table = utils.find_html_body_contents(resp, [{"class": ["content"]}, {"id": "network_node_list"}]) + utils.check_val_true(bool(nodes_table)) + + node_info = nodes_table.find_all("tr", recursive=True)[-1].find_all("td") + utils.check_val_equal(node_info[0].text, self.test_node_name) + utils.check_val_equal(node_info[1].text.strip(), "") + utils.check_val_equal(node_info[2].find("input").attrs.get("name"), + "create_node_link_{}".format(self.test_node_name)) + + @runner.MAGPIE_TEST_USERS + @runner.MAGPIE_TEST_LOGGED + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode(enable=False) + def test_UserAccount_NetworkInfo_NetworkModeOff(self): + """ + Test that node information is not visible when network mode is off. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "View network info.", "3.38.0", skip=True) + + self.login_test_user() + path = "/ui/users/{}".format(get_constant("MAGPIE_LOGGED_USER")) + resp = utils.TestSetup.check_UpStatus(self, method="GET", path=path) + + nodes_table = utils.find_html_body_contents(resp, [{"class": ["content"]}, {"id": "network_node_list"}]) + utils.check_val_false(bool(nodes_table)) + + @runner.MAGPIE_TEST_LOGGED + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_UserNetworkAuthorization(self): + """ + Test that authorization view is shown if there is a valid token in the request. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "View authorization page.", "3.38.0", skip=True) + + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + + with utils.TestSetup.valid_jwt(self, override_jwt_claims={"user_name": "test123"}) as token: + self.login_test_user() + path = "/ui/network/authorize?token={}&response_type=id_token&redirect_uri={}".format( + token, json.loads(self.test_redirect_uris)[0] + ) + resp = utils.TestSetup.check_UpStatus(self, method="GET", path=path, expected_title="Magpie") + node_form = utils.find_html_body_contents(resp, [{"class": ["content"]}, {"id": "authorize_node_link"}]) + utils.check_val_true(bool(node_form)) + utils.check_val_is_in( + "Magpie is requesting permission to link your account with the test123 user's account", + node_form.text + ) + utils.check_val_is_in( + 'on the Magpie instance named "{}"'.format(self.test_node_name), + node_form.text + ) + token_input = node_form.find_all("input", recursive=True)[0] + token = token_input.attrs.get("value") + utils.check_val_true(bool(token)) + claims = jwt.decode(token, options={"verify_signature": False}) + expected = { + "requesting_user_name": "test123", + "user_name": self.test_user_name, + "iss": get_constant("MAGPIE_NETWORK_INSTANCE_NAME"), + "aud": self.test_node_name + } + expiry = claims.pop("exp") + utils.check_val_type(expiry, int) + utils.check_val_equal(expected, claims) + + @runner.MAGPIE_TEST_LOGGED + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_UserNetworkAuthorization_BadResponseType(self): + """ + Test that authorization view is not shown if the response type is invalid. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "View authorization page.", "3.38.0", skip=True) + + with utils.TestSetup.valid_jwt(self, override_jwt_claims={"user_name": "test123"}) as token: + headers, cookies = self.login_test_user() + path = "/ui/network/authorize?token={}&response_type=other&redirect_uri={}".format( + token, json.loads(self.test_redirect_uris)[0] + ) + resp = utils.test_request(self, "GET", path, headers=headers, cookies=cookies, expect_errors=True) + utils.check_val_equal(resp.status_code, 400) + + @runner.MAGPIE_TEST_LOGGED + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_UserNetworkAuthorization_NoToken(self): + """ + Test that authorization view is not shown if the token is missing. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "View authorization page.", "3.38.0", skip=True) + + headers, cookies = self.login_test_user() + path = "/ui/network/authorize?response_type=id_token&redirect_uri={}".format( + json.loads(self.test_redirect_uris)[0]) + resp = utils.test_request(self, "GET", path, headers=headers, cookies=cookies, expect_errors=True) + utils.check_val_equal(resp.status_code, 400) + + @runner.MAGPIE_TEST_LOGGED + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_UserNetworkAuthorization_InvalidToken(self): + """ + Test that authorization view is not shown if the token is missing. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "View authorization page.", "3.38.0", skip=True) + + headers, cookies = self.login_test_user() + path = "/ui/network/authorize?token=123123&response_type=id_token&redirect_uri={}".format( + json.loads(self.test_redirect_uris)[0]) + resp = utils.test_request(self, "GET", path, headers=headers, cookies=cookies, expect_errors=True) + utils.check_val_equal(resp.status_code, 400) + + @runner.MAGPIE_TEST_LOGGED + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_UserNetworkAuthorization_BadRedirectURI(self): + """ + Test that authorization view is not shown if the response type is invalid. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "View authorization page.", "3.38.0", skip=True) + + with utils.TestSetup.valid_jwt(self, override_jwt_claims={"user_name": "test123"}) as token: + headers, cookies = self.login_test_user() + path = "/ui/network/authorize?token={}&response_type=id_token&redirect_uri=blah".format(token) + resp = utils.test_request(self, "GET", path, headers=headers, cookies=cookies, expect_errors=True) + utils.check_val_equal(resp.status_code, 400) @runner.MAGPIE_TEST_UI @six.add_metaclass(ABCMeta) @@ -9249,6 +9435,70 @@ def test_EditUser_PageStatus(self): path = "/ui/users/{}/default".format(self.test_user_name) utils.TestSetup.check_UpStatus(self, method="GET", path=path, expected_type=CONTENT_TYPE_HTML) + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_EditUser_NetworkInfo_WithRemoteUser(self): + """ + Test that remote user information is visible in a table + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "View network info.", "3.38.0", skip=True) + + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True) + path = "/ui/users/{}/default".format(self.test_user_name) + resp = utils.TestSetup.check_UpStatus(self, method="GET", path=path, expected_type=CONTENT_TYPE_HTML) + + nodes_table = utils.find_html_body_contents(resp, [{"class": ["content"]}, {"id": "network_node_list"}]) + utils.check_val_true(bool(nodes_table)) + + node_info = nodes_table.find_all("tr", recursive=True)[-1].find_all("td") + utils.check_val_equal(node_info[0].text, self.test_node_name) + utils.check_val_equal(node_info[1].text, self.test_remote_user_name) + utils.check_val_equal(node_info[2].find("input").attrs.get("name"), + "delete_node_link_{}".format(self.test_node_name)) + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode + def test_EditUser_NetworkInfo_WithoutRemoteUser(self): + """ + Test that remote user information is not visible when there are no remote users for the given user + but that the table is still visible. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "View network info.", "3.38.0", skip=True) + + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + utils.TestSetup.create_TestNetworkNode(self, override_exist=True) + path = "/ui/users/{}/default".format(self.test_user_name) + resp = utils.TestSetup.check_UpStatus(self, method="GET", path=path, expected_type=CONTENT_TYPE_HTML) + + nodes_table = utils.find_html_body_contents(resp, [{"class": ["content"]}, {"id": "network_node_list"}]) + utils.check_val_true(bool(nodes_table)) + utils.check_val_equal(len(nodes_table.find_all("tr", recursive=True)), 1) # header row only + + @runner.MAGPIE_TEST_NETWORK + @utils.check_network_mode(enable=False) + def test_EditUser_NetworkInfo_NetworkModeOff(self): + """ + Test that remote user information and table is not visible when network mode is off. + + .. versionadded:: 3.38 + """ + utils.warn_version(self, "View network info.", "3.38.0", skip=True) + utils.TestSetup.create_TestGroup(self, override_exist=True) + utils.TestSetup.create_TestUser(self, override_exist=True) + path = "/ui/users/{}/default".format(self.test_user_name) + resp = utils.TestSetup.check_UpStatus(self, method="GET", path=path, expected_type=CONTENT_TYPE_HTML) + + nodes_table = utils.find_html_body_contents(resp, [{"class": ["content"]}, {"id": "network_node_list"}]) + utils.check_val_false(bool(nodes_table)) + @runner.MAGPIE_TEST_USERS def test_EditUser_DisplayedInheritedPermissionPriority(self): """ diff --git a/tests/test_magpie_ui.py b/tests/test_magpie_ui.py index 0dc5a44b0..0e54cf2e3 100644 --- a/tests/test_magpie_ui.py +++ b/tests/test_magpie_ui.py @@ -85,6 +85,7 @@ def setUpClass(cls): raise_missing=False, raise_not_set=False) cls.test_group_name = get_constant("MAGPIE_TEST_GROUP", default_value="unittest-user-auth_ui-group-local", raise_missing=False, raise_not_set=False) + cls.setup_network_attrs() @runner.MAGPIE_TEST_UI @@ -125,6 +126,7 @@ def setUpClass(cls): cls.test_service_parent_resource_name = "magpie-unittest-ui-tree-parent" cls.test_service_child_resource_type = Route.resource_type_name cls.test_service_child_resource_name = "magpie-unittest-ui-tree-child" + cls.setup_network_attrs() @runner.MAGPIE_TEST_STATUS @runner.MAGPIE_TEST_FUNCTIONAL @@ -826,6 +828,7 @@ def setUpClass(cls): raise_missing=False, raise_not_set=False) cls.test_group_name = get_constant("MAGPIE_TEST_GROUP", default_value="unittest-user-auth_ui-group-remote", raise_missing=False, raise_not_set=False) + cls.setup_network_attrs() @runner.MAGPIE_TEST_UI @@ -855,3 +858,4 @@ def setUpClass(cls): raise_missing=False, raise_not_set=False) cls.test_service_type = ServiceAPI.service_type cls.test_service_name = "magpie-unittest-ui-admin-remote-service" + cls.setup_network_attrs() diff --git a/tests/utils.py b/tests/utils.py index a615c2ee8..709ca92d4 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1779,12 +1779,12 @@ def find_html_body_contents(response_or_body, html_search=None): # ignore 'dont care' items like comments hasattr(item, "attrs") and # filter by class names, any one matched if element as any - (len(item.attrs.get("class", [])) and - any(str(css_cls) in search_element.get("class", []) for css_cls in item.attrs.get("class", []))) or - # filter by html element name - (search_element.get("name", "") != "" and str(search_element.get("name")) == item.name) or - # filter by html element id - (search_element.get("id", "") != "" and str(search_element.get("id")) == item.attrs.get("id", "")) + ((len(item.attrs.get("class", [])) and + any(str(css_cls) in search_element.get("class", []) for css_cls in item.attrs.get("class", []))) or + # filter by html element name + (search_element.get("name", "") != "" and str(search_element.get("name")) == item.name) or + # filter by html element id + (search_element.get("id", "") != "" and str(search_element.get("id")) == item.attrs.get("id", ""))) ] if i + 1 != len(html_search): elem_idx = search_element.get("index") From d4b01b1e2206e6dab0f03dfc0a6166e382512d98 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Tue, 23 Apr 2024 23:49:03 -0400 Subject: [PATCH 37/55] style fixes --- Makefile | 14 ++++++++ magpie/api/login/login.py | 2 +- magpie/api/management/network/__init__.py | 7 ++-- .../api/management/network/network_utils.py | 4 +-- .../api/management/network/network_views.py | 3 +- .../network/node/network_node_utils.py | 5 ++- .../network/node/network_node_views.py | 3 +- magpie/api/schemas.py | 2 +- magpie/utils.py | 6 ++-- requirements-dev.txt | 2 +- requirements.txt | 2 +- tests/interfaces.py | 34 +++++++++---------- tests/test_constants.py | 10 +++--- tests/test_magpie_api.py | 1 - tests/test_magpie_cli.py | 6 ++-- tests/utils.py | 15 ++++---- 16 files changed, 62 insertions(+), 54 deletions(-) diff --git a/Makefile b/Makefile index cc828aacf..4fbec7327 100644 --- a/Makefile +++ b/Makefile @@ -540,6 +540,13 @@ check-security-only: check-security-code-only check-security-deps-only ## run s # 42194: https://github.com/kvesteri/sqlalchemy-utils/issues/166 # not fixed since 2015 # 51668: https://github.com/sqlalchemy/sqlalchemy/pull/8563 # still in beta + major version change sqlalchemy 2.0.0b1 # 51021: This is patched in jwcrypto>=1.4.0 but that version is not available for python version < 3.6 +# 66713: This is patched in jwcrypto>=1.5.1 but that version is not available for python version < 3.6 +# 63154: This is patched in jwcrypto>=1.5.1 but that version is not available for python version < 3.6 +# 64484: This is patched in bandit>=1.7.8 but that version is not available for python version < 3.8 +# 43407: This is an advisory that support for python < 3.6 is no longer supported in the next version +# 43452: This is a duplicate of 43407 +# 43450: This is a duplicate of 43407 +# 43451: This is a duplicate of 43407 .PHONY: check-security-deps-only check-security-deps-only: mkdir-reports ## run security checks on package dependencies @echo "Running security checks of dependencies..." @@ -553,6 +560,13 @@ check-security-deps-only: mkdir-reports ## run security checks on package depen -i 42194 \ -i 51668 \ -i 51021 \ + -i 66713 \ + -i 63154 \ + -i 64484 \ + -i 43407 \ + -i 43452 \ + -i 43450 \ + -i 43451 \ 1> >(tee "$(REPORTS_DIR)/check-security-deps.txt")' .PHONY: check-security-code-only diff --git a/magpie/api/login/login.py b/magpie/api/login/login.py index 18932ff70..47a866b4b 100644 --- a/magpie/api/login/login.py +++ b/magpie/api/login/login.py @@ -34,7 +34,7 @@ from magpie.api import schemas as s from magpie.api.management.user.user_formats import format_user from magpie.api.management.user.user_utils import create_user -from magpie.constants import get_constant, protected_user_email_regex, protected_user_name_regex, network_enabled +from magpie.constants import get_constant, network_enabled, protected_user_email_regex, protected_user_name_regex from magpie.security import authomatic_setup, get_providers from magpie.utils import ( CONTENT_TYPE_JSON, diff --git a/magpie/api/management/network/__init__.py b/magpie/api/management/network/__init__.py index 2565b9165..000c95069 100644 --- a/magpie/api/management/network/__init__.py +++ b/magpie/api/management/network/__init__.py @@ -4,15 +4,16 @@ def includeme(config): - from magpie.api import schemas as s - from magpie import utils from pyramid.exceptions import ConfigurationError + from magpie import utils + from magpie.api import schemas as s + LOGGER.info("Adding API network ...") try: utils.check_network_configured(config) except ConfigurationError as exc: - LOGGER.error("API network failed with following configuration error: {}".format(exc)) + LOGGER.error("API network failed with following configuration error: %s", exc) raise config.add_route(**s.service_api_route_info(s.NetworkTokenAPI)) config.add_route(**s.service_api_route_info(s.NetworkJSONWebKeySetAPI)) diff --git a/magpie/api/management/network/network_utils.py b/magpie/api/management/network/network_utils.py index 06687c067..84f1c5771 100644 --- a/magpie/api/management/network/network_utils.py +++ b/magpie/api/management/network/network_utils.py @@ -92,7 +92,7 @@ def create_private_key(filename, password=None, settings_container=None): if os.path.realpath(pem_file) == os.path.realpath(filename): password = pem_password - LOGGER.info("Creating a valid PEM file at '{}'.".format(filename)) + LOGGER.info("Creating a valid PEM file at '%s'.", filename) private_key = rsa.generate_private_key(public_exponent=65537, key_size=2048) if password: encryption_algorithm = serialization.BestAvailableEncryption(password) @@ -100,7 +100,7 @@ def create_private_key(filename, password=None, settings_container=None): encryption_algorithm = serialization.NoEncryption() private_bytes = private_key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.TraditionalOpenSSL, encryption_algorithm) - with open(filename, mode='wb') as f: + with open(filename, mode="wb") as f: f.write(private_bytes) diff --git a/magpie/api/management/network/network_views.py b/magpie/api/management/network/network_views.py index bc7e5c289..9caf29a26 100644 --- a/magpie/api/management/network/network_views.py +++ b/magpie/api/management/network/network_views.py @@ -36,7 +36,7 @@ def post_network_token_view(request): @view_config(route_name=s.NetworkTokenAPI.name, request_method="DELETE", decorator=check_network_mode_enabled, permission=NO_PERMISSION_REQUIRED) def delete_network_token_view(request): - node, network_remote_user = get_network_models_from_request_token(request) + _, network_remote_user = get_network_models_from_request_token(request) if network_remote_user and network_remote_user.network_token: request.db.delete(network_remote_user.network_token) if network_remote_user.user is None and sqlalchemy.inspect(network_remote_user).persistent: @@ -53,7 +53,6 @@ def delete_network_tokens_view(request): deleted = models.NetworkToken.delete_expired(request.db) else: deleted = request.db.query(NetworkToken).delete() - anonymous_network_user_ids = [n.anonymous_user(request.db).id for n in request.db.query(models.NetworkNode).all()] # clean up unused records in the database (no need to keep records associated with anonymous network users) (request.db.query(models.NetworkRemoteUser) .filter(models.NetworkRemoteUser.user_id == None) # noqa: E711 # pylint: disable=singleton-comparison diff --git a/magpie/api/management/network/node/network_node_utils.py b/magpie/api/management/network/node/network_node_utils.py index e01017419..81bdf1fd2 100644 --- a/magpie/api/management/network/node/network_node_utils.py +++ b/magpie/api/management/network/node/network_node_utils.py @@ -14,7 +14,7 @@ if TYPE_CHECKING: from pyramid.request import Request - from magpie.typedefs import AnyRequestType, JSON, List, Optional, Session, Str + from magpie.typedefs import JSON, AnyRequestType, List, Optional, Session, Str NAME_REGEX = r"^[\w-]+$" @@ -121,5 +121,4 @@ def load_redirect_uris(uris, request): fallback=lambda: request.db.rollback(), msg_on_fail=s.NetworkNodes_CheckInfo_RedirectURIsValue_BadRequestResponseSchema.description ) - else: - return uris + return uris diff --git a/magpie/api/management/network/node/network_node_views.py b/magpie/api/management/network/node/network_node_views.py index 660c2c0fa..24a6237fc 100644 --- a/magpie/api/management/network/node/network_node_views.py +++ b/magpie/api/management/network/node/network_node_views.py @@ -22,7 +22,8 @@ check_network_node_info, create_associated_user_groups, delete_network_node, - update_associated_user_groups, load_redirect_uris + load_redirect_uris, + update_associated_user_groups ) from magpie.api.requests import check_network_mode_enabled from magpie.constants import get_constant diff --git a/magpie/api/schemas.py b/magpie/api/schemas.py index 0ec62129c..af842c247 100644 --- a/magpie/api/schemas.py +++ b/magpie/api/schemas.py @@ -17,8 +17,8 @@ HTTPInternalServerError, HTTPMethodNotAllowed, HTTPNotAcceptable, - HTTPNotImplemented, HTTPNotFound, + HTTPNotImplemented, HTTPOk, HTTPUnauthorized, HTTPUnprocessableEntity diff --git a/magpie/utils.py b/magpie/utils.py index dcaf7f427..34dc02d46 100644 --- a/magpie/utils.py +++ b/magpie/utils.py @@ -1194,7 +1194,7 @@ def check_network_configured(settings_container=None): raise ConfigurationError("MAGPIE_NETWORK_INSTANCE_NAME is required when network mode is enabled.") from exc # import here to avoid a potential cyclical import - from magpie.api.management.network.network_utils import jwks, pem_files, create_private_key + from magpie.api.management.network.network_utils import create_private_key, jwks, pem_files try: jwks(settings_container=settings_container) except Exception as exc: @@ -1202,8 +1202,8 @@ def check_network_configured(settings_container=None): get_constant("MAGPIE_NETWORK_CREATE_MISSING_PEM_FILE", settings_container=settings_container)) pem_files_ = pem_files(settings_container) if isinstance(exc, FileNotFoundError) and create_missing and len(pem_files_) == 1: - LOGGER.warning("No network PEM files found") - create_private_key(pem_files_[0], settings_container=settings_container) + LOGGER.warning("No network PEM files found") + create_private_key(pem_files_[0], settings_container=settings_container) else: msg = ("Error occurred when loading PEM keys which are required when network mode is enabled. " "Check that the MAGPIE_NETWORK_PEM_FILES and MAGPIE_NETWORK_PEM_PASSWORDS are set properly. " diff --git a/requirements-dev.txt b/requirements-dev.txt index 8ee56618f..2d655b575 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,7 +5,7 @@ autopep8>=1.5.4; python_version >= "3.6" backports.tempfile; python_version < "3" bandit==1.7.1; python_version < "3.7" # pyup: ignore bandit==1.7.5; python_version == "3.7" # pyup: ignore -bandit==1.7.7; python_version >= "3.8" +bandit==1.7.8; python_version >= "3.8" bump2version==1.0.1 codacy-coverage>=1.3.11 coverage==5.5; python_version < "3" # pyup: ignore diff --git a/requirements.txt b/requirements.txt index 31f6c7f7d..55bcc929c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,7 +27,7 @@ gunicorn>=20; python_version >= "3" humanize jsonschema<4; python_version < "3.6" jsonschema>=4; python_version >= "3.6" -jwcrypto==1.5.0; python_version >= "3.6" +jwcrypto==1.5.6; python_version >= "3.6" jwcrypto==0.8; python_version < "3.6" # pyup: ignore lxml>=3.7 mako # controlled by pyramid_mako diff --git a/tests/interfaces.py b/tests/interfaces.py index f300d4846..ea4e2e48b 100644 --- a/tests/interfaces.py +++ b/tests/interfaces.py @@ -61,10 +61,10 @@ # pylint: disable=W0611,unused-import from typing import Dict, List, Optional, Set, Tuple, Union + from pytest_httpserver import HTTPServer from sqlalchemy.orm.session import Session from magpie.typedefs import JSON, CookiesType, HeadersType, PermissionDict, SettingsType, Str - from pytest_httpserver import HTTPServer @six.add_metaclass(ABCMeta) @@ -284,13 +284,13 @@ def setup_network_attrs(cls): cls.test_node_name = "node2" cls.test_remote_user_name = "remote_user_1" cls.test_node_host = get_constant("MAGPIE_TEST_REMOTE_NODE_SERVER_HOST", default_value="localhost", - raise_missing=False, raise_not_set=False) + raise_missing=False, raise_not_set=False) cls.test_node_port = int(get_constant("MAGPIE_TEST_REMOTE_NODE_SERVER_PORT", default_value=2002, raise_missing=False, raise_not_set=False)) cls.test_node_jwks_url = "http://{}:{}/network/jwks".format(cls.test_node_host, cls.test_node_port) cls.test_node_token_url = "http://{}:{}/network/token".format(cls.test_node_host, cls.test_node_port) cls.test_authorization_url = "http://{}:{}/ui/network/authorize".format(cls.test_node_host, cls.test_node_port) - cls.test_redirect_uris = '["http://{}:{}/network/link"]'.format(cls.test_node_host, cls.test_node_port) + cls.test_redirect_uris = "[\"http://{}:{}/network/link\"]".format(cls.test_node_host, cls.test_node_port) @classmethod def login_admin(cls): @@ -1403,7 +1403,6 @@ def test_DeleteNetworkToken_And_AnonymousUser(self): json_body = utils.get_json_body(resp) utils.check_val_false(bool(json_body["remote_users"])) - @runner.MAGPIE_TEST_NETWORK @utils.check_network_mode def test_DeleteNetworkToken_InvalidNode(self): @@ -2482,7 +2481,6 @@ def test_GetNetworkLink_DifferentUser(self): cookies=self.cookies, headers=self.headers, expect_errors=True) utils.check_response_basic_info(resp, expected_code=404) - @runner.MAGPIE_TEST_NETWORK @utils.check_network_mode def test_GetNetworkLink_MissingClaim_user_name(self): @@ -8191,7 +8189,7 @@ def test_GetDecodeJWT_NoToken(self): headers=self.headers, expect_errors=True) utils.check_response_basic_info(resp, expected_code=400) json_body = utils.get_json_body(resp) - utils.check_val_is_in("Missing token", json_body.get("detail", '')) + utils.check_val_is_in("Missing token", json_body.get("detail", "")) @runner.MAGPIE_TEST_NETWORK @utils.check_network_mode @@ -8207,7 +8205,7 @@ def test_GetDecodeJWT_BadToken(self): headers=self.headers, expect_errors=True) utils.check_response_basic_info(resp, expected_code=400) json_body = utils.get_json_body(resp) - utils.check_val_is_in("Token is improperly formatted", json_body.get("detail", '')) + utils.check_val_is_in("Token is improperly formatted", json_body.get("detail", "")) @runner.MAGPIE_TEST_NETWORK @utils.check_network_mode @@ -8224,7 +8222,7 @@ def test_GetDecodeJWT_BadIssuer(self): headers=self.headers, expect_errors=True) utils.check_response_basic_info(resp, expected_code=400) json_body = utils.get_json_body(resp) - utils.check_val_is_in("invalid or missing issuer claim", json_body.get("detail", '')) + utils.check_val_is_in("invalid or missing issuer claim", json_body.get("detail", "")) @runner.MAGPIE_TEST_NETWORK @utils.check_network_mode @@ -8339,7 +8337,6 @@ def test_PostNetworkNode_CreateAssociatedRecords(self): headers=self.headers, expect_errors=True) utils.check_response_basic_info(resp) - @runner.MAGPIE_TEST_NETWORK @utils.check_network_mode def test_PostNetworkNode_MissingParamName(self): @@ -8357,7 +8354,7 @@ def test_PostNetworkNode_MissingParamName(self): "redirect_uris": ["http://uri.test.some.example.com"] } - for param, value in node_info.items(): + for param in node_info: json_body = utils.TestSetup.create_TestNetworkNode(self, override_exist=True, override_data={**node_info, param: None}, expect_errors=True) @@ -8380,7 +8377,7 @@ def test_PostNetworkNode_InvalidParam(self): "redirect_uris": ["http://uri.test.some.example.com"] } - for param in node_info.keys(): + for param in node_info: json_body = utils.TestSetup.create_TestNetworkNode(self, override_exist=True, override_data={**node_info, param: ""}, expect_errors=True) @@ -8482,7 +8479,7 @@ def test_PatchNetworkNode_InvalidName(self): "redirect_uris": ["http://uri.test.some.example.com"] } - for param in node_info.keys(): + for param in node_info: resp = utils.test_request(self, "PATCH", "/network/nodes/{}".format(self.test_node_name), cookies=self.cookies, headers=self.headers, data={**node_info, param: ""}, expect_errors=True) @@ -8597,9 +8594,9 @@ def test_GetNetworkRemoteUser(self): utils.TestSetup.create_TestNetworkNode(self, override_exist=True) utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True) - resp = utils.test_request(self, "GET", - "/network/nodes/{}/remote_users/{}".format(self.test_node_name, - self.test_remote_user_name), + resp = utils.test_request(self, "GET", + "/network/nodes/{}/remote_users/{}".format(self.test_node_name, + self.test_remote_user_name), cookies=self.cookies, headers=self.headers) json_body = utils.check_response_basic_info(resp) @@ -8642,7 +8639,7 @@ def test_PostNetworkRemoteUser_MissingRequiredParams(self): "remote_user_name": self.test_remote_user_name, "node_name": self.test_node_name } - for key in data.keys(): + for key in data: resp = utils.test_request(self, "POST", "/network/remote_users", json={**data, key: None}, expect_errors=True, headers=self.headers, cookies=self.cookies) utils.check_response_basic_info(resp, expected_method="POST", expected_code=400) @@ -9254,7 +9251,7 @@ def test_UserNetworkAuthorization(self): utils.warn_version(self, "View authorization page.", "3.38.0", skip=True) utils.TestSetup.create_TestNetworkNode(self, override_exist=True) - + with utils.TestSetup.valid_jwt(self, override_jwt_claims={"user_name": "test123"}) as token: self.login_test_user() path = "/ui/network/authorize?token={}&response_type=id_token&redirect_uri={}".format( @@ -9268,7 +9265,7 @@ def test_UserNetworkAuthorization(self): node_form.text ) utils.check_val_is_in( - 'on the Magpie instance named "{}"'.format(self.test_node_name), + 'on the Magpie instance named "{}"'.format(self.test_node_name), # pylint: disable=C4001 node_form.text ) token_input = node_form.find_all("input", recursive=True)[0] @@ -9355,6 +9352,7 @@ def test_UserNetworkAuthorization_BadRedirectURI(self): resp = utils.test_request(self, "GET", path, headers=headers, cookies=cookies, expect_errors=True) utils.check_val_equal(resp.status_code, 400) + @runner.MAGPIE_TEST_UI @six.add_metaclass(ABCMeta) class Interface_MagpieUI_AdminAuth(AdminTestCase, BaseTestCase): diff --git a/tests/test_constants.py b/tests/test_constants.py index 3d581ce7a..6208f1c19 100644 --- a/tests/test_constants.py +++ b/tests/test_constants.py @@ -168,7 +168,7 @@ def test_no_network_network_mode_on(self): @runner.MAGPIE_TEST_NETWORK @utils.check_network_mode(enable=False) - def test_include_network(self): + def test_include_network_network_mode_off(self): c.protected_user_name_regex.cache_clear() assert re.search(c.protected_user_name_regex(), c.get_constant("MAGPIE_NETWORK_NAME_PREFIX") + "test") is None @@ -210,7 +210,7 @@ def test_no_network_network_mode_on(self): @runner.MAGPIE_TEST_NETWORK @utils.check_network_mode(enable=False) - def test_include_network(self): + def test_include_network_network_mode_off(self): c.protected_user_email_regex.cache_clear() assert re.search(c.protected_user_email_regex(), c.get_constant("MAGPIE_NETWORK_ANONYMOUS_EMAIL_FORMAT").format("test")) is None @@ -233,7 +233,8 @@ def test_include_network(self): assert re.search(c.protected_group_name_regex(), c.get_constant("MAGPIE_NETWORK_NAME_PREFIX") + "test") def test_no_admin(self): - assert re.search(c.protected_group_name_regex(include_admin=False), c.get_constant("MAGPIE_ADMIN_GROUP")) is None + assert re.search(c.protected_group_name_regex(include_admin=False), + c.get_constant("MAGPIE_ADMIN_GROUP")) is None def test_no_anonymous(self): assert re.search(c.protected_group_name_regex(include_anonymous=False), @@ -247,8 +248,7 @@ def test_no_network_network_mode_on(self): @runner.MAGPIE_TEST_NETWORK @utils.check_network_mode(enable=False) - def test_include_network(self): + def test_include_network_network_mode_off(self): c.protected_group_name_regex.cache_clear() assert re.search(c.protected_group_name_regex(), c.get_constant("MAGPIE_NETWORK_NAME_PREFIX") + "test") is None - diff --git a/tests/test_magpie_api.py b/tests/test_magpie_api.py index 01a840b35..a12ee1bb8 100644 --- a/tests/test_magpie_api.py +++ b/tests/test_magpie_api.py @@ -570,7 +570,6 @@ def test_DeleteNetworkTokens_AndAnonymousNetworkUsers_ExpiredOnly(self): with patch_datetime({"utcnow": datetime.datetime.utcnow() - datetime.timedelta(days=365)}): utils.TestSetup.create_TestNetworkToken(self, override_remote_user_name="test2", override_node_name="test2") - utils.test_request(self, "DELETE", "/network/tokens", data={"expired_only": True}, cookies=self.cookies, headers=self.headers) diff --git a/tests/test_magpie_cli.py b/tests/test_magpie_cli.py index 8dd49fe3c..4882a9d2e 100644 --- a/tests/test_magpie_cli.py +++ b/tests/test_magpie_cli.py @@ -294,16 +294,16 @@ def test_purge_expired_network_tokens(): test_password = "qwertyqwerty" def mocked_request(*args, **_kwargs): - method, url, *_ = args + method, *_ = args response = requests.Response() response.status_code = 200 if method == "DELETE": - response._content = '{"deleted": 101}'.encode() + response._content = '{"deleted": 101}'.encode() # pylint: disable=C4001,W0212 return response with mock.patch("requests.Session.request", side_effect=mocked_request) as session_mock: cmd = ["api", test_url, test_username, test_password] assert purge_expired_network_tokens.main(cmd) == 0, "failed execution due to invalid arguments" session_mock.assert_any_call("POST", "{}/signin".format(test_url), data=None, - json={'user_name': 'test_username', 'password': 'qwertyqwerty'}) + json={"user_name": "test_username", "password": "qwertyqwerty"}) session_mock.assert_any_call("DELETE", "{}/network/tokens?expired_only=true".format(test_url)) diff --git a/tests/utils.py b/tests/utils.py index 709ca92d4..27d4f66f2 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -22,7 +22,7 @@ import six from beaker.cache import cache_managers, cache_regions from cryptography.hazmat.primitives.asymmetric import rsa -from cryptography.hazmat.primitives.serialization import Encoding, PrivateFormat, NoEncryption +from cryptography.hazmat.primitives.serialization import Encoding, NoEncryption, PrivateFormat from jwt.utils import to_base64url_uint from pyramid.config import Configurator from pyramid.httpexceptions import HTTPException @@ -776,14 +776,12 @@ def wrapper(*args, **kwargs): elif not enable and remote_enabled: test.skipTest("Test requires remote to be running without network mode enabled.") return test_func(*args, **kwargs) - else: - # assume local test - return mocked_get_settings(settings=settings)(test_func)(*args, **kwargs) + # assume local test + return mocked_get_settings(settings=settings)(test_func)(*args, **kwargs) return wrapper if _test_func is None: return decorator_func - else: - return decorator_func(_test_func) + return decorator_func(_test_func) def mock_request(request_path_query="", # type: Str @@ -1533,10 +1531,9 @@ def patch_datetime(patches): https://williambert.online/2011/07/how-to-unit-testing-in-django-with-mocking-and-patching/ """ - from datetime import datetime as org_datetime - class FakeDatetime(org_datetime): + class FakeDatetime(datetime): def __new__(cls, *args, **kwargs): - return org_datetime.__new__(org_datetime, *args, **kwargs) + return datetime.__new__(datetime, *args, **kwargs) with mock.patch("datetime.datetime", FakeDatetime): for method, return_value in patches.items(): From cd9c048121fc1f50abc0d228b3883e13f97c5e0b Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Wed, 24 Apr 2024 08:53:43 -0400 Subject: [PATCH 38/55] fix dependency versions --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 55bcc929c..32b47b99f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,7 +27,8 @@ gunicorn>=20; python_version >= "3" humanize jsonschema<4; python_version < "3.6" jsonschema>=4; python_version >= "3.6" -jwcrypto==1.5.6; python_version >= "3.6" +jwcrypto==1.5.6; python_version >= "3.8" +jwcrypto==1.5.1; python_version < "3.8" and python_version >="3.6" jwcrypto==0.8; python_version < "3.6" # pyup: ignore lxml>=3.7 mako # controlled by pyramid_mako From 8107ebb6e21dda71bde960d9c955c9747bb17b4a Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Wed, 24 Apr 2024 09:29:07 -0400 Subject: [PATCH 39/55] fix requirements file: old range not parsed properly by pip? --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 32b47b99f..0b752361d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,7 +28,7 @@ humanize jsonschema<4; python_version < "3.6" jsonschema>=4; python_version >= "3.6" jwcrypto==1.5.6; python_version >= "3.8" -jwcrypto==1.5.1; python_version < "3.8" and python_version >="3.6" +jwcrypto==1.5.1; python_version == "3.6" or python_version == "3.7" jwcrypto==0.8; python_version < "3.6" # pyup: ignore lxml>=3.7 mako # controlled by pyramid_mako From 18bc104136c7b6ac79ebb8d4f9077b12f7e031ad Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Wed, 24 Apr 2024 10:12:47 -0400 Subject: [PATCH 40/55] test fixes --- .github/workflows/tests.yml | 6 ++++-- tests/interfaces.py | 9 ++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ea5e7b1cf..66c0a89c5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -52,7 +52,7 @@ jobs: allow-failure: [false] test-case: [test-local] # can use below option to set environment variables or makefile settings applied during test execution - test-option: [""] + test-option: ["MAGPIE_NETWORK_CREATE_MISSING_PEM_FILE=True"] include: # linter tests - os: ubuntu-latest @@ -62,7 +62,7 @@ jobs: python-version: "3.10" allow-failure: false test-case: check - # remote test + # remote test (note that network tests are skipped since the remote server won't have network mode enabled) - os: ubuntu-latest python-version: "3.11" allow-failure: true @@ -89,10 +89,12 @@ jobs: python-version: "3.5" allow-failure: true test-case: test-local + test-option: PYTEST_ADDOPTS='-m "not network"' # network mode is not supported for python version < 3.6 - os: ubuntu-20.04 python-version: "3.6" allow-failure: true test-case: test-local + test-option: "MAGPIE_NETWORK_CREATE_MISSING_PEM_FILE=True" # test allowed failing for recent versions # - os: ubuntu-latest # python-version: 3.12 diff --git a/tests/interfaces.py b/tests/interfaces.py index ea4e2e48b..67d41769e 100644 --- a/tests/interfaces.py +++ b/tests/interfaces.py @@ -8805,13 +8805,14 @@ def test_PostNetworkRemoteUser_MultipleAnonymous(self): utils.warn_version(self, "Create a remote user", "3.38.0", skip=True) utils.TestSetup.create_TestNetworkNode(self, override_exist=True) - anonymous_username = "{}{}".format(get_constant("MAGPIE_NETWORK_NAME_PREFIX"), self.test_node_name) utils.TestSetup.create_TestNetworkRemoteUser(self, override_exist=True, - override_user_name=anonymous_username) + override_data={ + "remote_user_name": self.test_remote_user_name, + "node_name": self.test_node_name + }) data = { "remote_user_name": "other_remote_user_name", - "user_name": anonymous_username, "node_name": self.test_node_name } resp = utils.test_request(self, "POST", "/network/remote_users", json=data, headers=self.headers, @@ -8828,6 +8829,8 @@ def test_PostNetworkRemoteUser_MultipleAnonymous(self): json_body = utils.check_response_basic_info(resp) actual = {k: v for k, v in json_body.items() if k in data} utils.check_val_equal(actual, data) + anonymous_username = "{}{}".format(get_constant("MAGPIE_NETWORK_NAME_PREFIX"), self.test_node_name) + utils.check_val_equal(json_body["user_name"], anonymous_username) @runner.MAGPIE_TEST_NETWORK @utils.check_network_mode From add132eab6a810c86f594b18bb6bda540d45d929 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Wed, 24 Apr 2024 10:52:20 -0400 Subject: [PATCH 41/55] support for python 3.5 --- magpie/constants.py | 13 ++++++++++++- tests/interfaces.py | 45 +++++++++++++++++++++++---------------------- tests/utils.py | 4 ++-- 3 files changed, 37 insertions(+), 25 deletions(-) diff --git a/magpie/constants.py b/magpie/constants.py index 710871bff..c8205f621 100644 --- a/magpie/constants.py +++ b/magpie/constants.py @@ -256,9 +256,20 @@ def protected_group_name_regex(include_admin=True, return re.compile("^({})$".format("|".join(patterns))) +def network_mode_supported(): + # type: () -> bool + """ + Return whether the current python version supports network mode. + """ + return (sys.version_info.major, sys.version_info.minor) > (3, 5) + + def network_enabled(settings_container=None): # type: (Optional[AnySettingsContainer]) -> bool - if sys.version_info.major < 3 or sys.version_info.minor < 6: + """ + Return whether network mode is enabled. + """ + if not network_mode_supported(): return False return asbool(get_constant("MAGPIE_NETWORK_ENABLED", settings_container=settings_container)) diff --git a/tests/interfaces.py b/tests/interfaces.py index 67d41769e..6cf99d74e 100644 --- a/tests/interfaces.py +++ b/tests/interfaces.py @@ -23,7 +23,7 @@ from magpie import __meta__ from magpie.api import schemas as s from magpie.api.webhooks import webhook_update_error_status -from magpie.constants import MAGPIE_ROOT, get_constant +from magpie.constants import MAGPIE_ROOT, get_constant, network_mode_supported from magpie.models import RESOURCE_TYPE_DICT, Directory, File, Layer, Process, Route, Service, UserStatuses, Workspace from magpie.permissions import ( PERMISSION_REASON_DEFAULT, @@ -204,27 +204,28 @@ def cleanup(cls): for res in list(cls.extra_resource_ids): # copy to update removed ones utils.TestSetup.delete_TestResource(cls, res) cls.extra_resource_ids.discard(res) - for node in list(cls.extra_node_names): - check_network_mode(utils.TestSetup.delete_TestNetworkNode, - enable=True)(cls, override_name=node, allow_missing=True) - cls.extra_node_names.discard(node) - for remote_user_name, node_name in list(cls.extra_network_tokens): - check_network_mode(utils.TestSetup.delete_TestNetworkToken, - enable=True)(cls, - override_remote_user_name=remote_user_name, - override_node_name=node_name, - allow_missing=True) # should already be deleted with associated models - cls.extra_network_tokens.discard((remote_user_name, node_name)) - for remote_user_name, node_name in list(cls.extra_remote_user_names): - check_network_mode(utils.TestSetup.delete_TestNetworkRemoteUser, - enable=True)(cls, override_remote_user_name=remote_user_name, - override_node_name=node_name, - allow_missing=True) # should already be deleted with associated models - cls.extra_remote_user_names.discard((remote_user_name, node_name)) - for host_port, server in list(cls.extra_remote_servers.items()): - if server.is_running(): - server.stop() - cls.extra_remote_servers.pop(host_port) + if network_mode_supported(): # This check is required when the python version is < 3.6 + for node in list(cls.extra_node_names): + check_network_mode(utils.TestSetup.delete_TestNetworkNode, + enable=True)(cls, override_name=node, allow_missing=True) + cls.extra_node_names.discard(node) + for remote_user_name, node_name in list(cls.extra_network_tokens): + check_network_mode(utils.TestSetup.delete_TestNetworkToken, + enable=True)(cls, + override_remote_user_name=remote_user_name, + override_node_name=node_name, + allow_missing=True) # should already be deleted with associated models + cls.extra_network_tokens.discard((remote_user_name, node_name)) + for remote_user_name, node_name in list(cls.extra_remote_user_names): + check_network_mode(utils.TestSetup.delete_TestNetworkRemoteUser, + enable=True)(cls, override_remote_user_name=remote_user_name, + override_node_name=node_name, + allow_missing=True) # should already be deleted with associated models + cls.extra_remote_user_names.discard((remote_user_name, node_name)) + for host_port, server in list(cls.extra_remote_servers.items()): + if server.is_running(): + server.stop() + cls.extra_remote_servers.pop(host_port) @property def update_method(self): diff --git a/tests/utils.py b/tests/utils.py index 27d4f66f2..96b5c4dc7 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -42,7 +42,7 @@ from magpie import __meta__, app, services from magpie.api import schemas from magpie.compat import LooseVersion -from magpie.constants import get_constant +from magpie.constants import get_constant, network_mode_supported from magpie.permissions import Access, PermissionSet, Scope from magpie.services import SERVICE_TYPE_DICT, ServiceAccess from magpie.utils import ( @@ -759,7 +759,7 @@ def check_network_mode(_test_func=None, enable=True): :param _test_func: Test function being decorated. :param enable: Boolean value indicating whether ``_test_func`` expects network mode to be enabled in order to pass. """ - if enable: + if enable and network_mode_supported(): settings = {"magpie.network_enabled": True} else: settings = {"magpie.network_enabled": False} From a14078368f86d9afd6e545c77a16bfb1042a4256 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Wed, 24 Apr 2024 13:44:50 -0400 Subject: [PATCH 42/55] add option to create private key through makefile --- .github/workflows/tests.yml | 7 ++- Makefile | 4 ++ magpie/cli/create_private_key.py | 79 ++++++++++++++++++++++++++++ setup.py | 3 +- tests/test_magpie_cli.py | 90 ++++++++++++++++++++++++++++++-- 5 files changed, 175 insertions(+), 8 deletions(-) create mode 100644 magpie/cli/create_private_key.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 66c0a89c5..9abb204b9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -50,9 +50,9 @@ jobs: os: [ubuntu-latest] python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] allow-failure: [false] - test-case: [test-local] + test-case: ["create-private-key test-local"] # can use below option to set environment variables or makefile settings applied during test execution - test-option: ["MAGPIE_NETWORK_CREATE_MISSING_PEM_FILE=True"] + test-option: [""] include: # linter tests - os: ubuntu-latest @@ -93,8 +93,7 @@ jobs: - os: ubuntu-20.04 python-version: "3.6" allow-failure: true - test-case: test-local - test-option: "MAGPIE_NETWORK_CREATE_MISSING_PEM_FILE=True" + test-case: create-private-key test-local # test allowed failing for recent versions # - os: ubuntu-latest # python-version: 3.12 diff --git a/Makefile b/Makefile index 4fbec7327..b865c367f 100644 --- a/Makefile +++ b/Makefile @@ -818,3 +818,7 @@ conda-env: conda-base ## create conda environment if missing and required echo "Creating conda environment at '$(CONDA_ENV_PATH)'..." && \ "$(CONDA_BIN)" create -y -n "$(CONDA_ENV_NAME)" python=$(PYTHON_VERSION)) \ ) + +.PHONY: create-private-key +create-private-key: ## create a private key file according to the MAGPIE_NETWORK_PEM_FILES and MAGPIE_NETWORK_PEM_PASSWORDS settings + @bash -c '$(CONDA_CMD) magpie_create_private_key --config "$(APP_INI)"' diff --git a/magpie/cli/create_private_key.py b/magpie/cli/create_private_key.py new file mode 100644 index 000000000..e132fea86 --- /dev/null +++ b/magpie/cli/create_private_key.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 + +""" +Create a private key file used to generate a JSON Web Key. + +This file is required when network mode is enabled in order to sign JSON Web Tokens. +""" + +import argparse +import os.path +import sys +from typing import TYPE_CHECKING + +from magpie.api.management.network.network_utils import pem_files, create_private_key +from magpie.cli.utils import make_logging_options, setup_logger_from_options +from magpie.constants import get_constant +from magpie.utils import get_logger, get_settings_from_config_ini + +if TYPE_CHECKING: + from typing import Optional, Sequence + + from magpie.typedefs import Str + +LOGGER = get_logger(__name__, + message_format="%(asctime)s - %(levelname)s - %(message)s", + datetime_format="%d-%b-%y %H:%M:%S", force_stdout=False) + + +def make_parser(): + # type: () -> argparse.ArgumentParser + parser = argparse.ArgumentParser(description="Create a private key used to generate a JSON Web Key.") + parser.add_argument("--config", "--ini", metavar="CONFIG", dest="ini_config", + default=get_constant("MAGPIE_INI_FILE_PATH"), + help="Configuration INI file to retrieve database connection settings (default: %(default)s).") + parser.add_argument("--key-file", + help="Location to write key file to. Default is to use the first file listed in the " + "MAGPIE_NETWORK_PEM_FILES variable.") + parser.add_argument("--password", + help="Password used to encrypt the key file. Default is to not encrypt the key file unless the " + "the --key-file argument is not set and there is an associated password in the " + "MAGPIE_NETWORK_PEM_PASSWORDS variable.") + parser.add_argument("--force", action="store_true", help="Recreate the key file if it already exists.") + make_logging_options(parser) + return parser + + +def main(args=None, parser=None, namespace=None): + # type: (Optional[Sequence[Str]], Optional[argparse.ArgumentParser], Optional[argparse.Namespace]) -> int + if not parser: + parser = make_parser() + args = parser.parse_args(args=args, namespace=namespace) + setup_logger_from_options(LOGGER, args) + settings_container = get_settings_from_config_ini(args.ini_config) + + if args.key_file: + key_file = args.key_file + else: + pem_files_ = pem_files(settings_container) + if pem_files_: + key_file = pem_files_[0] + else: + LOGGER.error( + "No network PEM files specified. Either set MAGPIE_NETWORK_PEM_FILES or use the --key-file argument") + return 1 + + if os.path.isfile(key_file) and not args.force: + LOGGER.warning("File %s already exists. To overwrite this file use the --force option.", key_file) + return 2 + + password = args.password + if password is not None: + password = password.encode() + + create_private_key(key_file, password=password, settings_container=settings_container) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/setup.py b/setup.py index cc2d3eb04..bddd16202 100644 --- a/setup.py +++ b/setup.py @@ -255,7 +255,8 @@ def _extra_requirements(base_requirements, other_requirements): "magpie_run_db_migration = magpie.cli.run_db_migration:main", "magpie_send_email = magpie.cli.send_email:main", "magpie_sync_resources = magpie.cli.sync_resources:main", - "magpie_purge_expired_network_tokens = magpie.cli.purge_expired_network_tokens:main" + "magpie_purge_expired_network_tokens = magpie.cli.purge_expired_network_tokens:main", + "magpie_create_private_key = magpie.cli.create_private_key:main" ], } ) diff --git a/tests/test_magpie_cli.py b/tests/test_magpie_cli.py index 4882a9d2e..c0223790b 100644 --- a/tests/test_magpie_cli.py +++ b/tests/test_magpie_cli.py @@ -37,7 +37,7 @@ ] -def run_and_get_output(command, trim=True): +def run_and_get_output(command, trim=True, expect_output=True): cmd = " ".join(command) if isinstance(command, (list, tuple)) else command env = {"PATH": os.path.expandvars(os.environ["PATH"])} # when debugging, explicit expand of install path required pipe = subprocess.PIPE @@ -45,9 +45,10 @@ def run_and_get_output(command, trim=True): out, err = proc.communicate() assert not err, "process returned with error code {}".format(err) # when no output is present, it is either because CLI was not installed correctly, or caused by some other error - assert out != "", "process did not execute as expected, no output available" out_lines = [line for line in out.splitlines() if not trim or (line and not line.startswith(" "))] - assert len(out_lines), "could not retrieve any console output" + if expect_output: + assert out != "", "process did not execute as expected, no output available" + assert len(out_lines), "could not retrieve any console output" return out_lines @@ -288,6 +289,9 @@ def test_purge_expired_network_tokens_help_directly(): assert "Delete all expired network tokens." in out_lines[1] +@runner.MAGPIE_TEST_CLI +@runner.MAGPIE_TEST_LOCAL +@runner.MAGPIE_TEST_NETWORK def test_purge_expired_network_tokens(): test_url = "http://localhost" test_username = "test_username" @@ -307,3 +311,83 @@ def mocked_request(*args, **_kwargs): session_mock.assert_any_call("POST", "{}/signin".format(test_url), data=None, json={"user_name": "test_username", "password": "qwertyqwerty"}) session_mock.assert_any_call("DELETE", "{}/network/tokens?expired_only=true".format(test_url)) + +@runner.MAGPIE_TEST_CLI +@runner.MAGPIE_TEST_LOCAL +@runner.MAGPIE_TEST_NETWORK +def test_create_private_key_help_via_magpie_helper(): + out_lines = run_and_get_output("magpie_helper create_private_key --help") + assert "usage: magpie_helper create_private_key" in out_lines[0] + assert "Create a private key used to generate a JSON Web Key." in out_lines[1] + + +@runner.MAGPIE_TEST_CLI +@runner.MAGPIE_TEST_LOCAL +@runner.MAGPIE_TEST_NETWORK +def test_create_private_key_help_directly(): + out_lines = run_and_get_output("magpie_create_private_key --help") + assert "usage: magpie_create_private_key" in out_lines[0] + assert "Create a private key used to generate a JSON Web Key." in out_lines[1] + + +@runner.MAGPIE_TEST_CLI +@runner.MAGPIE_TEST_LOCAL +@runner.MAGPIE_TEST_NETWORK +def test_create_private_key_create_file(): + tmp_file = tempfile.NamedTemporaryFile(mode="w") + tmp_file.close() + try: + run_and_get_output("magpie_create_private_key --key-file {}".format(tmp_file.name), expect_output=False) + with open(tmp_file.name) as f: + key_content = f.read() + assert key_content.startswith("-----BEGIN RSA PRIVATE KEY-----") + assert "ENCRYPTED" not in key_content + finally: + os.unlink(tmp_file.name) + + +@runner.MAGPIE_TEST_CLI +@runner.MAGPIE_TEST_LOCAL +@runner.MAGPIE_TEST_NETWORK +def test_create_private_key_create_file_no_force_exists(): + tmp_file = tempfile.NamedTemporaryFile(mode="w", delete=False) + tmp_file.close() + try: + out = run_and_get_output("magpie_create_private_key --key-file {}".format(tmp_file.name), expect_output=False) + assert "File {} already exists.".format(tmp_file.name) in out[0] + finally: + os.unlink(tmp_file.name) + + +@runner.MAGPIE_TEST_CLI +@runner.MAGPIE_TEST_LOCAL +@runner.MAGPIE_TEST_NETWORK +def test_create_private_key_create_file_force_exists(): + tmp_file = tempfile.NamedTemporaryFile(mode="w") + tmp_file.close() + try: + run_and_get_output("magpie_create_private_key --force --key-file {}".format(tmp_file.name), expect_output=False) + with open(tmp_file.name) as f: + key_content = f.read() + assert key_content.startswith("-----BEGIN RSA PRIVATE KEY-----") + assert "ENCRYPTED" not in key_content + + finally: + os.unlink(tmp_file.name) + + +@runner.MAGPIE_TEST_CLI +@runner.MAGPIE_TEST_LOCAL +@runner.MAGPIE_TEST_NETWORK +def test_create_private_key_create_file_with_password(): + tmp_file = tempfile.NamedTemporaryFile(mode="w") + tmp_file.close() + try: + run_and_get_output("magpie_create_private_key --password qwertqwerty --key-file {}".format(tmp_file.name), + expect_output=False) + with open(tmp_file.name) as f: + key_content = f.read() + assert key_content.startswith("-----BEGIN RSA PRIVATE KEY-----") + assert "ENCRYPTED" in key_content + finally: + os.unlink(tmp_file.name) From 34a1dd0928c5a4c5ec2c5778720faa6f2861d24c Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Wed, 24 Apr 2024 13:50:51 -0400 Subject: [PATCH 43/55] style fixes --- tests/test_magpie_cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_magpie_cli.py b/tests/test_magpie_cli.py index c0223790b..b2bb1033e 100644 --- a/tests/test_magpie_cli.py +++ b/tests/test_magpie_cli.py @@ -312,6 +312,7 @@ def mocked_request(*args, **_kwargs): json={"user_name": "test_username", "password": "qwertyqwerty"}) session_mock.assert_any_call("DELETE", "{}/network/tokens?expired_only=true".format(test_url)) + @runner.MAGPIE_TEST_CLI @runner.MAGPIE_TEST_LOCAL @runner.MAGPIE_TEST_NETWORK From 7002b2d6621ae02dd0eaca2fb566f8997c6cd9aa Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Wed, 24 Apr 2024 14:07:02 -0400 Subject: [PATCH 44/55] style fixes --- magpie/cli/create_private_key.py | 2 +- tests/test_magpie_cli.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/magpie/cli/create_private_key.py b/magpie/cli/create_private_key.py index e132fea86..02f6a1dc0 100644 --- a/magpie/cli/create_private_key.py +++ b/magpie/cli/create_private_key.py @@ -11,7 +11,7 @@ import sys from typing import TYPE_CHECKING -from magpie.api.management.network.network_utils import pem_files, create_private_key +from magpie.api.management.network.network_utils import create_private_key, pem_files from magpie.cli.utils import make_logging_options, setup_logger_from_options from magpie.constants import get_constant from magpie.utils import get_logger, get_settings_from_config_ini diff --git a/tests/test_magpie_cli.py b/tests/test_magpie_cli.py index b2bb1033e..c954d8a53 100644 --- a/tests/test_magpie_cli.py +++ b/tests/test_magpie_cli.py @@ -335,11 +335,11 @@ def test_create_private_key_help_directly(): @runner.MAGPIE_TEST_LOCAL @runner.MAGPIE_TEST_NETWORK def test_create_private_key_create_file(): - tmp_file = tempfile.NamedTemporaryFile(mode="w") + tmp_file = tempfile.NamedTemporaryFile(mode="w") # pylint: disable=R1732 tmp_file.close() try: run_and_get_output("magpie_create_private_key --key-file {}".format(tmp_file.name), expect_output=False) - with open(tmp_file.name) as f: + with open(tmp_file.name, encoding="utf-8") as f: key_content = f.read() assert key_content.startswith("-----BEGIN RSA PRIVATE KEY-----") assert "ENCRYPTED" not in key_content @@ -351,7 +351,7 @@ def test_create_private_key_create_file(): @runner.MAGPIE_TEST_LOCAL @runner.MAGPIE_TEST_NETWORK def test_create_private_key_create_file_no_force_exists(): - tmp_file = tempfile.NamedTemporaryFile(mode="w", delete=False) + tmp_file = tempfile.NamedTemporaryFile(mode="w", delete=False) # pylint: disable=R1732 tmp_file.close() try: out = run_and_get_output("magpie_create_private_key --key-file {}".format(tmp_file.name), expect_output=False) @@ -364,11 +364,11 @@ def test_create_private_key_create_file_no_force_exists(): @runner.MAGPIE_TEST_LOCAL @runner.MAGPIE_TEST_NETWORK def test_create_private_key_create_file_force_exists(): - tmp_file = tempfile.NamedTemporaryFile(mode="w") + tmp_file = tempfile.NamedTemporaryFile(mode="w") # pylint: disable=R1732 tmp_file.close() try: run_and_get_output("magpie_create_private_key --force --key-file {}".format(tmp_file.name), expect_output=False) - with open(tmp_file.name) as f: + with open(tmp_file.name, encoding="utf-8") as f: key_content = f.read() assert key_content.startswith("-----BEGIN RSA PRIVATE KEY-----") assert "ENCRYPTED" not in key_content @@ -381,12 +381,12 @@ def test_create_private_key_create_file_force_exists(): @runner.MAGPIE_TEST_LOCAL @runner.MAGPIE_TEST_NETWORK def test_create_private_key_create_file_with_password(): - tmp_file = tempfile.NamedTemporaryFile(mode="w") + tmp_file = tempfile.NamedTemporaryFile(mode="w") # pylint: disable=R1732 tmp_file.close() try: run_and_get_output("magpie_create_private_key --password qwertqwerty --key-file {}".format(tmp_file.name), expect_output=False) - with open(tmp_file.name) as f: + with open(tmp_file.name, encoding="utf-8") as f: key_content = f.read() assert key_content.startswith("-----BEGIN RSA PRIVATE KEY-----") assert "ENCRYPTED" in key_content From 336e5c3c9a19c63cdf42f34525baa8ba24d8e239 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Wed, 24 Apr 2024 15:14:25 -0400 Subject: [PATCH 45/55] try to ignore some test code --- .github/.gitleaks.toml | 1 + tests/test_magpie_cli.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/.gitleaks.toml b/.github/.gitleaks.toml index f4d48fd0c..bfb936c4e 100644 --- a/.github/.gitleaks.toml +++ b/.github/.gitleaks.toml @@ -116,3 +116,4 @@ title = "gitleaks config" # old commit files with false positives or dummy data '''magpie/login/login.py''', '''.+(.js.map)$'''] + regexes = ['''#\s*nogitleaks'''] diff --git a/tests/test_magpie_cli.py b/tests/test_magpie_cli.py index c954d8a53..abeac8eac 100644 --- a/tests/test_magpie_cli.py +++ b/tests/test_magpie_cli.py @@ -341,7 +341,7 @@ def test_create_private_key_create_file(): run_and_get_output("magpie_create_private_key --key-file {}".format(tmp_file.name), expect_output=False) with open(tmp_file.name, encoding="utf-8") as f: key_content = f.read() - assert key_content.startswith("-----BEGIN RSA PRIVATE KEY-----") + assert key_content.startswith("-----BEGIN RSA PRIVATE KEY-----") # nogitleaks assert "ENCRYPTED" not in key_content finally: os.unlink(tmp_file.name) @@ -370,7 +370,7 @@ def test_create_private_key_create_file_force_exists(): run_and_get_output("magpie_create_private_key --force --key-file {}".format(tmp_file.name), expect_output=False) with open(tmp_file.name, encoding="utf-8") as f: key_content = f.read() - assert key_content.startswith("-----BEGIN RSA PRIVATE KEY-----") + assert key_content.startswith("-----BEGIN RSA PRIVATE KEY-----") # nogitleaks assert "ENCRYPTED" not in key_content finally: @@ -388,7 +388,7 @@ def test_create_private_key_create_file_with_password(): expect_output=False) with open(tmp_file.name, encoding="utf-8") as f: key_content = f.read() - assert key_content.startswith("-----BEGIN RSA PRIVATE KEY-----") + assert key_content.startswith("-----BEGIN RSA PRIVATE KEY-----") # nogitleaks assert "ENCRYPTED" in key_content finally: os.unlink(tmp_file.name) From 704f2bc3c20731522b6984cad687d3ff8741b96e Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Wed, 24 Apr 2024 15:21:52 -0400 Subject: [PATCH 46/55] give up and just do something different --- .github/.gitleaks.toml | 1 - tests/test_magpie_cli.py | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/.gitleaks.toml b/.github/.gitleaks.toml index bfb936c4e..f4d48fd0c 100644 --- a/.github/.gitleaks.toml +++ b/.github/.gitleaks.toml @@ -116,4 +116,3 @@ title = "gitleaks config" # old commit files with false positives or dummy data '''magpie/login/login.py''', '''.+(.js.map)$'''] - regexes = ['''#\s*nogitleaks'''] diff --git a/tests/test_magpie_cli.py b/tests/test_magpie_cli.py index abeac8eac..bf9bd444a 100644 --- a/tests/test_magpie_cli.py +++ b/tests/test_magpie_cli.py @@ -341,7 +341,7 @@ def test_create_private_key_create_file(): run_and_get_output("magpie_create_private_key --key-file {}".format(tmp_file.name), expect_output=False) with open(tmp_file.name, encoding="utf-8") as f: key_content = f.read() - assert key_content.startswith("-----BEGIN RSA PRIVATE KEY-----") # nogitleaks + assert "PRIVATE KEY" in key_content.splitlines()[0] assert "ENCRYPTED" not in key_content finally: os.unlink(tmp_file.name) @@ -370,7 +370,7 @@ def test_create_private_key_create_file_force_exists(): run_and_get_output("magpie_create_private_key --force --key-file {}".format(tmp_file.name), expect_output=False) with open(tmp_file.name, encoding="utf-8") as f: key_content = f.read() - assert key_content.startswith("-----BEGIN RSA PRIVATE KEY-----") # nogitleaks + assert "PRIVATE KEY" in key_content.splitlines()[0] assert "ENCRYPTED" not in key_content finally: @@ -388,7 +388,7 @@ def test_create_private_key_create_file_with_password(): expect_output=False) with open(tmp_file.name, encoding="utf-8") as f: key_content = f.read() - assert key_content.startswith("-----BEGIN RSA PRIVATE KEY-----") # nogitleaks + assert "PRIVATE KEY" in key_content.splitlines()[0] assert "ENCRYPTED" in key_content finally: os.unlink(tmp_file.name) From 5f587deb1889a02ac9742057c18a4db7b971e674 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Wed, 24 Apr 2024 16:01:18 -0400 Subject: [PATCH 47/55] try to fix gitleaks again --- .github/.gitleaks.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/.gitleaks.toml b/.github/.gitleaks.toml index f4d48fd0c..90bb8db5f 100644 --- a/.github/.gitleaks.toml +++ b/.github/.gitleaks.toml @@ -49,6 +49,10 @@ title = "gitleaks config" description = "Asymmetric Private Key" regex = '''-----BEGIN ((EC|PGP|DSA|RSA|OPENSSH) )?PRIVATE KEY( BLOCK)?-----''' tags = ["key", "AsymmetricPrivateKey"] + [rules.allowlist] + description = "test for presence of private keys" + paths = ['''tests/test_magpie_cli.py'''] + commits = ["704f2bc3c20731522b6984cad687d3ff8741b96e", "336e5c3c9a19c63cdf42f34525baa8ba24d8e239", "34a1dd0928c5a4c5ec2c5778720faa6f2861d24c"] [[rules]] description = "Generic Credential" regex = '''(?i)(api_key|apikey|secret)(.{0,20})?['|"][0-9a-zA-Z]{16,45}['|"]''' From 76944c4e9e11050d885bdeb3b35e6e28ab0936c1 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Thu, 25 Apr 2024 09:48:19 -0400 Subject: [PATCH 48/55] fix test exception when python < 3.7 --- .github/.gitleaks.toml | 1 - tests/interfaces.py | 10 +++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/.gitleaks.toml b/.github/.gitleaks.toml index 90bb8db5f..944677e7e 100644 --- a/.github/.gitleaks.toml +++ b/.github/.gitleaks.toml @@ -52,7 +52,6 @@ title = "gitleaks config" [rules.allowlist] description = "test for presence of private keys" paths = ['''tests/test_magpie_cli.py'''] - commits = ["704f2bc3c20731522b6984cad687d3ff8741b96e", "336e5c3c9a19c63cdf42f34525baa8ba24d8e239", "34a1dd0928c5a4c5ec2c5778720faa6f2861d24c"] [[rules]] description = "Generic Credential" regex = '''(?i)(api_key|apikey|secret)(.{0,20})?['|"][0-9a-zA-Z]{16,45}['|"]''' diff --git a/tests/interfaces.py b/tests/interfaces.py index 6cf99d74e..b80acc675 100644 --- a/tests/interfaces.py +++ b/tests/interfaces.py @@ -5,6 +5,7 @@ import os import secrets import string +import sys import unittest import uuid from abc import ABCMeta, abstractmethod @@ -1340,7 +1341,14 @@ def test_PostNetworkToken_InvalidNodeCredentials(self): token = utils.TestSetup.create_TestNetworkToken(self, expect_errors=True) utils.check_val_equal(token.get("token"), None) utils.check_val_equal(token.get("code"), 500) - utils.check_val_equal(token.get("call", {}).get("exception"), "PyJWKClientConnectionError") + # python version < 3.7 uses pyjwt version < 2.5.0 + # pyjwt 2.5.0 changed the error type from HTTPError to PyJWKClientConnectionError if a valid JWKS could not + # be found at the given uri. + if (sys.version_info.major, sys.version_info.minor) < (3, 7): + error_type = "HTTPError" + else: + error_type = "PyJWKClientConnectionError" + utils.check_val_equal(token.get("call", {}).get("exception"), error_type) @runner.MAGPIE_TEST_NETWORK @utils.check_network_mode From 8c03aabe6c14073516087909143cddfef6400047 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Thu, 25 Apr 2024 11:21:44 -0400 Subject: [PATCH 49/55] don't mess with required test names --- .github/workflows/tests.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9abb204b9..7e1c56172 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -50,7 +50,7 @@ jobs: os: [ubuntu-latest] python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] allow-failure: [false] - test-case: ["create-private-key test-local"] + test-case: [test-local] # can use below option to set environment variables or makefile settings applied during test execution test-option: [""] include: @@ -127,6 +127,9 @@ jobs: run: | hash -r env | sort + - name: Create private key for network testing + if: ${{ matrix.test-case == 'test_local' }} + run: ${{ matrix.test-option }} make create-private-key # run '-only' test variations since dependencies are preinstalled, skip some resolution time - name: Run Tests run: ${{ matrix.test-option }} make stop ${{ matrix.test-case }}-only From a34a06075d574b18395eba83f4946df8f59ea39a Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Thu, 25 Apr 2024 12:10:08 -0400 Subject: [PATCH 50/55] ooops --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7e1c56172..1bbc00f80 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -128,7 +128,7 @@ jobs: hash -r env | sort - name: Create private key for network testing - if: ${{ matrix.test-case == 'test_local' }} + if: ${{ matrix.test-case == 'test-local' }} run: ${{ matrix.test-option }} make create-private-key # run '-only' test variations since dependencies are preinstalled, skip some resolution time - name: Run Tests From ae9caf67246839ec9f9303e3e090cd94c1acb8a6 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Thu, 25 Apr 2024 15:29:20 -0400 Subject: [PATCH 51/55] run the tests that I want. Env variable is overridden by command line arg --- .github/workflows/tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1bbc00f80..c11f7660e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -71,7 +71,7 @@ jobs: - os: ubuntu-latest python-version: "3.11" allow-failure: true - test-case: start test-remote + test-case: start test-all # the tests that are run are limited by the PYTEST_ADDOPTS in test-option below test-option: >- PYTEST_ADDOPTS='-m "remote and network"' MAGPIE_NETWORK_CREATE_MISSING_PEM_FILE=True MAGPIE_NETWORK_ENABLED=on MAGPIE_NETWORK_INSTANCE_NAME=example # coverage test @@ -88,8 +88,8 @@ jobs: - os: ubuntu-20.04 python-version: "3.5" allow-failure: true - test-case: test-local - test-option: PYTEST_ADDOPTS='-m "not network"' # network mode is not supported for python version < 3.6 + test-case: test-all # the tests that are run are limited by the PYTEST_ADDOPTS in test-option below + test-option: PYTEST_ADDOPTS='-m "local and not network"' # network mode is not supported for python version < 3.6 - os: ubuntu-20.04 python-version: "3.6" allow-failure: true From 37b9f62d91e9125cc56b1cdcd87e737e99f8e8bc Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Thu, 25 Apr 2024 16:26:44 -0400 Subject: [PATCH 52/55] try this again with the proper name since it adds a suffix I guess --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c11f7660e..1a4e6ea47 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -71,7 +71,7 @@ jobs: - os: ubuntu-latest python-version: "3.11" allow-failure: true - test-case: start test-all # the tests that are run are limited by the PYTEST_ADDOPTS in test-option below + test-case: start test # the tests that are run are limited by the PYTEST_ADDOPTS in test-option below test-option: >- PYTEST_ADDOPTS='-m "remote and network"' MAGPIE_NETWORK_CREATE_MISSING_PEM_FILE=True MAGPIE_NETWORK_ENABLED=on MAGPIE_NETWORK_INSTANCE_NAME=example # coverage test @@ -88,7 +88,7 @@ jobs: - os: ubuntu-20.04 python-version: "3.5" allow-failure: true - test-case: test-all # the tests that are run are limited by the PYTEST_ADDOPTS in test-option below + test-case: test # the tests that are run are limited by the PYTEST_ADDOPTS in test-option below test-option: PYTEST_ADDOPTS='-m "local and not network"' # network mode is not supported for python version < 3.6 - os: ubuntu-20.04 python-version: "3.6" From bd2da7627b1d3936f7175affc76c6b0fa972fce6 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Fri, 26 Apr 2024 14:23:59 -0400 Subject: [PATCH 53/55] remove code to support python versions < 3.8 --- Makefile | 16 -------------- magpie/constants.py | 10 --------- tests/interfaces.py | 54 +++++++++++++++++++-------------------------- tests/utils.py | 4 ++-- 4 files changed, 25 insertions(+), 59 deletions(-) diff --git a/Makefile b/Makefile index 284b518e0..51a9a81f4 100644 --- a/Makefile +++ b/Makefile @@ -539,14 +539,6 @@ check-security-only: check-security-code-only check-security-deps-only ## run s # ignored codes: # 42194: https://github.com/kvesteri/sqlalchemy-utils/issues/166 # not fixed since 2015 # 51668: https://github.com/sqlalchemy/sqlalchemy/pull/8563 # still in beta + major version change sqlalchemy 2.0.0b1 -# 51021: This is patched in jwcrypto>=1.4.0 but that version is not available for python version < 3.6 -# 66713: This is patched in jwcrypto>=1.5.1 but that version is not available for python version < 3.6 -# 63154: This is patched in jwcrypto>=1.5.1 but that version is not available for python version < 3.6 -# 64484: This is patched in bandit>=1.7.8 but that version is not available for python version < 3.8 -# 43407: This is an advisory that support for python < 3.6 is no longer supported in the next version -# 43452: This is a duplicate of 43407 -# 43450: This is a duplicate of 43407 -# 43451: This is a duplicate of 43407 .PHONY: check-security-deps-only check-security-deps-only: mkdir-reports ## run security checks on package dependencies @echo "Running security checks of dependencies..." @@ -559,14 +551,6 @@ check-security-deps-only: mkdir-reports ## run security checks on package depen -r "$(APP_ROOT)/requirements-sys.txt" \ -i 42194 \ -i 51668 \ - -i 51021 \ - -i 66713 \ - -i 63154 \ - -i 64484 \ - -i 43407 \ - -i 43452 \ - -i 43450 \ - -i 43451 \ 1> >(tee "$(REPORTS_DIR)/check-security-deps.txt")' .PHONY: check-security-code-only diff --git a/magpie/constants.py b/magpie/constants.py index c8205f621..3977157e7 100644 --- a/magpie/constants.py +++ b/magpie/constants.py @@ -256,21 +256,11 @@ def protected_group_name_regex(include_admin=True, return re.compile("^({})$".format("|".join(patterns))) -def network_mode_supported(): - # type: () -> bool - """ - Return whether the current python version supports network mode. - """ - return (sys.version_info.major, sys.version_info.minor) > (3, 5) - - def network_enabled(settings_container=None): # type: (Optional[AnySettingsContainer]) -> bool """ Return whether network mode is enabled. """ - if not network_mode_supported(): - return False return asbool(get_constant("MAGPIE_NETWORK_ENABLED", settings_container=settings_container)) diff --git a/tests/interfaces.py b/tests/interfaces.py index b80acc675..300b2044a 100644 --- a/tests/interfaces.py +++ b/tests/interfaces.py @@ -24,7 +24,7 @@ from magpie import __meta__ from magpie.api import schemas as s from magpie.api.webhooks import webhook_update_error_status -from magpie.constants import MAGPIE_ROOT, get_constant, network_mode_supported +from magpie.constants import MAGPIE_ROOT, get_constant from magpie.models import RESOURCE_TYPE_DICT, Directory, File, Layer, Process, Route, Service, UserStatuses, Workspace from magpie.permissions import ( PERMISSION_REASON_DEFAULT, @@ -205,28 +205,27 @@ def cleanup(cls): for res in list(cls.extra_resource_ids): # copy to update removed ones utils.TestSetup.delete_TestResource(cls, res) cls.extra_resource_ids.discard(res) - if network_mode_supported(): # This check is required when the python version is < 3.6 - for node in list(cls.extra_node_names): - check_network_mode(utils.TestSetup.delete_TestNetworkNode, - enable=True)(cls, override_name=node, allow_missing=True) - cls.extra_node_names.discard(node) - for remote_user_name, node_name in list(cls.extra_network_tokens): - check_network_mode(utils.TestSetup.delete_TestNetworkToken, - enable=True)(cls, - override_remote_user_name=remote_user_name, - override_node_name=node_name, - allow_missing=True) # should already be deleted with associated models - cls.extra_network_tokens.discard((remote_user_name, node_name)) - for remote_user_name, node_name in list(cls.extra_remote_user_names): - check_network_mode(utils.TestSetup.delete_TestNetworkRemoteUser, - enable=True)(cls, override_remote_user_name=remote_user_name, - override_node_name=node_name, - allow_missing=True) # should already be deleted with associated models - cls.extra_remote_user_names.discard((remote_user_name, node_name)) - for host_port, server in list(cls.extra_remote_servers.items()): - if server.is_running(): - server.stop() - cls.extra_remote_servers.pop(host_port) + for node in list(cls.extra_node_names): + check_network_mode(utils.TestSetup.delete_TestNetworkNode, + enable=True)(cls, override_name=node, allow_missing=True) + cls.extra_node_names.discard(node) + for remote_user_name, node_name in list(cls.extra_network_tokens): + check_network_mode(utils.TestSetup.delete_TestNetworkToken, + enable=True)(cls, + override_remote_user_name=remote_user_name, + override_node_name=node_name, + allow_missing=True) # should already be deleted with associated models + cls.extra_network_tokens.discard((remote_user_name, node_name)) + for remote_user_name, node_name in list(cls.extra_remote_user_names): + check_network_mode(utils.TestSetup.delete_TestNetworkRemoteUser, + enable=True)(cls, override_remote_user_name=remote_user_name, + override_node_name=node_name, + allow_missing=True) # should already be deleted with associated models + cls.extra_remote_user_names.discard((remote_user_name, node_name)) + for host_port, server in list(cls.extra_remote_servers.items()): + if server.is_running(): + server.stop() + cls.extra_remote_servers.pop(host_port) @property def update_method(self): @@ -1341,14 +1340,7 @@ def test_PostNetworkToken_InvalidNodeCredentials(self): token = utils.TestSetup.create_TestNetworkToken(self, expect_errors=True) utils.check_val_equal(token.get("token"), None) utils.check_val_equal(token.get("code"), 500) - # python version < 3.7 uses pyjwt version < 2.5.0 - # pyjwt 2.5.0 changed the error type from HTTPError to PyJWKClientConnectionError if a valid JWKS could not - # be found at the given uri. - if (sys.version_info.major, sys.version_info.minor) < (3, 7): - error_type = "HTTPError" - else: - error_type = "PyJWKClientConnectionError" - utils.check_val_equal(token.get("call", {}).get("exception"), error_type) + utils.check_val_equal(token.get("call", {}).get("exception"), "PyJWKClientConnectionError") @runner.MAGPIE_TEST_NETWORK @utils.check_network_mode diff --git a/tests/utils.py b/tests/utils.py index 96b5c4dc7..27d4f66f2 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -42,7 +42,7 @@ from magpie import __meta__, app, services from magpie.api import schemas from magpie.compat import LooseVersion -from magpie.constants import get_constant, network_mode_supported +from magpie.constants import get_constant from magpie.permissions import Access, PermissionSet, Scope from magpie.services import SERVICE_TYPE_DICT, ServiceAccess from magpie.utils import ( @@ -759,7 +759,7 @@ def check_network_mode(_test_func=None, enable=True): :param _test_func: Test function being decorated. :param enable: Boolean value indicating whether ``_test_func`` expects network mode to be enabled in order to pass. """ - if enable and network_mode_supported(): + if enable: settings = {"magpie.network_enabled": True} else: settings = {"magpie.network_enabled": False} From c5057cba3d1313a1679b49f0693caa5633d4f57f Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Fri, 26 Apr 2024 14:50:58 -0400 Subject: [PATCH 54/55] remove unused import --- magpie/constants.py | 1 - 1 file changed, 1 deletion(-) diff --git a/magpie/constants.py b/magpie/constants.py index 3977157e7..4a2cadb88 100644 --- a/magpie/constants.py +++ b/magpie/constants.py @@ -16,7 +16,6 @@ import os import re import shutil -import sys import warnings from typing import TYPE_CHECKING From 9a839b73cf906b7cd3332682954fb0d19e43927b Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Fri, 26 Apr 2024 14:55:30 -0400 Subject: [PATCH 55/55] another one --- tests/interfaces.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/interfaces.py b/tests/interfaces.py index 300b2044a..67d41769e 100644 --- a/tests/interfaces.py +++ b/tests/interfaces.py @@ -5,7 +5,6 @@ import os import secrets import string -import sys import unittest import uuid from abc import ABCMeta, abstractmethod