From 05b3199e8bfd55af175b75f9c790915a63c01a9e Mon Sep 17 00:00:00 2001 From: Holder Roberts Date: Mon, 6 Mar 2023 02:27:08 +0800 Subject: [PATCH 01/32] Update SQL Server engine as mssql-django (#446) --- CHANGELOG.rst | 8 ++++++++ environ/environ.py | 2 +- tests/test_db.py | 4 ++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6b310499..ed60d065 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file. The format is inspired by `Keep a Changelog `_ and this project adheres to `Semantic Versioning `_. +`v0.10.1`_ - 5-March-2023 +------------------------------ +Changed ++++++++ +- Used `mssql-django` as engine for SQL Server. + `#446 `_. + + `v0.10.0`_ - 2-March-2023 ------------------------------- Added diff --git a/environ/environ.py b/environ/environ.py index 9ca00a8e..f4acdcfc 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -121,7 +121,7 @@ class Env: 'mysql2': 'django.db.backends.mysql', 'mysql-connector': 'mysql.connector.django', 'mysqlgis': 'django.contrib.gis.db.backends.mysql', - 'mssql': 'sql_server.pyodbc', + 'mssql': 'mssql', 'oracle': 'django.db.backends.oracle', 'pyodbc': 'sql_server.pyodbc', 'redshift': 'django_redshift_backend', diff --git a/tests/test_db.py b/tests/test_db.py index c4074131..fe502845 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -118,7 +118,7 @@ # mysql://user:password@host/dbname ('mssql://enigma:secret@example.com/dbname' '?driver=ODBC Driver 13 for SQL Server', - 'sql_server.pyodbc', + 'mssql', 'dbname', 'example.com', 'enigma', @@ -127,7 +127,7 @@ # mysql://user:password@host:port/dbname ('mssql://enigma:secret@amazonaws.com\\insnsnss:12345/dbname' '?driver=ODBC Driver 13 for SQL Server', - 'sql_server.pyodbc', + 'mssql', 'dbname', 'amazonaws.com\\insnsnss', 'enigma', From d4fa3588d9e97dec987752d474160d69d0334a0a Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Sun, 5 Mar 2023 19:40:34 +0100 Subject: [PATCH 02/32] Fix changelog formatting and links --- CHANGELOG.rst | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ed60d065..419fe7c7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,16 +5,16 @@ All notable changes to this project will be documented in this file. The format is inspired by `Keep a Changelog `_ and this project adheres to `Semantic Versioning `_. -`v0.10.1`_ - 5-March-2023 ------------------------------- +`v0.11.0`_ - 00-Unreleased-2023 +------------------------------- Changed +++++++ -- Used `mssql-django` as engine for SQL Server. +- Used ``mssql-django`` as engine for SQL Server `#446 `_. `v0.10.0`_ - 2-March-2023 -------------------------------- +------------------------- Added +++++ - Use the core redis library by default if running Django >= 4.0 @@ -37,7 +37,7 @@ Deprecated Changed +++++++ - Used UTF-8 as a encoding when open ``.env`` file. -- Provided access to ```DB_SCHEMES`` through ``cls`` rather than +- Provided access to ``DB_SCHEMES`` through ``cls`` rather than ``Env`` in ``db_url_config`` `#414 `_. - Correct CI workflow to use supported Python versions/OS matrix @@ -349,7 +349,8 @@ Added - Initial release. -.. _v0.10.0: https://github.com/joke2k/django-environ/compare/v0.9.0...develop +.. _v0.11.0: https://github.com/joke2k/django-environ/compare/v0.10.0...develop +.. _v0.10.0: https://github.com/joke2k/django-environ/compare/v0.9.0...v0.10.0 .. _v0.9.0: https://github.com/joke2k/django-environ/compare/v0.8.1...v0.9.0 .. _v0.8.1: https://github.com/joke2k/django-environ/compare/v0.8.0...v0.8.1 .. _v0.8.0: https://github.com/joke2k/django-environ/compare/v0.7.0...v0.8.0 From d3ae91d1a935a3a8ff283c16e83e49c9e61ef7cb Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Sun, 5 Mar 2023 19:41:02 +0100 Subject: [PATCH 03/32] Update links in contributing guide --- CONTRIBUTING.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 29d555ef..549b7f84 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -74,5 +74,5 @@ Resources --------- * `How to Contribute to Open Source `_ -* `Using Pull Requests `_ -* `Writing good commit messages `_ +* `Using Pull Requests `_ +* `Writing good commit messages `_ From 337d3bc6b45f466c0b8b81b479d715309f598e04 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Sun, 5 Mar 2023 19:41:19 +0100 Subject: [PATCH 04/32] Fix example for proxy value --- docs/tips.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tips.rst b/docs/tips.rst index 1915fac9..20b8c3e1 100644 --- a/docs/tips.rst +++ b/docs/tips.rst @@ -236,7 +236,7 @@ Values that being with a ``$`` may be interpolated. Pass ``interpolate=True`` to # BAR=FOO # PROXY=$BAR - >>> print env.str('PROXY') + >>> print(env.str('PROXY')) FOO From f8f59da1e841a574d1974816e805c81538da4328 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Sun, 5 Mar 2023 19:41:30 +0100 Subject: [PATCH 05/32] Bump version --- environ/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/environ/__init__.py b/environ/__init__.py index c79736cc..54e9465c 100644 --- a/environ/__init__.py +++ b/environ/__init__.py @@ -1,6 +1,6 @@ # This file is part of the django-environ. # -# Copyright (c) 2021-2022, Serghei Iakovlev +# Copyright (c) 2021-2023, Serghei Iakovlev # Copyright (c) 2013-2021, Daniele Faraglia # # For the full copyright and license information, please view @@ -21,7 +21,7 @@ __copyright__ = 'Copyright (C) 2013-2022 Daniele Faraglia' """The copyright notice of the package.""" -__version__ = '0.10.0' +__version__ = '0.11.0' """The version of the package.""" __license__ = 'MIT' From 3d92784a9a1d369999588ba300b4bec6cf85bf98 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Sun, 5 Mar 2023 19:42:35 +0100 Subject: [PATCH 06/32] Removed support of Python 3.5 --- .github/workflows/ci.yml | 2 -- CHANGELOG.rst | 4 ++++ README.rst | 2 +- setup.py | 26 ++++++++++++++------------ tox.ini | 3 +-- 5 files changed, 20 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3261d728..13ed2a3f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,8 +59,6 @@ jobs: # versions by django-environ will continue for as long as possible, # and may be discontinued at any time. include: - - python: '3.5' - os: ubuntu-20.04 - python: '3.6' os: ubuntu-20.04 - python: '3.7' diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 419fe7c7..5167dddd 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,6 +12,10 @@ Changed - Used ``mssql-django`` as engine for SQL Server `#446 `_. +Removed ++++++++ +- Removed support of Python 3.5. + `v0.10.0`_ - 2-March-2023 ------------------------- diff --git a/README.rst b/README.rst index cf999e23..8cd42629 100644 --- a/README.rst +++ b/README.rst @@ -126,7 +126,7 @@ its documentation lives at `Read the Docs `_, and the latest release on `PyPI `_. -It’s rigorously tested on Python 3.5+, and officially supports +It’s rigorously tested on Python 3.6+, and officially supports Django 1.11, 2.2, 3., 3.1, 3.2, 4.0 and 4.1. If you'd like to contribute to ``django-environ`` you're most welcome! diff --git a/setup.py b/setup.py index d317bc0a..43147e3e 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ # # This file is part of the django-environ. # -# Copyright (c) 2021, Serghei Iakovlev +# Copyright (c) 2021-2023, Serghei Iakovlev # Copyright (c) 2013-2021, Daniele Faraglia # # For the full copyright and license information, please view @@ -10,19 +10,22 @@ import codecs import re -import sys -import warnings from os import path from setuptools import find_packages, setup -if sys.version_info < (3, 6): - warnings.warn( - "Support of Python < 3.6 is deprecated" - "and will be removed in a future release.", - DeprecationWarning - ) +# Use this code block for future deprecations of Python version: +# +# import warnings +# import sys +# +# if sys.version_info < (3, 6): +# warnings.warn( +# "Support of Python < 3.6 is deprecated" +# "and will be removed in a future release.", +# DeprecationWarning +# ) def read_file(filepath): @@ -120,7 +123,7 @@ def get_version_string(): return version_string -# What does this project relate to. +# What does this project relate to? KEYWORDS = [ 'environment', 'django', @@ -150,7 +153,6 @@ def get_version_string(): 'Programming Language :: Python', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', @@ -228,7 +230,7 @@ def get_version_string(): platforms=['any'], include_package_data=True, zip_safe=False, - python_requires='>=3.5,<4', + python_requires='>=3.6,<4', install_requires=INSTALL_REQUIRES, dependency_links=DEPENDENCY_LINKS, extras_require=EXTRAS_REQUIRE, diff --git a/tox.ini b/tox.ini index 6251d4ab..fad8dcf9 100644 --- a/tox.ini +++ b/tox.ini @@ -18,14 +18,13 @@ envlist = docs lint manifest - py{35,36,37,38,39,310,311}-django{111,22} + py{36,37,38,39,310,311}-django{111,22} py{36,37,38,39,310,311}-django{30,31,32} py{38,39,310,311}-django{40,41} pypy-django{111,22,30,31,32} [gh-actions] python = - 3.5: py35 3.6: py36 3.7: py37 3.8: py38 From faf7a4ee9a748fc74c10510a80453d4eb1481a1d Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Sun, 5 Mar 2023 19:49:09 +0100 Subject: [PATCH 07/32] Update sphinx configuration --- docs/conf.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 765f9947..8beac1f4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,6 +1,6 @@ # This file is part of the django-environ. # -# Copyright (c) 2021-2022, Serghei Iakovlev +# Copyright (c) 2021-2023, Serghei Iakovlev # Copyright (c) 2013-2021, Daniele Faraglia # # For the full copyright and license information, please view @@ -12,12 +12,10 @@ import codecs import os -import sys import re - +import sys from datetime import date - PROJECT_DIR = os.path.abspath('..') sys.path.insert(0, PROJECT_DIR) @@ -71,7 +69,7 @@ def find_version(meta_file): # The suffix of source filenames. source_suffix = ".rst" -# Allow non-local URIs so we can have images in CHANGELOG etc. +# Allow non-local URIs, so we can have images in CHANGELOG etc. suppress_warnings = [ "image.nonlocal_uri", ] From bec18530b41745f4ecf1f0a39d690c1e3c41d4a4 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Sun, 5 Mar 2023 20:01:34 +0100 Subject: [PATCH 08/32] Using f-strings --- environ/environ.py | 43 ++++++++++++++++++++----------------------- tox.ini | 5 +---- 2 files changed, 21 insertions(+), 27 deletions(-) diff --git a/environ/environ.py b/environ/environ.py index f4acdcfc..67b7f7d8 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -69,7 +69,7 @@ class NoValue: """Represent of no value object.""" def __repr__(self): - return '<{}>'.format(self.__class__.__name__) + return f'<{self.__class__.__name__}>' class Env: @@ -361,7 +361,7 @@ def get_value(self, var, cast=None, default=NOTSET, parse_default=False): "get '%s' casted as '%s' with default '%s'", var, cast, default) - var_name = "{}{}".format(self.prefix, var) + var_name = f'{self.prefix}{var}' if var_name in self.scheme: var_info = self.scheme[var_name] @@ -387,7 +387,7 @@ def get_value(self, var, cast=None, default=NOTSET, parse_default=False): value = self.ENVIRON[var_name] except KeyError as exc: if default is self.NOTSET: - error_msg = "Set the {} environment variable".format(var) + error_msg = f'Set the {var} environment variable' raise ImproperlyConfigured(error_msg) from exc value = default @@ -467,7 +467,7 @@ def parse_value(cls, value, cast): if len(parts) == 1: float_str = parts[0] else: - float_str = "{}.{}".format(''.join(parts[0:-1]), parts[-1]) + float_str = f"{''.join(parts[0:-1])}.{parts[-1]}" value = float(float_str) else: value = cast(value) @@ -524,15 +524,15 @@ def db_url_config(cls, url, engine=None): # sqlalchemy) path = ':memory:' if url.netloc: - warnings.warn('SQLite URL contains host component %r, ' - 'it will be ignored' % url.netloc, stacklevel=3) + warnings.warn( + f'SQLite URL contains host component {url.netloc!r}, ' + 'it will be ignored', + stacklevel=3 + ) if url.scheme == 'ldap': - path = '{scheme}://{hostname}'.format( - scheme=url.scheme, - hostname=url.hostname, - ) + path = f'{url.scheme}://{url.hostname}' if url.port: - path += ':{port}'.format(port=url.port) + path += f':{url.port}' user_host = url.netloc.rsplit('@', 1) if url.scheme in cls.POSTGRES_FAMILY and ',' in user_host[-1]: @@ -595,7 +595,7 @@ def db_url_config(cls, url, engine=None): config['ENGINE'] = cls.DB_SCHEMES[config['ENGINE']] if not config.get('ENGINE', False): - warnings.warn("Engine not recognized from url: {}".format(config)) + warnings.warn(f'Engine not recognized from url: {config}') return {} return config @@ -617,9 +617,7 @@ def cache_url_config(cls, url, backend=None): url = urlparse(url) if url.scheme not in cls.CACHE_SCHEMES: - raise ImproperlyConfigured( - 'Invalid cache schema {}'.format(url.scheme) - ) + raise ImproperlyConfigured(f'Invalid cache schema {url.scheme}') location = url.netloc.split(',') if len(location) == 1: @@ -707,7 +705,7 @@ def email_url_config(cls, url, backend=None): if backend: config['EMAIL_BACKEND'] = backend elif url.scheme not in cls.EMAIL_SCHEMES: - raise ImproperlyConfigured('Invalid email schema %s' % url.scheme) + raise ImproperlyConfigured(f'Invalid email schema {url.scheme}') elif url.scheme in cls.EMAIL_SCHEMES: config['EMAIL_BACKEND'] = cls.EMAIL_SCHEMES[url.scheme] @@ -749,13 +747,11 @@ def search_url_config(cls, url, engine=None): path = unquote_plus(path.split('?', 2)[0]) if url.scheme not in cls.SEARCH_SCHEMES: - raise ImproperlyConfigured( - 'Invalid search schema %s' % url.scheme - ) + raise ImproperlyConfigured(f'Invalid search schema {url.scheme}') config["ENGINE"] = cls.SEARCH_SCHEMES[url.scheme] # check commons params - params = {} # type: dict + params = {} if url.query: params = parse_qs(url.query) if 'EXCLUDED_INDEXES' in params: @@ -779,7 +775,7 @@ def search_url_config(cls, url, engine=None): config['KWARGS'] = params['KWARGS'][0] # remove trailing slash - if path.endswith("/"): + if path.endswith('/'): path = path[:-1] if url.scheme == 'solr': @@ -1023,7 +1019,7 @@ def __contains__(self, item): return item.__root__.startswith(base_path) def __repr__(self): - return "".format(self.__root__) + return f'' def __str__(self): return self.__root__ @@ -1050,5 +1046,6 @@ def _absolute_join(base, *paths, **kwargs): absolute_path = os.path.abspath(os.path.join(base, *paths)) if kwargs.get('required', False) and not os.path.exists(absolute_path): raise ImproperlyConfigured( - "Create required path: {}".format(absolute_path)) + f'Create required path: {absolute_path}' + ) return absolute_path diff --git a/tox.ini b/tox.ini index fad8dcf9..0d26af4b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ # This file is part of the django-environ. # -# Copyright (c) 2021, Serghei Iakovlev +# Copyright (c) 2021-2023, Serghei Iakovlev # Copyright (c) 2013-2021, Daniele Faraglia # # For the full copyright and license information, please view @@ -74,15 +74,12 @@ commands_pre = python -m pip install . commands = flake8 environ setup.py - # Format ("f") strings have not been introduced before Python 3.6, - # thus disable "consider-using-f-string" at this moment. pylint \ --logging-format-style=old \ --good-names-rgxs=m[0-9],f,v \ --disable=too-few-public-methods \ --disable=import-error \ --disable=unused-import \ - --disable=consider-using-f-string \ --disable=too-many-locals \ --disable=too-many-branches \ --disable=too-many-public-methods \ From 359c37dfffa1fafc0e38b7af3fc7accdf3901854 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9ni=20Gauffier?= Date: Fri, 10 Mar 2023 17:47:22 +0100 Subject: [PATCH 09/32] Prioritize django-redis for REDIS_DRIVER --- .gitignore | 4 ++++ environ/compat.py | 11 +++++++---- tests/test_cache.py | 15 ++++++++------- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 58629014..e400255c 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,8 @@ /*.egg-info /htmlcov /docs/_build +/.venv +/.vscode # Python cache. *.py[cod] @@ -29,3 +31,5 @@ __pycache__ # Ignore codecoverage stuff. .coverage* coverage.xml + + diff --git a/environ/compat.py b/environ/compat.py index 0cec1e5e..22236f1b 100644 --- a/environ/compat.py +++ b/environ/compat.py @@ -9,6 +9,7 @@ """This module handles import compatibility issues.""" from pkgutil import find_loader +import warnings if find_loader('simplejson'): @@ -28,15 +29,17 @@ class ImproperlyConfigured(Exception): def choose_rediscache_driver(): """Backward compatibility for RedisCache driver.""" + + # django-redis library takes precedence + if find_loader('django_redis'): + return 'django_redis.cache.RedisCache' + # use built-in support if Django 4+ if DJANGO_VERSION is not None and DJANGO_VERSION >= (4, 0): return 'django.core.cache.backends.redis.RedisCache' # back compatibility with redis_cache package - if find_loader('redis_cache'): - return 'redis_cache.RedisCache' - return 'django_redis.cache.RedisCache' - + return 'redis_cache.RedisCache' def choose_postgres_driver(): """Backward compatibility for postgresql driver.""" diff --git a/tests/test_cache.py b/tests/test_cache.py index a762c1b1..37d7b6eb 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -117,21 +117,22 @@ def test_pymemcache_compat(django_version, pymemcache_installed): @pytest.mark.parametrize('django_version', ((4, 0), (3, 2), None)) -@pytest.mark.parametrize('redis_cache_installed', (True, False)) -def test_rediscache_compat(django_version, redis_cache_installed): +@pytest.mark.parametrize('django_redis_installed', (True, False)) +def test_rediscache_compat(django_version, django_redis_installed): django_new = 'django.core.cache.backends.redis.RedisCache' redis_cache = 'redis_cache.RedisCache' - django_old = 'django_redis.cache.RedisCache' + django_redis = 'django_redis.cache.RedisCache' with mock.patch.object(environ.compat, 'DJANGO_VERSION', django_version): with mock.patch('environ.compat.find_loader') as mock_find_loader: - mock_find_loader.return_value = redis_cache_installed + mock_find_loader.return_value = django_redis_installed driver = environ.compat.choose_rediscache_driver() - if django_version and django_version >= (4, 0): + if django_redis_installed: + assert driver == django_redis + elif django_version and django_version >= (4, 0): assert driver == django_new else: - assert driver == redis_cache if redis_cache_installed else django_old - + assert driver == redis_cache def test_redis_parsing(): url = ('rediscache://127.0.0.1:6379/1?client_class=' From d88db95f07344d7f7aee9aa3992b35f422586d45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9ni=20Gauffier?= Date: Mon, 13 Mar 2023 13:40:28 +0100 Subject: [PATCH 10/32] lint --- .gitignore | 4 ---- environ/compat.py | 5 ++--- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index e400255c..58629014 100644 --- a/.gitignore +++ b/.gitignore @@ -21,8 +21,6 @@ /*.egg-info /htmlcov /docs/_build -/.venv -/.vscode # Python cache. *.py[cod] @@ -31,5 +29,3 @@ __pycache__ # Ignore codecoverage stuff. .coverage* coverage.xml - - diff --git a/environ/compat.py b/environ/compat.py index 22236f1b..b4ffee4e 100644 --- a/environ/compat.py +++ b/environ/compat.py @@ -9,8 +9,6 @@ """This module handles import compatibility issues.""" from pkgutil import find_loader -import warnings - if find_loader('simplejson'): import simplejson as json @@ -33,7 +31,7 @@ def choose_rediscache_driver(): # django-redis library takes precedence if find_loader('django_redis'): return 'django_redis.cache.RedisCache' - + # use built-in support if Django 4+ if DJANGO_VERSION is not None and DJANGO_VERSION >= (4, 0): return 'django.core.cache.backends.redis.RedisCache' @@ -41,6 +39,7 @@ def choose_rediscache_driver(): # back compatibility with redis_cache package return 'redis_cache.RedisCache' + def choose_postgres_driver(): """Backward compatibility for postgresql driver.""" old_django = DJANGO_VERSION is not None and DJANGO_VERSION < (2, 0) From 1e834a19e6f671bbe584687a13b70f57c5e684b6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Mar 2023 17:57:39 +0000 Subject: [PATCH 11/32] Bump actions/checkout from 3.3.0 to 3.4.0 Bumps [actions/checkout](https://github.com/actions/checkout) from 3.3.0 to 3.4.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3.3.0...v3.4.0) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 4 ++-- .github/workflows/ci.yml | 2 +- .github/workflows/codeql.yml | 2 +- .github/workflows/cs.yml | 2 +- .github/workflows/docs.yml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 636ad602..2791399f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,7 +24,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.3.0 + uses: actions/checkout@v3.4.0 - name: Set up Python 3.10 uses: actions/setup-python@v4.5.0 @@ -64,7 +64,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.3.0 + uses: actions/checkout@v3.4.0 - name: Set up Python 3.10 uses: actions/setup-python@v4.5.0 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 13ed2a3f..1687e0d1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,7 +66,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.3.0 + uses: actions/checkout@v3.4.0 with: fetch-depth: 5 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 4cf741a6..57a59f03 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -45,7 +45,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3.3.0 + uses: actions/checkout@v3.4.0 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/cs.yml b/.github/workflows/cs.yml index 795a1533..9143a2dd 100644 --- a/.github/workflows/cs.yml +++ b/.github/workflows/cs.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.3.0 + uses: actions/checkout@v3.4.0 - name: Set up Python 3.10 uses: actions/setup-python@v4.5.0 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index e079bee6..3e4bc5e9 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.3.0 + uses: actions/checkout@v3.4.0 - name: Set up Python 3.10 uses: actions/setup-python@v4.5.0 From f43deca453370829e0c711cfd70534990ef9fa74 Mon Sep 17 00:00:00 2001 From: "KimSia, Sim" <245021+simkimsia@users.noreply.github.com> Date: Fri, 24 Mar 2023 23:48:25 +0800 Subject: [PATCH 12/32] Add support for Django 4.2 --- README.rst | 2 +- setup.py | 2 +- tox.ini | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 8cd42629..29b8ec72 100644 --- a/README.rst +++ b/README.rst @@ -127,7 +127,7 @@ the code on `GitHub `_, and the latest release on `PyPI `_. It’s rigorously tested on Python 3.6+, and officially supports -Django 1.11, 2.2, 3., 3.1, 3.2, 4.0 and 4.1. +Django 1.11, 2.2, 3., 3.1, 3.2, 4.0, 4.1, and 4.2. If you'd like to contribute to ``django-environ`` you're most welcome! diff --git a/setup.py b/setup.py index 43147e3e..30525752 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,6 @@ from setuptools import find_packages, setup - # Use this code block for future deprecations of Python version: # # import warnings @@ -145,6 +144,7 @@ def get_version_string(): 'Framework :: Django :: 3.2', 'Framework :: Django :: 4.0', 'Framework :: Django :: 4.1', + 'Framework :: Django :: 4.2', 'Operating System :: OS Independent', diff --git a/tox.ini b/tox.ini index 0d26af4b..ca438d33 100644 --- a/tox.ini +++ b/tox.ini @@ -20,7 +20,7 @@ envlist = manifest py{36,37,38,39,310,311}-django{111,22} py{36,37,38,39,310,311}-django{30,31,32} - py{38,39,310,311}-django{40,41} + py{38,39,310,311}-django{40,41,42} pypy-django{111,22,30,31,32} [gh-actions] @@ -44,6 +44,7 @@ deps = django32: Django>=3.2,<3.3 django40: Django>=4.0,<4.1 django41: Django>=4.1,<4.2 + django42: Django>=4.2rc1,<5.0 commands_pre = python -m pip install --upgrade pip python -m pip install . From 4294fb56d4a13850527ede0089573584b641c624 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Mar 2023 17:57:26 +0000 Subject: [PATCH 13/32] Bump actions/checkout from 3.4.0 to 3.5.0 Bumps [actions/checkout](https://github.com/actions/checkout) from 3.4.0 to 3.5.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3.4.0...v3.5.0) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 4 ++-- .github/workflows/ci.yml | 2 +- .github/workflows/codeql.yml | 2 +- .github/workflows/cs.yml | 2 +- .github/workflows/docs.yml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2791399f..4738a07d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,7 +24,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.4.0 + uses: actions/checkout@v3.5.0 - name: Set up Python 3.10 uses: actions/setup-python@v4.5.0 @@ -64,7 +64,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.4.0 + uses: actions/checkout@v3.5.0 - name: Set up Python 3.10 uses: actions/setup-python@v4.5.0 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1687e0d1..150373e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,7 +66,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.4.0 + uses: actions/checkout@v3.5.0 with: fetch-depth: 5 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 57a59f03..4a560a7c 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -45,7 +45,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3.4.0 + uses: actions/checkout@v3.5.0 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/cs.yml b/.github/workflows/cs.yml index 9143a2dd..640b5f95 100644 --- a/.github/workflows/cs.yml +++ b/.github/workflows/cs.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.4.0 + uses: actions/checkout@v3.5.0 - name: Set up Python 3.10 uses: actions/setup-python@v4.5.0 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 3e4bc5e9..a58efbba 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.4.0 + uses: actions/checkout@v3.5.0 - name: Set up Python 3.10 uses: actions/setup-python@v4.5.0 From d1494c605e0d5639e18a8c66a71737ddcb920873 Mon Sep 17 00:00:00 2001 From: MayGrass <15877196+MayGrass@users.noreply.github.com> Date: Fri, 31 Mar 2023 15:06:30 +0800 Subject: [PATCH 14/32] Added support for parsing connection strings with special characters in the password and handling invalid IPv6 URLs. (#461) * Added new tests for connection strings with special characters in password and invalid IPv6 password. * Fix code style issue --------- Co-authored-by: Serghei Iakovlev --- environ/environ.py | 16 +++++++++++++++- tests/test_db.py | 18 ++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/environ/environ.py b/environ/environ.py index 67b7f7d8..cbbf1077 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -21,6 +21,7 @@ from urllib.parse import ( parse_qs, ParseResult, + quote, unquote, unquote_plus, urlparse, @@ -65,6 +66,10 @@ def _cast_urlstr(v): return unquote(v) if isinstance(v, str) else v +def _urlparse_quote(url): + return urlparse(quote(url, safe=':/?&=@')) + + class NoValue: """Represent of no value object.""" @@ -474,7 +479,9 @@ def parse_value(cls, value, cast): return value @classmethod + # pylint: disable=too-many-statements def db_url_config(cls, url, engine=None): + # pylint: enable-msg=too-many-statements """Parse an arbitrary database URL. Supports the following URL schemas: @@ -509,10 +516,17 @@ def db_url_config(cls, url, engine=None): 'NAME': ':memory:' } # note: no other settings are required for sqlite - url = urlparse(url) + try: + url = urlparse(url) + # handle Invalid IPv6 URL + except ValueError: + url = _urlparse_quote(url) config = {} + # handle unexpected URL schemes with special characters + if not url.path: + url = _urlparse_quote(urlunparse(url)) # Remove query strings. path = url.path[1:] path = unquote_plus(path.split('?', 2)[0]) diff --git a/tests/test_db.py b/tests/test_db.py index fe502845..8101a4a7 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -133,6 +133,22 @@ 'enigma', 'secret', 12345), + # mysql://user:password@host:port/dbname + ('mysql://enigma:><{~!@#$%^&*}[]@example.com:1234/dbname', + 'django.db.backends.mysql', + 'dbname', + 'example.com', + 'enigma', + '><{~!@#$%^&*}[]', + 1234), + # mysql://user:password@host/dbname + ('mysql://enigma:]password]@example.com/dbname', + 'django.db.backends.mysql', + 'dbname', + 'example.com', + 'enigma', + ']password]', + ''), ], ids=[ 'postgres', @@ -149,6 +165,8 @@ 'ldap', 'mssql', 'mssql_port', + 'mysql_password_special_chars', + 'mysql_invalid_ipv6_password', ], ) def test_db_parsing(url, engine, name, host, user, passwd, port): From 6cee1463a73cba15ec788166437ec3c04c39e472 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Apr 2023 17:57:15 +0000 Subject: [PATCH 15/32] Bump actions/checkout from 3.5.0 to 3.5.2 Bumps [actions/checkout](https://github.com/actions/checkout) from 3.5.0 to 3.5.2. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3.5.0...v3.5.2) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 4 ++-- .github/workflows/ci.yml | 2 +- .github/workflows/codeql.yml | 2 +- .github/workflows/cs.yml | 2 +- .github/workflows/docs.yml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4738a07d..f964a680 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,7 +24,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.5.0 + uses: actions/checkout@v3.5.2 - name: Set up Python 3.10 uses: actions/setup-python@v4.5.0 @@ -64,7 +64,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.5.0 + uses: actions/checkout@v3.5.2 - name: Set up Python 3.10 uses: actions/setup-python@v4.5.0 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 150373e8..e77660f6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,7 +66,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.5.0 + uses: actions/checkout@v3.5.2 with: fetch-depth: 5 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 4a560a7c..a9b4c0ff 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -45,7 +45,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3.5.0 + uses: actions/checkout@v3.5.2 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/cs.yml b/.github/workflows/cs.yml index 640b5f95..5bec0e53 100644 --- a/.github/workflows/cs.yml +++ b/.github/workflows/cs.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.5.0 + uses: actions/checkout@v3.5.2 - name: Set up Python 3.10 uses: actions/setup-python@v4.5.0 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index a58efbba..e455c2f2 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.5.0 + uses: actions/checkout@v3.5.2 - name: Set up Python 3.10 uses: actions/setup-python@v4.5.0 From 428c3cbc7f03ffe97871980be56cc63531fd53b0 Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 22 Apr 2023 17:12:52 -0600 Subject: [PATCH 16/32] Added support for secure Elasticsearch connections (#463) --- CHANGELOG.rst | 5 +++++ docs/types.rst | 8 ++++---- environ/environ.py | 16 ++++++++++++---- tests/test_search.py | 33 +++++++++++++++++++++++++++------ 4 files changed, 48 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5167dddd..682b1072 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ and this project adheres to `Semantic Versioning `_. + Changed +++++++ - Used ``mssql-django`` as engine for SQL Server diff --git a/docs/types.rst b/docs/types.rst index 3fcdcbbd..c099bf91 100644 --- a/docs/types.rst +++ b/docs/types.rst @@ -156,10 +156,10 @@ For more detailed example see ":ref:`complex_dict_format`". :py:meth:`~.environ.Env.search_url` supports the following URL schemas: -* Elasticsearch: ``elasticsearch://`` -* Elasticsearch2: ``elasticsearch2://`` -* Elasticsearch5: ``elasticsearch5://`` -* Elasticsearch7: ``elasticsearch7://`` +* Elasticsearch: ``elasticsearch://`` (http) or ``elasticsearchs://`` (https) +* Elasticsearch2: ``elasticsearch2://`` (http) or ``elasticsearch2s://`` (https) +* Elasticsearch5: ``elasticsearch5://`` (http) or ``elasticsearch5s://`` (https) +* Elasticsearch7: ``elasticsearch7://`` (http) or ``elasticsearch7s://`` (https) * Solr: ``solr://`` * Whoosh: ``whoosh://`` * Xapian: ``xapian://`` diff --git a/environ/environ.py b/environ/environ.py index cbbf1077..f72f5a3c 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -113,7 +113,6 @@ class Env: URL_CLASS = ParseResult POSTGRES_FAMILY = ['postgres', 'postgresql', 'psql', 'pgsql', 'postgis'] - ELASTICSEARCH_FAMILY = ['elasticsearch' + x for x in ['', '2', '5', '7']] DEFAULT_DATABASE_ENV = 'DATABASE_URL' DB_SCHEMES = { @@ -191,6 +190,9 @@ class Env: "xapian": "haystack.backends.xapian_backend.XapianEngine", "simple": "haystack.backends.simple_backend.SimpleEngine", } + ELASTICSEARCH_FAMILY = [scheme + s for scheme in SEARCH_SCHEMES + if scheme.startswith("elasticsearch") + for s in ('', 's')] CLOUDSQL = 'cloudsql' def __init__(self, **scheme): @@ -760,9 +762,15 @@ def search_url_config(cls, url, engine=None): path = url.path[1:] path = unquote_plus(path.split('?', 2)[0]) - if url.scheme not in cls.SEARCH_SCHEMES: + scheme = url.scheme + secure = False + # elasticsearch supports secure schemes, similar to http -> https + if scheme in cls.ELASTICSEARCH_FAMILY and scheme.endswith('s'): + scheme = scheme[:-1] + secure = True + if scheme not in cls.SEARCH_SCHEMES: raise ImproperlyConfigured(f'Invalid search schema {url.scheme}') - config["ENGINE"] = cls.SEARCH_SCHEMES[url.scheme] + config["ENGINE"] = cls.SEARCH_SCHEMES[scheme] # check commons params params = {} @@ -811,7 +819,7 @@ def search_url_config(cls, url, engine=None): index = split[0] config['URL'] = urlunparse( - ('http',) + url[1:2] + (path,) + ('', '', '') + ('https' if secure else 'http', url[1], path, '', '', '') ) if 'TIMEOUT' in params: config['TIMEOUT'] = cls.parse_value(params['TIMEOUT'][0], int) diff --git a/tests/test_search.py b/tests/test_search.py index 0992bf98..a6d8f061 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -33,25 +33,45 @@ def test_solr_multicore_parsing(solr_url): @pytest.mark.parametrize( - 'url,engine', + 'url,engine,scheme', [ ('elasticsearch://127.0.0.1:9200/index', - 'elasticsearch_backend.ElasticsearchSearchEngine'), + 'elasticsearch_backend.ElasticsearchSearchEngine', + 'http',), + ('elasticsearchs://127.0.0.1:9200/index', + 'elasticsearch_backend.ElasticsearchSearchEngine', + 'https',), ('elasticsearch2://127.0.0.1:9200/index', - 'elasticsearch2_backend.Elasticsearch2SearchEngine'), + 'elasticsearch2_backend.Elasticsearch2SearchEngine', + 'http',), + ('elasticsearch2s://127.0.0.1:9200/index', + 'elasticsearch2_backend.Elasticsearch2SearchEngine', + 'https',), ('elasticsearch5://127.0.0.1:9200/index', - 'elasticsearch5_backend.Elasticsearch5SearchEngine'), + 'elasticsearch5_backend.Elasticsearch5SearchEngine', + 'http'), + ('elasticsearch5s://127.0.0.1:9200/index', + 'elasticsearch5_backend.Elasticsearch5SearchEngine', + 'https'), ('elasticsearch7://127.0.0.1:9200/index', - 'elasticsearch7_backend.Elasticsearch7SearchEngine'), + 'elasticsearch7_backend.Elasticsearch7SearchEngine', + 'http'), + ('elasticsearch7s://127.0.0.1:9200/index', + 'elasticsearch7_backend.Elasticsearch7SearchEngine', + 'https'), ], ids=[ 'elasticsearch', + 'elasticsearchs', 'elasticsearch2', + 'elasticsearch2s', 'elasticsearch5', + 'elasticsearch5s', 'elasticsearch7', + 'elasticsearch7s', ] ) -def test_elasticsearch_parsing(url, engine): +def test_elasticsearch_parsing(url, engine, scheme): """Ensure all supported Elasticsearch engines are recognized.""" timeout = 360 url = '{}?TIMEOUT={}'.format(url, timeout) @@ -63,6 +83,7 @@ def test_elasticsearch_parsing(url, engine): assert 'TIMEOUT' in url.keys() assert url['TIMEOUT'] == timeout assert 'PATH' not in url + assert url["URL"].startswith(scheme + ":") @pytest.mark.parametrize('storage', ['file', 'ram']) From 52455fb5e8d16a6429e02e5900da9c9f04f6fa06 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Sun, 23 Apr 2023 02:05:43 +0200 Subject: [PATCH 17/32] Refactor search_url_config: split into smaller functions for readability This commit refactors the search_url_config function to improve its readability and maintainability by reducing its cyclomatic complexity. The function has been split into smaller, more focused functions, each responsible for handling a specific search scheme or task. This refactoring makes the code more modular and easier to understand, while also simplifying testing and further development. --- environ/environ.py | 131 ++++++++++++++++++++++++++------------------- 1 file changed, 77 insertions(+), 54 deletions(-) diff --git a/environ/environ.py b/environ/environ.py index f72f5a3c..fad85ab3 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -742,6 +742,72 @@ def email_url_config(cls, url, backend=None): return config + @classmethod + def _parse_common_search_params(cls, url): + cfg = {} + prs = {} + + if not url.query or str(url.query) == '': + return cfg, prs + + prs = parse_qs(url.query) + if 'EXCLUDED_INDEXES' in prs: + cfg['EXCLUDED_INDEXES'] = prs['EXCLUDED_INDEXES'][0].split(',') + if 'INCLUDE_SPELLING' in prs: + val = prs['INCLUDE_SPELLING'][0] + cfg['INCLUDE_SPELLING'] = cls.parse_value(val, bool) + if 'BATCH_SIZE' in prs: + cfg['BATCH_SIZE'] = cls.parse_value(prs['BATCH_SIZE'][0], int) + return cfg, prs + + @classmethod + def _parse_elasticsearch_search_params(cls, url, path, secure, params): + cfg = {} + split = path.rsplit('/', 1) + + if len(split) > 1: + path = '/'.join(split[:-1]) + index = split[-1] + else: + path = "" + index = split[0] + + cfg['URL'] = urlunparse( + ('https' if secure else 'http', url[1], path, '', '', '') + ) + if 'TIMEOUT' in params: + cfg['TIMEOUT'] = cls.parse_value(params['TIMEOUT'][0], int) + if 'KWARGS' in params: + cfg['KWARGS'] = params['KWARGS'][0] + cfg['INDEX_NAME'] = index + return cfg + + @classmethod + def _parse_solr_search_params(cls, url, path, params): + cfg = {} + cfg['URL'] = urlunparse(('http',) + url[1:2] + (path,) + ('', '', '')) + if 'TIMEOUT' in params: + cfg['TIMEOUT'] = cls.parse_value(params['TIMEOUT'][0], int) + if 'KWARGS' in params: + cfg['KWARGS'] = params['KWARGS'][0] + return cfg + + @classmethod + def _parse_whoosh_search_params(cls, params): + cfg = {} + if 'STORAGE' in params: + cfg['STORAGE'] = params['STORAGE'][0] + if 'POST_LIMIT' in params: + cfg['POST_LIMIT'] = cls.parse_value(params['POST_LIMIT'][0], int) + return cfg + + @classmethod + def _parse_xapian_search_params(cls, params): + cfg = {} + if 'FLAGS' in params: + cfg['FLAGS'] = params['FLAGS'][0] + return cfg + @classmethod def search_url_config(cls, url, engine=None): """Parse an arbitrary search URL. @@ -753,14 +819,11 @@ def search_url_config(cls, url, engine=None): :return: Parsed search URL. :rtype: dict """ - config = {} - url = urlparse(url) if not isinstance(url, cls.URL_CLASS) else url # Remove query strings. - path = url.path[1:] - path = unquote_plus(path.split('?', 2)[0]) + path = unquote_plus(url.path[1:].split('?', 2)[0]) scheme = url.scheme secure = False @@ -769,76 +832,36 @@ def search_url_config(cls, url, engine=None): scheme = scheme[:-1] secure = True if scheme not in cls.SEARCH_SCHEMES: - raise ImproperlyConfigured(f'Invalid search schema {url.scheme}') - config["ENGINE"] = cls.SEARCH_SCHEMES[scheme] + raise ImproperlyConfigured( + 'Invalid search schema ' + url.scheme) + config['ENGINE'] = cls.SEARCH_SCHEMES[scheme] # check commons params - params = {} - if url.query: - params = parse_qs(url.query) - if 'EXCLUDED_INDEXES' in params: - config['EXCLUDED_INDEXES'] \ - = params['EXCLUDED_INDEXES'][0].split(',') - if 'INCLUDE_SPELLING' in params: - config['INCLUDE_SPELLING'] = cls.parse_value( - params['INCLUDE_SPELLING'][0], - bool - ) - if 'BATCH_SIZE' in params: - config['BATCH_SIZE'] = cls.parse_value( - params['BATCH_SIZE'][0], - int - ) + cfg, params = cls._parse_common_search_params(url) + config.update(cfg) if url.scheme == 'simple': return config - if url.scheme in ['solr'] + cls.ELASTICSEARCH_FAMILY: - if 'KWARGS' in params: - config['KWARGS'] = params['KWARGS'][0] # remove trailing slash if path.endswith('/'): path = path[:-1] if url.scheme == 'solr': - config['URL'] = urlunparse( - ('http',) + url[1:2] + (path,) + ('', '', '') - ) - if 'TIMEOUT' in params: - config['TIMEOUT'] = cls.parse_value(params['TIMEOUT'][0], int) + config.update(cls._parse_solr_search_params(url, path, params)) return config if url.scheme in cls.ELASTICSEARCH_FAMILY: - split = path.rsplit("/", 1) - - if len(split) > 1: - path = "/".join(split[:-1]) - index = split[-1] - else: - path = "" - index = split[0] - - config['URL'] = urlunparse( - ('https' if secure else 'http', url[1], path, '', '', '') - ) - if 'TIMEOUT' in params: - config['TIMEOUT'] = cls.parse_value(params['TIMEOUT'][0], int) - config['INDEX_NAME'] = index + config.update(cls._parse_elasticsearch_search_params( + url, path, secure, params)) return config config['PATH'] = '/' + path if url.scheme == 'whoosh': - if 'STORAGE' in params: - config['STORAGE'] = params['STORAGE'][0] - if 'POST_LIMIT' in params: - config['POST_LIMIT'] = cls.parse_value( - params['POST_LIMIT'][0], - int - ) + config.update(cls._parse_whoosh_search_params(params)) elif url.scheme == 'xapian': - if 'FLAGS' in params: - config['FLAGS'] = params['FLAGS'][0] + config.update(cls._parse_xapian_search_params(params)) if engine: config['ENGINE'] = engine From 69b4bc9a5aa855040ca9e8da19aea35a08dcd639 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Sun, 23 Apr 2023 02:37:54 +0200 Subject: [PATCH 18/32] Simplify codebase by removing Python 3.5 support --- environ/environ.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/environ/environ.py b/environ/environ.py index fad85ab3..8d09258e 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -37,13 +37,7 @@ ) from .fileaware_mapping import FileAwareMapping -try: - from os import PathLike -except ImportError: # Python 3.5 support - from pathlib import PurePath as PathLike - -Openable = (str, PathLike) - +Openable = (str, os.PathLike) logger = logging.getLogger(__name__) @@ -832,8 +826,7 @@ def search_url_config(cls, url, engine=None): scheme = scheme[:-1] secure = True if scheme not in cls.SEARCH_SCHEMES: - raise ImproperlyConfigured( - 'Invalid search schema ' + url.scheme) + raise ImproperlyConfigured(f'Invalid search schema {url.scheme}') config['ENGINE'] = cls.SEARCH_SCHEMES[scheme] # check commons params From c61bfb070c4704eb2e3049c0eb153bbff20e5c0e Mon Sep 17 00:00:00 2001 From: Vasiliy Polyakov Date: Mon, 24 Apr 2023 17:56:16 +0500 Subject: [PATCH 19/32] Add variable expansion (fix #421) - Expand variables referenced as `$VAR` or `${VAR}`. - Detect infinite recursion in expansion (self-reference). --- CHANGELOG.rst | 2 ++ docs/quickstart.rst | 22 ++++++++++++++ environ/environ.py | 66 ++++++++++++++++++++++++++++++++-------- tests/test_expansion.py | 27 ++++++++++++++++ tests/test_expansion.txt | 9 ++++++ 5 files changed, 114 insertions(+), 12 deletions(-) create mode 100755 tests/test_expansion.py create mode 100755 tests/test_expansion.txt diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 682b1072..b8daeff1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,8 @@ Added +++++ - Added support for secure Elasticsearch connections `#463 `_. +- Added variable expansion + `#468 `_. Changed +++++++ diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 62473838..2ab100fd 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -23,6 +23,28 @@ And use it with ``settings.py`` as follows: :start-after: -code-begin- :end-before: -overview- +Variables can contain references to another variables: ``$VAR`` or ``${VAR}``. +Referenced variables are searched in the environment and within all definitions +in the ``.env`` file. References are checked for recursion (self-reference). +Exception is thrown if any reference results in infinite loop on any level +of recursion. Variable values are substituted similar to shell parameter +expansion. Example: + +.. code-block:: shell + + # shell + export POSTGRES_USERNAME='user' POSTGRES_PASSWORD='SECRET' + +.. code-block:: shell + + # .env + POSTGRES_HOSTNAME='example.com' + POSTGRES_DB='database' + DATABASE_URL="postgres://${POSTGRES_USERNAME}:${POSTGRES_PASSWORD}@${POSTGRES_HOSTNAME}:5432/${POSTGRES_DB}" + +The value of ``DATABASE_URL`` variable will become +``postgres://user:SECRET@example.com:5432/database``. + The ``.env`` file should be specific to the environment and not checked into version control, it is best practice documenting the ``.env`` file with an example. For example, you can also add ``.env.dist`` with a template of your variables to diff --git a/environ/environ.py b/environ/environ.py index 8d09258e..f045639e 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -17,7 +17,9 @@ import os import re import sys +import threading import warnings +from os.path import expandvars from urllib.parse import ( parse_qs, ParseResult, @@ -40,6 +42,9 @@ Openable = (str, os.PathLike) logger = logging.getLogger(__name__) +# Variables which values should not be expanded +NOT_EXPANDED = 'DJANGO_SECRET_KEY', 'CACHE_URL' + def _cast(value): # Safely evaluate an expression node or a string containing a Python @@ -189,7 +194,11 @@ class Env: for s in ('', 's')] CLOUDSQL = 'cloudsql' + VAR = re.compile(r'(?[A-Z_][0-9A-Z_]*)}?', + re.IGNORECASE) + def __init__(self, **scheme): + self._local = threading.local() self.smart_cast = True self.escape_proxy = False self.prefix = "" @@ -343,9 +352,13 @@ def path(self, var, default=NOTSET, **kwargs): """ return Path(self.get_value(var, default=default), **kwargs) - def get_value(self, var, cast=None, default=NOTSET, parse_default=False): + def get_value(self, var, cast=None, # pylint: disable=R0913 + default=NOTSET, parse_default=False, add_prefix=True): """Return value for given environment variable. + - Expand variables referenced as ``$VAR`` or ``${VAR}``. + - Detect infinite recursion in expansion (self-reference). + :param str var: Name of variable. :param collections.abc.Callable or None cast: @@ -354,15 +367,33 @@ def get_value(self, var, cast=None, default=NOTSET, parse_default=False): If var not present in environ, return this instead. :param bool parse_default: Force to parse default. + :param bool add_prefix: + Whether to add prefix to variable name. :returns: Value from environment or default (if set). :rtype: typing.IO[typing.Any] """ - + var_name = f'{self.prefix}{var}' if add_prefix else var + if not hasattr(self._local, 'vars'): + self._local.vars = set() + if var_name in self._local.vars: + error_msg = f"Environment variable '{var_name}' recursively "\ + "references itself (eventually)" + raise ImproperlyConfigured(error_msg) + + self._local.vars.add(var_name) + try: + return self._get_value( + var_name, cast=cast, default=default, + parse_default=parse_default) + finally: + self._local.vars.remove(var_name) + + def _get_value(self, var_name, cast=None, default=NOTSET, + parse_default=False): logger.debug( "get '%s' casted as '%s' with default '%s'", - var, cast, default) + var_name, cast, default) - var_name = f'{self.prefix}{var}' if var_name in self.scheme: var_info = self.scheme[var_name] @@ -388,26 +419,37 @@ def get_value(self, var, cast=None, default=NOTSET, parse_default=False): value = self.ENVIRON[var_name] except KeyError as exc: if default is self.NOTSET: - error_msg = f'Set the {var} environment variable' + error_msg = f'Set the {var_name} environment variable' raise ImproperlyConfigured(error_msg) from exc value = default + # Expand variables + if isinstance(value, (bytes, str)) and var_name not in NOT_EXPANDED: + def repl(match_): + return self.get_value( + match_.group('name'), cast=cast, default=default, + parse_default=parse_default, add_prefix=False) + + is_bytes = isinstance(value, bytes) + if is_bytes: + value = value.decode('utf-8') + value = self.VAR.sub(repl, value) + value = expandvars(value) + if is_bytes: + value = value.encode('utf-8') + # Resolve any proxied values prefix = b'$' if isinstance(value, bytes) else '$' escape = rb'\$' if isinstance(value, bytes) else r'\$' - if hasattr(value, 'startswith') and value.startswith(prefix): - value = value.lstrip(prefix) - value = self.get_value(value, cast=cast, default=default) if self.escape_proxy and hasattr(value, 'replace'): value = value.replace(escape, prefix) # Smart casting - if self.smart_cast: - if cast is None and default is not None and \ - not isinstance(default, NoValue): - cast = type(default) + if self.smart_cast and cast is None and default is not None \ + and not isinstance(default, NoValue): + cast = type(default) value = None if default is None and value == '' else value diff --git a/tests/test_expansion.py b/tests/test_expansion.py new file mode 100755 index 00000000..757ee9a2 --- /dev/null +++ b/tests/test_expansion.py @@ -0,0 +1,27 @@ +import pytest + +from environ import Env, Path +from environ.compat import ImproperlyConfigured + + +class TestExpansion: + def setup_method(self, method): + Env.ENVIRON = {} + self.env = Env() + self.env.read_env(Path(__file__, is_file=True)('test_expansion.txt')) + + def test_expansion(self): + assert self.env('HELLO') == 'Hello, world!' + + def test_braces(self): + assert self.env('BRACES') == 'Hello, world!' + + def test_recursion(self): + with pytest.raises(ImproperlyConfigured) as excinfo: + self.env('RECURSIVE') + assert str(excinfo.value) == "Environment variable 'RECURSIVE' recursively references itself (eventually)" + + def test_transitive(self): + with pytest.raises(ImproperlyConfigured) as excinfo: + self.env('R4') + assert str(excinfo.value) == "Environment variable 'R4' recursively references itself (eventually)" diff --git a/tests/test_expansion.txt b/tests/test_expansion.txt new file mode 100755 index 00000000..8290e45f --- /dev/null +++ b/tests/test_expansion.txt @@ -0,0 +1,9 @@ +VAR1='Hello' +VAR2='world' +HELLO="$VAR1, $VAR2!" +BRACES="${VAR1}, ${VAR2}!" +RECURSIVE="This variable is $RECURSIVE" +R1="$R2" +R2="$R3" +R3="$R4" +R4="$R1" From 6c1daf905a26c9fe733a83ea280433cc538e13dc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Apr 2023 17:57:40 +0000 Subject: [PATCH 20/32] Bump actions/setup-python from 4.5.0 to 4.6.0 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4.5.0 to 4.6.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4.5.0...v4.6.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 4 ++-- .github/workflows/ci.yml | 2 +- .github/workflows/cs.yml | 2 +- .github/workflows/docs.yml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f964a680..98606b21 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,7 +27,7 @@ jobs: uses: actions/checkout@v3.5.2 - name: Set up Python 3.10 - uses: actions/setup-python@v4.5.0 + uses: actions/setup-python@v4.6.0 with: python-version: '3.10' @@ -67,7 +67,7 @@ jobs: uses: actions/checkout@v3.5.2 - name: Set up Python 3.10 - uses: actions/setup-python@v4.5.0 + uses: actions/setup-python@v4.6.0 with: python-version: '3.10' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e77660f6..c7def1ff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,7 +71,7 @@ jobs: fetch-depth: 5 - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v4.5.0 + uses: actions/setup-python@v4.6.0 with: python-version: ${{ matrix.python }} diff --git a/.github/workflows/cs.yml b/.github/workflows/cs.yml index 5bec0e53..0d1e1cef 100644 --- a/.github/workflows/cs.yml +++ b/.github/workflows/cs.yml @@ -28,7 +28,7 @@ jobs: uses: actions/checkout@v3.5.2 - name: Set up Python 3.10 - uses: actions/setup-python@v4.5.0 + uses: actions/setup-python@v4.6.0 with: python-version: '3.10' diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index e455c2f2..10ae2554 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -29,7 +29,7 @@ jobs: uses: actions/checkout@v3.5.2 - name: Set up Python 3.10 - uses: actions/setup-python@v4.5.0 + uses: actions/setup-python@v4.6.0 with: python-version: '3.10' From 688a4d0b941ee9e5bc7d3e8ce8087bcca9489d50 Mon Sep 17 00:00:00 2001 From: "KimSia, Sim" <245021+simkimsia@users.noreply.github.com> Date: Tue, 23 May 2023 18:07:29 +0800 Subject: [PATCH 21/32] =?UTF-8?q?=E2=99=BB=EF=B8=8FRefactor=20the=20README?= =?UTF-8?q?=20and=20tox.ini?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit to correct a typo in README and to make the tox.ini consistent by dropping the rc1 for Django 4.2 --- README.rst | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 29b8ec72..7fcedadb 100644 --- a/README.rst +++ b/README.rst @@ -127,7 +127,7 @@ the code on `GitHub `_, and the latest release on `PyPI `_. It’s rigorously tested on Python 3.6+, and officially supports -Django 1.11, 2.2, 3., 3.1, 3.2, 4.0, 4.1, and 4.2. +Django 1.11, 2.2, 3.0, 3.1, 3.2, 4.0, 4.1, and 4.2. If you'd like to contribute to ``django-environ`` you're most welcome! diff --git a/tox.ini b/tox.ini index ca438d33..d2367374 100644 --- a/tox.ini +++ b/tox.ini @@ -44,7 +44,7 @@ deps = django32: Django>=3.2,<3.3 django40: Django>=4.0,<4.1 django41: Django>=4.1,<4.2 - django42: Django>=4.2rc1,<5.0 + django42: Django>=4.2,<5.0 commands_pre = python -m pip install --upgrade pip python -m pip install . From 58bb0db85c83b1ad1656a0f57478920df98355a3 Mon Sep 17 00:00:00 2001 From: "KimSia, Sim" <245021+simkimsia@users.noreply.github.com> Date: Tue, 23 May 2023 18:26:33 +0800 Subject: [PATCH 22/32] =?UTF-8?q?=F0=9F=90=9BFIX=20broken=20link=20in=20BA?= =?UTF-8?q?CKERS.rst?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fixed #473 --- BACKERS.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BACKERS.rst b/BACKERS.rst index 4ac3b73c..dc014446 100644 --- a/BACKERS.rst +++ b/BACKERS.rst @@ -21,7 +21,7 @@ Thank you to all our backers! |ocbackerimage| .. |ocsponsor0| image:: https://opencollective.com/django-environ/sponsor/0/avatar.svg - :target: https://triplebyte.com/ + :target: https://opencollective.com/triplebyte :alt: Sponsor .. |ocsponsor1| image:: https://images.opencollective.com/static/images/become_sponsor.svg :target: https://opencollective.com/django-environ/contribute/sponsors-3474/checkout From e1486b2c0051bf81991cb1429e76e7b003a56889 Mon Sep 17 00:00:00 2001 From: Daniele Faraglia Date: Wed, 24 May 2023 14:38:25 +0100 Subject: [PATCH 23/32] Update CHANGELOG.rst update changelog for Django 4.2 --- CHANGELOG.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b8daeff1..1fc31243 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,8 @@ and this project adheres to `Semantic Versioning `_. - Added support for secure Elasticsearch connections `#463 `_. - Added variable expansion From 92062c1eb0dbf3b2fd69274ddd30144b3c15d803 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jun 2023 17:57:18 +0000 Subject: [PATCH 24/32] Bump actions/checkout from 3.5.2 to 3.5.3 Bumps [actions/checkout](https://github.com/actions/checkout) from 3.5.2 to 3.5.3. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3.5.2...v3.5.3) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 4 ++-- .github/workflows/ci.yml | 2 +- .github/workflows/codeql.yml | 2 +- .github/workflows/cs.yml | 2 +- .github/workflows/docs.yml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 98606b21..c30b26b9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,7 +24,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.5.2 + uses: actions/checkout@v3.5.3 - name: Set up Python 3.10 uses: actions/setup-python@v4.6.0 @@ -64,7 +64,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.5.2 + uses: actions/checkout@v3.5.3 - name: Set up Python 3.10 uses: actions/setup-python@v4.6.0 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c7def1ff..3d620d52 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,7 +66,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.5.2 + uses: actions/checkout@v3.5.3 with: fetch-depth: 5 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index a9b4c0ff..78cd7c05 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -45,7 +45,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3.5.2 + uses: actions/checkout@v3.5.3 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/cs.yml b/.github/workflows/cs.yml index 0d1e1cef..cf0b6650 100644 --- a/.github/workflows/cs.yml +++ b/.github/workflows/cs.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.5.2 + uses: actions/checkout@v3.5.3 - name: Set up Python 3.10 uses: actions/setup-python@v4.6.0 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 10ae2554..a1b1652b 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.5.2 + uses: actions/checkout@v3.5.3 - name: Set up Python 3.10 uses: actions/setup-python@v4.6.0 From 7f31fcae1b6a173bebe56c026ac3b66b272b7ddf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 Jun 2023 22:02:58 +0000 Subject: [PATCH 25/32] Bump actions/setup-python from 4.6.0 to 4.6.1 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4.6.0 to 4.6.1. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4.6.0...v4.6.1) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 4 ++-- .github/workflows/ci.yml | 2 +- .github/workflows/cs.yml | 2 +- .github/workflows/docs.yml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c30b26b9..783d742f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,7 +27,7 @@ jobs: uses: actions/checkout@v3.5.3 - name: Set up Python 3.10 - uses: actions/setup-python@v4.6.0 + uses: actions/setup-python@v4.6.1 with: python-version: '3.10' @@ -67,7 +67,7 @@ jobs: uses: actions/checkout@v3.5.3 - name: Set up Python 3.10 - uses: actions/setup-python@v4.6.0 + uses: actions/setup-python@v4.6.1 with: python-version: '3.10' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d620d52..f7b5d671 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,7 +71,7 @@ jobs: fetch-depth: 5 - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v4.6.0 + uses: actions/setup-python@v4.6.1 with: python-version: ${{ matrix.python }} diff --git a/.github/workflows/cs.yml b/.github/workflows/cs.yml index cf0b6650..2e4c6402 100644 --- a/.github/workflows/cs.yml +++ b/.github/workflows/cs.yml @@ -28,7 +28,7 @@ jobs: uses: actions/checkout@v3.5.3 - name: Set up Python 3.10 - uses: actions/setup-python@v4.6.0 + uses: actions/setup-python@v4.6.1 with: python-version: '3.10' diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index a1b1652b..ff76054e 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -29,7 +29,7 @@ jobs: uses: actions/checkout@v3.5.3 - name: Set up Python 3.10 - uses: actions/setup-python@v4.6.0 + uses: actions/setup-python@v4.6.1 with: python-version: '3.10' From 2750dd54e0e4b1b591a31409cfb1dc16a82b958d Mon Sep 17 00:00:00 2001 From: Leon van der Ree Date: Thu, 6 Jul 2023 23:17:36 +0200 Subject: [PATCH 26/32] Handle inline comments, and ignore spaces outside quotes (#475) * handle inline comments, and ignore spaces outside quotes * fixed codestyle issues * updated release notes * fixed new linting issue, improved example in changelog * included suggestions --------- Co-authored-by: Serghei Iakovlev --- CHANGELOG.rst | 4 ++++ environ/environ.py | 12 ++++++++++-- tests/fixtures.py | 7 ++++++- tests/test_env.py | 8 ++++++-- tests/test_env.txt | 7 ++++++- 5 files changed, 32 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1fc31243..aa7b3ff8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -15,11 +15,15 @@ Added `#463 `_. - Added variable expansion `#468 `_. +- Added capability to handle comments after #, after quoted values, like ``KEY= 'part1 # part2' # comment`` + `#475 `_. Changed +++++++ - Used ``mssql-django`` as engine for SQL Server `#446 `_. +- Changed handling bool values, stripping whitespace around value + `#475 `_. Removed +++++++ diff --git a/environ/environ.py b/environ/environ.py index f045639e..644286e1 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -473,7 +473,7 @@ def parse_value(cls, value, cast): try: value = int(value) != 0 except ValueError: - value = value.lower() in cls.BOOLEAN_TRUE_STRINGS + value = value.lower().strip() in cls.BOOLEAN_TRUE_STRINGS elif isinstance(cast, list): value = list(map(cast[0], [x for x in value.split(',') if x])) elif isinstance(cast, tuple): @@ -970,9 +970,17 @@ def _keep_escaped_format_characters(match): m1 = re.match(r'\A(?:export )?([A-Za-z_0-9]+)=(.*)\Z', line) if m1: key, val = m1.group(1), m1.group(2) - m2 = re.match(r"\A'(.*)'\Z", val) + # Look for value in quotes, ignore post-# comments + # (outside quotes) + m2 = re.match(r"\A\s*'(? Date: Mon, 26 Sep 2022 15:19:47 +0200 Subject: [PATCH 27/32] Add interpolate argument to avoid resolving proxied values. The argument is already documented but not implemented yet. Fixes #415 --- CHANGELOG.rst | 2 ++ docs/tips.rst | 2 +- environ/environ.py | 6 ++++-- tests/test_env.py | 4 ++++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index aa7b3ff8..8e181b19 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -17,6 +17,8 @@ Added `#468 `_. - Added capability to handle comments after #, after quoted values, like ``KEY= 'part1 # part2' # comment`` `#475 `_. +- Added support for ``interpolate`` parameter + `#419 `_. Changed +++++++ diff --git a/docs/tips.rst b/docs/tips.rst index 20b8c3e1..ab59f691 100644 --- a/docs/tips.rst +++ b/docs/tips.rst @@ -226,7 +226,7 @@ Proxy value =========== Values that being with a ``$`` may be interpolated. Pass ``interpolate=True`` to -``environ.Env()`` to enable this feature: +``environ.Env()`` to enable this feature (``True`` by default): .. code-block:: python diff --git a/environ/environ.py b/environ/environ.py index 644286e1..f35470c5 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -197,11 +197,12 @@ class Env: VAR = re.compile(r'(?[A-Z_][0-9A-Z_]*)}?', re.IGNORECASE) - def __init__(self, **scheme): + def __init__(self, interpolate=True, **scheme): self._local = threading.local() self.smart_cast = True self.escape_proxy = False self.prefix = "" + self.interpolate = interpolate self.scheme = scheme def __call__(self, var, cast=None, default=NOTSET, parse_default=False): @@ -425,7 +426,8 @@ def _get_value(self, var_name, cast=None, default=NOTSET, value = default # Expand variables - if isinstance(value, (bytes, str)) and var_name not in NOT_EXPANDED: + if self.interpolate and isinstance(value, (bytes, str)) \ + and var_name not in NOT_EXPANDED: def repl(match_): return self.get_value( match_.group('name'), cast=cast, default=default, diff --git a/tests/test_env.py b/tests/test_env.py index 0b105f55..1f2df156 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -134,6 +134,10 @@ def test_bool_true(self, value, variable): def test_proxied_value(self): assert self.env('PROXIED_VAR') == 'bar' + def test_not_interpolated_proxied_value(self): + env = Env(interpolate=False) + assert env('PROXIED_VAR') == '$STR_VAR' + def test_escaped_dollar_sign(self): self.env.escape_proxy = True assert self.env('ESCAPED_VAR') == '$baz' From 5fe1c7f2da203dece3d01c0e17530fb947e25057 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Jul 2023 17:03:27 +0000 Subject: [PATCH 28/32] Bump actions/setup-python from 4.6.1 to 4.7.0 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4.6.1 to 4.7.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4.6.1...v4.7.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 4 ++-- .github/workflows/ci.yml | 2 +- .github/workflows/cs.yml | 2 +- .github/workflows/docs.yml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 783d742f..61102608 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,7 +27,7 @@ jobs: uses: actions/checkout@v3.5.3 - name: Set up Python 3.10 - uses: actions/setup-python@v4.6.1 + uses: actions/setup-python@v4.7.0 with: python-version: '3.10' @@ -67,7 +67,7 @@ jobs: uses: actions/checkout@v3.5.3 - name: Set up Python 3.10 - uses: actions/setup-python@v4.6.1 + uses: actions/setup-python@v4.7.0 with: python-version: '3.10' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f7b5d671..86eefcb9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,7 +71,7 @@ jobs: fetch-depth: 5 - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v4.6.1 + uses: actions/setup-python@v4.7.0 with: python-version: ${{ matrix.python }} diff --git a/.github/workflows/cs.yml b/.github/workflows/cs.yml index 2e4c6402..f2c9e4b4 100644 --- a/.github/workflows/cs.yml +++ b/.github/workflows/cs.yml @@ -28,7 +28,7 @@ jobs: uses: actions/checkout@v3.5.3 - name: Set up Python 3.10 - uses: actions/setup-python@v4.6.1 + uses: actions/setup-python@v4.7.0 with: python-version: '3.10' diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index ff76054e..6b91877b 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -29,7 +29,7 @@ jobs: uses: actions/checkout@v3.5.3 - name: Set up Python 3.10 - uses: actions/setup-python@v4.6.1 + uses: actions/setup-python@v4.7.0 with: python-version: '3.10' From 638720cf3cc4f409a16e65df02b3bd4a49ca19d0 Mon Sep 17 00:00:00 2001 From: wongcht Date: Sun, 27 Aug 2023 20:35:08 +0100 Subject: [PATCH 29/32] use importlib.util.find_spec --- environ/compat.py | 10 +++++----- tests/test_cache.py | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/environ/compat.py b/environ/compat.py index b4ffee4e..49b5b480 100644 --- a/environ/compat.py +++ b/environ/compat.py @@ -8,14 +8,14 @@ """This module handles import compatibility issues.""" -from pkgutil import find_loader +from importlib.util import find_spec -if find_loader('simplejson'): +if find_spec('simplejson'): import simplejson as json else: import json -if find_loader('django'): +if find_spec('django'): from django import VERSION as DJANGO_VERSION from django.core.exceptions import ImproperlyConfigured else: @@ -29,7 +29,7 @@ def choose_rediscache_driver(): """Backward compatibility for RedisCache driver.""" # django-redis library takes precedence - if find_loader('django_redis'): + if find_spec('django_redis'): return 'django_redis.cache.RedisCache' # use built-in support if Django 4+ @@ -51,7 +51,7 @@ def choose_postgres_driver(): def choose_pymemcache_driver(): """Backward compatibility for pymemcache.""" old_django = DJANGO_VERSION is not None and DJANGO_VERSION < (3, 2) - if old_django or not find_loader('pymemcache'): + if old_django or not find_spec('pymemcache'): # The original backend choice for the 'pymemcache' scheme is # unfortunately 'pylibmc'. return 'django.core.cache.backends.memcached.PyLibMCCache' diff --git a/tests/test_cache.py b/tests/test_cache.py index 37d7b6eb..a8aff161 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -107,8 +107,8 @@ def test_pymemcache_compat(django_version, pymemcache_installed): old = 'django.core.cache.backends.memcached.PyLibMCCache' new = 'django.core.cache.backends.memcached.PyMemcacheCache' with mock.patch.object(environ.compat, 'DJANGO_VERSION', django_version): - with mock.patch('environ.compat.find_loader') as mock_find_loader: - mock_find_loader.return_value = pymemcache_installed + with mock.patch('environ.compat.find_spec') as mock_find_spec: + mock_find_spec.return_value = pymemcache_installed driver = environ.compat.choose_pymemcache_driver() if django_version and django_version < (3, 2): assert driver == old @@ -124,8 +124,8 @@ def test_rediscache_compat(django_version, django_redis_installed): django_redis = 'django_redis.cache.RedisCache' with mock.patch.object(environ.compat, 'DJANGO_VERSION', django_version): - with mock.patch('environ.compat.find_loader') as mock_find_loader: - mock_find_loader.return_value = django_redis_installed + with mock.patch('environ.compat.find_spec') as mock_find_spec: + mock_find_spec.return_value = django_redis_installed driver = environ.compat.choose_rediscache_driver() if django_redis_installed: assert driver == django_redis From 89ac873e6eb588641e5517a31ca1cbcd65b82e29 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Aug 2023 17:16:18 +0000 Subject: [PATCH 30/32] Bump actions/checkout from 3.5.3 to 3.6.0 Bumps [actions/checkout](https://github.com/actions/checkout) from 3.5.3 to 3.6.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3.5.3...v3.6.0) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 4 ++-- .github/workflows/ci.yml | 2 +- .github/workflows/codeql.yml | 2 +- .github/workflows/cs.yml | 2 +- .github/workflows/docs.yml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 61102608..0779431c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,7 +24,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - name: Set up Python 3.10 uses: actions/setup-python@v4.7.0 @@ -64,7 +64,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - name: Set up Python 3.10 uses: actions/setup-python@v4.7.0 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 86eefcb9..5ecdb2fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,7 +66,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 with: fetch-depth: 5 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 78cd7c05..07dd2008 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -45,7 +45,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/cs.yml b/.github/workflows/cs.yml index f2c9e4b4..fc596beb 100644 --- a/.github/workflows/cs.yml +++ b/.github/workflows/cs.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - name: Set up Python 3.10 uses: actions/setup-python@v4.7.0 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 6b91877b..e03e0d8b 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - name: Set up Python 3.10 uses: actions/setup-python@v4.7.0 From ff289de49edae2123915244c5ce63b7a2a21738f Mon Sep 17 00:00:00 2001 From: Wong Chun Hong Date: Tue, 29 Aug 2023 21:27:31 +0100 Subject: [PATCH 31/32] remove comma --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 7fcedadb..528d1df1 100644 --- a/README.rst +++ b/README.rst @@ -127,7 +127,7 @@ the code on `GitHub `_, and the latest release on `PyPI `_. It’s rigorously tested on Python 3.6+, and officially supports -Django 1.11, 2.2, 3.0, 3.1, 3.2, 4.0, 4.1, and 4.2. +Django 1.11, 2.2, 3.0, 3.1, 3.2, 4.0, 4.1 and 4.2. If you'd like to contribute to ``django-environ`` you're most welcome! From cbff150e80a8b741041e600de66f2991d4807662 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Wed, 30 Aug 2023 12:43:01 +0200 Subject: [PATCH 32/32] Update change log --- CHANGELOG.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8e181b19..78b6a0bf 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is inspired by `Keep a Changelog `_ and this project adheres to `Semantic Versioning `_. -`v0.11.0`_ - 00-Unreleased-2023 +`v0.11.0`_ - 30-August-2023 ------------------------------- Added +++++ @@ -15,10 +15,11 @@ Added `#463 `_. - Added variable expansion `#468 `_. -- Added capability to handle comments after #, after quoted values, like ``KEY= 'part1 # part2' # comment`` +- Added capability to handle comments after #, after quoted values, + like ``KEY= 'part1 # part2' # comment`` `#475 `_. - Added support for ``interpolate`` parameter - `#419 `_. + `#415 `_. Changed +++++++ @@ -26,6 +27,9 @@ Changed `#446 `_. - Changed handling bool values, stripping whitespace around value `#475 `_. +- Use ``importlib.util.find_spec`` to ``replace pkgutil.find_loader`` + `#482 `_. + Removed +++++++