diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3b3f5576..0637b00b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,33 @@ All notable changes to this project will be documented in this file. The format is inspired by `Keep a Changelog `_ and this project adheres to `Semantic Versioning `_. +`v0.8.0`_ - 17-October-2021 +------------------------------ +Added ++++++ +- Log invalid lines when parse .env file + `#283 `_. +- Added docker-style file variable support + `#189 `_. +- Added option to override existing variables with ``read_env`` + `#103 `_, + `#249 `_. +- Added support for empty var with None default value + `#209 `_. +- Added ``pymemcache`` cache backend for Django 3.2+ + `#335 `_. + + +Fixed ++++++ +- Keep newline/tab escapes in quoted strings + `#296 `_. +- Handle escaped dollar sign in values + `#271 `_. +- Fixed incorrect parsing of ``DATABASES_URL`` for Google Cloud MySQL + `#294 `_. + + `v0.7.0`_ - 11-September-2021 ------------------------------ Added @@ -219,6 +246,7 @@ Added - Initial release. +.. _v0.8.0: https://github.com/joke2k/django-environ/compare/v0.7.0...v0.8.0 .. _v0.7.0: https://github.com/joke2k/django-environ/compare/v0.6.0...v0.7.0 .. _v0.6.0: https://github.com/joke2k/django-environ/compare/v0.5.0...v0.6.0 .. _v0.5.0: https://github.com/joke2k/django-environ/compare/v0.4.5...v0.5.0 diff --git a/docs/tips.rst b/docs/tips.rst index 777d5589..dbb762fb 100644 --- a/docs/tips.rst +++ b/docs/tips.rst @@ -128,15 +128,54 @@ You can use something like this to handle similar cases. Multiline value =============== -You can set a multiline variable value: +To get multiline value pass ``multiline=True`` to ```str()```. + +.. note:: + + You shouldn't escape newline/tab characters yourself if you want to preserve + the formatting. + +The following example demonstrates the above: + +**.env file**: + +.. code-block:: shell + + # .env file contents + UNQUOTED_CERT=---BEGIN---\r\n---END--- + QUOTED_CERT="---BEGIN---\r\n---END---" + ESCAPED_CERT=---BEGIN---\\n---END--- + +**settings.py file**: .. code-block:: python - # MULTILINE_TEXT=Hello\\nWorld - >>> print env.str('MULTILINE_TEXT', multiline=True) - Hello - World + # settings.py file contents + import environ + + + env = environ.Env() + + print(env.str('UNQUOTED_CERT', multiline=True)) + # ---BEGIN--- + # ---END--- + + print(env.str('UNQUOTED_CERT', multiline=False)) + # ---BEGIN---\r\n---END--- + + print(env.str('QUOTED_CERT', multiline=True)) + # ---BEGIN--- + # ---END--- + + print(env.str('QUOTED_CERT', multiline=False)) + # ---BEGIN---\r\n---END--- + print(env.str('ESCAPED_CERT', multiline=True)) + # ---BEGIN---\ + # ---END--- + + print(env.str('ESCAPED_CERT', multiline=False)) + # ---BEGIN---\\n---END--- Proxy value =========== @@ -156,10 +195,30 @@ Values that being with a ``$`` may be interpolated. Pass ``interpolate=True`` to FOO +Escape Proxy +============ + +If you're having trouble with values starting with dollar sign ($) without the intention of proxying the value to +another, You should enbale the ``escape_proxy`` and prepend a backslash to it. + +.. code-block:: python + + import environ + + env = environ.Env() + env.escape_proxy = True + + # ESCAPED_VAR=\$baz + env.str('ESCAPED_VAR') # $baz + + +Reading env files +================= + .. _multiple-env-files-label: Multiple env files -================== +------------------ There is an ability point to the .env file location using an environment variable. This feature may be convenient in a production systems with a @@ -188,7 +247,7 @@ while ``./manage.py runserver`` uses ``.env``. Using Path objects when reading env -=================================== +----------------------------------- It is possible to use of ``pathlib.Path`` objects when reading environment file from the filesystem: @@ -210,3 +269,16 @@ It is possible to use of ``pathlib.Path`` objects when reading environment file env.read_env(os.path.join(BASE_DIR, '.env')) env.read_env(pathlib.Path(str(BASE_DIR)).joinpath('.env')) env.read_env(pathlib.Path(str(BASE_DIR)) / '.env') + + +Overwriting existing environment values from env files +------------------------------------------------------ + +If you want variables set within your env files to take higher precidence than +an existing set environment variable, use the ``overwrite=True`` argument of +``read_env``. For example: + +.. code-block:: python + + env = environ.Env() + env.read_env(BASE_DIR('.env'), overwrite=True) diff --git a/docs/types.rst b/docs/types.rst index 1e80e582..292ed64f 100644 --- a/docs/types.rst +++ b/docs/types.rst @@ -34,8 +34,10 @@ Supported types * Dummy: ``dummycache://`` * File: ``filecache://`` * Memory: ``locmemcache://`` - * Memcached: ``memcache://`` - * Python memory: ``pymemcache://`` + * Memcached: + * ``memcache://`` (uses ``python-memcached`` backend, deprecated in Django 3.2) + * ``pymemcache://`` (uses ``pymemcache`` backend if Django >=3.2 and package is installed, otherwise will use ``pylibmc`` backend to keep config backwards compatibility) + * ``pylibmc://`` * Redis: ``rediscache://``, ``redis://``, or ``rediss://`` * ``search_url`` diff --git a/environ/__init__.py b/environ/__init__.py index bf8689a0..e37b6e65 100644 --- a/environ/__init__.py +++ b/environ/__init__.py @@ -31,7 +31,7 @@ __copyright__ = 'Copyright (C) 2021 Daniele Faraglia' -__version__ = '0.7.0' +__version__ = '0.8.0' __license__ = 'MIT' __author__ = 'Daniele Faraglia' __author_email__ = 'daniele.faraglia@gmail.com' diff --git a/environ/compat.py b/environ/compat.py index 0f9eb9a5..8c259f85 100644 --- a/environ/compat.py +++ b/environ/compat.py @@ -8,15 +8,15 @@ """This module handles import compatibility issues.""" -import pkgutil +from pkgutil import find_loader -if pkgutil.find_loader('simplejson'): +if find_loader('simplejson'): import simplejson as json else: import json -if pkgutil.find_loader('django'): +if find_loader('django'): from django import VERSION as DJANGO_VERSION from django.core.exceptions import ImproperlyConfigured else: @@ -33,7 +33,20 @@ class ImproperlyConfigured(Exception): DJANGO_POSTGRES = 'django.db.backends.postgresql' # back compatibility with redis_cache package -if pkgutil.find_loader('redis_cache'): +if find_loader('redis_cache'): REDIS_DRIVER = 'redis_cache.RedisCache' else: REDIS_DRIVER = 'django_redis.cache.RedisCache' + + +# back compatibility for pymemcache +def choose_pymemcache_driver(): + old_django = DJANGO_VERSION is not None and DJANGO_VERSION < (3, 2) + if old_django or not find_loader('pymemcache'): + # The original backend choice for the 'pymemcache' scheme is + # unfortunately 'pylibmc'. + return 'django.core.cache.backends.memcached.PyLibMCCache' + return 'django.core.cache.backends.memcached.PyMemcacheCache' + + +PYMEMCACHE_DRIVER = choose_pymemcache_driver() diff --git a/environ/environ.py b/environ/environ.py index d312c296..505577fc 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -26,15 +26,21 @@ urlunparse, ) -from .compat import DJANGO_POSTGRES, ImproperlyConfigured, json, REDIS_DRIVER +from .compat import ( + DJANGO_POSTGRES, + ImproperlyConfigured, + json, + PYMEMCACHE_DRIVER, + REDIS_DRIVER, +) from .fileaware_mapping import FileAwareMapping try: from os import PathLike +except ImportError: # Python 3.5 support + from pathlib import PurePath as PathLike - Openable = (str, PathLike) -except ImportError: - Openable = (str,) +Openable = (str, PathLike) logger = logging.getLogger(__name__) @@ -116,7 +122,8 @@ class Env: 'filecache': 'django.core.cache.backends.filebased.FileBasedCache', 'locmemcache': 'django.core.cache.backends.locmem.LocMemCache', 'memcache': 'django.core.cache.backends.memcached.MemcachedCache', - 'pymemcache': 'django.core.cache.backends.memcached.PyLibMCCache', + 'pymemcache': PYMEMCACHE_DRIVER, + 'pylibmc': 'django.core.cache.backends.memcached.PyLibMCCache', 'rediscache': REDIS_DRIVER, 'redis': REDIS_DRIVER, 'rediss': REDIS_DRIVER, @@ -157,9 +164,11 @@ class Env: "xapian": "haystack.backends.xapian_backend.XapianEngine", "simple": "haystack.backends.simple_backend.SimpleEngine", } + CLOUDSQL = 'cloudsql' def __init__(self, **scheme): self.smart_cast = True + self.escape_proxy = False self.scheme = scheme def __call__(self, var, cast=None, default=NOTSET, parse_default=False): @@ -181,7 +190,7 @@ def str(self, var, default=NOTSET, multiline=False): """ value = self.get_value(var, cast=str, default=default) if multiline: - return value.replace('\\n', '\n') + return re.sub(r'(\\r)?\\n', r'\n', value) return value def unicode(self, var, default=NOTSET): @@ -365,16 +374,22 @@ def get_value(self, var, cast=None, default=NOTSET, parse_default=False): # Resolve any proxied values prefix = b'$' if isinstance(value, bytes) else '$' + escape = rb'\$' if isinstance(value, bytes) else r'\$' if hasattr(value, 'startswith') and value.startswith(prefix): value = value.lstrip(prefix) value = self.get_value(value, cast=cast, default=default) + if self.escape_proxy and hasattr(value, 'replace'): + value = value.replace(escape, prefix) + # Smart casting if self.smart_cast: if cast is None and default is not None and \ not isinstance(default, NoValue): cast = type(default) + value = None if default is None and value == '' else value + if value != default or (parse_default and value): value = self.parse_value(value, cast) @@ -495,7 +510,10 @@ def db_url_config(cls, url, engine=None): 'PORT': _cast_int(url.port) or '', }) - if url.scheme in cls.POSTGRES_FAMILY and path.startswith('/'): + if ( + url.scheme in cls.POSTGRES_FAMILY and path.startswith('/') + or cls.CLOUDSQL in path and path.startswith('/') + ): config['HOST'], config['NAME'] = path.rsplit('/', 1) if url.scheme == 'oracle' and path == '': @@ -732,16 +750,29 @@ def search_url_config(cls, url, engine=None): return config @classmethod - def read_env(cls, env_file=None, **overrides): + def read_env(cls, env_file=None, overwrite=False, **overrides): """Read a .env file into os.environ. If not given a path to a dotenv path, does filthy magic stack backtracking to find the dotenv in the same directory as the file that called read_env. + Existing environment variables take precedent and are NOT overwritten + by the file content. ``overwrite=True`` will force an overwrite of + existing environment variables. + Refs: - https://wellfire.co/learn/easier-12-factor-django - https://gist.github.com/bennylope/2999704 + + :param env_file: The path to the `.env` file your application should + use. If a path is not provided, `read_env` will attempt to import + the Django settings module from the Django project root. + :param overwrite: ``overwrite=True`` will force an overwrite of + existing environment variables. + :param **overrides: Any additional keyword arguments provided directly + to read_env will be added to the environment. If the key matches an + existing environment variable, the value will be overridden. """ if env_file is None: frame = sys._getframe() @@ -757,7 +788,8 @@ def read_env(cls, env_file=None, **overrides): try: if isinstance(env_file, Openable): - with open(env_file) as f: + # Python 3.5 support (wrap path with str). + with open(str(env_file)) as f: content = f.read() else: with env_file as f: @@ -770,6 +802,13 @@ def read_env(cls, env_file=None, **overrides): logger.debug('Read environment variables from: {}'.format(env_file)) + def _keep_escaped_format_characters(match): + """Keep escaped newline/tabs in quoted strings""" + escaped_char = match.group(1) + if escaped_char in 'rnt': + return '\\' + escaped_char + return escaped_char + for line in content.splitlines(): m1 = re.match(r'\A(?:export )?([A-Za-z_0-9]+)=(.*)\Z', line) if m1: @@ -779,12 +818,25 @@ def read_env(cls, env_file=None, **overrides): val = m2.group(1) m3 = re.match(r'\A"(.*)"\Z', val) if m3: - val = re.sub(r'\\(.)', r'\1', m3.group(1)) - cls.ENVIRON.setdefault(key, str(val)) + val = re.sub(r'\\(.)', _keep_escaped_format_characters, + m3.group(1)) + overrides[key] = str(val) + else: + logger.warning('Invalid line: %s', line) + + def set_environ(envval): + """Return lambda to set environ. + + Use setdefault unless overwrite is specified. + """ + if overwrite: + return lambda k, v: envval.update({k: str(v)}) + return lambda k, v: envval.setdefault(k, str(v)) + + setenv = set_environ(cls.ENVIRON) - # set defaults for key, value in overrides.items(): - cls.ENVIRON.setdefault(key, value) + setenv(key, value) class FileAwareEnv(Env): diff --git a/tests/fixtures.py b/tests/fixtures.py index f685f5cb..25213ac7 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -13,6 +13,7 @@ class FakeEnv: URL = 'http://www.google.com/' POSTGRES = 'postgres://uf07k1:wegauwhg@ec2-107-21-253-135.compute-1.amazonaws.com:5431/d8r82722' MYSQL = 'mysql://bea6eb0:69772142@us-cdbr-east.cleardb.com/heroku_97681?reconnect=true' + MYSQL_CLOUDSQL_URL = 'mysql://djuser:hidden-password@//cloudsql/arvore-codelab:us-central1:mysqlinstance/mydatabase' MYSQLGIS = 'mysqlgis://user:password@127.0.0.1/some_database' SQLITE = 'sqlite:////full/path/to/your/database/file.sqlite' ORACLE_TNS = 'oracle://user:password@sid/' @@ -31,6 +32,8 @@ class FakeEnv: def generate_data(cls): return dict(STR_VAR='bar', MULTILINE_STR_VAR='foo\\nbar', + MULTILINE_QUOTED_STR_VAR='---BEGIN---\\r\\n---END---', + MULTILINE_ESCAPED_STR_VAR='---BEGIN---\\\\n---END---', INT_VAR='42', FLOAT_VAR='33.3', FLOAT_COMMA_VAR='33,3', @@ -51,6 +54,7 @@ def generate_data(cls): BOOL_FALSE_STRING_LIKE_BOOL='False', BOOL_FALSE_BOOL=False, PROXIED_VAR='$STR_VAR', + ESCAPED_VAR=r'\$baz', INT_LIST='42,33', INT_TUPLE='(42,33)', STR_LIST_WITH_SPACES=' foo, bar', @@ -64,6 +68,7 @@ def generate_data(cls): DATABASE_ORACLE_TNS_URL=cls.ORACLE_TNS, DATABASE_REDSHIFT_URL=cls.REDSHIFT, DATABASE_CUSTOM_BACKEND_URL=cls.CUSTOM_BACKEND, + DATABASE_MYSQL_CLOUDSQL_URL=cls.MYSQL_CLOUDSQL_URL, CACHE_URL=cls.MEMCACHE, CACHE_REDIS=cls.REDIS, EMAIL_URL=cls.EMAIL, diff --git a/tests/test_cache.py b/tests/test_cache.py index dd4e4626..44ef46c9 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -6,10 +6,13 @@ # For the full copyright and license information, please view # the LICENSE.txt file that was distributed with this source code. +from unittest import mock + import pytest +import environ.compat from environ import Env -from environ.compat import REDIS_DRIVER, ImproperlyConfigured +from environ.compat import PYMEMCACHE_DRIVER, REDIS_DRIVER, ImproperlyConfigured def test_base_options_parsing(): @@ -63,7 +66,7 @@ def test_base_options_parsing(): 'django.core.cache.backends.memcached.MemcachedCache', '127.0.0.1:11211'), ('pymemcache://127.0.0.1:11211', - 'django.core.cache.backends.memcached.PyLibMCCache', + PYMEMCACHE_DRIVER, '127.0.0.1:11211'), ], ids=[ @@ -90,6 +93,21 @@ def test_cache_parsing(url, backend, location): assert url['LOCATION'] == location +@pytest.mark.parametrize('django_version', ((3, 2), (3, 1), None)) +@pytest.mark.parametrize('pymemcache_installed', (True, False)) +def test_pymemcache_compat(django_version, pymemcache_installed): + old = 'django.core.cache.backends.memcached.PyLibMCCache' + new = 'django.core.cache.backends.memcached.PyMemcacheCache' + with mock.patch.object(environ.compat, 'DJANGO_VERSION', django_version): + with mock.patch('environ.compat.find_loader') as mock_find_loader: + mock_find_loader.return_value = pymemcache_installed + driver = environ.compat.choose_pymemcache_driver() + if django_version and django_version < (3, 2): + assert driver == old + else: + assert driver == new if pymemcache_installed else old + + def test_redis_parsing(): url = ('rediscache://127.0.0.1:6379/1?client_class=' 'django_redis.client.DefaultClient&password=secret') diff --git a/tests/test_env.py b/tests/test_env.py index efb09fbc..0be9a60a 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -57,6 +57,10 @@ def test_contains(self): ('STR_VAR', 'bar', False), ('MULTILINE_STR_VAR', 'foo\\nbar', False), ('MULTILINE_STR_VAR', 'foo\nbar', True), + ('MULTILINE_QUOTED_STR_VAR', '---BEGIN---\\r\\n---END---', False), + ('MULTILINE_QUOTED_STR_VAR', '---BEGIN---\n---END---', True), + ('MULTILINE_ESCAPED_STR_VAR', '---BEGIN---\\\\n---END---', False), + ('MULTILINE_ESCAPED_STR_VAR', '---BEGIN---\\\n---END---', True), ], ) def test_str(self, var, val, multiline): @@ -82,6 +86,7 @@ def test_int(self): def test_int_with_none_default(self): assert self.env('NOT_PRESENT_VAR', cast=int, default=None) is None + assert self.env('EMPTY_INT_VAR', cast=int, default=None) is None @pytest.mark.parametrize( 'value,variable', @@ -122,6 +127,14 @@ def test_bool_true(self, value, variable): def test_proxied_value(self): assert self.env('PROXIED_VAR') == 'bar' + def test_escaped_dollar_sign(self): + self.env.escape_proxy = True + assert self.env('ESCAPED_VAR') == '$baz' + + def test_escaped_dollar_sign_disabled(self): + self.env.escape_proxy = False + assert self.env('ESCAPED_VAR') == r'\$baz' + def test_int_list(self): assert_type_and_value(list, [42, 33], self.env('INT_LIST', cast=[int])) assert_type_and_value(list, [42, 33], self.env.list('INT_LIST', int)) @@ -148,6 +161,7 @@ def test_dict_value(self): [ ('a=1', dict, {'a': '1'}), ('a=1', dict(value=int), {'a': 1}), + ('a=1', dict(value=float), {'a': 1.0}), ('a=1,2,3', dict(value=[str]), {'a': ['1', '2', '3']}), ('a=1,2,3', dict(value=[int]), {'a': [1, 2, 3]}), ('a=1;b=1.1,2.2;c=3', dict(value=int, cast=dict(b=[float])), @@ -159,6 +173,7 @@ def test_dict_value(self): ids=[ 'dict', 'dict_int', + 'dict_float', 'dict_str_list', 'dict_int_list', 'dict_int_cast', @@ -203,6 +218,8 @@ def test_url_encoded_parts(self): '/full/path/to/your/database/file.sqlite', '', '', '', ''), ('DATABASE_CUSTOM_BACKEND_URL', 'custom.backend', 'database', 'example.com', 'user', 'password', 5430), + ('DATABASE_MYSQL_CLOUDSQL_URL', 'django.db.backends.mysql', 'mydatabase', + '/cloudsql/arvore-codelab:us-central1:mysqlinstance', 'djuser', 'hidden-password', ''), ], ids=[ 'postgres', @@ -213,6 +230,7 @@ def test_url_encoded_parts(self): 'redshift', 'sqlite', 'custom', + 'cloudsql', ], ) def test_db_url_value(self, var, engine, name, host, user, passwd, port): @@ -303,34 +321,43 @@ def setup_method(self, method): PATH_VAR=Path(__file__, is_file=True).__root__ ) - def test_read_env_path_like(self): + def create_temp_env_file(self, name): import pathlib import tempfile - path_like = (pathlib.Path(tempfile.gettempdir()) / 'test_pathlib.env') + env_file_path = (pathlib.Path(tempfile.gettempdir()) / name) try: - path_like.unlink() + env_file_path.unlink() except FileNotFoundError: pass - assert not path_like.exists() + assert not env_file_path.exists() + return env_file_path + + def test_read_env_path_like(self): + env_file_path = self.create_temp_env_file('test_pathlib.env') env_key = 'SECRET' env_val = 'enigma' env_str = env_key + '=' + env_val # open() doesn't take path-like on Python < 3.6 - try: - with open(path_like, 'w', encoding='utf-8') as f: - f.write(env_str + '\n') - except TypeError: - return + with open(str(env_file_path), 'w', encoding='utf-8') as f: + f.write(env_str + '\n') - assert path_like.exists() - self.env.read_env(path_like) + self.env.read_env(env_file_path) assert env_key in self.env.ENVIRON assert self.env.ENVIRON[env_key] == env_val + @pytest.mark.parametrize("overwrite", [True, False]) + def test_existing_overwrite(self, overwrite): + env_file_path = self.create_temp_env_file('test_existing.env') + with open(str(env_file_path), 'w') as f: + f.write("EXISTING=b") + self.env.ENVIRON['EXISTING'] = "a" + self.env.read_env(env_file_path, overwrite=overwrite) + assert self.env.ENVIRON["EXISTING"] == ("b" if overwrite else "a") + class TestSubClass(TestEnv): def setup_method(self, method): diff --git a/tests/test_env.txt b/tests/test_env.txt index dfbd61ef..c6363ed3 100644 --- a/tests/test_env.txt +++ b/tests/test_env.txt @@ -1,5 +1,6 @@ DICT_VAR=foo=bar,test=on DATABASE_MYSQL_URL=mysql://bea6eb0:69772142@us-cdbr-east.cleardb.com/heroku_97681?reconnect=true +DATABASE_MYSQL_CLOUDSQL_URL=mysql://djuser:hidden-password@//cloudsql/arvore-codelab:us-central1:mysqlinstance/mydatabase DATABASE_MYSQL_GIS_URL=mysqlgis://user:password@127.0.0.1/some_database CACHE_URL=memcache://127.0.0.1:11211 CACHE_REDIS=rediscache://127.0.0.1:6379/1?client_class=django_redis.client.DefaultClient&password=secret @@ -28,11 +29,15 @@ FLOAT_STRANGE_VAR1=123,420,333.3 FLOAT_STRANGE_VAR2=123.420.333,3 FLOAT_NEGATIVE_VAR=-1.0 PROXIED_VAR=$STR_VAR +ESCAPED_VAR=\$baz EMPTY_LIST= +EMPTY_INT_VAR= INT_VAR=42 STR_LIST_WITH_SPACES= foo, bar STR_VAR=bar MULTILINE_STR_VAR=foo\nbar +MULTILINE_QUOTED_STR_VAR="---BEGIN---\r\n---END---" +MULTILINE_ESCAPED_STR_VAR=---BEGIN---\\n---END--- INT_LIST=42,33 CYRILLIC_VAR=фуубар INT_TUPLE=(42,33) diff --git a/tests/test_fileaware.py b/tests/test_fileaware.py index 11e9d8a1..b4733bc2 100644 --- a/tests/test_fileaware.py +++ b/tests/test_fileaware.py @@ -1,3 +1,11 @@ +# This file is part of the django-environ. +# +# Copyright (c) 2021, Serghei Iakovlev +# Copyright (c) 2013-2021, Daniele Faraglia +# +# For the full copyright and license information, please view +# the LICENSE.txt file that was distributed with this source code. + import os import tempfile from contextlib import contextmanager