From 03a9a2a89e039c607bb80ec2742d450ac8493165 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Wed, 2 Sep 2015 19:52:06 +0000 Subject: [PATCH 001/113] SimpleFS plugin (fixes #742) --- letsencrypt/plugins/common.py | 4 ++ letsencrypt/plugins/simplefs.py | 81 +++++++++++++++++++++++++++++++++ setup.py | 1 + 3 files changed, 86 insertions(+) create mode 100644 letsencrypt/plugins/simplefs.py diff --git a/letsencrypt/plugins/common.py b/letsencrypt/plugins/common.py index bef8b4d81da..3ec1f1f7c42 100644 --- a/letsencrypt/plugins/common.py +++ b/letsencrypt/plugins/common.py @@ -49,6 +49,10 @@ def dest_namespace(self): """ArgumentParser dest namespace (prefix of all destinations).""" return dest_namespace(self.name) + def option_name(self, name): + """Option name (include plugin namespace).""" + return self.option_namespace + name + def dest(self, var): """Find a destination for given variable ``var``.""" # this should do exactly the same what ArgumentParser(arg), diff --git a/letsencrypt/plugins/simplefs.py b/letsencrypt/plugins/simplefs.py new file mode 100644 index 00000000000..8bff1946e71 --- /dev/null +++ b/letsencrypt/plugins/simplefs.py @@ -0,0 +1,81 @@ +"""SimpleFS plugin.""" +import errno +import logging +import os + +import zope.interface + +from acme import challenges + +from letsencrypt import errors +from letsencrypt import interfaces +from letsencrypt.plugins import common + + +logger = logging.getLogger(__name__) + + +class Authenticator(common.Plugin): + """SimpleFS Authenticator.""" + zope.interface.implements(interfaces.IAuthenticator) + zope.interface.classProvides(interfaces.IPluginFactory) + + description = "SimpleFS Authenticator" + + def more_info(self): # pylint: disable=missing-docstring,no-self-use + return """\ +Authenticator plugin that performs SimpleHTTP challenge by saving +necessary validation resources to appropriate paths on the file +system. It expects that there is some other HTTP server configured +to serve all files under specified web root ({0}).""".format( + self.option_name("root")) + + @classmethod + def add_parser_arguments(cls, add): + add("root", help="public_html / webroot path") + + def get_chall_pref(self, domain): + # pylint: disable=missing-docstring,no-self-use + return [challenges.SimpleHTTP] + + def __init__(self, *args, **kwargs): + super(Authenticator, self).__init__(*args, **kwargs) + + root = self.conf("root") + if root is None: + raise errors.Error("--{0} must be set".format( + self.option_name("root"))) + if not os.path.isdir(root): + raise errors.Error(root + " does not exist or is not a directory") + self.full_root = os.path.join( + root, challenges.SimpleHTTPResponse.URI_ROOT_PATH) + + def prepare(self): # pylint: disable=missing-docstring + logger.debug("Creating root challenges validation dir at %s", + self.full_root) + try: + os.makedirs(self.full_root) + except OSError as exception: + if exception.errno != errno.EEXIST: + raise + + def perform(self, achalls): # pylint: disable=missing-docstring + return [self._perform_single(achall) for achall in achalls] + + def _path_for_achall(self, achall): + return os.path.join(self.full_root, achall.chall.encode("token")) + + def _perform_single(self, achall): + response, validation = achall.gen_response_and_validation( + tls=(not self.config.no_simple_http_tls)) + path = self._path_for_achall(achall) + logger.debug("Attempting to save validation to %s", path) + with open(path, "w") as validation_file: + validation_file.write(validation.json_dumps()) + return response + + def cleanup(self, achalls): + for achall in achalls: + path = self._path_for_achall(achall) + logger.debug("Removing %s", path) + os.remove(path) diff --git a/setup.py b/setup.py index f816c6c56e0..bd954a5d67a 100644 --- a/setup.py +++ b/setup.py @@ -118,6 +118,7 @@ def read_file(filename, encoding='utf8'): 'manual = letsencrypt.plugins.manual:ManualAuthenticator', # TODO: null should probably not be presented to the user 'null = letsencrypt.plugins.null:Installer', + 'simplefs = letsencrypt.plugins.simplefs:Authenticator', 'standalone = letsencrypt.plugins.standalone.authenticator' ':StandaloneAuthenticator', ], From 058a85eafdb80d2a442ed07f8a24ed43e84c37f5 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 6 Sep 2015 13:17:29 +0000 Subject: [PATCH 002/113] satisfy lint --- letsencrypt/plugins/simplefs.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/letsencrypt/plugins/simplefs.py b/letsencrypt/plugins/simplefs.py index 8bff1946e71..67e59983e7a 100644 --- a/letsencrypt/plugins/simplefs.py +++ b/letsencrypt/plugins/simplefs.py @@ -22,20 +22,21 @@ class Authenticator(common.Plugin): description = "SimpleFS Authenticator" - def more_info(self): # pylint: disable=missing-docstring,no-self-use - return """\ + MORE_INFO = """\ Authenticator plugin that performs SimpleHTTP challenge by saving necessary validation resources to appropriate paths on the file system. It expects that there is some other HTTP server configured -to serve all files under specified web root ({0}).""".format( - self.option_name("root")) +to serve all files under specified web root ({0}).""" + + def more_info(self): # pylint: disable=missing-docstring,no-self-use + return self.MORE_INFO.format(self.conf("root")) @classmethod def add_parser_arguments(cls, add): add("root", help="public_html / webroot path") def get_chall_pref(self, domain): - # pylint: disable=missing-docstring,no-self-use + # pylint: disable=missing-docstring,no-self-use,unused-argument return [challenges.SimpleHTTP] def __init__(self, *args, **kwargs): @@ -74,7 +75,7 @@ def _perform_single(self, achall): validation_file.write(validation.json_dumps()) return response - def cleanup(self, achalls): + def cleanup(self, achalls): # pylint: disable=missing-docstring for achall in achalls: path = self._path_for_achall(achall) logger.debug("Removing %s", path) From 57f6979f67a7f6e5765cb3d02c268853a90dd10d Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 6 Sep 2015 14:03:21 +0000 Subject: [PATCH 003/113] Add tests for SimpleFS plugin. --- letsencrypt/plugins/common.py | 8 +-- letsencrypt/plugins/common_test.py | 3 + letsencrypt/plugins/simplefs.py | 15 +++-- letsencrypt/plugins/simplefs_test.py | 82 ++++++++++++++++++++++++++++ 4 files changed, 99 insertions(+), 9 deletions(-) create mode 100644 letsencrypt/plugins/simplefs_test.py diff --git a/letsencrypt/plugins/common.py b/letsencrypt/plugins/common.py index 3ec1f1f7c42..8c4d618b843 100644 --- a/letsencrypt/plugins/common.py +++ b/letsencrypt/plugins/common.py @@ -44,15 +44,15 @@ def option_namespace(self): """ArgumentParser options namespace (prefix of all options).""" return option_namespace(self.name) + def option_name(self, name): + """Option name (include plugin namespace).""" + return self.option_namespace + name + @property def dest_namespace(self): """ArgumentParser dest namespace (prefix of all destinations).""" return dest_namespace(self.name) - def option_name(self, name): - """Option name (include plugin namespace).""" - return self.option_namespace + name - def dest(self, var): """Find a destination for given variable ``var``.""" # this should do exactly the same what ArgumentParser(arg), diff --git a/letsencrypt/plugins/common_test.py b/letsencrypt/plugins/common_test.py index fa761839c27..9c6df8c9e05 100644 --- a/letsencrypt/plugins/common_test.py +++ b/letsencrypt/plugins/common_test.py @@ -50,6 +50,9 @@ def test_init(self): def test_option_namespace(self): self.assertEqual("mock-", self.plugin.option_namespace) + def test_option_name(self): + self.assertEqual("mock-foo_bar", self.plugin.option_name("foo_bar")) + def test_dest_namespace(self): self.assertEqual("mock_", self.plugin.dest_namespace) diff --git a/letsencrypt/plugins/simplefs.py b/letsencrypt/plugins/simplefs.py index 67e59983e7a..ad83c13d7f8 100644 --- a/letsencrypt/plugins/simplefs.py +++ b/letsencrypt/plugins/simplefs.py @@ -35,32 +35,37 @@ def more_info(self): # pylint: disable=missing-docstring,no-self-use def add_parser_arguments(cls, add): add("root", help="public_html / webroot path") - def get_chall_pref(self, domain): + def get_chall_pref(self, domain): # pragma: no cover # pylint: disable=missing-docstring,no-self-use,unused-argument return [challenges.SimpleHTTP] def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) + self.full_root = None + def prepare(self): # pylint: disable=missing-docstring root = self.conf("root") if root is None: - raise errors.Error("--{0} must be set".format( + raise errors.PluginError("--{0} must be set".format( self.option_name("root"))) if not os.path.isdir(root): - raise errors.Error(root + " does not exist or is not a directory") + raise errors.PluginError( + root + " does not exist or is not a directory") self.full_root = os.path.join( root, challenges.SimpleHTTPResponse.URI_ROOT_PATH) - def prepare(self): # pylint: disable=missing-docstring logger.debug("Creating root challenges validation dir at %s", self.full_root) try: os.makedirs(self.full_root) except OSError as exception: if exception.errno != errno.EEXIST: - raise + raise errors.PluginError( + "Couldn't create root for SimpleHTTP " + "challenge responses: {0}", exception) def perform(self, achalls): # pylint: disable=missing-docstring + assert self.full_root is not None return [self._perform_single(achall) for achall in achalls] def _path_for_achall(self, achall): diff --git a/letsencrypt/plugins/simplefs_test.py b/letsencrypt/plugins/simplefs_test.py new file mode 100644 index 00000000000..f80e8b29a62 --- /dev/null +++ b/letsencrypt/plugins/simplefs_test.py @@ -0,0 +1,82 @@ +"""Tests for letsencrypt.plugins.simplefs.""" +import os +import shutil +import tempfile +import unittest + +import mock + +from acme import jose + +from letsencrypt import achallenges +from letsencrypt import errors + +from letsencrypt.tests import acme_util +from letsencrypt.tests import test_util + + +KEY = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem")) + + +class AuthenticatorTest(unittest.TestCase): + """Tests for letsencrypt.plugins.simplefs.Authenticator.""" + + achall = achallenges.SimpleHTTP( + challb=acme_util.SIMPLE_HTTP_P, domain=None, account_key=KEY) + + def setUp(self): + from letsencrypt.plugins.simplefs import Authenticator + self.root = tempfile.mkdtemp() + self.validation_path = os.path.join( + self.root, ".well-known", "acme-challenge", + "ZXZhR3hmQURzNnBTUmIyTEF2OUlaZjE3RHQzanV4R0orUEN0OTJ3citvQQ") + self.config = mock.MagicMock(simplefs_root=self.root) + self.auth = Authenticator(self.config, "simplefs") + self.auth.prepare() + + def tearDown(self): + shutil.rmtree(self.root) + + def test_more_info(self): + more_info = self.auth.more_info() + self.assertTrue(isinstance(more_info, str)) + self.assertTrue(self.root in more_info) + + def test_add_parser_arguments(self): + add = mock.MagicMock() + self.auth.add_parser_arguments(add) + self.assertEqual(1, add.call_count) + + def test_prepare_bad_root(self): + self.config.simplefs_root = os.path.join(self.root, "null") + self.assertRaises(errors.PluginError, self.auth.prepare) + + def test_prepare_missing_root(self): + self.config.simplefs_root = None + self.assertRaises(errors.PluginError, self.auth.prepare) + + def test_prepare_full_root_exists(self): + # prepare() has already been called once in setUp() + self.auth.prepare() # shouldn't raise any exceptions + + def test_prepare_reraises_other_errors(self): + self.auth.full_root = os.path.join(self.root, "null") + os.chmod(self.root, 0o000) + self.assertRaises(errors.PluginError, self.auth.prepare) + os.chmod(self.root, 0o700) + + def test_perform_cleanup(self): + responses = self.auth.perform([self.achall]) + self.assertEqual(1, len(responses)) + self.assertTrue(os.path.exists(self.validation_path)) + with open(self.validation_path) as validation_f: + validation = jose.JWS.json_loads(validation_f.read()) + self.assertTrue(responses[0].check_validation( + validation, self.achall.chall, KEY.public_key())) + + self.auth.cleanup([self.achall]) + self.assertFalse(os.path.exists(self.validation_path)) + + +if __name__ == "__main__": + unittest.main() # pragma: no cover From 8c7b8b835117fe5cca92367d9244c4dd480c4ec6 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Mon, 7 Sep 2015 06:33:16 +0000 Subject: [PATCH 004/113] Add docs for SimpleFS plugin --- docs/api/plugins/simplefs.rst | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/api/plugins/simplefs.rst diff --git a/docs/api/plugins/simplefs.rst b/docs/api/plugins/simplefs.rst new file mode 100644 index 00000000000..7165b6aca81 --- /dev/null +++ b/docs/api/plugins/simplefs.rst @@ -0,0 +1,5 @@ +:mod:`letsencrypt.plugins.simplefs` +----------------------------------- + +.. automodule:: letsencrypt.plugins.simplefs + :members: From 33f1c301924cc6b5c022bf8c9113b04290d2da0a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 22 Sep 2015 16:49:07 -0700 Subject: [PATCH 005/113] Updated a2enmod.sh --- .../configurators/apache/a2enmod.sh | 34 +++++-------------- 1 file changed, 9 insertions(+), 25 deletions(-) diff --git a/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/a2enmod.sh b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/a2enmod.sh index f822a1f7ba6..0a8ade4c2a1 100755 --- a/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/a2enmod.sh +++ b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/a2enmod.sh @@ -3,31 +3,15 @@ # httpd docker image. First argument is the server_root and the second is the # module to be enabled. -APACHE_CONFDIR=$1 +confdir=$1 +module=$2 -enable () { - echo "LoadModule "$1"_module /usr/local/apache2/modules/mod_"$1".so" >> \ - $APACHE_CONFDIR"/test.conf" - available_base="/mods-available/"$1".conf" - available_conf=$APACHE_CONFDIR$available_base - enabled_dir=$APACHE_CONFDIR"/mods-enabled" - enabled_conf=$enabled_dir"/"$1".conf" - if [ -e "$available_conf" -a -d "$enabled_dir" -a ! -e "$enabled_conf" ] - then - ln -s "..$available_base" $enabled_conf - fi -} - -if [ $2 == "ssl" ] -then - # Enables ssl and all its dependencies - enable "setenvif" - enable "mime" - enable "socache_shmcb" - enable "ssl" -elif [ $2 == "rewrite" ] +echo "LoadModule ${module}_module " \ + "/usr/local/apache2/modules/mod_${module}.so" >> "${confdir}/test.conf" +available_conf=$APACHE_CONFDIR"/mods-available/${module}.conf" +enabled_dir=$APACHE_CONFDIR"/mods-enabled" +enabled_conf=$enabled_dir"/"$1".conf" +if [ -e "$available_conf" -a -d "$enabled_dir" -a ! -e "$enabled_conf" ] then - enable "rewrite" -else - exit 1 + ln -s "..$available_base" $enabled_conf fi From d73b600eeb4cf4ea602352fd7b786f26cbeb471a Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sat, 26 Sep 2015 14:55:27 +0000 Subject: [PATCH 006/113] acme: _serve_sni -> SSLSocket --- acme/acme/crypto_util.py | 112 ++++++++++++++++++++++++---------- acme/acme/crypto_util_test.py | 44 +++++++------ 2 files changed, 101 insertions(+), 55 deletions(-) diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index 030946f8244..32533630ba8 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -26,47 +26,95 @@ _DEFAULT_DVSNI_SSL_METHOD = OpenSSL.SSL.SSLv23_METHOD -def _serve_sni(certs, sock, reuseaddr=True, method=_DEFAULT_DVSNI_SSL_METHOD, - accept=None): - """Start SNI-enabled server, that drops connection after handshake. +class SSLSocket(object): # pylint: disable=too-few-public-methods + """SSL wrapper for sockets.""" - :param certs: Mapping from SNI name to ``(key, cert)`` `tuple`. - :param sock: Already bound socket. - :param bool reuseaddr: Should `socket.SO_REUSEADDR` be set? - :param method: See `OpenSSL.SSL.Context` for allowed values. - :param accept: Callable that doesn't take any arguments and - returns ``True`` if more connections should be served. + def __init__(self, sock, certs, method=_DEFAULT_DVSNI_SSL_METHOD): + self.sock = sock + self.certs = certs + self.method = method - """ - def _pick_certificate(connection): + def __getattr__(self, name): + return getattr(self.sock, name) + + def _pick_certificate_cb(self, connection): + """SNI certificate callback. + + This method will set a new OpenSSL context object for this + connection when an incoming connection provides an SNI name + (in order to serve the appropriate certificate, if any). + + :param connection: The TLS connection object on which the SNI + extension was received. + :type connection: :class:`OpenSSL.Connection` + + """ + server_name = connection.get_servername() try: - key, cert = certs[connection.get_servername()] + key, cert = self.certs[server_name] except KeyError: + logger.debug("Server name (%s) not recognized, dropping SSL", + server_name) return - new_context = OpenSSL.SSL.Context(method) + new_context = OpenSSL.SSL.Context(self.method) new_context.use_privatekey(key) new_context.use_certificate(cert) connection.set_context(new_context) - if reuseaddr: - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.listen(1) # TODO: add func arg? - - while accept is None or accept(): - server, addr = sock.accept() - logger.debug('Received connection from %s', addr) - - with contextlib.closing(server): - context = OpenSSL.SSL.Context(method) - context.set_tlsext_servername_callback(_pick_certificate) - - server_ssl = OpenSSL.SSL.Connection(context, server) - server_ssl.set_accept_state() - try: - server_ssl.do_handshake() - server_ssl.shutdown() - except OpenSSL.SSL.Error as error: - raise errors.Error(error) + class FakeConnection(object): + """Fake OpenSSL.SSL.Connection.""" + + # pylint: disable=missing-docstring + + def __init__(self, connection): + self._wrapped = connection + self._makefile_refs = 0 + + def __getattr__(self, name): + return getattr(self._wrapped, name) + + def shutdown(self, *unused_args): + # OpenSSL.SSL.Connection.shutdown doesn't accept any args + return self._wrapped.shutdown() + + # stuff below ripped off from + # https://hg.python.org/cpython/file/2.7/Lib/ssl.py + # XXX: this uses Python's internal API + + def makefile(self, mode='r', bufsize=-1): + self._makefile_refs += 1 + # SocketServer.StreamRequesthandler.finish will try to + # close the wfile/rfile. close=True causes curl: (56) + # GnuTLS recv error (-110): The TLS connection was + # non-properly terminated. + # TODO: doesn't work in Python3 + # pylint: disable=protected-access + return socket._fileobject(self._wrapped, mode, bufsize, close=False) + + def close(self): + if self._makefile_refs < 1: + self._wrapped.close() + else: + self._makefile_refs -= 1 + + def accept(self): # pylint: disable=missing-docstring + sock, addr = self.sock.accept() + + context = OpenSSL.SSL.Context(self.method) + context.set_tlsext_servername_callback(self._pick_certificate_cb) + + ssl_sock = self.FakeConnection(OpenSSL.SSL.Connection(context, sock)) + ssl_sock.set_accept_state() + + logger.debug("Performing handshake with %s", addr) + try: + ssl_sock.do_handshake() + except OpenSSL.SSL.Error as error: + # _pick_certificate_cb might have returned without + # creating SSL context (wrong server name) + raise socket.error(error) + + return ssl_sock, addr def probe_sni(name, host, port=443, timeout=300, diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 64c7cb55287..bfd16388ce3 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -4,45 +4,43 @@ import time import unittest -import mock -import OpenSSL +from six.moves import socketserver # pylint: disable=import-error from acme import errors from acme import jose from acme import test_util -class ServeProbeSNITest(unittest.TestCase): - """Tests for acme.crypto_util._serve_sni/probe_sni.""" +class SSLSocketAndProbeSNITest(unittest.TestCase): + """Tests for acme.crypto_util.SSLSocket/probe_sni.""" def setUp(self): self.cert = test_util.load_cert('cert.pem') - key = OpenSSL.crypto.load_privatekey( - OpenSSL.crypto.FILETYPE_PEM, - test_util.load_vector('rsa512_key.pem')) + key = test_util.load_pyopenssl_private_key('rsa512_key.pem') # pylint: disable=protected-access certs = {b'foo': (key, self.cert._wrapped)} - sock = socket.socket() - sock.bind(('', 0)) # pick random port - self.port = sock.getsockname()[1] + from acme.crypto_util import SSLSocket - self.server = threading.Thread(target=self._run_server, args=(certs, sock)) - self.server.start() - time.sleep(1) # TODO: avoid race conditions in other way + class _TestServer(socketserver.TCPServer): - @classmethod - def _run_server(cls, certs, sock): - from acme.crypto_util import _serve_sni - # TODO: improve testing of server errors and their conditions - try: - return _serve_sni( - certs, sock, accept=mock.Mock(side_effect=[True, False])) - except errors.Error: - pass + # pylint: disable=too-few-public-methods + # six.moves.* | pylint: disable=attribute-defined-outside-init,no-init + + def server_bind(self): # pylint: disable=missing-docstring + self.socket = SSLSocket(socket.socket(), certs=certs) + socketserver.TCPServer.server_bind(self) + + self.server = _TestServer(('', 0), socketserver.BaseRequestHandler) + self.port = self.server.socket.getsockname()[1] + self.server_thread = threading.Thread( + # pylint: disable=no-member + target=self.server.handle_request) + self.server_thread.start() + time.sleep(1) # TODO: avoid race conditions in other way def tearDown(self): - self.server.join() + self.server_thread.join() def _probe(self, name): from acme.crypto_util import probe_sni From 1b24fdae84b7fa17823df4338901b1712950e7ae Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sat, 26 Sep 2015 14:56:44 +0000 Subject: [PATCH 007/113] acme: challenges helpers --- acme/acme/challenges.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 13186cc4f8a..a16cc6f8991 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -54,6 +54,9 @@ class SimpleHTTP(DVChallenge): TOKEN_SIZE = 128 / 8 # Based on the entropy value from the spec """Minimum size of the :attr:`token` in bytes.""" + URI_ROOT_PATH = ".well-known/acme-challenge" + """URI root path for the server provisioned resource.""" + # TODO: acme-spec doesn't specify token as base64-encoded value token = jose.Field( "token", encoder=jose.encode_b64jose, decoder=functools.partial( @@ -72,6 +75,11 @@ def good_token(self): # XXX: @token.decoder # URI_ROOT_PATH! return b'..' not in self.token and b'/' not in self.token + @property + def path(self): + """Path (starting with '/') for provisioned resource.""" + return '/' + self.URI_ROOT_PATH + '/' + self.encode('token') + @ChallengeResponse.register class SimpleHTTPResponse(ChallengeResponse): @@ -83,12 +91,12 @@ class SimpleHTTPResponse(ChallengeResponse): typ = "simpleHttp" tls = jose.Field("tls", default=True, omitempty=True) - URI_ROOT_PATH = ".well-known/acme-challenge" - """URI root path for the server provisioned resource.""" - + URI_ROOT_PATH = SimpleHTTP.URI_ROOT_PATH _URI_TEMPLATE = "{scheme}://{domain}/" + URI_ROOT_PATH + "/{token}" CONTENT_TYPE = "application/jose+json" + PORT = 80 + TLS_PORT = 443 @property def scheme(self): @@ -98,7 +106,7 @@ def scheme(self): @property def port(self): """Port that the ACME client should be listening for validation.""" - return 443 if self.tls else 80 + return self.TLS_PORT if self.tls else self.PORT def uri(self, domain, chall): """Create an URI to the provisioned resource. From daa459f2778816225107a6e6cddc3e66ef17318d Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sat, 26 Sep 2015 14:57:07 +0000 Subject: [PATCH 008/113] Add acme.standalone --- acme/acme/standalone.py | 188 ++++++++++++++++++++ acme/acme/standalone_test.py | 128 +++++++++++++ acme/examples/standalone/README | 2 + acme/examples/standalone/localhost/cert.pem | 1 + acme/examples/standalone/localhost/key.pem | 1 + docs/pkgs/acme/index.rst | 7 + 6 files changed, 327 insertions(+) create mode 100644 acme/acme/standalone.py create mode 100644 acme/acme/standalone_test.py create mode 100644 acme/examples/standalone/README create mode 120000 acme/examples/standalone/localhost/cert.pem create mode 120000 acme/examples/standalone/localhost/key.pem diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py new file mode 100644 index 00000000000..8f604a4395c --- /dev/null +++ b/acme/acme/standalone.py @@ -0,0 +1,188 @@ +"""Support for standalone client challenge solvers. """ +import argparse +import collections +import functools +import logging +import os +import sys + +import six +from six.moves import BaseHTTPServer # pylint: disable=import-error +from six.moves import http_client # pylint: disable=import-error +from six.moves import socketserver # pylint: disable=import-error + +import OpenSSL + +from acme import challenges +from acme import crypto_util + + +logger = logging.getLogger(__name__) + +# six.moves.* | pylint: disable=no-member,attribute-defined-outside-init +# pylint: disable=too-few-public-methods,no-init + + +class TLSServer(socketserver.TCPServer): + """Generic TLS Server.""" + + def __init__(self, *args, **kwargs): + self.certs = kwargs.pop("certs", {}) + self.method = kwargs.pop( + # pylint: disable=protected-access + "method", crypto_util._DEFAULT_DVSNI_SSL_METHOD) + self.allow_reuse_address = kwargs.pop("allow_reuse_address", True) + socketserver.TCPServer.__init__(self, *args, **kwargs) + + def _wrap_sock(self): + self.socket = crypto_util.SSLSocket( + self.socket, certs=self.certs, method=self.method) + + def server_bind(self): # pylint: disable=missing-docstring + self._wrap_sock() + return socketserver.TCPServer.server_bind(self) + + +class HTTPSServer(TLSServer, BaseHTTPServer.HTTPServer): + """HTTPS Server.""" + + def server_bind(self): + self._wrap_sock() + BaseHTTPServer.HTTPServer.server_bind(self) + + +class ACMEServerMixin: # pylint: disable=old-style-class,no-init + """ACME server common settings mixin.""" + server_version = "ACME standalone client" + allow_reuse_address = True + + +class ACMETLSServer(HTTPSServer, ACMEServerMixin): + """ACME TLS Server.""" + + +class ACMEServer(BaseHTTPServer.HTTPServer, ACMEServerMixin): + """ACME Server (non-TLS).""" + + +class SimpleHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): + """SimpleHTTP challenge handler. + + Adheres to the stdlib"s `socketserver.BaseRequestHandler` interface. + + :ivar set simple_http_resources: A set of `SimpleHTTPResource` + objects. TODO: better name? + + """ + SimpleHTTPResource = collections.namedtuple( + "SimpleHTTPResource", "chall response validation") + + def __init__(self, *args, **kwargs): + self.simple_http_resources = kwargs.pop("simple_http_resources", set()) + socketserver.BaseRequestHandler.__init__(self, *args, **kwargs) + + def do_GET(self): # pylint: disable=invalid-name,missing-docstring + if self.path == "/": + self.handle_index() + elif self.path.startswith("/" + challenges.SimpleHTTP.URI_ROOT_PATH): + self.handle_simple_http_resource() + else: + self.handle_404() + + def handle_index(self): + """Handle index page.""" + self.send_response(200) + self.send_header("Content-type", "text/html") + self.end_headers() + self.wfile.write(self.server.server_version) + + def handle_404(self): + """Handler 404 Not Found errors.""" + self.send_response(http_client.NOT_FOUND, message="Not Found") + self.send_header("Content-type", "text/html") + self.end_headers() + self.wfile.write("404") + + def handle_simple_http_resource(self): + """Handle SimpleHTTP provisioned resources.""" + for resource in self.simple_http_resources: + if resource.chall.path == self.path: + logger.debug("Serving SimpleHTTP with token %r", + resource.chall.encode("token")) + self.send_response(http_client.OK) + self.send_header("Content-type", resource.response.CONTENT_TYPE) + self.end_headers() + self.wfile.write(resource.validation.json_dumps().encode()) + return + else: # pylint: disable=useless-else-on-loop + logger.debug("No resources to serve") + logger.debug("%s does not correspond to any resource. ignoring", + self.path) + + @classmethod + def partial_init(cls, simple_http_resources): + """Partially initialize this handler. + + This is useful because `socketserver.BaseServer` takes + uninitialized handler and initializes it with the current + request. + + """ + return functools.partial( + cls, simple_http_resources=simple_http_resources) + + +class ACMERequestHandler(SimpleHTTPRequestHandler): + """ACME request handler.""" + + def handle_one_request(self): + """Handle single request. + + Makes sure that DVSNI probers are ignored. + + """ + try: + return SimpleHTTPRequestHandler.handle_one_request(self) + except OpenSSL.SSL.ZeroReturnError: + logger.debug("Client prematurely closed connection (prober?). " + "Ignoring request.") + + +def simple_server(cli_args, forever=True): + """Run simple standalone client server.""" + logging.basicConfig(level=logging.DEBUG) + + parser = argparse.ArgumentParser() + parser.add_argument( + "-p", "--port", default=0, help="Port to serve at. By default " + "picks random free port.") + args = parser.parse_args(cli_args[1:]) + + certs = {} + resources = {} + + _, hosts, _ = next(os.walk('.')) + for host in hosts: + with open(os.path.join(host, "cert.pem")) as cert_file: + cert_contents = cert_file.read() + with open(os.path.join(host, "key.pem")) as key_file: + key_contents = key_file.read() + certs[host] = ( + OpenSSL.crypto.load_privatekey( + OpenSSL.crypto.FILETYPE_PEM, key_contents), + OpenSSL.crypto.load_certificate( + OpenSSL.crypto.FILETYPE_PEM, cert_contents)) + + handler = ACMERequestHandler.partial_init( + simple_http_resources=resources) + server = ACMETLSServer(('', int(args.port)), handler, certs=certs) + six.print_("Serving at https://localhost:{0}...".format( + server.socket.getsockname()[1])) + if forever: # pragma: no cover + server.serve_forever() + else: + server.handle_request() + + +if __name__ == "__main__": + sys.exit(simple_server(sys.argv)) # pragma: no cover diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py new file mode 100644 index 00000000000..794fb1a6e86 --- /dev/null +++ b/acme/acme/standalone_test.py @@ -0,0 +1,128 @@ +"""Tests for acme.standalone.""" +import os +import threading +import time +import unittest + +from six.moves import http_client # pylint: disable=import-error +from six.moves import socketserver # pylint: disable=import-error + +import requests + +from acme import challenges +from acme import crypto_util +from acme import jose +from acme import test_util + + +class TLSServerTest(unittest.TestCase): + """Tests for acme.standalone.TLSServer.""" + + def test_bind(self): # pylint: disable=no-self-use + from acme.standalone import TLSServer + server = TLSServer( + ('', 0), socketserver.BaseRequestHandler, bind_and_activate=True) + server.server_close() # pylint: disable=no-member + + +class ACMESimpleHTTPTLSServerTestEndToEnd(unittest.TestCase): + """End-to-end test for ACME TLS server with SimpleHTTP.""" + + def setUp(self): + self.certs = { + b'localhost': (test_util.load_pyopenssl_private_key('rsa512_key.pem'), + # pylint: disable=protected-access + test_util.load_cert('cert.pem')._wrapped), + } + self.account_key = jose.JWK.load( + test_util.load_vector('rsa1024_key.pem')) + + from acme.standalone import ACMETLSServer + from acme.standalone import ACMERequestHandler + self.resources = set() + handler = ACMERequestHandler.partial_init( + simple_http_resources=self.resources) + self.server = ACMETLSServer(('', 0), handler, certs=self.certs) + self.server_thread = threading.Thread( + # pylint: disable=no-member + target=self.server.handle_request) + self.server_thread.start() + + self.port = self.server.socket.getsockname()[1] + + def tearDown(self): + self.server_thread.join() + + def test_index(self): + response = requests.get( + 'https://localhost:{0}'.format(self.port), verify=False) + self.assertEqual(response.text, 'ACME standalone client') + self.assertTrue(response.ok) + + def test_404(self): + response = requests.get( + 'https://localhost:{0}/foo'.format(self.port), verify=False) + self.assertEqual(response.status_code, http_client.NOT_FOUND) + + def test_dvsni(self): + cert = crypto_util.probe_sni( + b'localhost', *self.server.socket.getsockname()) + self.assertEqual(jose.ComparableX509(cert), + jose.ComparableX509(self.certs[b'localhost'][1])) + + def _test_simple_http(self, add): + chall = challenges.SimpleHTTP(token=(b'x' * 16)) + response = challenges.SimpleHTTPResponse(tls=True) + + from acme.standalone import SimpleHTTPRequestHandler + resource = SimpleHTTPRequestHandler.SimpleHTTPResource( + chall=chall, response=response, validation=response.gen_validation( + chall, self.account_key)) + if add: + self.resources.add(resource) + return resource.response.simple_verify( + resource.chall, 'localhost', self.account_key.public_key(), + port=self.port) + + def test_simple_http_found(self): + self.assertTrue(self._test_simple_http(add=True)) + + def test_simple_http_not_found(self): + self.assertFalse(self._test_simple_http(add=False)) + + +class TestSimpleServer(unittest.TestCase): + """Tests for acme.standalone.simple_server.""" + + TEST_CWD = os.path.join(os.path.dirname(__file__), '..', 'examples', 'standalone') + + def setUp(self): + from acme.standalone import simple_server + self.thread = threading.Thread(target=simple_server, kwargs={ + 'cli_args': ('xxx', '--port', '1234'), + 'forever': False, + }) + self.old_cwd = os.getcwd() + os.chdir(self.TEST_CWD) + self.thread.start() + + def tearDown(self): + os.chdir(self.old_cwd) + self.thread.join() + + def test_it(self): + max_attempts = 5 + while max_attempts: + max_attempts -= 1 + try: + response = requests.get('https://localhost:1234', verify=False) + except requests.ConnectionError: + self.assertTrue(max_attempts > 0, "Timeout!") + time.sleep(1) # wait until thread starts + else: + self.assertEqual(response.text, 'ACME standalone client') + break + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/acme/examples/standalone/README b/acme/examples/standalone/README new file mode 100644 index 00000000000..89bc5d74ec8 --- /dev/null +++ b/acme/examples/standalone/README @@ -0,0 +1,2 @@ +python -m acme.standalone -p 1234 +curl -k https://localhost:1234 \ No newline at end of file diff --git a/acme/examples/standalone/localhost/cert.pem b/acme/examples/standalone/localhost/cert.pem new file mode 120000 index 00000000000..569366af991 --- /dev/null +++ b/acme/examples/standalone/localhost/cert.pem @@ -0,0 +1 @@ +../../../acme/testdata/cert.pem \ No newline at end of file diff --git a/acme/examples/standalone/localhost/key.pem b/acme/examples/standalone/localhost/key.pem new file mode 120000 index 00000000000..870f4f87669 --- /dev/null +++ b/acme/examples/standalone/localhost/key.pem @@ -0,0 +1 @@ +../../../acme/testdata/rsa512_key.pem \ No newline at end of file diff --git a/docs/pkgs/acme/index.rst b/docs/pkgs/acme/index.rst index 04194c353b7..23c5a3284a6 100644 --- a/docs/pkgs/acme/index.rst +++ b/docs/pkgs/acme/index.rst @@ -47,3 +47,10 @@ Errors .. automodule:: acme.errors :members: + + +Standalone +---------- + +.. automodule:: acme.standalone + :members: From a874654e34d18ab57158f117e23693ba8d955d4c Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sat, 26 Sep 2015 16:14:00 +0000 Subject: [PATCH 009/113] NamespaceConfig.simple_http_port. --- letsencrypt/configuration.py | 18 ++++++++++++++++++ letsencrypt/tests/configuration_test.py | 3 ++- letsencrypt/tests/renewer_test.py | 3 ++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/letsencrypt/configuration.py b/letsencrypt/configuration.py index 20774e5cca5..5f965cb6d7c 100644 --- a/letsencrypt/configuration.py +++ b/letsencrypt/configuration.py @@ -4,6 +4,8 @@ import zope.interface +from acme import challenges + from letsencrypt import constants from letsencrypt import interfaces @@ -34,6 +36,13 @@ class NamespaceConfig(object): def __init__(self, namespace): self.namespace = namespace + # XXX: breaks renewer in some bizarre way + #if self.no_simple_http_tls and ( + # self.simple_http_port == self.dvsni_port): + # raise errors.Error( + # "Trying to run SimpleHTTP non-TLS and DVSNI " + # "on the same port ({0})".format(self.dvsni_port)) + def __getattr__(self, name): return getattr(self.namespace, name) @@ -69,6 +78,15 @@ def temp_checkpoint_dir(self): # pylint: disable=missing-docstring return os.path.join( self.namespace.work_dir, constants.TEMP_CHECKPOINT_DIR) + @property + def simple_http_port(self): # pylint: disable=missing-docstring + if self.namespace.simple_http_port is not None: + return self.namespace.simple_http_port + if self.no_simple_http_tls: + return challenges.SimpleHTTPResponse.PORT + else: + return challenges.SimpleHTTPResponse.TLS_PORT + class RenewerConfiguration(object): """Configuration wrapper for renewer.""" diff --git a/letsencrypt/tests/configuration_test.py b/letsencrypt/tests/configuration_test.py index 546834c05fb..91a3dfe37f2 100644 --- a/letsencrypt/tests/configuration_test.py +++ b/letsencrypt/tests/configuration_test.py @@ -11,7 +11,8 @@ class NamespaceConfigTest(unittest.TestCase): def setUp(self): self.namespace = mock.MagicMock( config_dir='/tmp/config', work_dir='/tmp/foo', foo='bar', - server='https://acme-server.org:443/new') + server='https://acme-server.org:443/new', + dvsni_port='1234', simple_http_port='4321') from letsencrypt.configuration import NamespaceConfig self.config = NamespaceConfig(self.namespace) diff --git a/letsencrypt/tests/renewer_test.py b/letsencrypt/tests/renewer_test.py index 293b09537e4..84d4cf6ecab 100644 --- a/letsencrypt/tests/renewer_test.py +++ b/letsencrypt/tests/renewer_test.py @@ -39,7 +39,8 @@ def setUp(self): self.tempdir = tempfile.mkdtemp() self.cli_config = configuration.RenewerConfiguration( - namespace=mock.MagicMock(config_dir=self.tempdir)) + namespace=mock.MagicMock( + config_dir=self.tempdir, no_simple_http_tls=False)) # TODO: maybe provide RenewerConfiguration.make_dirs? os.makedirs(os.path.join(self.tempdir, "live", "example.org")) os.makedirs(os.path.join(self.tempdir, "archive", "example.org")) From ef3605730c349073f1f461bf59cbd8d448f7487d Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sat, 26 Sep 2015 16:26:36 +0000 Subject: [PATCH 010/113] Move already_listening to plugins.util --- docs/api/plugins/util.rst | 5 + .../plugins/standalone/authenticator.py | 44 +------- .../standalone/tests/authenticator_test.py | 103 ------------------ letsencrypt/plugins/util.py | 49 +++++++++ letsencrypt/plugins/util_test.py | 103 ++++++++++++++++++ 5 files changed, 158 insertions(+), 146 deletions(-) create mode 100644 docs/api/plugins/util.rst create mode 100644 letsencrypt/plugins/util.py create mode 100644 letsencrypt/plugins/util_test.py diff --git a/docs/api/plugins/util.rst b/docs/api/plugins/util.rst new file mode 100644 index 00000000000..6bc8995db15 --- /dev/null +++ b/docs/api/plugins/util.rst @@ -0,0 +1,5 @@ +:mod:`letsencrypt.plugins.util` +------------------------------- + +.. automodule:: letsencrypt.plugins.util + :members: diff --git a/letsencrypt/plugins/standalone/authenticator.py b/letsencrypt/plugins/standalone/authenticator.py index 968063781e9..996f41cdcb3 100644 --- a/letsencrypt/plugins/standalone/authenticator.py +++ b/letsencrypt/plugins/standalone/authenticator.py @@ -1,6 +1,5 @@ """Standalone authenticator.""" import os -import psutil import signal import socket import sys @@ -289,47 +288,6 @@ def start_listener(self, port): # should terminate via sys.exit(). return self.do_child_process(port) - def already_listening(self, port): # pylint: disable=no-self-use - """Check if a process is already listening on the port. - - If so, also tell the user via a display notification. - - .. warning:: - On some operating systems, this function can only usefully be - run as root. - - :param int port: The TCP port in question. - :returns: True or False.""" - - listeners = [conn.pid for conn in psutil.net_connections() - if conn.status == 'LISTEN' and - conn.type == socket.SOCK_STREAM and - conn.laddr[1] == port] - try: - if listeners and listeners[0] is not None: - # conn.pid may be None if the current process doesn't have - # permission to identify the listening process! Additionally, - # listeners may have more than one element if separate - # sockets have bound the same port on separate interfaces. - # We currently only have UI to notify the user about one - # of them at a time. - pid = listeners[0] - name = psutil.Process(pid).name() - display = zope.component.getUtility(interfaces.IDisplay) - display.notification( - "The program {0} (process ID {1}) is already listening " - "on TCP port {2}. This will prevent us from binding to " - "that port. Please stop the {0} program temporarily " - "and then try again.".format(name, pid, port)) - return True - except (psutil.NoSuchProcess, psutil.AccessDenied): - # Perhaps the result of a race where the process could have - # exited or relinquished the port (NoSuchProcess), or the result - # of an OS policy where we're not allowed to look up the process - # name (AccessDenied). - pass - return False - # IAuthenticator method implementations follow def get_chall_pref(self, unused_domain): # pylint: disable=no-self-use @@ -383,7 +341,7 @@ def perform(self, achalls): if not self.tasks: raise ValueError("nothing for .perform() to do") - if self.already_listening(self.config.dvsni_port): + if util.already_listening(self.config.dvsni_port): # If we know a process is already listening on this port, # tell the user, and don't even attempt to bind it. (This # test is Linux-specific and won't indicate that the port diff --git a/letsencrypt/plugins/standalone/tests/authenticator_test.py b/letsencrypt/plugins/standalone/tests/authenticator_test.py index 7ff2c03e1d4..955426533e3 100644 --- a/letsencrypt/plugins/standalone/tests/authenticator_test.py +++ b/letsencrypt/plugins/standalone/tests/authenticator_test.py @@ -187,109 +187,6 @@ def test_subproc_signal_handler_trouble(self, mock_exit, mock_kill): mock_exit.assert_called_once_with(0) -class AlreadyListeningTest(unittest.TestCase): - """Tests for already_listening() method.""" - def setUp(self): - from letsencrypt.plugins.standalone.authenticator import \ - StandaloneAuthenticator - self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None) - - @mock.patch("letsencrypt.plugins.standalone.authenticator.psutil." - "net_connections") - @mock.patch("letsencrypt.plugins.standalone.authenticator.psutil.Process") - @mock.patch("letsencrypt.plugins.standalone.authenticator." - "zope.component.getUtility") - def test_race_condition(self, mock_get_utility, mock_process, mock_net): - # This tests a race condition, or permission problem, or OS - # incompatibility in which, for some reason, no process name can be - # found to match the identified listening PID. - from psutil._common import sconn - conns = [ - sconn(fd=-1, family=2, type=1, laddr=("0.0.0.0", 30), - raddr=(), status="LISTEN", pid=None), - sconn(fd=3, family=2, type=1, laddr=("192.168.5.10", 32783), - raddr=("20.40.60.80", 22), status="ESTABLISHED", pid=1234), - sconn(fd=-1, family=10, type=1, laddr=("::1", 54321), - raddr=("::1", 111), status="CLOSE_WAIT", pid=None), - sconn(fd=3, family=2, type=1, laddr=("0.0.0.0", 17), - raddr=(), status="LISTEN", pid=4416)] - mock_net.return_value = conns - mock_process.side_effect = psutil.NoSuchProcess("No such PID") - # We simulate being unable to find the process name of PID 4416, - # which results in returning False. - self.assertFalse(self.authenticator.already_listening(17)) - self.assertEqual(mock_get_utility.generic_notification.call_count, 0) - mock_process.assert_called_once_with(4416) - - @mock.patch("letsencrypt.plugins.standalone.authenticator.psutil." - "net_connections") - @mock.patch("letsencrypt.plugins.standalone.authenticator.psutil.Process") - @mock.patch("letsencrypt.plugins.standalone.authenticator." - "zope.component.getUtility") - def test_not_listening(self, mock_get_utility, mock_process, mock_net): - from psutil._common import sconn - conns = [ - sconn(fd=-1, family=2, type=1, laddr=("0.0.0.0", 30), - raddr=(), status="LISTEN", pid=None), - sconn(fd=3, family=2, type=1, laddr=("192.168.5.10", 32783), - raddr=("20.40.60.80", 22), status="ESTABLISHED", pid=1234), - sconn(fd=-1, family=10, type=1, laddr=("::1", 54321), - raddr=("::1", 111), status="CLOSE_WAIT", pid=None)] - mock_net.return_value = conns - mock_process.name.return_value = "inetd" - self.assertFalse(self.authenticator.already_listening(17)) - self.assertEqual(mock_get_utility.generic_notification.call_count, 0) - self.assertEqual(mock_process.call_count, 0) - - @mock.patch("letsencrypt.plugins.standalone.authenticator.psutil." - "net_connections") - @mock.patch("letsencrypt.plugins.standalone.authenticator.psutil.Process") - @mock.patch("letsencrypt.plugins.standalone.authenticator." - "zope.component.getUtility") - def test_listening_ipv4(self, mock_get_utility, mock_process, mock_net): - from psutil._common import sconn - conns = [ - sconn(fd=-1, family=2, type=1, laddr=("0.0.0.0", 30), - raddr=(), status="LISTEN", pid=None), - sconn(fd=3, family=2, type=1, laddr=("192.168.5.10", 32783), - raddr=("20.40.60.80", 22), status="ESTABLISHED", pid=1234), - sconn(fd=-1, family=10, type=1, laddr=("::1", 54321), - raddr=("::1", 111), status="CLOSE_WAIT", pid=None), - sconn(fd=3, family=2, type=1, laddr=("0.0.0.0", 17), - raddr=(), status="LISTEN", pid=4416)] - mock_net.return_value = conns - mock_process.name.return_value = "inetd" - result = self.authenticator.already_listening(17) - self.assertTrue(result) - self.assertEqual(mock_get_utility.call_count, 1) - mock_process.assert_called_once_with(4416) - - @mock.patch("letsencrypt.plugins.standalone.authenticator.psutil." - "net_connections") - @mock.patch("letsencrypt.plugins.standalone.authenticator.psutil.Process") - @mock.patch("letsencrypt.plugins.standalone.authenticator." - "zope.component.getUtility") - def test_listening_ipv6(self, mock_get_utility, mock_process, mock_net): - from psutil._common import sconn - conns = [ - sconn(fd=-1, family=2, type=1, laddr=("0.0.0.0", 30), - raddr=(), status="LISTEN", pid=None), - sconn(fd=3, family=2, type=1, laddr=("192.168.5.10", 32783), - raddr=("20.40.60.80", 22), status="ESTABLISHED", pid=1234), - sconn(fd=-1, family=10, type=1, laddr=("::1", 54321), - raddr=("::1", 111), status="CLOSE_WAIT", pid=None), - sconn(fd=3, family=10, type=1, laddr=("::", 12345), raddr=(), - status="LISTEN", pid=4420), - sconn(fd=3, family=2, type=1, laddr=("0.0.0.0", 17), - raddr=(), status="LISTEN", pid=4416)] - mock_net.return_value = conns - mock_process.name.return_value = "inetd" - result = self.authenticator.already_listening(12345) - self.assertTrue(result) - self.assertEqual(mock_get_utility.call_count, 1) - mock_process.assert_called_once_with(4420) - - class PerformTest(unittest.TestCase): """Tests for perform() method.""" def setUp(self): diff --git a/letsencrypt/plugins/util.py b/letsencrypt/plugins/util.py new file mode 100644 index 00000000000..42c64b05235 --- /dev/null +++ b/letsencrypt/plugins/util.py @@ -0,0 +1,49 @@ +"""Plugin utilities.""" +import socket + +import psutil +import zope.component + +from letsencrypt import interfaces + + +def already_listening(port): + """Check if a process is already listening on the port. + + If so, also tell the user via a display notification. + + .. warning:: + On some operating systems, this function can only usefully be + run as root. + + :param int port: The TCP port in question. + :returns: True or False.""" + + listeners = [conn.pid for conn in psutil.net_connections() + if conn.status == 'LISTEN' and + conn.type == socket.SOCK_STREAM and + conn.laddr[1] == port] + try: + if listeners and listeners[0] is not None: + # conn.pid may be None if the current process doesn't have + # permission to identify the listening process! Additionally, + # listeners may have more than one element if separate + # sockets have bound the same port on separate interfaces. + # We currently only have UI to notify the user about one + # of them at a time. + pid = listeners[0] + name = psutil.Process(pid).name() + display = zope.component.getUtility(interfaces.IDisplay) + display.notification( + "The program {0} (process ID {1}) is already listening " + "on TCP port {2}. This will prevent us from binding to " + "that port. Please stop the {0} program temporarily " + "and then try again.".format(name, pid, port)) + return True + except (psutil.NoSuchProcess, psutil.AccessDenied): + # Perhaps the result of a race where the process could have + # exited or relinquished the port (NoSuchProcess), or the result + # of an OS policy where we're not allowed to look up the process + # name (AccessDenied). + pass + return False diff --git a/letsencrypt/plugins/util_test.py b/letsencrypt/plugins/util_test.py new file mode 100644 index 00000000000..14cbcf7a891 --- /dev/null +++ b/letsencrypt/plugins/util_test.py @@ -0,0 +1,103 @@ +"""Tests for letsencrypt.plugins.util.""" +import unittest + +import mock +import psutil + + +class AlreadyListeningTest(unittest.TestCase): + """Tests for letsencrypt.plugins.already_listening.""" + def _call(self, *args, **kwargs): + from letsencrypt.plugins.util import already_listening + return already_listening(*args, **kwargs) + + @mock.patch("letsencrypt.plugins.util.psutil.net_connections") + @mock.patch("letsencrypt.plugins.util.psutil.Process") + @mock.patch("letsencrypt.plugins.util.zope.component.getUtility") + def test_race_condition(self, mock_get_utility, mock_process, mock_net): + # This tests a race condition, or permission problem, or OS + # incompatibility in which, for some reason, no process name can be + # found to match the identified listening PID. + from psutil._common import sconn + conns = [ + sconn(fd=-1, family=2, type=1, laddr=("0.0.0.0", 30), + raddr=(), status="LISTEN", pid=None), + sconn(fd=3, family=2, type=1, laddr=("192.168.5.10", 32783), + raddr=("20.40.60.80", 22), status="ESTABLISHED", pid=1234), + sconn(fd=-1, family=10, type=1, laddr=("::1", 54321), + raddr=("::1", 111), status="CLOSE_WAIT", pid=None), + sconn(fd=3, family=2, type=1, laddr=("0.0.0.0", 17), + raddr=(), status="LISTEN", pid=4416)] + mock_net.return_value = conns + mock_process.side_effect = psutil.NoSuchProcess("No such PID") + # We simulate being unable to find the process name of PID 4416, + # which results in returning False. + self.assertFalse(self._call(17)) + self.assertEqual(mock_get_utility.generic_notification.call_count, 0) + mock_process.assert_called_once_with(4416) + + @mock.patch("letsencrypt.plugins.util.psutil.net_connections") + @mock.patch("letsencrypt.plugins.util.psutil.Process") + @mock.patch("letsencrypt.plugins.util.zope.component.getUtility") + def test_not_listening(self, mock_get_utility, mock_process, mock_net): + from psutil._common import sconn + conns = [ + sconn(fd=-1, family=2, type=1, laddr=("0.0.0.0", 30), + raddr=(), status="LISTEN", pid=None), + sconn(fd=3, family=2, type=1, laddr=("192.168.5.10", 32783), + raddr=("20.40.60.80", 22), status="ESTABLISHED", pid=1234), + sconn(fd=-1, family=10, type=1, laddr=("::1", 54321), + raddr=("::1", 111), status="CLOSE_WAIT", pid=None)] + mock_net.return_value = conns + mock_process.name.return_value = "inetd" + self.assertFalse(self._call(17)) + self.assertEqual(mock_get_utility.generic_notification.call_count, 0) + self.assertEqual(mock_process.call_count, 0) + + @mock.patch("letsencrypt.plugins.util.psutil.net_connections") + @mock.patch("letsencrypt.plugins.util.psutil.Process") + @mock.patch("letsencrypt.plugins.util.zope.component.getUtility") + def test_listening_ipv4(self, mock_get_utility, mock_process, mock_net): + from psutil._common import sconn + conns = [ + sconn(fd=-1, family=2, type=1, laddr=("0.0.0.0", 30), + raddr=(), status="LISTEN", pid=None), + sconn(fd=3, family=2, type=1, laddr=("192.168.5.10", 32783), + raddr=("20.40.60.80", 22), status="ESTABLISHED", pid=1234), + sconn(fd=-1, family=10, type=1, laddr=("::1", 54321), + raddr=("::1", 111), status="CLOSE_WAIT", pid=None), + sconn(fd=3, family=2, type=1, laddr=("0.0.0.0", 17), + raddr=(), status="LISTEN", pid=4416)] + mock_net.return_value = conns + mock_process.name.return_value = "inetd" + result = self._call(17) + self.assertTrue(result) + self.assertEqual(mock_get_utility.call_count, 1) + mock_process.assert_called_once_with(4416) + + @mock.patch("letsencrypt.plugins.util.psutil.net_connections") + @mock.patch("letsencrypt.plugins.util.psutil.Process") + @mock.patch("letsencrypt.plugins.util.zope.component.getUtility") + def test_listening_ipv6(self, mock_get_utility, mock_process, mock_net): + from psutil._common import sconn + conns = [ + sconn(fd=-1, family=2, type=1, laddr=("0.0.0.0", 30), + raddr=(), status="LISTEN", pid=None), + sconn(fd=3, family=2, type=1, laddr=("192.168.5.10", 32783), + raddr=("20.40.60.80", 22), status="ESTABLISHED", pid=1234), + sconn(fd=-1, family=10, type=1, laddr=("::1", 54321), + raddr=("::1", 111), status="CLOSE_WAIT", pid=None), + sconn(fd=3, family=10, type=1, laddr=("::", 12345), raddr=(), + status="LISTEN", pid=4420), + sconn(fd=3, family=2, type=1, laddr=("0.0.0.0", 17), + raddr=(), status="LISTEN", pid=4416)] + mock_net.return_value = conns + mock_process.name.return_value = "inetd" + result = self._call(12345) + self.assertTrue(result) + self.assertEqual(mock_get_utility.call_count, 1) + mock_process.assert_called_once_with(4420) + + +if __name__ == "__main__": + unittest.main() # pragma: no cover From faa6cbdd71ec72616b44a6c2abc8dd9cdff2172e Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sat, 26 Sep 2015 16:47:29 +0000 Subject: [PATCH 011/113] Standalone 2.0 --- letsencrypt/achallenges.py | 22 +- letsencrypt/errors.py | 10 + letsencrypt/plugins/disco_test.py | 12 +- letsencrypt/plugins/standalone.py | 200 ++++++++ letsencrypt/plugins/standalone/__init__.py | 1 - .../plugins/standalone/authenticator.py | 394 -------------- .../plugins/standalone/tests/__init__.py | 1 - .../standalone/tests/authenticator_test.py | 483 ------------------ letsencrypt/tests/achallenges_test.py | 8 +- setup.py | 4 +- 10 files changed, 228 insertions(+), 907 deletions(-) create mode 100644 letsencrypt/plugins/standalone.py delete mode 100644 letsencrypt/plugins/standalone/__init__.py delete mode 100644 letsencrypt/plugins/standalone/authenticator.py delete mode 100644 letsencrypt/plugins/standalone/tests/__init__.py delete mode 100644 letsencrypt/plugins/standalone/tests/authenticator_test.py diff --git a/letsencrypt/achallenges.py b/letsencrypt/achallenges.py index 958a2973313..e86f51a3f30 100644 --- a/letsencrypt/achallenges.py +++ b/letsencrypt/achallenges.py @@ -19,8 +19,6 @@ """ import logging -import OpenSSL - from acme import challenges from acme import jose @@ -56,10 +54,10 @@ class DVSNI(AnnotatedChallenge): __slots__ = ('challb', 'domain', 'account_key') acme_type = challenges.DVSNI - def gen_cert_and_response(self, key_pem=None, bits=2048, alg=jose.RS256): + def gen_cert_and_response(self, key=None, bits=2048, alg=jose.RS256): """Generate a DVSNI cert and response. - :param bytes key_pem: Private PEM-encoded key used for + :param OpenSSL.crypto.PKey key: Private key used for certificate generation. If none provided, a fresh key will be generated. :param int bits: Number of bits for fresh key generation. @@ -67,23 +65,15 @@ def gen_cert_and_response(self, key_pem=None, bits=2048, alg=jose.RS256): :returns: ``(response, cert_pem, key_pem)`` tuple, where ``response`` is an instance of - `acme.challenges.DVSNIResponse`, ``cert_pem`` is the - PEM-encoded certificate and ``key_pem`` is PEM-encoded - private key. + `acme.challenges.DVSNIResponse`, ``cert`` is a certificate + (`OpenSSL.crypto.X509`) and ``key`` is a private key + (`OpenSSL.crypto.PKey`). :rtype: tuple """ - key = None if key_pem is None else OpenSSL.crypto.load_privatekey( - OpenSSL.crypto.FILETYPE_PEM, key_pem) response = self.challb.chall.gen_response(self.account_key, alg=alg) cert, key = response.gen_cert(key=key, bits=bits) - - cert_pem = OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, cert) - key_pem = OpenSSL.crypto.dump_privatekey( - OpenSSL.crypto.FILETYPE_PEM, key) - - return response, cert_pem, key_pem + return response, cert, key class SimpleHTTP(AnnotatedChallenge): diff --git a/letsencrypt/errors.py b/letsencrypt/errors.py index ba0601d2986..98c24bf50f3 100644 --- a/letsencrypt/errors.py +++ b/letsencrypt/errors.py @@ -80,3 +80,13 @@ class NotSupportedError(PluginError): class RevokerError(Error): """Let's Encrypt Revoker error.""" + + +class StandaloneBindError(Error): + """Standalone plugin bind error.""" + + def __init__(self, socket_error, port): + super(StandaloneBindError, self).__init__( + "Problem binding to port {0}: {1}".format(port, socket_error)) + self.socket_error = socket_error + self.port = port diff --git a/letsencrypt/plugins/disco_test.py b/letsencrypt/plugins/disco_test.py index 41699d1eff1..8660d94a1cf 100644 --- a/letsencrypt/plugins/disco_test.py +++ b/letsencrypt/plugins/disco_test.py @@ -8,11 +8,11 @@ from letsencrypt import errors from letsencrypt import interfaces -from letsencrypt.plugins.standalone import authenticator +from letsencrypt.plugins import standalone EP_SA = pkg_resources.EntryPoint( - "sa", "letsencrypt.plugins.standalone.authenticator", - attrs=("StandaloneAuthenticator",), + "sa", "letsencrypt.plugins.standalone", + attrs=("Authenticator",), dist=mock.MagicMock(key="letsencrypt")) @@ -71,8 +71,7 @@ def test__init__(self): self.assertTrue(self.plugin_ep.entry_point is EP_SA) self.assertEqual("sa", self.plugin_ep.name) - self.assertTrue( - self.plugin_ep.plugin_cls is authenticator.StandaloneAuthenticator) + self.assertTrue(self.plugin_ep.plugin_cls is standalone.Authenticator) def test_init(self): config = mock.MagicMock() @@ -174,8 +173,7 @@ def test_find_all(self): with mock.patch("letsencrypt.plugins.disco.pkg_resources") as mock_pkg: mock_pkg.iter_entry_points.return_value = iter([EP_SA]) plugins = PluginsRegistry.find_all() - self.assertTrue(plugins["sa"].plugin_cls - is authenticator.StandaloneAuthenticator) + self.assertTrue(plugins["sa"].plugin_cls is standalone.Authenticator) self.assertTrue(plugins["sa"].entry_point is EP_SA) def test_getitem(self): diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py new file mode 100644 index 00000000000..8e5f1e77dca --- /dev/null +++ b/letsencrypt/plugins/standalone.py @@ -0,0 +1,200 @@ +"""Standalone Authenticator.""" +import collections +import functools +import logging +import random +import socket +import threading + +from six.moves import BaseHTTPServer # pylint: disable=import-error + +import OpenSSL +import zope.interface + +from acme import challenges +from acme import crypto_util as acme_crypto_util +from acme import standalone as acme_standalone + +from letsencrypt import achallenges +from letsencrypt import errors +from letsencrypt import interfaces + +from letsencrypt.plugins import common +from letsencrypt.plugins import util + +logger = logging.getLogger(__name__) + + +class ServerManager(object): + """Standalone servers manager.""" + + def __init__(self, certs, simple_http_resources): + self.servers = {} + self.certs = certs + self.simple_http_resources = simple_http_resources + + def run(self, port, tls): + """Run ACME server on specified ``port``.""" + if port in self.servers: + return self.servers[port] + + logger.debug("Starting new server at %s (tls=%s)", port, tls) + handler = acme_standalone.ACMERequestHandler.partial_init( + self.simple_http_resources) + + if tls: + cls = functools.partial( + acme_standalone.HTTPSServer, certs=self.certs) + else: + cls = BaseHTTPServer.HTTPServer + + try: + server = cls(('', port), handler) + except socket.error as error: + errors.StandaloneBindError(error, port) + + stop = threading.Event() + thread = threading.Thread( + target=self._serve, + args=(server, stop), + ) + thread.start() + self.servers[port] = (server, thread, stop) + return self.servers[port] + + def _serve(self, server, stop): + while not stop.is_set(): + server.handle_request() + + def stop(self, port): + """Stop ACME server running on the specified ``port``.""" + server, thread, stop = self.servers[port] + stop.set() + + # dummy request to terminate last handle_request() + sock = socket.socket() + try: + sock.connect(server.socket.getsockname()) + except socket.error: + pass # thread is probably already finished + finally: + sock.close() + + thread.join() + del self.servers[port] + + def items(self): + """Return a list of all port, server tuples.""" + return self.servers.items() + + +class Authenticator(common.Plugin): + """Standalone Authenticator. + + This authenticator creates its own ephemeral TCP listener on the + necessary port in order to respond to incoming DVSNI and SimpleHTTP + challenges from the certificate authority. Therefore, it does not + rely on any existing server program. + + """ + zope.interface.implements(interfaces.IAuthenticator) + zope.interface.classProvides(interfaces.IPluginFactory) + + description = "Standalone Authenticator" + supported_challenges = set([challenges.SimpleHTTP, challenges.DVSNI]) + + def __init__(self, *args, **kwargs): + super(Authenticator, self).__init__(*args, **kwargs) + + # one self-signed key for all DVSNI and SimpleHTTP certificates + self.key = OpenSSL.crypto.PKey() + self.key.generate_key(OpenSSL.crypto.TYPE_RSA, bits=2048) + # TODO: generate only when the first SimpleHTTP challenge is solved + self.simple_http_cert = acme_crypto_util.gen_ss_cert( + self.key, domains=["temp server"]) + + self.responses = {} + self.servers = {} + self.served = collections.defaultdict(set) + + # Stuff below is shared across threads (i.e. servers read + # values, main thread writes). Due to the nature of Cython's + # GIL, the operations are safe, c.f. + # https://docs.python.org/2/faq/library.html#what-kinds-of-global-value-mutation-are-thread-safe + self.certs = {} + self.simple_http_resources = set() + + self.servers = ServerManager(self.certs, self.simple_http_resources) + + def more_info(self): # pylint: disable=missing-docstring + return self.__doc__ + + def prepare(self): # pylint: disable=missing-docstring + if any(util.already_listening(port) for port in + (self.config.dvsni_port, self.config.simple_http_port)): + raise errors.MisconfigurationError( + "One of the (possibly) required ports is already taken taken.") + + # TODO: add --chall-pref flag + def get_chall_pref(self, domain): + # pylint: disable=unused-argument,missing-docstring + chall_pref = list(self.supported_challenges) + random.shuffle(chall_pref) # 50% for each challenge + return chall_pref + + def perform(self, achalls): # pylint: disable=missing-docstring + try: + return self.perform2(achalls) + except errors.StandaloneBindError as error: + display = zope.component.getUtility(interfaces.IDisplay) + + if error.socket_error.errno == socket.errno.EACCES: + display.notification( + "Could not bind TCP port {0} because you don't have " + "the appropriate permissions (for example, you " + "aren't running this program as " + "root).".format(error.port)) + elif error.socket_error.errno == socket.errno.EADDRINUSE: + display.notification( + "Could not bind TCP port {0} because it is already in " + "use by another process on this system (such as a web " + "server). Please stop the program in question and then " + "try again.".format(error.port)) + else: + raise # XXX: How to handle unknown errors in binding? + + def perform2(self, achalls): + """Perform achallenges without IDisplay interaction.""" + responses = [] + tls = not self.config.no_simple_http_tls + + for achall in achalls: + if isinstance(achall, achallenges.SimpleHTTP): + server, _, _ = self.servers.run(self.config.simple_http_port, tls=tls) + response, validation = achall.gen_response_and_validation(tls=tls) + self.simple_http_resources.add( + acme_standalone.SimpleHTTPRequestHandler.SimpleHTTPResource( + chall=achall.chall, response=response, + validation=validation)) + cert = self.simple_http_cert + domain = achall.domain + else: # DVSNI + server, _, _ = self.servers.run(self.config.dvsni_port, tls=True) + response, cert, _ = achall.gen_cert_and_response(self.key) + domain = response.z_domain + self.certs[domain] = (self.key, cert) + self.responses[achall] = response + self.served[server].add(achall) + responses.append(response) + + return responses + + def cleanup(self, achalls): # pylint: disable=missing-docstring + # reduce self.served and close servers if none challenges are served + for server, server_achalls in self.served.items(): + for achall in achalls: + if achall in server_achalls: + server_achalls.remove(achall) + for port, (server, _, _) in self.servers.items(): + if not self.served[server]: + self.servers.stop(port) diff --git a/letsencrypt/plugins/standalone/__init__.py b/letsencrypt/plugins/standalone/__init__.py deleted file mode 100644 index 972c484edd4..00000000000 --- a/letsencrypt/plugins/standalone/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Let's Encrypt Standalone Authenticator plugin.""" diff --git a/letsencrypt/plugins/standalone/authenticator.py b/letsencrypt/plugins/standalone/authenticator.py deleted file mode 100644 index 996f41cdcb3..00000000000 --- a/letsencrypt/plugins/standalone/authenticator.py +++ /dev/null @@ -1,394 +0,0 @@ -"""Standalone authenticator.""" -import os -import signal -import socket -import sys -import time - -import OpenSSL -import zope.component -import zope.interface - -from acme import challenges - -from letsencrypt import achallenges -from letsencrypt import crypto_util -from letsencrypt import interfaces - -from letsencrypt.plugins import common - - -class StandaloneAuthenticator(common.Plugin): - # pylint: disable=too-many-instance-attributes - """Standalone authenticator. - - This authenticator creates its own ephemeral TCP listener on the - specified port in order to respond to incoming DVSNI challenges from - the certificate authority. Therefore, it does not rely on any - existing server program. - - :param OpenSSL.crypto.PKey private_key: DVSNI challenge certificate - key. - :param sni_names: Mapping from z_domain (`bytes`) to PEM-encoded - certificate (`bytes`). - - """ - zope.interface.implements(interfaces.IAuthenticator) - zope.interface.classProvides(interfaces.IPluginFactory) - - description = "Standalone Authenticator" - - def __init__(self, *args, **kwargs): - super(StandaloneAuthenticator, self).__init__(*args, **kwargs) - self.child_pid = None - self.parent_pid = os.getpid() - self.subproc_state = None - self.tasks = {} - self.sni_names = {} - self.sock = None - self.connection = None - self.key_pem = crypto_util.make_key(bits=2048) - self.private_key = OpenSSL.crypto.load_privatekey( - OpenSSL.crypto.FILETYPE_PEM, self.key_pem) - self.ssl_conn = None - - def prepare(self): - """There is nothing left to setup. - - .. todo:: This should probably do the port check - - """ - - def client_signal_handler(self, sig, unused_frame): - """Signal handler for the parent process. - - This handler receives inter-process communication from the - child process in the form of Unix signals. - - :param int sig: Which signal the process received. - - """ - # subprocess to client READY: SIGIO - # subprocess to client INUSE: SIGUSR1 - # subprocess to client CANTBIND: SIGUSR2 - if sig == signal.SIGIO: - self.subproc_state = "ready" - elif sig == signal.SIGUSR1: - self.subproc_state = "inuse" - elif sig == signal.SIGUSR2: - self.subproc_state = "cantbind" - else: - # NOTREACHED - raise ValueError("Unexpected signal in signal handler") - - def subproc_signal_handler(self, sig, unused_frame): - """Signal handler for the child process. - - This handler receives inter-process communication from the parent - process in the form of Unix signals. - - :param int sig: Which signal the process received. - - """ - # client to subprocess CLEANUP : SIGINT - if sig == signal.SIGINT: - try: - self.ssl_conn.shutdown() - self.ssl_conn.close() - except BaseException: - # There might not even be any currently active SSL connection. - pass - try: - self.connection.close() - except BaseException: - # There might not even be any currently active connection. - pass - try: - self.sock.close() - except BaseException: - # Various things can go wrong in the course of closing these - # connections, but none of them can clearly be usefully - # reported here and none of them should impede us from - # exiting as gracefully as possible. - pass - - os.kill(self.parent_pid, signal.SIGUSR1) - sys.exit(0) - - def sni_callback(self, connection): - """Used internally to respond to incoming SNI names. - - This method will set a new OpenSSL context object for this - connection when an incoming connection provides an SNI name - (in order to serve the appropriate certificate, if any). - - :param connection: The TLS connection object on which the SNI - extension was received. - :type connection: :class:`OpenSSL.Connection` - - """ - sni_name = connection.get_servername() - if sni_name in self.sni_names: - pem_cert = self.sni_names[sni_name] - else: - # TODO: Should we really present a certificate if we get an - # unexpected SNI name? Or should we just disconnect? - pem_cert = next(self.sni_names.itervalues()) - cert = OpenSSL.crypto.load_certificate( - OpenSSL.crypto.FILETYPE_PEM, pem_cert) - new_ctx = OpenSSL.SSL.Context(OpenSSL.SSL.TLSv1_METHOD) - new_ctx.set_verify(OpenSSL.SSL.VERIFY_NONE, lambda: False) - new_ctx.use_certificate(cert) - new_ctx.use_privatekey(self.private_key) - connection.set_context(new_ctx) - - def do_parent_process(self, port, delay_amount=5): - """Perform the parent process side of the TCP listener task. - - This should only be called by :meth:`start_listener`. We will - wait up to delay_amount seconds to hear from the child process - via a signal. - - :param int port: Which TCP port to bind. - :param float delay_amount: How long in seconds to wait for the - subprocess to notify us whether it succeeded. - - :returns: ``True`` or ``False`` according to whether we were notified - that the child process succeeded or failed in binding the port. - :rtype: bool - - """ - display = zope.component.getUtility(interfaces.IDisplay) - - start_time = time.time() - while time.time() < start_time + delay_amount: - if self.subproc_state == "ready": - return True - elif self.subproc_state == "inuse": - display.notification( - "Could not bind TCP port {0} because it is already in " - "use by another process on this system (such as a web " - "server). Please stop the program in question and then " - "try again.".format(port)) - return False - elif self.subproc_state == "cantbind": - display.notification( - "Could not bind TCP port {0} because you don't have " - "the appropriate permissions (for example, you " - "aren't running this program as " - "root).".format(port)) - return False - time.sleep(0.1) - - display.notification( - "Subprocess unexpectedly timed out while trying to bind TCP " - "port {0}.".format(port)) - - return False - - def do_child_process(self, port): - """Perform the child process side of the TCP listener task. - - This should only be called by :meth:`start_listener`. - - Normally does not return; instead, the child process exits from - within this function or from within the child process signal - handler. - - :param int port: Which TCP port to bind. - - """ - signal.signal(signal.SIGINT, self.subproc_signal_handler) - self.sock = socket.socket() - # SO_REUSEADDR flag tells the kernel to reuse a local socket - # in TIME_WAIT state, without waiting for its natural timeout - # to expire. - self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - try: - self.sock.bind(("0.0.0.0", port)) - except socket.error, error: - if error.errno == socket.errno.EACCES: - # Signal permissions denied to bind TCP port - os.kill(self.parent_pid, signal.SIGUSR2) - elif error.errno == socket.errno.EADDRINUSE: - # Signal TCP port is already in use - os.kill(self.parent_pid, signal.SIGUSR1) - else: - # XXX: How to handle unknown errors in binding? - raise error - sys.exit(1) - # XXX: We could use poll mechanism to handle simultaneous - # XXX: rather than sequential inbound TCP connections here - self.sock.listen(1) - # Signal that we've successfully bound TCP port - os.kill(self.parent_pid, signal.SIGIO) - - while True: - self.connection, _ = self.sock.accept() - - # The code below uses the PyOpenSSL bindings to respond to - # the client. This may expose us to bugs and vulnerabilities - # in OpenSSL (and creates additional dependencies). - ctx = OpenSSL.SSL.Context(OpenSSL.SSL.TLSv1_METHOD) - ctx.set_verify(OpenSSL.SSL.VERIFY_NONE, lambda: False) - pem_cert = self.tasks.values()[0] - first_cert = OpenSSL.crypto.load_certificate( - OpenSSL.crypto.FILETYPE_PEM, pem_cert) - ctx.use_certificate(first_cert) - ctx.use_privatekey(self.private_key) - ctx.set_cipher_list("HIGH") - ctx.set_tlsext_servername_callback(self.sni_callback) - self.ssl_conn = OpenSSL.SSL.Connection(ctx, self.connection) - self.ssl_conn.set_accept_state() - self.ssl_conn.do_handshake() - self.ssl_conn.shutdown() - self.ssl_conn.close() - - def start_listener(self, port): - """Start listener. - - Create a child process which will start a TCP listener on the - specified port to perform the specified DVSNI challenges. - - :param int port: The TCP port to bind. - - :returns: ``True`` or ``False`` to indicate success or failure creating - the subprocess. - :rtype: bool - - """ - # In order to avoid a race condition, we set the signal handler - # that will be needed by the parent process now, and undo this - # action if we turn out to be the child process. (This needs - # to happen before the fork because the child will send one of - # these signals to the parent almost immediately after the - # fork, and the parent must already be ready to receive it.) - signal.signal(signal.SIGIO, self.client_signal_handler) - signal.signal(signal.SIGUSR1, self.client_signal_handler) - signal.signal(signal.SIGUSR2, self.client_signal_handler) - - sys.stdout.flush() - fork_result = os.fork() - if fork_result: - # PARENT process (still the Let's Encrypt client process) - self.child_pid = fork_result - # do_parent_process() can return True or False to indicate - # reported success or failure creating the listener. - return self.do_parent_process(port) - else: - # CHILD process (the TCP listener subprocess) - # Undo the parent's signal handler settings, which aren't - # applicable to us. - signal.signal(signal.SIGIO, signal.SIG_DFL) - signal.signal(signal.SIGUSR1, signal.SIG_DFL) - signal.signal(signal.SIGUSR2, signal.SIG_DFL) - - self.child_pid = os.getpid() - # do_child_process() is normally not expected to return but - # should terminate via sys.exit(). - return self.do_child_process(port) - - # IAuthenticator method implementations follow - - def get_chall_pref(self, unused_domain): # pylint: disable=no-self-use - """Get challenge preferences. - - IAuthenticator interface method get_chall_pref. - Return a list of challenge types that this authenticator - can perform for this domain. In the case of the - StandaloneAuthenticator, the only challenge type that can ever - be performed is dvsni. - - :returns: A list containing only 'dvsni'. - - """ - return [challenges.DVSNI] - - def perform(self, achalls): - """Perform the challenge. - - .. warning:: - For the StandaloneAuthenticator, because there is no convenient - way to add additional requests, this should only be invoked - once; subsequent invocations are an error. To perform - validations for multiple independent sets of domains, a separate - StandaloneAuthenticator should be instantiated. - - """ - if self.child_pid or self.tasks: - # We should not be willing to continue with perform - # if there were existing pending challenges. - raise ValueError(".perform() was called with pending tasks!") - results_if_success = [] - results_if_failure = [] - if not achalls or not isinstance(achalls, list): - raise ValueError(".perform() was called without challenge list") - # TODO: "bits" should be user-configurable - for achall in achalls: - if isinstance(achall, achallenges.DVSNI): - # We will attempt to do it - response, cert_pem, _ = achall.gen_cert_and_response( - key_pem=self.key_pem) - self.sni_names[response.z_domain] = cert_pem - self.tasks[achall.token] = cert_pem - results_if_success.append(response) - results_if_failure.append(None) - else: - # We will not attempt to do this challenge because it - # is not a type we can handle - results_if_success.append(False) - results_if_failure.append(False) - if not self.tasks: - raise ValueError("nothing for .perform() to do") - - if util.already_listening(self.config.dvsni_port): - # If we know a process is already listening on this port, - # tell the user, and don't even attempt to bind it. (This - # test is Linux-specific and won't indicate that the port - # is bound if invoked on a different operating system.) - return results_if_failure - # Try to do the authentication; note that this creates - # the listener subprocess via os.fork() - if self.start_listener(self.config.dvsni_port): - return results_if_success - else: - # TODO: This should probably raise a DVAuthError exception - # rather than returning a list of None objects. - return results_if_failure - - def cleanup(self, achalls): - """Clean up. - - If some challenges are removed from the list, the authenticator - socket will still respond to those challenges. Once all - challenges have been removed from the list, the listener is - deactivated and stops listening. - - """ - # Remove this from pending tasks list - for achall in achalls: - assert isinstance(achall, achallenges.DVSNI) - if achall.token in self.tasks: - del self.tasks[achall.token] - else: - # Could not find the challenge to remove! - raise ValueError("could not find the challenge to remove") - if self.child_pid and not self.tasks: - # There are no remaining challenges, so - # try to shutdown self.child_pid cleanly. - # TODO: ignore any signals from child during this process - os.kill(self.child_pid, signal.SIGINT) - time.sleep(1) - # TODO: restore original signal handlers in parent process - # by resetting their actions to SIG_DFL - # print "TCP listener subprocess has been told to shut down" - - def more_info(self): # pylint: disable=no-self-use - """Human-readable string that describes the Authenticator.""" - return ("The Standalone Authenticator uses PyOpenSSL to listen " - "on port {port} and perform DVSNI challenges. Once a " - "certificate is attained, it will be saved in the " - "(TODO) current working directory.{linesep}{linesep}" - "TCP port {port} must be available in order to use the " - "Standalone Authenticator.".format( - linesep=os.linesep, port=self.config.dvsni_port)) diff --git a/letsencrypt/plugins/standalone/tests/__init__.py b/letsencrypt/plugins/standalone/tests/__init__.py deleted file mode 100644 index 059cd2780d8..00000000000 --- a/letsencrypt/plugins/standalone/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Let's Encrypt Standalone Tests""" diff --git a/letsencrypt/plugins/standalone/tests/authenticator_test.py b/letsencrypt/plugins/standalone/tests/authenticator_test.py deleted file mode 100644 index 955426533e3..00000000000 --- a/letsencrypt/plugins/standalone/tests/authenticator_test.py +++ /dev/null @@ -1,483 +0,0 @@ -"""Tests for letsencrypt.plugins.standalone.authenticator.""" -import os -import psutil -import signal -import socket -import unittest - -import mock -import OpenSSL - -from acme import challenges -from acme import jose - -from letsencrypt import achallenges - -from letsencrypt.tests import acme_util -from letsencrypt.tests import test_util - - -ACCOUNT_KEY = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem")) -CHALL_KEY_PEM = test_util.load_vector("rsa512_key_2.pem") -CHALL_KEY = OpenSSL.crypto.load_privatekey( - OpenSSL.crypto.FILETYPE_PEM, CHALL_KEY_PEM) -CONFIG = mock.Mock(dvsni_port=5001) - - -# Classes based on to allow interrupting infinite loop under test -# after one iteration, based on. -# http://igorsobreira.com/2013/03/17/testing-infinite-loops.html - -class _SocketAcceptOnlyNTimes(object): - # pylint: disable=too-few-public-methods - """ - Callable that will raise `CallableExhausted` - exception after `limit` calls, modified to also return - a tuple simulating the return values of a socket.accept() - call - """ - def __init__(self, limit): - self.limit = limit - self.calls = 0 - - def __call__(self): - self.calls += 1 - if self.calls > self.limit: - raise CallableExhausted - # Modified here for a single use as socket.accept() - return (mock.MagicMock(), "ignored") - - -class CallableExhausted(Exception): - # pylint: disable=too-few-public-methods - """Exception raised when a method is called more than the - specified number of times.""" - - -class ChallPrefTest(unittest.TestCase): - """Tests for chall_pref() method.""" - def setUp(self): - from letsencrypt.plugins.standalone.authenticator import \ - StandaloneAuthenticator - self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None) - - def test_chall_pref(self): - self.assertEqual(self.authenticator.get_chall_pref("example.com"), - [challenges.DVSNI]) - - -class SNICallbackTest(unittest.TestCase): - """Tests for sni_callback() method.""" - def setUp(self): - from letsencrypt.plugins.standalone.authenticator import \ - StandaloneAuthenticator - self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None) - self.cert = achallenges.DVSNI( - challb=acme_util.DVSNI_P, - domain="example.com", - account_key=ACCOUNT_KEY - ).gen_cert_and_response(key_pem=CHALL_KEY_PEM)[1] - self.authenticator.private_key = CHALL_KEY - self.authenticator.sni_names = {"abcdef.acme.invalid": self.cert} - self.authenticator.child_pid = 12345 - - def test_real_servername(self): - connection = mock.MagicMock() - connection.get_servername.return_value = "abcdef.acme.invalid" - self.authenticator.sni_callback(connection) - self.assertEqual(connection.set_context.call_count, 1) - called_ctx = connection.set_context.call_args[0][0] - self.assertTrue(isinstance(called_ctx, OpenSSL.SSL.Context)) - - def test_fake_servername(self): - """Test behavior of SNI callback when an unexpected name is received. - - (Currently the expected behavior in this case is to return the - "first" certificate with which the listener was configured, - although they are stored in an unordered data structure so - this might not be the one that was first in the challenge list - passed to the perform method. In the future, this might result - in dropping the connection instead.)""" - connection = mock.MagicMock() - connection.get_servername.return_value = "example.com" - self.authenticator.sni_callback(connection) - self.assertEqual(connection.set_context.call_count, 1) - called_ctx = connection.set_context.call_args[0][0] - self.assertTrue(isinstance(called_ctx, OpenSSL.SSL.Context)) - - -class ClientSignalHandlerTest(unittest.TestCase): - """Tests for client_signal_handler() method.""" - def setUp(self): - from letsencrypt.plugins.standalone.authenticator import \ - StandaloneAuthenticator - self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None) - self.authenticator.tasks = {"footoken.acme.invalid": "stuff"} - self.authenticator.child_pid = 12345 - - def test_client_signal_handler(self): - self.assertTrue(self.authenticator.subproc_state is None) - self.authenticator.client_signal_handler(signal.SIGIO, None) - self.assertEqual(self.authenticator.subproc_state, "ready") - - self.authenticator.client_signal_handler(signal.SIGUSR1, None) - self.assertEqual(self.authenticator.subproc_state, "inuse") - - self.authenticator.client_signal_handler(signal.SIGUSR2, None) - self.assertEqual(self.authenticator.subproc_state, "cantbind") - - # Testing the unreached path for a signal other than these - # specified (which can't occur in normal use because this - # function is only set as a signal handler for the above three - # signals). - self.assertRaises( - ValueError, self.authenticator.client_signal_handler, - signal.SIGPIPE, None) - - -class SubprocSignalHandlerTest(unittest.TestCase): - """Tests for subproc_signal_handler() method.""" - def setUp(self): - from letsencrypt.plugins.standalone.authenticator import \ - StandaloneAuthenticator - self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None) - self.authenticator.tasks = {"footoken.acme.invalid": "stuff"} - self.authenticator.child_pid = 12345 - self.authenticator.parent_pid = 23456 - - @mock.patch("letsencrypt.plugins.standalone.authenticator.os.kill") - @mock.patch("letsencrypt.plugins.standalone.authenticator.sys.exit") - def test_subproc_signal_handler(self, mock_exit, mock_kill): - self.authenticator.ssl_conn = mock.MagicMock() - self.authenticator.connection = mock.MagicMock() - self.authenticator.sock = mock.MagicMock() - self.authenticator.subproc_signal_handler(signal.SIGINT, None) - self.assertEquals(self.authenticator.ssl_conn.shutdown.call_count, 1) - self.assertEquals(self.authenticator.ssl_conn.close.call_count, 1) - self.assertEquals(self.authenticator.connection.close.call_count, 1) - self.assertEquals(self.authenticator.sock.close.call_count, 1) - mock_kill.assert_called_once_with( - self.authenticator.parent_pid, signal.SIGUSR1) - mock_exit.assert_called_once_with(0) - - @mock.patch("letsencrypt.plugins.standalone.authenticator.os.kill") - @mock.patch("letsencrypt.plugins.standalone.authenticator.sys.exit") - def test_subproc_signal_handler_trouble(self, mock_exit, mock_kill): - """Test attempting to shut down a non-existent connection. - - (This could occur because none was established or active at the - time the signal handler tried to perform the cleanup).""" - self.authenticator.ssl_conn = mock.MagicMock() - self.authenticator.connection = mock.MagicMock() - self.authenticator.sock = mock.MagicMock() - # AttributeError simulates the case where one of these properties - # is None because no connection exists. We raise it for - # ssl_conn.close() instead of ssl_conn.shutdown() for better code - # coverage. - self.authenticator.ssl_conn.close.side_effect = AttributeError("!") - self.authenticator.connection.close.side_effect = AttributeError("!") - self.authenticator.sock.close.side_effect = AttributeError("!") - self.authenticator.subproc_signal_handler(signal.SIGINT, None) - self.assertEquals(self.authenticator.ssl_conn.shutdown.call_count, 1) - self.assertEquals(self.authenticator.ssl_conn.close.call_count, 1) - self.assertEquals(self.authenticator.connection.close.call_count, 1) - self.assertEquals(self.authenticator.sock.close.call_count, 1) - mock_kill.assert_called_once_with( - self.authenticator.parent_pid, signal.SIGUSR1) - mock_exit.assert_called_once_with(0) - - -class PerformTest(unittest.TestCase): - """Tests for perform() method.""" - def setUp(self): - from letsencrypt.plugins.standalone.authenticator import \ - StandaloneAuthenticator - self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None) - - self.achall1 = achallenges.DVSNI( - challb=acme_util.chall_to_challb( - challenges.DVSNI(token=b"foo"), "pending"), - domain="foo.example.com", account_key=ACCOUNT_KEY) - self.achall2 = achallenges.DVSNI( - challb=acme_util.chall_to_challb( - challenges.DVSNI(token=b"bar"), "pending"), - domain="bar.example.com", account_key=ACCOUNT_KEY) - bad_achall = ("This", "Represents", "A Non-DVSNI", "Challenge") - self.achalls = [self.achall1, self.achall2, bad_achall] - - def test_perform_when_already_listening(self): - self.authenticator.already_listening = mock.Mock() - self.authenticator.already_listening.return_value = True - result = self.authenticator.perform([self.achall1]) - self.assertEqual(result, [None]) - - def test_can_perform(self): - """What happens if start_listener() returns True.""" - self.authenticator.start_listener = mock.Mock() - self.authenticator.start_listener.return_value = True - self.authenticator.already_listening = mock.Mock(return_value=False) - result = self.authenticator.perform(self.achalls) - self.assertEqual(len(self.authenticator.tasks), 2) - self.assertTrue(self.achall1.token in self.authenticator.tasks) - self.assertTrue(self.achall2.token in self.authenticator.tasks) - self.assertTrue(isinstance(result, list)) - self.assertEqual(len(result), 3) - self.assertTrue(isinstance(result[0], challenges.ChallengeResponse)) - self.assertTrue(isinstance(result[1], challenges.ChallengeResponse)) - self.assertFalse(result[2]) - self.authenticator.start_listener.assert_called_once_with( - CONFIG.dvsni_port) - - def test_cannot_perform(self): - """What happens if start_listener() returns False.""" - self.authenticator.start_listener = mock.Mock() - self.authenticator.start_listener.return_value = False - self.authenticator.already_listening = mock.Mock(return_value=False) - result = self.authenticator.perform(self.achalls) - self.assertEqual(len(self.authenticator.tasks), 2) - self.assertTrue(self.achall1.token in self.authenticator.tasks) - self.assertTrue(self.achall2.token in self.authenticator.tasks) - self.assertTrue(isinstance(result, list)) - self.assertEqual(len(result), 3) - self.assertEqual(result, [None, None, False]) - self.authenticator.start_listener.assert_called_once_with( - CONFIG.dvsni_port) - - def test_perform_with_pending_tasks(self): - self.authenticator.tasks = {"footoken.acme.invalid": "cert_data"} - extra_achall = acme_util.DVSNI_P - self.assertRaises( - ValueError, self.authenticator.perform, [extra_achall]) - - def test_perform_without_challenge_list(self): - extra_achall = acme_util.DVSNI_P - # This is wrong because a challenge must be specified. - self.assertRaises(ValueError, self.authenticator.perform, []) - # This is wrong because it must be a list, not a bare challenge. - self.assertRaises( - ValueError, self.authenticator.perform, extra_achall) - # This is wrong because the list must contain at least one challenge. - self.assertRaises( - ValueError, self.authenticator.perform, range(20)) - - -class StartListenerTest(unittest.TestCase): - """Tests for start_listener() method.""" - def setUp(self): - from letsencrypt.plugins.standalone.authenticator import \ - StandaloneAuthenticator - self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None) - - @mock.patch("letsencrypt.plugins.standalone.authenticator.os.fork") - def test_start_listener_fork_parent(self, mock_fork): - self.authenticator.do_parent_process = mock.Mock() - self.authenticator.do_parent_process.return_value = True - mock_fork.return_value = 22222 - result = self.authenticator.start_listener(1717) - # start_listener is expected to return the True or False return - # value from do_parent_process. - self.assertTrue(result) - self.assertEqual(self.authenticator.child_pid, 22222) - self.authenticator.do_parent_process.assert_called_once_with(1717) - - @mock.patch("letsencrypt.plugins.standalone.authenticator.os.fork") - def test_start_listener_fork_child(self, mock_fork): - self.authenticator.do_parent_process = mock.Mock() - self.authenticator.do_child_process = mock.Mock() - mock_fork.return_value = 0 - self.authenticator.start_listener(1717) - self.assertEqual(self.authenticator.child_pid, os.getpid()) - self.authenticator.do_child_process.assert_called_once_with(1717) - - -class DoParentProcessTest(unittest.TestCase): - """Tests for do_parent_process() method.""" - def setUp(self): - from letsencrypt.plugins.standalone.authenticator import \ - StandaloneAuthenticator - self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None) - - @mock.patch("letsencrypt.plugins.standalone.authenticator." - "zope.component.getUtility") - def test_do_parent_process_ok(self, mock_get_utility): - self.authenticator.subproc_state = "ready" - result = self.authenticator.do_parent_process(1717) - self.assertTrue(result) - self.assertEqual(mock_get_utility.call_count, 1) - - @mock.patch("letsencrypt.plugins.standalone.authenticator." - "zope.component.getUtility") - def test_do_parent_process_inuse(self, mock_get_utility): - self.authenticator.subproc_state = "inuse" - result = self.authenticator.do_parent_process(1717) - self.assertFalse(result) - self.assertEqual(mock_get_utility.call_count, 1) - - @mock.patch("letsencrypt.plugins.standalone.authenticator." - "zope.component.getUtility") - def test_do_parent_process_cantbind(self, mock_get_utility): - self.authenticator.subproc_state = "cantbind" - result = self.authenticator.do_parent_process(1717) - self.assertFalse(result) - self.assertEqual(mock_get_utility.call_count, 1) - - @mock.patch("letsencrypt.plugins.standalone.authenticator." - "zope.component.getUtility") - def test_do_parent_process_timeout(self, mock_get_utility): - # Normally times out in 5 seconds and returns False. We can - # now set delay_amount to a lower value so that it times out - # faster than it would under normal use. - result = self.authenticator.do_parent_process(1717, delay_amount=1) - self.assertFalse(result) - self.assertEqual(mock_get_utility.call_count, 1) - - -class DoChildProcessTest(unittest.TestCase): - """Tests for do_child_process() method.""" - def setUp(self): - from letsencrypt.plugins.standalone.authenticator import \ - StandaloneAuthenticator - self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None) - self.cert = achallenges.DVSNI( - challb=acme_util.chall_to_challb( - challenges.DVSNI(token=b"abcdef"), "pending"), - domain="example.com", account_key=ACCOUNT_KEY).gen_cert_and_response( - key_pem=CHALL_KEY_PEM)[1] - self.authenticator.private_key = CHALL_KEY - self.authenticator.tasks = {"abcdef.acme.invalid": self.cert} - self.authenticator.parent_pid = 12345 - - @mock.patch("letsencrypt.plugins.standalone.authenticator.socket.socket") - @mock.patch("letsencrypt.plugins.standalone.authenticator.os.kill") - @mock.patch("letsencrypt.plugins.standalone.authenticator.sys.exit") - def test_do_child_process_cantbind1( - self, mock_exit, mock_kill, mock_socket): - mock_exit.side_effect = IndentationError("subprocess would exit here") - eaccess = socket.error(socket.errno.EACCES, "Permission denied") - sample_socket = mock.MagicMock() - sample_socket.bind.side_effect = eaccess - mock_socket.return_value = sample_socket - # Using the IndentationError as an error that cannot easily be - # generated at runtime, to indicate the behavior of sys.exit has - # taken effect without actually causing the test process to exit. - # (Just replacing it with a no-op causes logic errors because the - # do_child_process code assumes that calling sys.exit() will - # cause subsequent code not to be executed.) - self.assertRaises( - IndentationError, self.authenticator.do_child_process, 1717) - mock_exit.assert_called_once_with(1) - mock_kill.assert_called_once_with(12345, signal.SIGUSR2) - - @mock.patch("letsencrypt.plugins.standalone.authenticator.socket.socket") - @mock.patch("letsencrypt.plugins.standalone.authenticator.os.kill") - @mock.patch("letsencrypt.plugins.standalone.authenticator.sys.exit") - def test_do_child_process_cantbind2(self, mock_exit, mock_kill, - mock_socket): - mock_exit.side_effect = IndentationError("subprocess would exit here") - eaccess = socket.error(socket.errno.EADDRINUSE, "Port already in use") - sample_socket = mock.MagicMock() - sample_socket.bind.side_effect = eaccess - mock_socket.return_value = sample_socket - self.assertRaises( - IndentationError, self.authenticator.do_child_process, 1717) - mock_exit.assert_called_once_with(1) - mock_kill.assert_called_once_with(12345, signal.SIGUSR1) - - @mock.patch("letsencrypt.plugins.standalone.authenticator." - "socket.socket") - def test_do_child_process_cantbind3(self, mock_socket): - """Test case where attempt to bind socket results in an unhandled - socket error. (The expected behavior is arguably wrong because it - will crash the program; the reason for the expected behavior is - that we don't have a way to report arbitrary socket errors.)""" - eio = socket.error(socket.errno.EIO, "Imaginary unhandled error") - sample_socket = mock.MagicMock() - sample_socket.bind.side_effect = eio - mock_socket.return_value = sample_socket - self.assertRaises( - socket.error, self.authenticator.do_child_process, 1717) - - @mock.patch("letsencrypt.plugins.standalone.authenticator." - "OpenSSL.SSL.Connection") - @mock.patch("letsencrypt.plugins.standalone.authenticator.socket.socket") - @mock.patch("letsencrypt.plugins.standalone.authenticator.os.kill") - def test_do_child_process_success( - self, mock_kill, mock_socket, mock_connection): - sample_socket = mock.MagicMock() - sample_socket.accept.side_effect = _SocketAcceptOnlyNTimes(2) - mock_socket.return_value = sample_socket - mock_connection.return_value = mock.MagicMock() - self.assertRaises( - CallableExhausted, self.authenticator.do_child_process, 1717) - mock_socket.assert_called_once_with() - sample_socket.bind.assert_called_once_with(("0.0.0.0", 1717)) - sample_socket.listen.assert_called_once_with(1) - self.assertEqual(sample_socket.accept.call_count, 3) - mock_kill.assert_called_once_with(12345, signal.SIGIO) - # TODO: We could have some tests about the fact that the listener - # asks OpenSSL to negotiate a TLS connection (and correctly - # sets the SNI callback function). - - -class CleanupTest(unittest.TestCase): - """Tests for cleanup() method.""" - def setUp(self): - from letsencrypt.plugins.standalone.authenticator import \ - StandaloneAuthenticator - self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None) - self.achall = achallenges.DVSNI( - challb=acme_util.chall_to_challb( - challenges.DVSNI(token=b"footoken"), "pending"), - domain="foo.example.com", account_key="key") - self.authenticator.tasks = {self.achall.token: "stuff"} - self.authenticator.child_pid = 12345 - - @mock.patch("letsencrypt.plugins.standalone.authenticator.os.kill") - @mock.patch("letsencrypt.plugins.standalone.authenticator.time.sleep") - def test_cleanup(self, mock_sleep, mock_kill): - mock_sleep.return_value = None - mock_kill.return_value = None - - self.authenticator.cleanup([self.achall]) - - mock_kill.assert_called_once_with(12345, signal.SIGINT) - mock_sleep.assert_called_once_with(1) - - def test_bad_cleanup(self): - self.assertRaises( - ValueError, self.authenticator.cleanup, [achallenges.DVSNI( - challb=acme_util.chall_to_challb( - challenges.DVSNI(token=b"badtoken"), "pending"), - domain="bad.example.com", account_key="key")]) - - -class MoreInfoTest(unittest.TestCase): - """Tests for more_info() method. (trivially)""" - def setUp(self): - from letsencrypt.plugins.standalone.authenticator import ( - StandaloneAuthenticator) - self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None) - - def test_more_info(self): - """Make sure exceptions aren't raised.""" - self.authenticator.more_info() - - -class InitTest(unittest.TestCase): - """Tests for more_info() method. (trivially)""" - def setUp(self): - from letsencrypt.plugins.standalone.authenticator import ( - StandaloneAuthenticator) - self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None) - - def test_prepare(self): - """Make sure exceptions aren't raised. - - .. todo:: Add on more once things are setup appropriately. - - """ - self.authenticator.prepare() - - -if __name__ == "__main__": - unittest.main() # pragma: no cover diff --git a/letsencrypt/tests/achallenges_test.py b/letsencrypt/tests/achallenges_test.py index 1cd2f167792..66b1a7ca744 100644 --- a/letsencrypt/tests/achallenges_test.py +++ b/letsencrypt/tests/achallenges_test.py @@ -1,6 +1,8 @@ """Tests for letsencrypt.achallenges.""" import unittest +import OpenSSL + from acme import challenges from acme import jose @@ -22,10 +24,10 @@ def test_proxy(self): self.assertEqual(self.challb.token, self.achall.token) def test_gen_cert_and_response(self): - response, cert_pem, key_pem = self.achall.gen_cert_and_response() + response, cert, key = self.achall.gen_cert_and_response() self.assertTrue(isinstance(response, challenges.DVSNIResponse)) - self.assertTrue(isinstance(cert_pem, bytes)) - self.assertTrue(isinstance(key_pem, bytes)) + self.assertTrue(isinstance(cert, OpenSSL.crypto.X509)) + self.assertTrue(isinstance(key, OpenSSL.crypto.PKey)) if __name__ == "__main__": diff --git a/setup.py b/setup.py index 6e1640e3e2f..5853c8ac097 100644 --- a/setup.py +++ b/setup.py @@ -42,6 +42,7 @@ def read_file(filename, encoding='utf8'): 'python2-pythondialog>=3.2.2rc1', # Debian squeeze support, cf. #280 'pytz', 'requests', + 'six', 'zope.component', 'zope.interface', ] @@ -119,8 +120,7 @@ def read_file(filename, encoding='utf8'): 'manual = letsencrypt.plugins.manual:ManualAuthenticator', # TODO: null should probably not be presented to the user 'null = letsencrypt.plugins.null:Installer', - 'standalone = letsencrypt.plugins.standalone.authenticator' - ':StandaloneAuthenticator', + 'standalone = letsencrypt.plugins.standalone:Authenticator', ], }, From a4d0188d215e6394edce152b750105e284d911e2 Mon Sep 17 00:00:00 2001 From: Brandon Kreisel Date: Sat, 3 Oct 2015 12:50:18 -0400 Subject: [PATCH 012/113] Add Mac compatibility to integration tests --- letsencrypt/plugins/manual.py | 1 + letsencrypt/plugins/standalone/authenticator.py | 11 +++++++---- tests/boulder-integration.sh | 6 +++++- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/letsencrypt/plugins/manual.py b/letsencrypt/plugins/manual.py index 3f7276725c5..f532b8fbc77 100644 --- a/letsencrypt/plugins/manual.py +++ b/letsencrypt/plugins/manual.py @@ -159,6 +159,7 @@ def _perform_single(self, achall): # don't care about setting stdout and stderr, # we're in test mode anyway shell=True, + executable="/bin/bash", # "preexec_fn" is UNIX specific, but so is "command" preexec_fn=os.setsid) except OSError as error: # ValueError should not happen! diff --git a/letsencrypt/plugins/standalone/authenticator.py b/letsencrypt/plugins/standalone/authenticator.py index 968063781e9..f7c24f5e511 100644 --- a/letsencrypt/plugins/standalone/authenticator.py +++ b/letsencrypt/plugins/standalone/authenticator.py @@ -301,11 +301,14 @@ def already_listening(self, port): # pylint: disable=no-self-use :param int port: The TCP port in question. :returns: True or False.""" - listeners = [conn.pid for conn in psutil.net_connections() - if conn.status == 'LISTEN' and - conn.type == socket.SOCK_STREAM and - conn.laddr[1] == port] try: + + # net_connections() can raise AccessDenied on certain OSs + listeners = [conn.pid for conn in psutil.net_connections() + if conn.status == 'LISTEN' and + conn.type == socket.SOCK_STREAM and + conn.laddr[1] == port] + if listeners and listeners[0] is not None: # conn.pid may be None if the current process doesn't have # permission to identify the listening process! Additionally, diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index ed877d136f0..efd16ef6cde 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -50,7 +50,11 @@ dir="$root/conf/archive/le1.wtf" for x in cert chain fullchain privkey; do latest="$(ls -1t $dir/ | grep -e "^${x}" | head -n1)" - live="$(readlink -f "$root/conf/live/le1.wtf/${x}.pem")" + if [ `uname` == 'Darwin' ]; then + live="$(greadlink -f "$root/conf/live/le1.wtf/${x}.pem")" + else + live="$(readlink -f "$root/conf/live/le1.wtf/${x}.pem")" + fi [ "${dir}/${latest}" = "$live" ] # renewer fails this test done From d88455a1b9529545318d113e82e6ab31fc7c72d7 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 09:11:37 +0000 Subject: [PATCH 013/113] Rename simplefs to webroot --- docs/api/plugins/simplefs.rst | 5 ----- docs/api/plugins/webroot.rst | 5 +++++ letsencrypt/plugins/{simplefs.py => webroot.py} | 6 +++--- .../plugins/{simplefs_test.py => webroot_test.py} | 14 +++++++------- setup.py | 2 +- 5 files changed, 16 insertions(+), 16 deletions(-) delete mode 100644 docs/api/plugins/simplefs.rst create mode 100644 docs/api/plugins/webroot.rst rename letsencrypt/plugins/{simplefs.py => webroot.py} (96%) rename letsencrypt/plugins/{simplefs_test.py => webroot_test.py} (85%) diff --git a/docs/api/plugins/simplefs.rst b/docs/api/plugins/simplefs.rst deleted file mode 100644 index 7165b6aca81..00000000000 --- a/docs/api/plugins/simplefs.rst +++ /dev/null @@ -1,5 +0,0 @@ -:mod:`letsencrypt.plugins.simplefs` ------------------------------------ - -.. automodule:: letsencrypt.plugins.simplefs - :members: diff --git a/docs/api/plugins/webroot.rst b/docs/api/plugins/webroot.rst new file mode 100644 index 00000000000..339d546a5f6 --- /dev/null +++ b/docs/api/plugins/webroot.rst @@ -0,0 +1,5 @@ +:mod:`letsencrypt.plugins.webroot` +---------------------------------- + +.. automodule:: letsencrypt.plugins.webroot + :members: diff --git a/letsencrypt/plugins/simplefs.py b/letsencrypt/plugins/webroot.py similarity index 96% rename from letsencrypt/plugins/simplefs.py rename to letsencrypt/plugins/webroot.py index ad83c13d7f8..d641855aea0 100644 --- a/letsencrypt/plugins/simplefs.py +++ b/letsencrypt/plugins/webroot.py @@ -1,4 +1,4 @@ -"""SimpleFS plugin.""" +"""Webroot plugin.""" import errno import logging import os @@ -16,11 +16,11 @@ class Authenticator(common.Plugin): - """SimpleFS Authenticator.""" + """Webroot Authenticator.""" zope.interface.implements(interfaces.IAuthenticator) zope.interface.classProvides(interfaces.IPluginFactory) - description = "SimpleFS Authenticator" + description = "Webroot Authenticator" MORE_INFO = """\ Authenticator plugin that performs SimpleHTTP challenge by saving diff --git a/letsencrypt/plugins/simplefs_test.py b/letsencrypt/plugins/webroot_test.py similarity index 85% rename from letsencrypt/plugins/simplefs_test.py rename to letsencrypt/plugins/webroot_test.py index f80e8b29a62..abd12e1527a 100644 --- a/letsencrypt/plugins/simplefs_test.py +++ b/letsencrypt/plugins/webroot_test.py @@ -1,4 +1,4 @@ -"""Tests for letsencrypt.plugins.simplefs.""" +"""Tests for letsencrypt.plugins.webroot.""" import os import shutil import tempfile @@ -19,19 +19,19 @@ class AuthenticatorTest(unittest.TestCase): - """Tests for letsencrypt.plugins.simplefs.Authenticator.""" + """Tests for letsencrypt.plugins.webroot.Authenticator.""" achall = achallenges.SimpleHTTP( challb=acme_util.SIMPLE_HTTP_P, domain=None, account_key=KEY) def setUp(self): - from letsencrypt.plugins.simplefs import Authenticator + from letsencrypt.plugins.webroot import Authenticator self.root = tempfile.mkdtemp() self.validation_path = os.path.join( self.root, ".well-known", "acme-challenge", "ZXZhR3hmQURzNnBTUmIyTEF2OUlaZjE3RHQzanV4R0orUEN0OTJ3citvQQ") - self.config = mock.MagicMock(simplefs_root=self.root) - self.auth = Authenticator(self.config, "simplefs") + self.config = mock.MagicMock(webroot_root=self.root) + self.auth = Authenticator(self.config, "webroot") self.auth.prepare() def tearDown(self): @@ -48,11 +48,11 @@ def test_add_parser_arguments(self): self.assertEqual(1, add.call_count) def test_prepare_bad_root(self): - self.config.simplefs_root = os.path.join(self.root, "null") + self.config.webroot_root = os.path.join(self.root, "null") self.assertRaises(errors.PluginError, self.auth.prepare) def test_prepare_missing_root(self): - self.config.simplefs_root = None + self.config.webroot_root = None self.assertRaises(errors.PluginError, self.auth.prepare) def test_prepare_full_root_exists(self): diff --git a/setup.py b/setup.py index a4897aadbf6..92dd39d46c7 100644 --- a/setup.py +++ b/setup.py @@ -119,7 +119,7 @@ def read_file(filename, encoding='utf8'): 'letsencrypt.plugins': [ 'manual = letsencrypt.plugins.manual:Authenticator', 'null = letsencrypt.plugins.null:Installer', - 'simplefs = letsencrypt.plugins.simplefs:Authenticator', + 'webroot = letsencrypt.plugins.webroot:Authenticator', 'standalone = letsencrypt.plugins.standalone.authenticator' ':StandaloneAuthenticator', ], From 63c080b05f2040d3507526efa1cbe10a01bfa565 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 09:15:17 +0000 Subject: [PATCH 014/113] --webroot-root -> --webroot-path --- letsencrypt/plugins/webroot.py | 16 ++++++++-------- letsencrypt/plugins/webroot_test.py | 20 ++++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/letsencrypt/plugins/webroot.py b/letsencrypt/plugins/webroot.py index d641855aea0..ed8991bc5a5 100644 --- a/letsencrypt/plugins/webroot.py +++ b/letsencrypt/plugins/webroot.py @@ -29,11 +29,11 @@ class Authenticator(common.Plugin): to serve all files under specified web root ({0}).""" def more_info(self): # pylint: disable=missing-docstring,no-self-use - return self.MORE_INFO.format(self.conf("root")) + return self.MORE_INFO.format(self.conf("path")) @classmethod def add_parser_arguments(cls, add): - add("root", help="public_html / webroot path") + add("path", help="public_html / webroot path") def get_chall_pref(self, domain): # pragma: no cover # pylint: disable=missing-docstring,no-self-use,unused-argument @@ -44,15 +44,15 @@ def __init__(self, *args, **kwargs): self.full_root = None def prepare(self): # pylint: disable=missing-docstring - root = self.conf("root") - if root is None: + path = self.conf("path") + if path is None: raise errors.PluginError("--{0} must be set".format( - self.option_name("root"))) - if not os.path.isdir(root): + self.option_name("path"))) + if not os.path.isdir(path): raise errors.PluginError( - root + " does not exist or is not a directory") + path + " does not exist or is not a directory") self.full_root = os.path.join( - root, challenges.SimpleHTTPResponse.URI_ROOT_PATH) + path, challenges.SimpleHTTPResponse.URI_ROOT_PATH) logger.debug("Creating root challenges validation dir at %s", self.full_root) diff --git a/letsencrypt/plugins/webroot_test.py b/letsencrypt/plugins/webroot_test.py index abd12e1527a..d8c0e2aa23c 100644 --- a/letsencrypt/plugins/webroot_test.py +++ b/letsencrypt/plugins/webroot_test.py @@ -26,21 +26,21 @@ class AuthenticatorTest(unittest.TestCase): def setUp(self): from letsencrypt.plugins.webroot import Authenticator - self.root = tempfile.mkdtemp() + self.path = tempfile.mkdtemp() self.validation_path = os.path.join( - self.root, ".well-known", "acme-challenge", + self.path, ".well-known", "acme-challenge", "ZXZhR3hmQURzNnBTUmIyTEF2OUlaZjE3RHQzanV4R0orUEN0OTJ3citvQQ") - self.config = mock.MagicMock(webroot_root=self.root) + self.config = mock.MagicMock(webroot_path=self.path) self.auth = Authenticator(self.config, "webroot") self.auth.prepare() def tearDown(self): - shutil.rmtree(self.root) + shutil.rmtree(self.path) def test_more_info(self): more_info = self.auth.more_info() self.assertTrue(isinstance(more_info, str)) - self.assertTrue(self.root in more_info) + self.assertTrue(self.path in more_info) def test_add_parser_arguments(self): add = mock.MagicMock() @@ -48,11 +48,11 @@ def test_add_parser_arguments(self): self.assertEqual(1, add.call_count) def test_prepare_bad_root(self): - self.config.webroot_root = os.path.join(self.root, "null") + self.config.webroot_path = os.path.join(self.path, "null") self.assertRaises(errors.PluginError, self.auth.prepare) def test_prepare_missing_root(self): - self.config.webroot_root = None + self.config.webroot_path = None self.assertRaises(errors.PluginError, self.auth.prepare) def test_prepare_full_root_exists(self): @@ -60,10 +60,10 @@ def test_prepare_full_root_exists(self): self.auth.prepare() # shouldn't raise any exceptions def test_prepare_reraises_other_errors(self): - self.auth.full_root = os.path.join(self.root, "null") - os.chmod(self.root, 0o000) + self.auth.full_path = os.path.join(self.path, "null") + os.chmod(self.path, 0o000) self.assertRaises(errors.PluginError, self.auth.prepare) - os.chmod(self.root, 0o700) + os.chmod(self.path, 0o700) def test_perform_cleanup(self): responses = self.auth.perform([self.achall]) From b4b7b020a28bca4618dc0be438054bc18e17c8c8 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 10:32:29 +0000 Subject: [PATCH 015/113] Add NamespaceConfigTest.test_simple_http_port --- letsencrypt/tests/configuration_test.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/letsencrypt/tests/configuration_test.py b/letsencrypt/tests/configuration_test.py index 24a84d888c8..acf9273d01e 100644 --- a/letsencrypt/tests/configuration_test.py +++ b/letsencrypt/tests/configuration_test.py @@ -12,7 +12,7 @@ def setUp(self): self.namespace = mock.MagicMock( config_dir='/tmp/config', work_dir='/tmp/foo', foo='bar', server='https://acme-server.org:443/new', - dvsni_port='1234', simple_http_port='4321') + dvsni_port='1234', simple_http_port=4321) from letsencrypt.configuration import NamespaceConfig self.config = NamespaceConfig(self.namespace) @@ -47,6 +47,16 @@ def test_dynamic_dirs(self, constants): self.assertEqual(self.config.key_dir, '/tmp/config/keys') self.assertEqual(self.config.temp_checkpoint_dir, '/tmp/foo/t') + def test_simple_http_port(self): + self.assertEqual(4321, self.config.simple_http_port) + + self.namespace.simple_http_port = None + self.namespace.no_simple_http_tls = True + self.assertEqual(80, self.config.simple_http_port) + + self.namespace.no_simple_http_tls = False + self.assertEqual(443, self.config.simple_http_port) + class RenewerConfigurationTest(unittest.TestCase): """Test for letsencrypt.configuration.RenewerConfiguration.""" From b660650a903343c72014969cf2a7d58d6ce1a3b0 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 10:37:47 +0000 Subject: [PATCH 016/113] Add StandaloneBindErrorTest --- letsencrypt/tests/errors_test.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/letsencrypt/tests/errors_test.py b/letsencrypt/tests/errors_test.py index a99d84719d2..5da7c0b7a80 100644 --- a/letsencrypt/tests/errors_test.py +++ b/letsencrypt/tests/errors_test.py @@ -1,6 +1,8 @@ """Tests for letsencrypt.errors.""" import unittest +import mock + from acme import messages from letsencrypt import achallenges @@ -22,5 +24,21 @@ def test_str(self): "Failed authorization procedure. example.com (dns): tls")) +class StandaloneBindErrorTest(unittest.TestCase): + """Tests for letsencrypt.errors.StandaloneBindError.""" + + def setUp(self): + from letsencrypt.errors import StandaloneBindError + self.error = StandaloneBindError(mock.sentinel.error, 1234) + + def test_instance_args(self): + self.assertEqual(mock.sentinel.error, self.error.socket_error) + self.assertEqual(1234, self.error.port) + + def test_str(self): + self.assertTrue(str(self.error).startswith( + "Problem binding to port 1234: ")) + + if __name__ == "__main__": unittest.main() # pragma: no cover From b0efc61f97e2a29adc1ac5c4c5439ec4cc70d77b Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 10:58:31 +0000 Subject: [PATCH 017/113] Add ServerManagerTest stub. --- letsencrypt/plugins/standalone.py | 2 +- letsencrypt/plugins/standalone_test.py | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 letsencrypt/plugins/standalone_test.py diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index 8e5f1e77dca..4444ed2c183 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -49,7 +49,7 @@ def run(self, port, tls): cls = BaseHTTPServer.HTTPServer try: - server = cls(('', port), handler) + server = cls(("", port), handler) except socket.error as error: errors.StandaloneBindError(error, port) diff --git a/letsencrypt/plugins/standalone_test.py b/letsencrypt/plugins/standalone_test.py new file mode 100644 index 00000000000..8c0dd6f9777 --- /dev/null +++ b/letsencrypt/plugins/standalone_test.py @@ -0,0 +1,21 @@ +"""Tests for letsencrypt.plugins.standalone.""" +import unittest + + +class ServerManagerTest(unittest.TestCase): + """Tests for letsencrypt.plugins.standalone.ServerManager.""" + + def setUp(self): + from letsencrypt.plugins.standalone import ServerManager + self.certs = {} + self.simple_http_resources = {} + self.mgr = ServerManager(self.certs, self.simple_http_resources) + + def test_init(self): + self.assertTrue(self.mgr.certs is self.certs) + self.assertTrue( + self.mgr.simple_http_resources is self.simple_http_resources) + + +if __name__ == "__main__": + unittest.main() # pragma: no cover From d1fcc422e0aa218d579028f0536c7214fe68d2ba Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 11:09:42 +0000 Subject: [PATCH 018/113] Use ACME(TLS)Server in plugins.standalone --- letsencrypt/plugins/standalone.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index 4444ed2c183..5f2bc9292b4 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -44,9 +44,9 @@ def run(self, port, tls): if tls: cls = functools.partial( - acme_standalone.HTTPSServer, certs=self.certs) + acme_standalone.ACMETLSServer, certs=self.certs) else: - cls = BaseHTTPServer.HTTPServer + cls = acme_standalone.ACMEServer try: server = cls(("", port), handler) From 22b1514f516b2404a7db748a7f0adbd40cf55fc4 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 11:01:20 +0000 Subject: [PATCH 019/113] server_forever2/shutdown2 --- acme/acme/standalone.py | 37 ++++++++++++++++++++++++++++++- acme/acme/standalone_test.py | 23 ++++++++++++++++++- letsencrypt/plugins/standalone.py | 32 ++++++-------------------- 3 files changed, 65 insertions(+), 27 deletions(-) diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index 8f604a4395c..501d239cbcc 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -4,6 +4,7 @@ import functools import logging import os +import socket import sys import six @@ -52,18 +53,52 @@ def server_bind(self): class ACMEServerMixin: # pylint: disable=old-style-class,no-init - """ACME server common settings mixin.""" + """ACME server common settings mixin. + + .. warning:: + Subclasses have to init ``_stopped = False`` (it's not done here, + because of old-style classes madness). + + """ server_version = "ACME standalone client" allow_reuse_address = True + def serve_forever2(self): + """Serve forever, until other thread calls `shutdown2`.""" + while not self._stopped: + self.handle_request() + + def shutdown2(self): + """Shutdown server loop from `serve_forever2`.""" + self._stopped = True + + # dummy request to terminate last server_forever2.handle_request() + sock = socket.socket() + try: + sock.connect(self.socket.getsockname()) + except socket.error: + pass # thread is probably already finished + finally: + sock.close() + + self.server_close() + class ACMETLSServer(HTTPSServer, ACMEServerMixin): """ACME TLS Server.""" + def __init__(self, *args, **kwargs): + self._stopped = False + HTTPSServer.__init__(self, *args, **kwargs) + class ACMEServer(BaseHTTPServer.HTTPServer, ACMEServerMixin): """ACME Server (non-TLS).""" + def __init__(self, *args, **kwargs): + self._stopped = False + BaseHTTPServer.HTTPServer.__init__(self, *args, **kwargs) + class SimpleHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): """SimpleHTTP challenge handler. diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py index 794fb1a6e86..9ff99f5fff7 100644 --- a/acme/acme/standalone_test.py +++ b/acme/acme/standalone_test.py @@ -25,6 +25,26 @@ def test_bind(self): # pylint: disable=no-self-use server.server_close() # pylint: disable=no-member +class ACMEServerMixinTest(unittest.TestCase): + """Tests for acme.standalone.ACMEServerMixin.""" + + def test_shutdown2_not_running(self): + from acme.standalone import ACMEServer + server = ACMEServer(("", 0), socketserver.BaseRequestHandler) + server.shutdown2() + server.shutdown2() + + +class ACMEServerTest(unittest.TestCase): + """Test for acme.standalone.ACMEServer.""" + + def test_init(self): + from acme.standalone import ACMEServer + server = ACMEServer(("", 0), socketserver.BaseRequestHandler) + # pylint: disable=protected-access + self.assertFalse(server._stopped) + + class ACMESimpleHTTPTLSServerTestEndToEnd(unittest.TestCase): """End-to-end test for ACME TLS server with SimpleHTTP.""" @@ -45,12 +65,13 @@ def setUp(self): self.server = ACMETLSServer(('', 0), handler, certs=self.certs) self.server_thread = threading.Thread( # pylint: disable=no-member - target=self.server.handle_request) + target=self.server.serve_forever2) self.server_thread.start() self.port = self.server.socket.getsockname()[1] def tearDown(self): + self.server.shutdown2() self.server_thread.join() def test_index(self): diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index 5f2bc9292b4..071fdbf4267 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -53,33 +53,15 @@ def run(self, port, tls): except socket.error as error: errors.StandaloneBindError(error, port) - stop = threading.Event() - thread = threading.Thread( - target=self._serve, - args=(server, stop), - ) + thread = threading.Thread(target=server.serve_forever2) thread.start() - self.servers[port] = (server, thread, stop) + self.servers[port] = (server, thread) return self.servers[port] - def _serve(self, server, stop): - while not stop.is_set(): - server.handle_request() - def stop(self, port): """Stop ACME server running on the specified ``port``.""" - server, thread, stop = self.servers[port] - stop.set() - - # dummy request to terminate last handle_request() - sock = socket.socket() - try: - sock.connect(server.socket.getsockname()) - except socket.error: - pass # thread is probably already finished - finally: - sock.close() - + server, thread = self.servers[port] + server.shutdown2() thread.join() del self.servers[port] @@ -170,7 +152,7 @@ def perform2(self, achalls): for achall in achalls: if isinstance(achall, achallenges.SimpleHTTP): - server, _, _ = self.servers.run(self.config.simple_http_port, tls=tls) + server, _ = self.servers.run(self.config.simple_http_port, tls=tls) response, validation = achall.gen_response_and_validation(tls=tls) self.simple_http_resources.add( acme_standalone.SimpleHTTPRequestHandler.SimpleHTTPResource( @@ -179,7 +161,7 @@ def perform2(self, achalls): cert = self.simple_http_cert domain = achall.domain else: # DVSNI - server, _, _ = self.servers.run(self.config.dvsni_port, tls=True) + server, _ = self.servers.run(self.config.dvsni_port, tls=True) response, cert, _ = achall.gen_cert_and_response(self.key) domain = response.z_domain self.certs[domain] = (self.key, cert) @@ -195,6 +177,6 @@ def cleanup(self, achalls): # pylint: disable=missing-docstring for achall in achalls: if achall in server_achalls: server_achalls.remove(achall) - for port, (server, _, _) in self.servers.items(): + for port, (server, _) in self.servers.items(): if not self.served[server]: self.servers.stop(port) From 7687ecd6e36dbb15fcc633bc3c98123cf9605859 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 11:41:19 +0000 Subject: [PATCH 020/113] 100% coverage for standalone.ServerManager --- letsencrypt/plugins/standalone.py | 9 ++++--- letsencrypt/plugins/standalone_test.py | 34 ++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index 071fdbf4267..b3fb92ad1ab 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -51,12 +51,15 @@ def run(self, port, tls): try: server = cls(("", port), handler) except socket.error as error: - errors.StandaloneBindError(error, port) + raise errors.StandaloneBindError(error, port) + + # if port == 0, then random free port on OS is taken + real_port = server.socket.getsockname() thread = threading.Thread(target=server.serve_forever2) thread.start() - self.servers[port] = (server, thread) - return self.servers[port] + self.servers[real_port] = (server, thread) + return self.servers[real_port] def stop(self, port): """Stop ACME server running on the specified ``port``.""" diff --git a/letsencrypt/plugins/standalone_test.py b/letsencrypt/plugins/standalone_test.py index 8c0dd6f9777..ad8122c78c9 100644 --- a/letsencrypt/plugins/standalone_test.py +++ b/letsencrypt/plugins/standalone_test.py @@ -1,6 +1,9 @@ """Tests for letsencrypt.plugins.standalone.""" +import socket import unittest +from letsencrypt import errors + class ServerManagerTest(unittest.TestCase): """Tests for letsencrypt.plugins.standalone.ServerManager.""" @@ -16,6 +19,37 @@ def test_init(self): self.assertTrue( self.mgr.simple_http_resources is self.simple_http_resources) + def test_run_stop_non_tls(self): + server, thread = self.mgr.run(port=0, tls=False) + self.mgr.stop(port=server.socket.getsockname()) + + def test_run_stop_tls(self): + server, thread = self.mgr.run(port=0, tls=True) + self.mgr.stop(port=server.socket.getsockname()) + + def test_run_idempotent(self): + server, thread = self.mgr.run(port=0, tls=False) + port = server.socket.getsockname() + server2, thread2 = self.mgr.run(port=port, tls=False) + self.assertTrue(server is server2) + self.assertTrue(thread2 is thread2) + self.mgr.stop(port) + + def test_run_bind_error(self): + some_server = socket.socket() + some_server.bind(("", 0)) + port = some_server.getsockname()[1] + self.assertRaises( + errors.StandaloneBindError, self.mgr.run, port, tls=False) + + def test_items(self): + server, thread = self.mgr.run(port=0, tls=True) + port = server.socket.getsockname() + self.assertEqual(port, self.mgr.items()[0][0]) + self.assertTrue(self.mgr.items()[0][1][0] is server) + self.assertTrue(self.mgr.items()[0][1][1] is thread) + self.mgr.stop(port=port) + if __name__ == "__main__": unittest.main() # pragma: no cover From bba0560c0ea0a368aa46a80ba173a3095011c991 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 13:24:30 +0000 Subject: [PATCH 021/113] Almost full coverage for plugins.standalone (perform2 left). --- letsencrypt/plugins/standalone.py | 3 +- letsencrypt/plugins/standalone_test.py | 85 ++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 2 deletions(-) diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index b3fb92ad1ab..bfe4a6606da 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -99,7 +99,6 @@ def __init__(self, *args, **kwargs): self.key, domains=["temp server"]) self.responses = {} - self.servers = {} self.served = collections.defaultdict(set) # Stuff below is shared across threads (i.e. servers read @@ -118,7 +117,7 @@ def prepare(self): # pylint: disable=missing-docstring if any(util.already_listening(port) for port in (self.config.dvsni_port, self.config.simple_http_port)): raise errors.MisconfigurationError( - "One of the (possibly) required ports is already taken taken.") + "One of the (possibly) required ports is already taken.") # TODO: add --chall-pref flag def get_chall_pref(self, domain): diff --git a/letsencrypt/plugins/standalone_test.py b/letsencrypt/plugins/standalone_test.py index ad8122c78c9..8b76633c931 100644 --- a/letsencrypt/plugins/standalone_test.py +++ b/letsencrypt/plugins/standalone_test.py @@ -2,7 +2,13 @@ import socket import unittest +import mock +import six + +from acme import challenges + from letsencrypt import errors +from letsencrypt import interfaces class ServerManagerTest(unittest.TestCase): @@ -51,5 +57,84 @@ def test_items(self): self.mgr.stop(port=port) +class AuthenticatorTest(unittest.TestCase): + """Tests for letsencrypt.plugins.standalone.Authenticator.""" + + def setUp(self): + from letsencrypt.plugins.standalone import Authenticator + config = mock.MagicMock(dvsni_port=1234, simple_http_port=4321) + self.auth = Authenticator(config, name="standalone") + + def test_more_info(self): + self.assertTrue(isinstance(self.auth.more_info(), six.string_types)) + + @mock.patch("letsencrypt.plugins.standalone.util") + def test_prepare_misconfiguration(self, mock_util): + mock_util.already_listening.return_value = True + self.assertRaises(errors.MisconfigurationError, self.auth.prepare) + mock_util.already_listening.assert_called_once_with(1234) + + def test_get_chall_pref(self): + self.assertEqual(set(self.auth.get_chall_pref(domain=None)), + set([challenges.SimpleHTTP, challenges.DVSNI])) + + @mock.patch("letsencrypt.plugins.standalone.zope.component.getUtility") + def test_perform(self, mock_get_utility): + achalls = [1, 2, 3] + self.auth.perform2 = mock.Mock(return_value=mock.sentinel.responses) + self.assertEqual(mock.sentinel.responses, self.auth.perform(achalls)) + self.auth.perform2.assert_called_once_with(achalls) + + @mock.patch("letsencrypt.plugins.standalone.zope.component.getUtility") + def _test_perform_bind_errors(self, errno, achalls, mock_get_utility): + def _perform2(achalls): + raise errors.StandaloneBindError(mock.Mock(errno=errno), 1234) + + self.auth.perform2 = mock.MagicMock(side_effect=_perform2) + self.auth.perform(achalls) + mock_get_utility.assert_called_once_with(interfaces.IDisplay) + notification = mock_get_utility.return_value.notification + self.assertEqual(1, notification.call_count) + self.assertTrue("1234" in notification.call_args[0][0]) + + def test_perform_eacces(self): + self._test_perform_bind_errors(socket.errno.EACCES, []) + + def test_perform_eaddrinuse(self): + self._test_perform_bind_errors(socket.errno.EADDRINUSE, []) + + def test_perfom_unknown_bind_error(self): + self.assertRaises( + errors.StandaloneBindError, self._test_perform_bind_errors, + socket.errno.ENOTCONN, []) + + def test_cleanup(self): + servers = {1: "server1", 2: "server2"} + self.auth.servers = mock.Mock() + self.auth.servers.items.return_value = [ + (1, ("server1", "thread1")), + (2, ("server2", "thread2")), + ] + self.auth.served["server1"].add("chall1") + self.auth.served["server2"].update(["chall2", "chall3"]) + + self.auth.cleanup(["chall1"]) + self.assertEqual(self.auth.served, { + "server1": set(), "server2": set(["chall2", "chall3"])}) + self.auth.servers.stop.assert_called_once_with(1) + + self.auth.servers.items.return_value = [ + (2, ("server2", "thread2")), + ] + self.auth.cleanup(["chall2"]) + self.assertEqual(self.auth.served, { + "server1": set(), "server2": set(["chall3"])}) + self.assertEqual(1, self.auth.servers.stop.call_count) + + self.auth.cleanup(["chall3"]) + self.assertEqual(self.auth.served, { + "server1": set(), "server2": set([])}) + self.auth.servers.stop.assert_called_with(2) + if __name__ == "__main__": unittest.main() # pragma: no cover From 517a74f432758b27615b1cc4361968dcbe77bd9a Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 17:21:13 +0000 Subject: [PATCH 022/113] standalone 2.0: lint, docs, cleanup. --- acme/acme/crypto_util.py | 8 ++- letsencrypt/plugins/standalone.py | 74 +++++++++++++++++++------- letsencrypt/plugins/standalone_test.py | 33 ++++++------ 3 files changed, 78 insertions(+), 37 deletions(-) diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index 32533630ba8..9ea5812b384 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -27,8 +27,14 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods - """SSL wrapper for sockets.""" + """SSL wrapper for sockets. + :ivar socket sock: Original wrapped socket. + :ivar dict certs: Mapping from domain names (`bytes`) to + `OpenSSL.crypto.X509`. + :ivar method: See `OpenSSL.SSL.Context` for allowed values. + + """ def __init__(self, sock, certs, method=_DEFAULT_DVSNI_SSL_METHOD): self.sock = sock self.certs = certs diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index bfe4a6606da..cde673b2cd3 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -6,8 +6,6 @@ import socket import threading -from six.moves import BaseHTTPServer # pylint: disable=import-error - import OpenSSL import zope.interface @@ -26,17 +24,39 @@ class ServerManager(object): - """Standalone servers manager.""" + """Standalone servers manager. + + Manager for `ACMEServer` and `ACMETLSServer` instances. + `certs` and `simple_http_resources` correspond to + `acme.crypto_util.SSLSocket.certs` and + `acme.crypto_util.SSLSocket.simple_http_resources` respectively. All + created servers share the same certificates and resources, so if + you're running both TLS and non-TLS instances, SimpleHTTP handlers + will serve the same URLs! + + """ def __init__(self, certs, simple_http_resources): - self.servers = {} + self._servers = {} self.certs = certs self.simple_http_resources = simple_http_resources def run(self, port, tls): - """Run ACME server on specified ``port``.""" - if port in self.servers: - return self.servers[port] + """Run ACME server on specified ``port``. + + This method is idempotent, i.e. all calls with the same pair of + ``(port, tls)`` will reuse the same server. + + :param int port: Port to run the server on. + :param bool tls: TLS or non-TLS? + + :returns: Server instance (`ACMEServerMixin`) and the + corresponding (already started) thread (`threading.Thread`). + :rtype: tuple + + """ + if port in self._servers: + return self._servers[port] logger.debug("Starting new server at %s (tls=%s)", port, tls) handler = acme_standalone.ACMERequestHandler.partial_init( @@ -54,23 +74,38 @@ def run(self, port, tls): raise errors.StandaloneBindError(error, port) # if port == 0, then random free port on OS is taken - real_port = server.socket.getsockname() + # pylint: disable=no-member + host, real_port = server.socket.getsockname() thread = threading.Thread(target=server.serve_forever2) + logger.debug("Starting server at %s:%d", host, real_port) thread.start() - self.servers[real_port] = (server, thread) - return self.servers[real_port] + + self._servers[real_port] = (server, thread) + return self._servers[real_port] def stop(self, port): - """Stop ACME server running on the specified ``port``.""" - server, thread = self.servers[port] + """Stop ACME server running on the specified ``port``. + + :param int port: + + """ + server, thread = self._servers[port] server.shutdown2() thread.join() - del self.servers[port] + del self._servers[port] + + def running(self): + """Return all running instances. + + Once the server is stopped using `stop`, it will not be + returned. + + :returns: ``(port, (server, thread))`` + :rtype: tuple - def items(self): - """Return a list of all port, server tuples.""" - return self.servers.items() + """ + return self._servers.items() class Authenticator(common.Plugin): @@ -98,7 +133,6 @@ def __init__(self, *args, **kwargs): self.simple_http_cert = acme_crypto_util.gen_ss_cert( self.key, domains=["temp server"]) - self.responses = {} self.served = collections.defaultdict(set) # Stuff below is shared across threads (i.e. servers read @@ -117,7 +151,8 @@ def prepare(self): # pylint: disable=missing-docstring if any(util.already_listening(port) for port in (self.config.dvsni_port, self.config.simple_http_port)): raise errors.MisconfigurationError( - "One of the (possibly) required ports is already taken.") + "At least one of the (possibly) required ports is " + "already taken.") # TODO: add --chall-pref flag def get_chall_pref(self, domain): @@ -167,7 +202,6 @@ def perform2(self, achalls): response, cert, _ = achall.gen_cert_and_response(self.key) domain = response.z_domain self.certs[domain] = (self.key, cert) - self.responses[achall] = response self.served[server].add(achall) responses.append(response) @@ -179,6 +213,6 @@ def cleanup(self, achalls): # pylint: disable=missing-docstring for achall in achalls: if achall in server_achalls: server_achalls.remove(achall) - for port, (server, _) in self.servers.items(): + for port, (server, _) in self.servers.running(): if not self.served[server]: self.servers.stop(port) diff --git a/letsencrypt/plugins/standalone_test.py b/letsencrypt/plugins/standalone_test.py index 8b76633c931..2cde623ac18 100644 --- a/letsencrypt/plugins/standalone_test.py +++ b/letsencrypt/plugins/standalone_test.py @@ -26,19 +26,19 @@ def test_init(self): self.mgr.simple_http_resources is self.simple_http_resources) def test_run_stop_non_tls(self): - server, thread = self.mgr.run(port=0, tls=False) - self.mgr.stop(port=server.socket.getsockname()) + server, _ = self.mgr.run(port=0, tls=False) + self.mgr.stop(port=server.socket.getsockname()[1]) def test_run_stop_tls(self): - server, thread = self.mgr.run(port=0, tls=True) - self.mgr.stop(port=server.socket.getsockname()) + server, _ = self.mgr.run(port=0, tls=True) + self.mgr.stop(port=server.socket.getsockname()[1]) def test_run_idempotent(self): server, thread = self.mgr.run(port=0, tls=False) - port = server.socket.getsockname() + port = server.socket.getsockname()[1] server2, thread2 = self.mgr.run(port=port, tls=False) self.assertTrue(server is server2) - self.assertTrue(thread2 is thread2) + self.assertTrue(thread is thread2) self.mgr.stop(port) def test_run_bind_error(self): @@ -48,12 +48,12 @@ def test_run_bind_error(self): self.assertRaises( errors.StandaloneBindError, self.mgr.run, port, tls=False) - def test_items(self): + def test_running(self): server, thread = self.mgr.run(port=0, tls=True) - port = server.socket.getsockname() - self.assertEqual(port, self.mgr.items()[0][0]) - self.assertTrue(self.mgr.items()[0][1][0] is server) - self.assertTrue(self.mgr.items()[0][1][1] is thread) + port = server.socket.getsockname()[1] + self.assertEqual(port, self.mgr.running()[0][0]) + self.assertTrue(self.mgr.running()[0][1][0] is server) + self.assertTrue(self.mgr.running()[0][1][1] is thread) self.mgr.stop(port=port) @@ -79,7 +79,7 @@ def test_get_chall_pref(self): set([challenges.SimpleHTTP, challenges.DVSNI])) @mock.patch("letsencrypt.plugins.standalone.zope.component.getUtility") - def test_perform(self, mock_get_utility): + def test_perform(self, unused_mock_get_utility): achalls = [1, 2, 3] self.auth.perform2 = mock.Mock(return_value=mock.sentinel.responses) self.assertEqual(mock.sentinel.responses, self.auth.perform(achalls)) @@ -87,7 +87,7 @@ def test_perform(self, mock_get_utility): @mock.patch("letsencrypt.plugins.standalone.zope.component.getUtility") def _test_perform_bind_errors(self, errno, achalls, mock_get_utility): - def _perform2(achalls): + def _perform2(unused_achalls): raise errors.StandaloneBindError(mock.Mock(errno=errno), 1234) self.auth.perform2 = mock.MagicMock(side_effect=_perform2) @@ -98,9 +98,11 @@ def _perform2(achalls): self.assertTrue("1234" in notification.call_args[0][0]) def test_perform_eacces(self): + # pylint: disable=no-value-for-parameter self._test_perform_bind_errors(socket.errno.EACCES, []) def test_perform_eaddrinuse(self): + # pylint: disable=no-value-for-parameter self._test_perform_bind_errors(socket.errno.EADDRINUSE, []) def test_perfom_unknown_bind_error(self): @@ -109,9 +111,8 @@ def test_perfom_unknown_bind_error(self): socket.errno.ENOTCONN, []) def test_cleanup(self): - servers = {1: "server1", 2: "server2"} self.auth.servers = mock.Mock() - self.auth.servers.items.return_value = [ + self.auth.servers.running.return_value = [ (1, ("server1", "thread1")), (2, ("server2", "thread2")), ] @@ -123,7 +124,7 @@ def test_cleanup(self): "server1": set(), "server2": set(["chall2", "chall3"])}) self.auth.servers.stop.assert_called_once_with(1) - self.auth.servers.items.return_value = [ + self.auth.servers.running.return_value = [ (2, ("server2", "thread2")), ] self.auth.cleanup(["chall2"]) From df04938f6ae628dfb32d98730dd7f51849ab59d9 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 17:45:49 +0000 Subject: [PATCH 023/113] Standalone 2.0: add detection for unsupported SimpleHTTP TLS platform. --- acme/acme/crypto_util.py | 12 +++++++++++- acme/acme/standalone.py | 7 +++++++ letsencrypt/plugins/standalone.py | 11 ++++++++--- letsencrypt/plugins/standalone_test.py | 19 +++++++++++++++---- 4 files changed, 41 insertions(+), 8 deletions(-) diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index 9ea5812b384..5829a511f22 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -70,6 +70,15 @@ def _pick_certificate_cb(self, connection): class FakeConnection(object): """Fake OpenSSL.SSL.Connection.""" + MAKEFILE_SUPPORT = hasattr(socket, "_fileobject") + """Is `makefile` supported on your platform? + + .. warning:: `makefile`, as currently implemented, is supported + on select platforms only, as it uses CPython's internal API. + You've been warned! + + """ + # pylint: disable=missing-docstring def __init__(self, connection): @@ -85,9 +94,10 @@ def shutdown(self, *unused_args): # stuff below ripped off from # https://hg.python.org/cpython/file/2.7/Lib/ssl.py - # XXX: this uses Python's internal API def makefile(self, mode='r', bufsize=-1): + assert self.MAKEFILE_SUPPORT, ( + "You need compatible version for makefile support") self._makefile_refs += 1 # SocketServer.StreamRequesthandler.finish will try to # close the wfile/rfile. close=True causes curl: (56) diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index 501d239cbcc..ce10407e69d 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -87,6 +87,13 @@ def shutdown2(self): class ACMETLSServer(HTTPSServer, ACMEServerMixin): """ACME TLS Server.""" + SIMPLE_HTTP_SUPPORT = crypto_util.SSLSocket.FakeConnection.MAKEFILE_SUPPORT + """Is SimpleHTTP supported on your platform. + + Please see a warning for `acme.crypto_util.SSLSocket.FakeConnection`. + + """ + def __init__(self, *args, **kwargs): self._stopped = False HTTPSServer.__init__(self, *args, **kwargs) diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index cde673b2cd3..08d1e5c634e 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -4,6 +4,7 @@ import logging import random import socket +import sys import threading import OpenSSL @@ -121,7 +122,6 @@ class Authenticator(common.Plugin): zope.interface.classProvides(interfaces.IPluginFactory) description = "Standalone Authenticator" - supported_challenges = set([challenges.SimpleHTTP, challenges.DVSNI]) def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) @@ -136,7 +136,7 @@ def __init__(self, *args, **kwargs): self.served = collections.defaultdict(set) # Stuff below is shared across threads (i.e. servers read - # values, main thread writes). Due to the nature of Cython's + # values, main thread writes). Due to the nature of CPython's # GIL, the operations are safe, c.f. # https://docs.python.org/2/faq/library.html#what-kinds-of-global-value-mutation-are-thread-safe self.certs = {} @@ -157,7 +157,12 @@ def prepare(self): # pylint: disable=missing-docstring # TODO: add --chall-pref flag def get_chall_pref(self, domain): # pylint: disable=unused-argument,missing-docstring - chall_pref = list(self.supported_challenges) + supported_challenges = set([challenges.SimpleHTTP, challenges.DVSNI]) + if not self.config.no_simple_http_tls and not ( + acme_standalone.ACMETLSServer.SIMPLE_HTTP_SUPPORT): + logger.debug("SimpleHTTPS not supported: %s", sys.version) + supported_challenges.discard(challenges.SimpleHTTP) + chall_pref = list(supported_challenges) random.shuffle(chall_pref) # 50% for each challenge return chall_pref diff --git a/letsencrypt/plugins/standalone_test.py b/letsencrypt/plugins/standalone_test.py index 2cde623ac18..7fff7b97265 100644 --- a/letsencrypt/plugins/standalone_test.py +++ b/letsencrypt/plugins/standalone_test.py @@ -62,8 +62,8 @@ class AuthenticatorTest(unittest.TestCase): def setUp(self): from letsencrypt.plugins.standalone import Authenticator - config = mock.MagicMock(dvsni_port=1234, simple_http_port=4321) - self.auth = Authenticator(config, name="standalone") + self.config = mock.MagicMock(dvsni_port=1234, simple_http_port=4321) + self.auth = Authenticator(self.config, name="standalone") def test_more_info(self): self.assertTrue(isinstance(self.auth.more_info(), six.string_types)) @@ -74,9 +74,20 @@ def test_prepare_misconfiguration(self, mock_util): self.assertRaises(errors.MisconfigurationError, self.auth.prepare) mock_util.already_listening.assert_called_once_with(1234) - def test_get_chall_pref(self): + @mock.patch("letsencrypt.plugins.standalone.acme_standalone") + def test_get_chall_pref_tls_supported(self, mock_astandalone): + mock_astandalone.ACMETLSServer.SIMPLE_HTTP_SUPPORT = True + for no_simple_http_tls in True, False: + self.config.no_simple_http_tls = no_simple_http_tls + self.assertEqual(set(self.auth.get_chall_pref(domain=None)), + set([challenges.DVSNI, challenges.SimpleHTTP])) + + @mock.patch("letsencrypt.plugins.standalone.acme_standalone") + def test_get_chall_pref_simple_tls_not_supported(self, mock_astandalone): + mock_astandalone.ACMETLSServer.SIMPLE_HTTP_SUPPORT = False + self.config.no_simple_http_tls = False self.assertEqual(set(self.auth.get_chall_pref(domain=None)), - set([challenges.SimpleHTTP, challenges.DVSNI])) + set([challenges.DVSNI])) @mock.patch("letsencrypt.plugins.standalone.zope.component.getUtility") def test_perform(self, unused_mock_get_utility): From 1a0f8889ad741f98e7669070f70c8339fc53f38a Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 17:54:58 +0000 Subject: [PATCH 024/113] ServerManager.running returns dict --- letsencrypt/plugins/standalone.py | 7 +++-- letsencrypt/plugins/standalone_test.py | 40 +++++++++++++------------- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index 08d1e5c634e..33eabbfab45 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -8,6 +8,7 @@ import threading import OpenSSL +import six import zope.interface from acme import challenges @@ -102,11 +103,11 @@ def running(self): Once the server is stopped using `stop`, it will not be returned. - :returns: ``(port, (server, thread))`` + :returns: Mapping from port to ``(server, thread)``. :rtype: tuple """ - return self._servers.items() + return self._servers.copy() class Authenticator(common.Plugin): @@ -218,6 +219,6 @@ def cleanup(self, achalls): # pylint: disable=missing-docstring for achall in achalls: if achall in server_achalls: server_achalls.remove(achall) - for port, (server, _) in self.servers.running(): + for port, (server, _) in six.iteritems(self.servers.running()): if not self.served[server]: self.servers.stop(port) diff --git a/letsencrypt/plugins/standalone_test.py b/letsencrypt/plugins/standalone_test.py index 7fff7b97265..a9f321cf39b 100644 --- a/letsencrypt/plugins/standalone_test.py +++ b/letsencrypt/plugins/standalone_test.py @@ -25,21 +25,28 @@ def test_init(self): self.assertTrue( self.mgr.simple_http_resources is self.simple_http_resources) - def test_run_stop_non_tls(self): - server, _ = self.mgr.run(port=0, tls=False) - self.mgr.stop(port=server.socket.getsockname()[1]) + def _test_run_stop(self, tls): + server, _ = self.mgr.run(port=0, tls=tls) + port = server.socket.getsockname()[1] + self.assertEqual(self.mgr.running(), {port: (server, mock.ANY)}) + self.mgr.stop(port=port) + self.assertEqual(self.mgr.running(), {}) def test_run_stop_tls(self): - server, _ = self.mgr.run(port=0, tls=True) - self.mgr.stop(port=server.socket.getsockname()[1]) + self._test_run_stop(tls=True) + + def test_run_stop_non_tls(self): + self._test_run_stop(tls=False) def test_run_idempotent(self): server, thread = self.mgr.run(port=0, tls=False) port = server.socket.getsockname()[1] server2, thread2 = self.mgr.run(port=port, tls=False) + self.assertEqual(self.mgr.running(), {port: (server, thread)}) self.assertTrue(server is server2) self.assertTrue(thread is thread2) self.mgr.stop(port) + self.assertEqual(self.mgr.running(), {}) def test_run_bind_error(self): some_server = socket.socket() @@ -47,14 +54,7 @@ def test_run_bind_error(self): port = some_server.getsockname()[1] self.assertRaises( errors.StandaloneBindError, self.mgr.run, port, tls=False) - - def test_running(self): - server, thread = self.mgr.run(port=0, tls=True) - port = server.socket.getsockname()[1] - self.assertEqual(port, self.mgr.running()[0][0]) - self.assertTrue(self.mgr.running()[0][1][0] is server) - self.assertTrue(self.mgr.running()[0][1][1] is thread) - self.mgr.stop(port=port) + self.assertEqual(self.mgr.running(), {}) class AuthenticatorTest(unittest.TestCase): @@ -123,10 +123,10 @@ def test_perfom_unknown_bind_error(self): def test_cleanup(self): self.auth.servers = mock.Mock() - self.auth.servers.running.return_value = [ - (1, ("server1", "thread1")), - (2, ("server2", "thread2")), - ] + self.auth.servers.running.return_value = { + 1: ("server1", "thread1"), + 2: ("server2", "thread2"), + } self.auth.served["server1"].add("chall1") self.auth.served["server2"].update(["chall2", "chall3"]) @@ -135,9 +135,9 @@ def test_cleanup(self): "server1": set(), "server2": set(["chall2", "chall3"])}) self.auth.servers.stop.assert_called_once_with(1) - self.auth.servers.running.return_value = [ - (2, ("server2", "thread2")), - ] + self.auth.servers.running.return_value = { + 2: ("server2", "thread2"), + } self.auth.cleanup(["chall2"]) self.assertEqual(self.auth.served, { "server1": set(), "server2": set(["chall3"])}) From 774dc7db9aaeacd9562429a98bdc4fc791ba7f2f Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 18:23:26 +0000 Subject: [PATCH 025/113] 100% coverage for Standalone 2.0 --- letsencrypt/plugins/standalone_test.py | 39 ++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/letsencrypt/plugins/standalone_test.py b/letsencrypt/plugins/standalone_test.py index a9f321cf39b..d5836eab401 100644 --- a/letsencrypt/plugins/standalone_test.py +++ b/letsencrypt/plugins/standalone_test.py @@ -6,10 +6,16 @@ import six from acme import challenges +from acme import jose +from acme import standalone as acme_standalone +from letsencrypt import achallenges from letsencrypt import errors from letsencrypt import interfaces +from letsencrypt.tests import acme_util +from letsencrypt.tests import test_util + class ServerManagerTest(unittest.TestCase): """Tests for letsencrypt.plugins.standalone.ServerManager.""" @@ -121,6 +127,39 @@ def test_perfom_unknown_bind_error(self): errors.StandaloneBindError, self._test_perform_bind_errors, socket.errno.ENOTCONN, []) + def test_perform2(self): + domain = b'localhost' + key = jose.JWK.load(test_util.load_vector('rsa512_key.pem')) + simple_http = achallenges.SimpleHTTP( + challb=acme_util.SIMPLE_HTTP_P, domain=domain, account_key=key) + dvsni = achallenges.DVSNI( + challb=acme_util.DVSNI_P, domain=domain, account_key=key) + + self.auth.servers = mock.MagicMock() + + def _run(port, tls): # pylint: disable=unused-argument + return "server{0}".format(port), "thread{0}".format(port) + + self.auth.servers.run.side_effect = _run + responses = self.auth.perform2([simple_http, dvsni]) + + self.assertTrue(isinstance(responses, list)) + self.assertEqual(2, len(responses)) + self.assertTrue(isinstance(responses[0], challenges.SimpleHTTPResponse)) + self.assertTrue(isinstance(responses[1], challenges.DVSNIResponse)) + + self.assertEqual(self.auth.servers.run.mock_calls, [ + mock.call(4321, tls=False), mock.call(1234, tls=True)]) + self.assertEqual(self.auth.served, { + "server1234": set([dvsni]), + "server4321": set([simple_http]), + }) + self.assertEqual(1, len(self.auth.simple_http_resources)) + self.assertEqual(2, len(self.auth.certs)) + self.assertEqual(list(self.auth.simple_http_resources), [ + acme_standalone.SimpleHTTPRequestHandler.SimpleHTTPResource( + acme_util.SIMPLE_HTTP, responses[0], mock.ANY)]) + def test_cleanup(self): self.auth.servers = mock.Mock() self.auth.servers.running.return_value = { From ea45fc6504af1c25e3faeab72f457839da51a5ea Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 18:32:49 +0000 Subject: [PATCH 026/113] TestSimpleServer: don't rely on symlinks --- acme/acme/standalone_test.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py index 9ff99f5fff7..a6f7502b636 100644 --- a/acme/acme/standalone_test.py +++ b/acme/acme/standalone_test.py @@ -1,6 +1,8 @@ """Tests for acme.standalone.""" import os +import shutil import threading +import tempfile import time import unittest @@ -115,21 +117,28 @@ def test_simple_http_not_found(self): class TestSimpleServer(unittest.TestCase): """Tests for acme.standalone.simple_server.""" - TEST_CWD = os.path.join(os.path.dirname(__file__), '..', 'examples', 'standalone') - def setUp(self): + # mirror ../examples/standalone + self.test_cwd = tempfile.mkdtemp() + localhost_dir = os.path.join(self.test_cwd, 'localhost') + os.makedirs(localhost_dir) + shutil.copy(test_util.vector_path('cert.pem'), localhost_dir) + shutil.copy(test_util.vector_path('rsa512_key.pem'), + os.path.join(localhost_dir, 'key.pem')) + from acme.standalone import simple_server self.thread = threading.Thread(target=simple_server, kwargs={ 'cli_args': ('xxx', '--port', '1234'), 'forever': False, }) self.old_cwd = os.getcwd() - os.chdir(self.TEST_CWD) + os.chdir(self.test_cwd) self.thread.start() def tearDown(self): os.chdir(self.old_cwd) self.thread.join() + shutil.rmtree(self.test_cwd) def test_it(self): max_attempts = 5 From 509af11a92ec84a94e03c75d6ffb86ac8f068940 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 18:53:36 +0000 Subject: [PATCH 027/113] --standalone-supported-chalenges --- letsencrypt/plugins/standalone.py | 37 ++++++++++++++++++++++++-- letsencrypt/plugins/standalone_test.py | 3 ++- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index 33eabbfab45..b76095f8934 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -1,4 +1,5 @@ """Standalone Authenticator.""" +import argparse import collections import functools import logging @@ -110,6 +111,27 @@ def running(self): return self._servers.copy() +SUPPORTED_CHALLENGES = set([challenges.DVSNI, challenges.SimpleHTTP]) + + +def supported_challenges_validator(data): + """Supported challenges validator.""" + challs = data.split(",") + unrecognized = [name for name in challs + if name not in challenges.Challenge.TYPES] + if unrecognized: + raise argparse.ArgumentTypeError( + "Unrecognized challenges: {0}".format(", ".join(unrecognized))) + + choices = set(chall.typ for chall in SUPPORTED_CHALLENGES) + if not set(challs).issubset(choices): + raise argparse.ArgumentTypeError( + "Plugin does not support the following (valid) " + "challenges: {0}".format(", ".join(challs - choices))) + + return data + + class Authenticator(common.Plugin): """Standalone Authenticator. @@ -145,6 +167,18 @@ def __init__(self, *args, **kwargs): self.servers = ServerManager(self.certs, self.simple_http_resources) + @classmethod + def add_parser_arguments(cls, add): + add("supported-challenges", help="Supported challenges, " + "order preferences are randomly chosen.", + type=supported_challenges_validator, default=",".join( + sorted(chall.typ for chall in SUPPORTED_CHALLENGES))) + + @property + def supported_challenges(self): + return set(challenges.Challenge.TYPES[name] for name in + self.conf("supported-challenges").split(",")) + def more_info(self): # pylint: disable=missing-docstring return self.__doc__ @@ -155,10 +189,9 @@ def prepare(self): # pylint: disable=missing-docstring "At least one of the (possibly) required ports is " "already taken.") - # TODO: add --chall-pref flag def get_chall_pref(self, domain): # pylint: disable=unused-argument,missing-docstring - supported_challenges = set([challenges.SimpleHTTP, challenges.DVSNI]) + supported_challenges = self.supported_challenges if not self.config.no_simple_http_tls and not ( acme_standalone.ACMETLSServer.SIMPLE_HTTP_SUPPORT): logger.debug("SimpleHTTPS not supported: %s", sys.version) diff --git a/letsencrypt/plugins/standalone_test.py b/letsencrypt/plugins/standalone_test.py index d5836eab401..4a077a36f97 100644 --- a/letsencrypt/plugins/standalone_test.py +++ b/letsencrypt/plugins/standalone_test.py @@ -68,7 +68,8 @@ class AuthenticatorTest(unittest.TestCase): def setUp(self): from letsencrypt.plugins.standalone import Authenticator - self.config = mock.MagicMock(dvsni_port=1234, simple_http_port=4321) + self.config = mock.MagicMock(dvsni_port=1234, simple_http_port=4321, + standalone_supported_challenges="dvsni,simpleHttp") self.auth = Authenticator(self.config, name="standalone") def test_more_info(self): From 827c66c6674b3a98beaaf31362de496333efd1ea Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 19:19:36 +0000 Subject: [PATCH 028/113] Integration tests: standalone dvsni and simpleHttp. --- tests/boulder-integration.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index ed877d136f0..210f13c2330 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -21,8 +21,8 @@ common() { "$@" } -common --domains le1.wtf auth -common --domains le2.wtf run +common --domains le1.wtf --standalone-supported-challenges dvsni auth +common --domains le2.wtf --standalone-supported-challenges simpleHttp run common -a manual -d le.wtf auth common -a manual -d le.wtf --no-simple-http-tls auth From 4d25cabfe2ae2a0e23324b9c53331cbfe658c51a Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 19:27:47 +0000 Subject: [PATCH 029/113] Add missing docstring --- letsencrypt/plugins/standalone.py | 1 + 1 file changed, 1 insertion(+) diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index b76095f8934..17e99073f2d 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -176,6 +176,7 @@ def add_parser_arguments(cls, add): @property def supported_challenges(self): + """Challenges supported by this plugin.""" return set(challenges.Challenge.TYPES[name] for name in self.conf("supported-challenges").split(",")) From ed70c948aaad5d56835eecd7e7a77fd4ba87ca88 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 19:50:27 +0000 Subject: [PATCH 030/113] Renewer: restore IConfig.simple_http_port data type. --- letsencrypt/renewer.py | 1 + letsencrypt/tests/renewer_test.py | 1 + 2 files changed, 2 insertions(+) diff --git a/letsencrypt/renewer.py b/letsencrypt/renewer.py index 1c9cddc95ab..e00fe5532c8 100644 --- a/letsencrypt/renewer.py +++ b/letsencrypt/renewer.py @@ -70,6 +70,7 @@ def renew(cert, old_version): # was an int, not a str) config.rsa_key_size = int(config.rsa_key_size) config.dvsni_port = int(config.dvsni_port) + config.namespace.simple_http_port = int(config.namespace.simple_http_port) zope.component.provideUtility(config) try: authenticator = plugins[renewalparams["authenticator"]] diff --git a/letsencrypt/tests/renewer_test.py b/letsencrypt/tests/renewer_test.py index 6941e084c99..518332a9415 100644 --- a/letsencrypt/tests/renewer_test.py +++ b/letsencrypt/tests/renewer_test.py @@ -625,6 +625,7 @@ def test_renew(self, mock_c, mock_acc_storage, mock_pd): self.test_rc.configfile["renewalparams"]["server"] = "acme.example.com" self.test_rc.configfile["renewalparams"]["authenticator"] = "fake" self.test_rc.configfile["renewalparams"]["dvsni_port"] = "4430" + self.test_rc.configfile["renewalparams"]["simple_http_port"] = "1234" self.test_rc.configfile["renewalparams"]["account"] = "abcde" mock_auth = mock.MagicMock() mock_pd.PluginsRegistry.find_all.return_value = {"apache": mock_auth} From 94c6e307c960c1abc5d7f25a72b0d4fd1821d359 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 20:13:00 +0000 Subject: [PATCH 031/113] Fix plugins.common.Dvsni._setup_challenge_cert. --- letsencrypt/plugins/common.py | 7 ++++++- letsencrypt/plugins/common_test.py | 11 ++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/letsencrypt/plugins/common.py b/letsencrypt/plugins/common.py index 95ad56a0a87..88394f56569 100644 --- a/letsencrypt/plugins/common.py +++ b/letsencrypt/plugins/common.py @@ -5,6 +5,7 @@ import shutil import tempfile +import OpenSSL import zope.interface from acme.jose import util as jose_util @@ -181,7 +182,11 @@ def _setup_challenge_cert(self, achall, s=None): self.configurator.reverter.register_file_creation(True, key_path) self.configurator.reverter.register_file_creation(True, cert_path) - response, cert_pem, key_pem = achall.gen_cert_and_response(s) + response, cert, key = achall.gen_cert_and_response(s) + cert_pem = OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_PEM, cert) + key_pem = OpenSSL.crypto.dump_privatekey( + OpenSSL.crypto.FILETYPE_PEM, key) # Write out challenge cert and key with open(cert_path, "wb") as cert_chall_fd: diff --git a/letsencrypt/plugins/common_test.py b/letsencrypt/plugins/common_test.py index fa761839c27..f1eb19094ef 100644 --- a/letsencrypt/plugins/common_test.py +++ b/letsencrypt/plugins/common_test.py @@ -2,6 +2,7 @@ import unittest import mock +import OpenSSL from acme import challenges from acme import jose @@ -144,7 +145,9 @@ def test_setup_challenge_cert(self): response = challenges.DVSNIResponse(validation=mock.Mock()) achall = mock.MagicMock() - achall.gen_cert_and_response.return_value = (response, "cert", "key") + key = test_util.load_pyopenssl_private_key("rsa512_key.pem") + achall.gen_cert_and_response.return_value = ( + response, test_util.load_cert("cert.pem"), key) with mock.patch("letsencrypt.plugins.common.open", mock_open, create=True): @@ -156,10 +159,12 @@ def test_setup_challenge_cert(self): # pylint: disable=no-member mock_open.assert_called_once_with(self.sni.get_cert_path(achall), "wb") - mock_open.return_value.write.assert_called_once_with("cert") + mock_open.return_value.write.assert_called_once_with( + test_util.load_vector("cert.pem")) mock_safe_open.assert_called_once_with( self.sni.get_key_path(achall), "wb", chmod=0o400) - mock_safe_open.return_value.write.assert_called_once_with("key") + mock_safe_open.return_value.write.assert_called_once_with( + OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, key)) if __name__ == "__main__": From 1774ab64c4590014f425ebfaa913ede3d586ad06 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 20:24:53 +0000 Subject: [PATCH 032/113] Add SupportedChallengesValidatorTest. --- letsencrypt/plugins/standalone.py | 8 ++++++-- letsencrypt/plugins/standalone_test.py | 27 ++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index 17e99073f2d..05a9f165574 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -115,7 +115,11 @@ def running(self): def supported_challenges_validator(data): - """Supported challenges validator.""" + """Supported challenges validator for the `argparse`. + + It should be passed as `type` argument to `add_argument`. + + """ challs = data.split(",") unrecognized = [name for name in challs if name not in challenges.Challenge.TYPES] @@ -127,7 +131,7 @@ def supported_challenges_validator(data): if not set(challs).issubset(choices): raise argparse.ArgumentTypeError( "Plugin does not support the following (valid) " - "challenges: {0}".format(", ".join(challs - choices))) + "challenges: {0}".format(", ".join(set(challs) - choices))) return data diff --git a/letsencrypt/plugins/standalone_test.py b/letsencrypt/plugins/standalone_test.py index 4a077a36f97..1cd31010701 100644 --- a/letsencrypt/plugins/standalone_test.py +++ b/letsencrypt/plugins/standalone_test.py @@ -1,4 +1,5 @@ """Tests for letsencrypt.plugins.standalone.""" +import argparse import socket import unittest @@ -63,6 +64,28 @@ def test_run_bind_error(self): self.assertEqual(self.mgr.running(), {}) +class SupportedChallengesValidatorTest(unittest.TestCase): + """Tests for plugins.standalone.supported_challenges_validator.""" + + def _call(self, data): + from letsencrypt.plugins.standalone import ( + supported_challenges_validator) + return supported_challenges_validator(data) + + def test_correct(self): + self.assertEqual("dvsni", self._call("dvsni")) + self.assertEqual("simpleHttp", self._call("simpleHttp")) + self.assertEqual("dvsni,simpleHttp", self._call("dvsni,simpleHttp")) + self.assertEqual("simpleHttp,dvsni", self._call("simpleHttp,dvsni")) + + def test_unrecognized(self): + assert "foo" not in challenges.Challenge.TYPES + self.assertRaises(argparse.ArgumentTypeError, self._call, "foo") + + def test_not_subset(self): + self.assertRaises(argparse.ArgumentTypeError, self._call, "dns") + + class AuthenticatorTest(unittest.TestCase): """Tests for letsencrypt.plugins.standalone.Authenticator.""" @@ -72,6 +95,10 @@ def setUp(self): standalone_supported_challenges="dvsni,simpleHttp") self.auth = Authenticator(self.config, name="standalone") + def test_supported_challenges(self): + self.assertEqual(self.auth.supported_challenges, + set([challenges.DVSNI, challenges.SimpleHTTP])) + def test_more_info(self): self.assertTrue(isinstance(self.auth.more_info(), six.string_types)) From 034af2003c45b50e4047b332f7bcdff725fe8fec Mon Sep 17 00:00:00 2001 From: Brandon Kreisel Date: Sun, 4 Oct 2015 20:07:00 -0400 Subject: [PATCH 033/113] Only ask for bash on OS X --- letsencrypt/plugins/manual.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/letsencrypt/plugins/manual.py b/letsencrypt/plugins/manual.py index f532b8fbc77..179a7b49f5e 100644 --- a/letsencrypt/plugins/manual.py +++ b/letsencrypt/plugins/manual.py @@ -154,12 +154,18 @@ def _perform_single(self, achall): if self.conf("test-mode"): logger.debug("Test mode. Executing the manual command: %s", command) try: + # sh shipped with OS X does't support echo -n + if sys.platform == "darwin": + executable = "/bin/bash" + else: + executable = "/bin/sh" + self._httpd = subprocess.Popen( command, # don't care about setting stdout and stderr, # we're in test mode anyway shell=True, - executable="/bin/bash", + executable=executable, # "preexec_fn" is UNIX specific, but so is "command" preexec_fn=os.setsid) except OSError as error: # ValueError should not happen! From 7420a78296371c63e61c242d75e0761eb153ffd6 Mon Sep 17 00:00:00 2001 From: Brandon Kreisel Date: Sun, 4 Oct 2015 20:08:15 -0400 Subject: [PATCH 034/113] Shrink AccessDenied error handler and inform --- .../plugins/standalone/authenticator.py | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/letsencrypt/plugins/standalone/authenticator.py b/letsencrypt/plugins/standalone/authenticator.py index f7c24f5e511..cdf9a6f0444 100644 --- a/letsencrypt/plugins/standalone/authenticator.py +++ b/letsencrypt/plugins/standalone/authenticator.py @@ -1,4 +1,5 @@ """Standalone authenticator.""" +import logging import os import psutil import signal @@ -19,6 +20,9 @@ from letsencrypt.plugins import common +logger = logging.getLogger(__name__) + + class StandaloneAuthenticator(common.Plugin): # pylint: disable=too-many-instance-attributes """Standalone authenticator. @@ -302,13 +306,21 @@ def already_listening(self, port): # pylint: disable=no-self-use :returns: True or False.""" try: + net_connections = psutil.net_connections() + except psutil.AccessDenied as error: + logger.info("Access denied when trying to list network " + "connections: %s. Are you root?", error) + # this function is just a pre-check that often causes false + # positives and problems in testing (c.f. #680 on Mac, #255 + # generally); we will fail later in bind() anyway + return False + + listeners = [conn.pid for conn in net_connections + if conn.status == 'LISTEN' and + conn.type == socket.SOCK_STREAM and + conn.laddr[1] == port] - # net_connections() can raise AccessDenied on certain OSs - listeners = [conn.pid for conn in psutil.net_connections() - if conn.status == 'LISTEN' and - conn.type == socket.SOCK_STREAM and - conn.laddr[1] == port] - + try: if listeners and listeners[0] is not None: # conn.pid may be None if the current process doesn't have # permission to identify the listening process! Additionally, From c9e28309ed2d3efb3c2c21301f88796fe87169ab Mon Sep 17 00:00:00 2001 From: Brandon Kreisel Date: Sun, 4 Oct 2015 20:08:38 -0400 Subject: [PATCH 035/113] Define constants for OS specific executables --- tests/boulder-integration.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index efd16ef6cde..633d426e34c 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -14,6 +14,12 @@ export PATH="/usr/sbin:$PATH" # /usr/sbin/nginx export GOPATH="${GOPATH:-/tmp/go}" export PATH="$GOPATH/bin:$PATH" +if [ `uname` == 'Darwin' ]; then + readlink="greadlink" +else + readlink="readlink" +fi + common() { letsencrypt_test \ --authenticator standalone \ @@ -50,11 +56,7 @@ dir="$root/conf/archive/le1.wtf" for x in cert chain fullchain privkey; do latest="$(ls -1t $dir/ | grep -e "^${x}" | head -n1)" - if [ `uname` == 'Darwin' ]; then - live="$(greadlink -f "$root/conf/live/le1.wtf/${x}.pem")" - else - live="$(readlink -f "$root/conf/live/le1.wtf/${x}.pem")" - fi + live="$($readlink -f "$root/conf/live/le1.wtf/${x}.pem")" [ "${dir}/${latest}" = "$live" ] # renewer fails this test done From 2737c3299d41cf8dd1a386626846e09316e27bab Mon Sep 17 00:00:00 2001 From: Brandon Kreisel Date: Sun, 4 Oct 2015 20:13:07 -0400 Subject: [PATCH 036/113] That shouldn't be in the try block --- letsencrypt/plugins/manual.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/letsencrypt/plugins/manual.py b/letsencrypt/plugins/manual.py index 179a7b49f5e..f1170eea7a4 100644 --- a/letsencrypt/plugins/manual.py +++ b/letsencrypt/plugins/manual.py @@ -153,13 +153,13 @@ def _perform_single(self, achall): ct=response.CONTENT_TYPE, port=port) if self.conf("test-mode"): logger.debug("Test mode. Executing the manual command: %s", command) + # sh shipped with OS X does't support echo -n + if sys.platform == "darwin": + executable = "/bin/bash" + else: + executable = "/bin/sh" + try: - # sh shipped with OS X does't support echo -n - if sys.platform == "darwin": - executable = "/bin/bash" - else: - executable = "/bin/sh" - self._httpd = subprocess.Popen( command, # don't care about setting stdout and stderr, From 973cd6ce42b837175087f4e2a4af4f978fbe9df2 Mon Sep 17 00:00:00 2001 From: Liam Marshall Date: Mon, 5 Oct 2015 18:19:36 -0500 Subject: [PATCH 037/113] Add instructions for submitting a PR. --- docs/contributing.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index 3959ccee10e..d2bb2ebd5a4 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -267,6 +267,17 @@ Please: .. _PEP 8 - Style Guide for Python Code: https://www.python.org/dev/peps/pep-0008 +Submitting a pull request +========================= +Steps: + +1. Write your code! +2. Make sure your environment is set up properly and that you're in your virtualenv (this is a **very important** step). +3. Run ``./pep8.travis.sh`` to do a cursory check of your code style. Fix any errors. +4. Run ``tox -e lint`` to check for pylint errors. Fix any errors. +5. Run ``tox`` to run the unit tests. Fix any errors. +6. :ref:`Run the integration tests `. +7. Submit the PR. Updating the documentation ========================== From a4e5f298565e9b21c4d551bcd3122a404f3fc729 Mon Sep 17 00:00:00 2001 From: Liam Marshall Date: Mon, 5 Oct 2015 18:25:33 -0500 Subject: [PATCH 038/113] Add link to instructions for running integration tests --- docs/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index d2bb2ebd5a4..a76d76cd89a 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -276,7 +276,7 @@ Steps: 3. Run ``./pep8.travis.sh`` to do a cursory check of your code style. Fix any errors. 4. Run ``tox -e lint`` to check for pylint errors. Fix any errors. 5. Run ``tox`` to run the unit tests. Fix any errors. -6. :ref:`Run the integration tests `. +6. Run the integration tests, see `integration`_. 7. Submit the PR. Updating the documentation From e8118c862b3b9d9dbe34b36a52c1d6990226652e Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Tue, 6 Oct 2015 18:54:05 +0000 Subject: [PATCH 039/113] Renewer logging setup (fixes #897) --- letsencrypt/cli.py | 50 ++++++++++++++++++------------- letsencrypt/renewer.py | 29 +++++++++++++++--- letsencrypt/tests/renewer_test.py | 34 +++++++++++++-------- 3 files changed, 76 insertions(+), 37 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 0bd5f537e4e..66254d2b0b5 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -841,9 +841,23 @@ def _plugins_parsing(helpful, plugins): helpful.add_plugin_args(plugins) -def _setup_logging(args): - level = -args.verbose_count * 10 - fmt = "%(asctime)s:%(levelname)s:%(name)s:%(message)s" +def setup_log_file_handler(args, logfile, fmt): + """Setup file debug logging.""" + log_file_path = os.path.join(args.logs_dir, logfile) + handler = logging.handlers.RotatingFileHandler( + log_file_path, maxBytes=2 ** 20, backupCount=10) + # rotate on each invocation, rollover only possible when maxBytes + # is nonzero and backupCount is nonzero, so we set maxBytes as big + # as possible not to overrun in single CLI invocation (1MB). + handler.doRollover() # TODO: creates empty letsencrypt.log.1 file + handler.setLevel(logging.DEBUG) + handler_formatter = logging.Formatter(fmt=fmt) + handler_formatter.converter = time.gmtime # don't use localtime + handler.setFormatter(handler_formatter) + return handler, log_file_path + + +def _cli_log_handler(args, level, fmt): if args.text_mode: handler = colored_logging.StreamHandler() handler.setFormatter(logging.Formatter(fmt)) @@ -852,30 +866,26 @@ def _setup_logging(args): # dialog box is small, display as less as possible handler.setFormatter(logging.Formatter("%(message)s")) handler.setLevel(level) + return handler - # TODO: use fileConfig? - # unconditionally log to file for debugging purposes - # TODO: change before release? - log_file_name = os.path.join(args.logs_dir, 'letsencrypt.log') - file_handler = logging.handlers.RotatingFileHandler( - log_file_name, maxBytes=2 ** 20, backupCount=10) - # rotate on each invocation, rollover only possible when maxBytes - # is nonzero and backupCount is nonzero, so we set maxBytes as big - # as possible not to overrun in single CLI invocation (1MB). - file_handler.doRollover() # TODO: creates empty letsencrypt.log.1 file - file_handler.setLevel(logging.DEBUG) - file_handler_formatter = logging.Formatter(fmt=fmt) - file_handler_formatter.converter = time.gmtime # don't use localtime - file_handler.setFormatter(file_handler_formatter) +def setup_logging(args, cli_handler_factory, logfile): + """Setup logging.""" + fmt = "%(asctime)s:%(levelname)s:%(name)s:%(message)s" + level = -args.verbose_count * 10 + file_handler, log_file_path = setup_log_file_handler( + args, logfile=logfile, fmt=fmt) + cli_handler = cli_handler_factory(args, level, fmt) + + # TODO: use fileConfig? root_logger = logging.getLogger() root_logger.setLevel(logging.DEBUG) # send all records to handlers - root_logger.addHandler(handler) + root_logger.addHandler(cli_handler) root_logger.addHandler(file_handler) logger.debug("Root logging level set at %d", level) - logger.info("Saving debug log to %s", log_file_name) + logger.info("Saving debug log to %s", log_file_path) def _handle_exception(exc_type, exc_value, trace, args): @@ -944,7 +954,7 @@ def main(cli_args=sys.argv[1:]): # private key! #525 le_util.make_or_verify_dir( args.logs_dir, 0o700, os.geteuid(), "--strict-permissions" in cli_args) - _setup_logging(args) + setup_logging(args, _cli_log_handler, logfile='letsencrypt.log') # do not log `args`, as it contains sensitive data (e.g. revoke --key)! logger.debug("Arguments: %r", cli_args) diff --git a/letsencrypt/renewer.py b/letsencrypt/renewer.py index 1c9cddc95ab..98ecc83b3e9 100644 --- a/letsencrypt/renewer.py +++ b/letsencrypt/renewer.py @@ -8,6 +8,7 @@ """ import argparse +import logging import os import sys @@ -17,10 +18,12 @@ from letsencrypt import account from letsencrypt import configuration +from letsencrypt import colored_logging from letsencrypt import cli from letsencrypt import client from letsencrypt import crypto_util from letsencrypt import errors +from letsencrypt import le_util from letsencrypt import notify from letsencrypt import storage @@ -28,6 +31,9 @@ from letsencrypt.plugins import disco as plugins_disco +logger = logging.getLogger(__name__) + + class _AttrDict(dict): """Attribute dictionary. @@ -104,6 +110,12 @@ def renew(cert, old_version): # (where fewer than all names were renewed) +def _cli_log_handler(args, level, fmt): # pylint: disable=unused-argument + handler = colored_logging.StreamHandler() + handler.setFormatter(logging.Formatter(fmt)) + return handler + + def _paths_parser(parser): add = parser.add_argument_group("paths").add_argument add("--config-dir", default=cli.flag_default("config_dir"), @@ -119,11 +131,16 @@ def _paths_parser(parser): def _create_parser(): parser = argparse.ArgumentParser() #parser.add_argument("--cron", action="store_true", help="Run as cronjob.") - # pylint: disable=protected-access + parser.add_argument( + "-v", "--verbose", dest="verbose_count", action="count", + default=cli.flag_default("verbose_count"), help="This flag can be used " + "multiple times to incrementally increase the verbosity of output, " + "e.g. -vvv.") + return _paths_parser(parser) -def main(config=None, args=sys.argv[1:]): +def main(config=None, cli_args=sys.argv[1:]): """Main function for autorenewer script.""" # TODO: Distinguish automated invocation from manual invocation, # perhaps by looking at sys.argv[0] and inhibiting automated @@ -133,8 +150,12 @@ def main(config=None, args=sys.argv[1:]): zope.component.provideUtility(display_util.FileDisplay(sys.stdout)) - cli_config = configuration.RenewerConfiguration( - _create_parser().parse_args(args)) + args = _create_parser().parse_args(cli_args) + + le_util.make_or_verify_dir(args.logs_dir, 0o700, os.geteuid()) + cli.setup_logging(args, _cli_log_handler, logfile='renewer.log') + + cli_config = configuration.RenewerConfiguration(args) config = storage.config_with_defaults(config) # Now attempt to read the renewer config file and augment or replace diff --git a/letsencrypt/tests/renewer_test.py b/letsencrypt/tests/renewer_test.py index 6f115abf9b7..bc8afbcb58d 100644 --- a/letsencrypt/tests/renewer_test.py +++ b/letsencrypt/tests/renewer_test.py @@ -44,8 +44,15 @@ def setUp(self): self.tempdir = tempfile.mkdtemp() self.cli_config = configuration.RenewerConfiguration( - namespace=mock.MagicMock(config_dir=self.tempdir)) + namespace=mock.MagicMock( + config_dir=self.tempdir, + work_dir=self.tempdir, + logs_dir=self.tempdir, + ) + ) + # TODO: maybe provide RenewerConfiguration.make_dirs? + # TODO: main() should create those dirs, c.f. #902 os.makedirs(os.path.join(self.tempdir, "live", "example.org")) os.makedirs(os.path.join(self.tempdir, "archive", "example.org")) os.makedirs(os.path.join(self.tempdir, "renewal")) @@ -62,6 +69,9 @@ def setUp(self): self.test_rc = storage.RenewableCert( self.config, self.defaults, self.cli_config) + def tearDown(self): + shutil.rmtree(self.tempdir) + def _write_out_ex_kinds(self): for kind in ALL_FOUR: where = getattr(self.test_rc, kind) @@ -79,11 +89,6 @@ def _write_out_ex_kinds(self): class RenewableCertTests(BaseRenewableCertTest): # pylint: disable=too-many-public-methods """Tests for letsencrypt.renewer.*.""" - def setUp(self): - super(RenewableCertTests, self).setUp() - - def tearDown(self): - shutil.rmtree(self.tempdir) def test_initialization(self): self.assertEqual(self.test_rc.lineagename, "example.org") @@ -665,11 +670,17 @@ def test_renew(self, mock_c, mock_acc_storage, mock_pd): # This should fail because the renewal itself appears to fail self.assertFalse(renewer.renew(self.test_rc, 1)) + def _common_cli_args(self): + return [ + "--config-dir", self.cli_config.config_dir, + "--work-dir", self.cli_config.work_dir, + "--logs-dir", self.cli_config.logs_dir, + ] + @mock.patch("letsencrypt.renewer.notify") @mock.patch("letsencrypt.storage.RenewableCert") @mock.patch("letsencrypt.renewer.renew") def test_main(self, mock_renew, mock_rc, mock_notify): - """Test for main() function.""" from letsencrypt import renewer mock_rc_instance = mock.MagicMock() mock_rc_instance.should_autodeploy.return_value = True @@ -691,8 +702,7 @@ def test_main(self, mock_renew, mock_rc, mock_notify): "example.com.conf"), "w") as f: f.write("cert = cert.pem\nprivkey = privkey.pem\n") f.write("chain = chain.pem\nfullchain = fullchain.pem\n") - renewer.main(self.defaults, args=[ - '--config-dir', self.cli_config.config_dir]) + renewer.main(self.defaults, cli_args=self._common_cli_args()) self.assertEqual(mock_rc.call_count, 2) self.assertEqual(mock_rc_instance.update_all_links_to.call_count, 2) self.assertEqual(mock_notify.notify.call_count, 4) @@ -705,8 +715,7 @@ def test_main(self, mock_renew, mock_rc, mock_notify): mock_happy_instance.should_autorenew.return_value = False mock_happy_instance.latest_common_version.return_value = 10 mock_rc.return_value = mock_happy_instance - renewer.main(self.defaults, args=[ - '--config-dir', self.cli_config.config_dir]) + renewer.main(self.defaults, cli_args=self._common_cli_args()) self.assertEqual(mock_rc.call_count, 4) self.assertEqual(mock_happy_instance.update_all_links_to.call_count, 0) self.assertEqual(mock_notify.notify.call_count, 4) @@ -717,8 +726,7 @@ def test_bad_config_file(self): with open(os.path.join(self.cli_config.renewal_configs_dir, "bad.conf"), "w") as f: f.write("incomplete = configfile\n") - renewer.main(self.defaults, args=[ - '--config-dir', self.cli_config.config_dir]) + renewer.main(self.defaults, cli_args=self._common_cli_args()) # The errors.CertStorageError is caught inside and nothing happens. From f0c11152d2eb39e67e4f8a5f43fffa9140e72bd0 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Wed, 7 Oct 2015 19:00:47 +0000 Subject: [PATCH 040/113] ACMEServerMixin.__init__ --- acme/acme/standalone.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index ce10407e69d..d4d6f0ea42f 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -53,16 +53,13 @@ def server_bind(self): class ACMEServerMixin: # pylint: disable=old-style-class,no-init - """ACME server common settings mixin. - - .. warning:: - Subclasses have to init ``_stopped = False`` (it's not done here, - because of old-style classes madness). - - """ + """ACME server common settings mixin.""" server_version = "ACME standalone client" allow_reuse_address = True + def __init__(self): + self._stopped = False + def serve_forever2(self): """Serve forever, until other thread calls `shutdown2`.""" while not self._stopped: @@ -95,7 +92,7 @@ class ACMETLSServer(HTTPSServer, ACMEServerMixin): """ def __init__(self, *args, **kwargs): - self._stopped = False + ACMEServerMixin.__init__(self) HTTPSServer.__init__(self, *args, **kwargs) @@ -103,7 +100,7 @@ class ACMEServer(BaseHTTPServer.HTTPServer, ACMEServerMixin): """ACME Server (non-TLS).""" def __init__(self, *args, **kwargs): - self._stopped = False + ACMEServerMixin.__init__(self) BaseHTTPServer.HTTPServer.__init__(self, *args, **kwargs) From 7102f9ef4b1bc4eaaa53e4e0eb03e98afe21ee36 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Wed, 7 Oct 2015 19:20:47 +0000 Subject: [PATCH 041/113] Don't expose threads from ServerManager. --- letsencrypt/plugins/standalone.py | 36 ++++++++++++++------------ letsencrypt/plugins/standalone_test.py | 23 ++++++++-------- 2 files changed, 30 insertions(+), 29 deletions(-) diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index 05a9f165574..551707263cd 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -39,8 +39,10 @@ class ServerManager(object): will serve the same URLs! """ + _Instance = collections.namedtuple("_Instance", "server thread") + def __init__(self, certs, simple_http_resources): - self._servers = {} + self._instances = {} self.certs = certs self.simple_http_resources = simple_http_resources @@ -53,13 +55,12 @@ def run(self, port, tls): :param int port: Port to run the server on. :param bool tls: TLS or non-TLS? - :returns: Server instance (`ACMEServerMixin`) and the - corresponding (already started) thread (`threading.Thread`). - :rtype: tuple + :returns: Server instance. + :rtype: ACMEServerMixin """ - if port in self._servers: - return self._servers[port] + if port in self._instances: + return self._instances[port].server logger.debug("Starting new server at %s (tls=%s)", port, tls) handler = acme_standalone.ACMERequestHandler.partial_init( @@ -84,8 +85,8 @@ def run(self, port, tls): logger.debug("Starting server at %s:%d", host, real_port) thread.start() - self._servers[real_port] = (server, thread) - return self._servers[real_port] + self._instances[real_port] = self._Instance(server, thread) + return server def stop(self, port): """Stop ACME server running on the specified ``port``. @@ -93,10 +94,10 @@ def stop(self, port): :param int port: """ - server, thread = self._servers[port] - server.shutdown2() - thread.join() - del self._servers[port] + instance = self._instances[port] + instance.server.shutdown2() + instance.thread.join() + del self._instances[port] def running(self): """Return all running instances. @@ -104,11 +105,12 @@ def running(self): Once the server is stopped using `stop`, it will not be returned. - :returns: Mapping from port to ``(server, thread)``. + :returns: Mapping from ``port`` to ``server``. :rtype: tuple """ - return self._servers.copy() + return dict((port, instance.server) for port, instance + in six.iteritems(self._instances)) SUPPORTED_CHALLENGES = set([challenges.DVSNI, challenges.SimpleHTTP]) @@ -233,7 +235,7 @@ def perform2(self, achalls): for achall in achalls: if isinstance(achall, achallenges.SimpleHTTP): - server, _ = self.servers.run(self.config.simple_http_port, tls=tls) + server = self.servers.run(self.config.simple_http_port, tls=tls) response, validation = achall.gen_response_and_validation(tls=tls) self.simple_http_resources.add( acme_standalone.SimpleHTTPRequestHandler.SimpleHTTPResource( @@ -242,7 +244,7 @@ def perform2(self, achalls): cert = self.simple_http_cert domain = achall.domain else: # DVSNI - server, _ = self.servers.run(self.config.dvsni_port, tls=True) + server = self.servers.run(self.config.dvsni_port, tls=True) response, cert, _ = achall.gen_cert_and_response(self.key) domain = response.z_domain self.certs[domain] = (self.key, cert) @@ -257,6 +259,6 @@ def cleanup(self, achalls): # pylint: disable=missing-docstring for achall in achalls: if achall in server_achalls: server_achalls.remove(achall) - for port, (server, _) in six.iteritems(self.servers.running()): + for port, server in six.iteritems(self.servers.running()): if not self.served[server]: self.servers.stop(port) diff --git a/letsencrypt/plugins/standalone_test.py b/letsencrypt/plugins/standalone_test.py index 1cd31010701..a45935d2b08 100644 --- a/letsencrypt/plugins/standalone_test.py +++ b/letsencrypt/plugins/standalone_test.py @@ -33,9 +33,9 @@ def test_init(self): self.mgr.simple_http_resources is self.simple_http_resources) def _test_run_stop(self, tls): - server, _ = self.mgr.run(port=0, tls=tls) - port = server.socket.getsockname()[1] - self.assertEqual(self.mgr.running(), {port: (server, mock.ANY)}) + server = self.mgr.run(port=0, tls=tls) + port = server.socket.getsockname()[1] # pylint: disable=no-member + self.assertEqual(self.mgr.running(), {port: server}) self.mgr.stop(port=port) self.assertEqual(self.mgr.running(), {}) @@ -46,12 +46,11 @@ def test_run_stop_non_tls(self): self._test_run_stop(tls=False) def test_run_idempotent(self): - server, thread = self.mgr.run(port=0, tls=False) - port = server.socket.getsockname()[1] - server2, thread2 = self.mgr.run(port=port, tls=False) - self.assertEqual(self.mgr.running(), {port: (server, thread)}) + server = self.mgr.run(port=0, tls=False) + port = server.socket.getsockname()[1] # pylint: disable=no-member + server2 = self.mgr.run(port=port, tls=False) + self.assertEqual(self.mgr.running(), {port: server}) self.assertTrue(server is server2) - self.assertTrue(thread is thread2) self.mgr.stop(port) self.assertEqual(self.mgr.running(), {}) @@ -166,7 +165,7 @@ def test_perform2(self): self.auth.servers = mock.MagicMock() def _run(port, tls): # pylint: disable=unused-argument - return "server{0}".format(port), "thread{0}".format(port) + return "server{0}".format(port) self.auth.servers.run.side_effect = _run responses = self.auth.perform2([simple_http, dvsni]) @@ -191,8 +190,8 @@ def _run(port, tls): # pylint: disable=unused-argument def test_cleanup(self): self.auth.servers = mock.Mock() self.auth.servers.running.return_value = { - 1: ("server1", "thread1"), - 2: ("server2", "thread2"), + 1: "server1", + 2: "server2", } self.auth.served["server1"].add("chall1") self.auth.served["server2"].update(["chall2", "chall3"]) @@ -203,7 +202,7 @@ def test_cleanup(self): self.auth.servers.stop.assert_called_once_with(1) self.auth.servers.running.return_value = { - 2: ("server2", "thread2"), + 2: "server2", } self.auth.cleanup(["chall2"]) self.assertEqual(self.auth.served, { From 5afb0ebd1c16442a2f94469ee3db4f27fc3a9852 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Wed, 7 Oct 2015 19:42:32 +0000 Subject: [PATCH 042/113] Remove SimpleHTTP TLS from standalone 2.0 --- letsencrypt/configuration.py | 7 ++----- letsencrypt/plugins/standalone.py | 15 +++++---------- letsencrypt/plugins/standalone_test.py | 15 ++------------- letsencrypt/tests/configuration_test.py | 5 ----- 4 files changed, 9 insertions(+), 33 deletions(-) diff --git a/letsencrypt/configuration.py b/letsencrypt/configuration.py index 5f965cb6d7c..a2eab5ecb73 100644 --- a/letsencrypt/configuration.py +++ b/letsencrypt/configuration.py @@ -37,8 +37,7 @@ def __init__(self, namespace): self.namespace = namespace # XXX: breaks renewer in some bizarre way - #if self.no_simple_http_tls and ( - # self.simple_http_port == self.dvsni_port): + #if self.simple_http_port == self.dvsni_port: # raise errors.Error( # "Trying to run SimpleHTTP non-TLS and DVSNI " # "on the same port ({0})".format(self.dvsni_port)) @@ -82,10 +81,8 @@ def temp_checkpoint_dir(self): # pylint: disable=missing-docstring def simple_http_port(self): # pylint: disable=missing-docstring if self.namespace.simple_http_port is not None: return self.namespace.simple_http_port - if self.no_simple_http_tls: - return challenges.SimpleHTTPResponse.PORT else: - return challenges.SimpleHTTPResponse.TLS_PORT + return challenges.SimpleHTTPResponse.PORT class RenewerConfiguration(object): diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index 551707263cd..97964e8cce6 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -5,7 +5,6 @@ import logging import random import socket -import sys import threading import OpenSSL @@ -198,12 +197,7 @@ def prepare(self): # pylint: disable=missing-docstring def get_chall_pref(self, domain): # pylint: disable=unused-argument,missing-docstring - supported_challenges = self.supported_challenges - if not self.config.no_simple_http_tls and not ( - acme_standalone.ACMETLSServer.SIMPLE_HTTP_SUPPORT): - logger.debug("SimpleHTTPS not supported: %s", sys.version) - supported_challenges.discard(challenges.SimpleHTTP) - chall_pref = list(supported_challenges) + chall_pref = list(self.supported_challenges) random.shuffle(chall_pref) # 50% for each challenge return chall_pref @@ -231,12 +225,13 @@ def perform(self, achalls): # pylint: disable=missing-docstring def perform2(self, achalls): """Perform achallenges without IDisplay interaction.""" responses = [] - tls = not self.config.no_simple_http_tls for achall in achalls: if isinstance(achall, achallenges.SimpleHTTP): - server = self.servers.run(self.config.simple_http_port, tls=tls) - response, validation = achall.gen_response_and_validation(tls=tls) + server = self.servers.run( + self.config.simple_http_port, tls=False) + response, validation = achall.gen_response_and_validation( + tls=False) self.simple_http_resources.add( acme_standalone.SimpleHTTPRequestHandler.SimpleHTTPResource( chall=achall.chall, response=response, diff --git a/letsencrypt/plugins/standalone_test.py b/letsencrypt/plugins/standalone_test.py index a45935d2b08..e99bd473a7f 100644 --- a/letsencrypt/plugins/standalone_test.py +++ b/letsencrypt/plugins/standalone_test.py @@ -107,20 +107,9 @@ def test_prepare_misconfiguration(self, mock_util): self.assertRaises(errors.MisconfigurationError, self.auth.prepare) mock_util.already_listening.assert_called_once_with(1234) - @mock.patch("letsencrypt.plugins.standalone.acme_standalone") - def test_get_chall_pref_tls_supported(self, mock_astandalone): - mock_astandalone.ACMETLSServer.SIMPLE_HTTP_SUPPORT = True - for no_simple_http_tls in True, False: - self.config.no_simple_http_tls = no_simple_http_tls - self.assertEqual(set(self.auth.get_chall_pref(domain=None)), - set([challenges.DVSNI, challenges.SimpleHTTP])) - - @mock.patch("letsencrypt.plugins.standalone.acme_standalone") - def test_get_chall_pref_simple_tls_not_supported(self, mock_astandalone): - mock_astandalone.ACMETLSServer.SIMPLE_HTTP_SUPPORT = False - self.config.no_simple_http_tls = False + def test_get_chall_pref(self): self.assertEqual(set(self.auth.get_chall_pref(domain=None)), - set([challenges.DVSNI])) + set([challenges.DVSNI, challenges.SimpleHTTP])) @mock.patch("letsencrypt.plugins.standalone.zope.component.getUtility") def test_perform(self, unused_mock_get_utility): diff --git a/letsencrypt/tests/configuration_test.py b/letsencrypt/tests/configuration_test.py index acf9273d01e..c1eba85703d 100644 --- a/letsencrypt/tests/configuration_test.py +++ b/letsencrypt/tests/configuration_test.py @@ -49,14 +49,9 @@ def test_dynamic_dirs(self, constants): def test_simple_http_port(self): self.assertEqual(4321, self.config.simple_http_port) - self.namespace.simple_http_port = None - self.namespace.no_simple_http_tls = True self.assertEqual(80, self.config.simple_http_port) - self.namespace.no_simple_http_tls = False - self.assertEqual(443, self.config.simple_http_port) - class RenewerConfigurationTest(unittest.TestCase): """Test for letsencrypt.configuration.RenewerConfiguration.""" From 744afe9cea0be2fbb5f8c155e65060d4277de524 Mon Sep 17 00:00:00 2001 From: Brandon Kreisel Date: Thu, 8 Oct 2015 16:15:09 -0400 Subject: [PATCH 043/113] PEP8 E128 up in here. Don't assume sh exists --- letsencrypt/plugins/manual.py | 2 +- letsencrypt/plugins/standalone/authenticator.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/letsencrypt/plugins/manual.py b/letsencrypt/plugins/manual.py index f19861d9d70..65d7d687687 100644 --- a/letsencrypt/plugins/manual.py +++ b/letsencrypt/plugins/manual.py @@ -133,7 +133,7 @@ def _perform_single(self, achall): if sys.platform == "darwin": executable = "/bin/bash" else: - executable = "/bin/sh" + executable = None try: self._httpd = subprocess.Popen( diff --git a/letsencrypt/plugins/standalone/authenticator.py b/letsencrypt/plugins/standalone/authenticator.py index cdf9a6f0444..a9972fba2b0 100644 --- a/letsencrypt/plugins/standalone/authenticator.py +++ b/letsencrypt/plugins/standalone/authenticator.py @@ -309,7 +309,7 @@ def already_listening(self, port): # pylint: disable=no-self-use net_connections = psutil.net_connections() except psutil.AccessDenied as error: logger.info("Access denied when trying to list network " - "connections: %s. Are you root?", error) + "connections: %s. Are you root?", error) # this function is just a pre-check that often causes false # positives and problems in testing (c.f. #680 on Mac, #255 # generally); we will fail later in bind() anyway From 304414a214730593e11520affcd99684f1bddd74 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Thu, 8 Oct 2015 21:10:12 +0000 Subject: [PATCH 044/113] Remove SimpleHTTP TLS from acme. --- acme/acme/crypto_util.py | 31 --------- acme/acme/standalone.py | 69 ++++++-------------- acme/acme/standalone_test.py | 104 ++++++++++++++++++------------ letsencrypt/plugins/standalone.py | 19 ++---- 4 files changed, 87 insertions(+), 136 deletions(-) diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index 5829a511f22..5f24e9d9e31 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -70,20 +70,10 @@ def _pick_certificate_cb(self, connection): class FakeConnection(object): """Fake OpenSSL.SSL.Connection.""" - MAKEFILE_SUPPORT = hasattr(socket, "_fileobject") - """Is `makefile` supported on your platform? - - .. warning:: `makefile`, as currently implemented, is supported - on select platforms only, as it uses CPython's internal API. - You've been warned! - - """ - # pylint: disable=missing-docstring def __init__(self, connection): self._wrapped = connection - self._makefile_refs = 0 def __getattr__(self, name): return getattr(self._wrapped, name) @@ -92,27 +82,6 @@ def shutdown(self, *unused_args): # OpenSSL.SSL.Connection.shutdown doesn't accept any args return self._wrapped.shutdown() - # stuff below ripped off from - # https://hg.python.org/cpython/file/2.7/Lib/ssl.py - - def makefile(self, mode='r', bufsize=-1): - assert self.MAKEFILE_SUPPORT, ( - "You need compatible version for makefile support") - self._makefile_refs += 1 - # SocketServer.StreamRequesthandler.finish will try to - # close the wfile/rfile. close=True causes curl: (56) - # GnuTLS recv error (-110): The TLS connection was - # non-properly terminated. - # TODO: doesn't work in Python3 - # pylint: disable=protected-access - return socket._fileobject(self._wrapped, mode, bufsize, close=False) - - def close(self): - if self._makefile_refs < 1: - self._wrapped.close() - else: - self._makefile_refs -= 1 - def accept(self): # pylint: disable=missing-docstring sock, addr = self.sock.accept() diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index d4d6f0ea42f..089c2ff18c3 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -44,15 +44,7 @@ def server_bind(self): # pylint: disable=missing-docstring return socketserver.TCPServer.server_bind(self) -class HTTPSServer(TLSServer, BaseHTTPServer.HTTPServer): - """HTTPS Server.""" - - def server_bind(self): - self._wrap_sock() - BaseHTTPServer.HTTPServer.server_bind(self) - - -class ACMEServerMixin: # pylint: disable=old-style-class,no-init +class ACMEServerMixin: # pylint: disable=old-style-class """ACME server common settings mixin.""" server_version = "ACME standalone client" allow_reuse_address = True @@ -81,27 +73,23 @@ def shutdown2(self): self.server_close() -class ACMETLSServer(HTTPSServer, ACMEServerMixin): - """ACME TLS Server.""" - - SIMPLE_HTTP_SUPPORT = crypto_util.SSLSocket.FakeConnection.MAKEFILE_SUPPORT - """Is SimpleHTTP supported on your platform. +class DVSNIServer(TLSServer, ACMEServerMixin): + """DVSNI Server.""" - Please see a warning for `acme.crypto_util.SSLSocket.FakeConnection`. - - """ - - def __init__(self, *args, **kwargs): + def __init__(self, server_address, certs): ACMEServerMixin.__init__(self) - HTTPSServer.__init__(self, *args, **kwargs) + TLSServer.__init__( + self, server_address, socketserver.BaseRequestHandler, certs=certs) -class ACMEServer(BaseHTTPServer.HTTPServer, ACMEServerMixin): - """ACME Server (non-TLS).""" +class SimpleHTTPServer(BaseHTTPServer.HTTPServer, ACMEServerMixin): + """SimpleHTTP Server.""" - def __init__(self, *args, **kwargs): + def __init__(self, server_address, resources): ACMEServerMixin.__init__(self) - BaseHTTPServer.HTTPServer.__init__(self, *args, **kwargs) + BaseHTTPServer.HTTPServer.__init__( + self, server_address, SimpleHTTPRequestHandler.partial_init( + simple_http_resources=resources)) class SimpleHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): @@ -133,14 +121,14 @@ def handle_index(self): self.send_response(200) self.send_header("Content-type", "text/html") self.end_headers() - self.wfile.write(self.server.server_version) + self.wfile.write(self.server.server_version.encode()) def handle_404(self): """Handler 404 Not Found errors.""" self.send_response(http_client.NOT_FOUND, message="Not Found") self.send_header("Content-type", "text/html") self.end_headers() - self.wfile.write("404") + self.wfile.write(b"404") def handle_simple_http_resource(self): """Handle SimpleHTTP provisioned resources.""" @@ -171,24 +159,8 @@ def partial_init(cls, simple_http_resources): cls, simple_http_resources=simple_http_resources) -class ACMERequestHandler(SimpleHTTPRequestHandler): - """ACME request handler.""" - - def handle_one_request(self): - """Handle single request. - - Makes sure that DVSNI probers are ignored. - - """ - try: - return SimpleHTTPRequestHandler.handle_one_request(self) - except OpenSSL.SSL.ZeroReturnError: - logger.debug("Client prematurely closed connection (prober?). " - "Ignoring request.") - - -def simple_server(cli_args, forever=True): - """Run simple standalone client server.""" +def simple_dvsni_server(cli_args, forever=True): + """Run simple standalone DVSNI server.""" logging.basicConfig(level=logging.DEBUG) parser = argparse.ArgumentParser() @@ -198,7 +170,6 @@ def simple_server(cli_args, forever=True): args = parser.parse_args(cli_args[1:]) certs = {} - resources = {} _, hosts, _ = next(os.walk('.')) for host in hosts: @@ -206,15 +177,13 @@ def simple_server(cli_args, forever=True): cert_contents = cert_file.read() with open(os.path.join(host, "key.pem")) as key_file: key_contents = key_file.read() - certs[host] = ( + certs[host.encode()] = ( OpenSSL.crypto.load_privatekey( OpenSSL.crypto.FILETYPE_PEM, key_contents), OpenSSL.crypto.load_certificate( OpenSSL.crypto.FILETYPE_PEM, cert_contents)) - handler = ACMERequestHandler.partial_init( - simple_http_resources=resources) - server = ACMETLSServer(('', int(args.port)), handler, certs=certs) + server = DVSNIServer(('', int(args.port)), certs=certs) six.print_("Serving at https://localhost:{0}...".format( server.socket.getsockname()[1])) if forever: # pragma: no cover @@ -224,4 +193,4 @@ def simple_server(cli_args, forever=True): if __name__ == "__main__": - sys.exit(simple_server(sys.argv)) # pragma: no cover + sys.exit(simple_dvsni_server(sys.argv)) # pragma: no cover diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py index a6f7502b636..9eb192c7489 100644 --- a/acme/acme/standalone_test.py +++ b/acme/acme/standalone_test.py @@ -13,6 +13,7 @@ from acme import challenges from acme import crypto_util +from acme import errors from acme import jose from acme import test_util @@ -30,25 +31,27 @@ def test_bind(self): # pylint: disable=no-self-use class ACMEServerMixinTest(unittest.TestCase): """Tests for acme.standalone.ACMEServerMixin.""" - def test_shutdown2_not_running(self): - from acme.standalone import ACMEServer - server = ACMEServer(("", 0), socketserver.BaseRequestHandler) - server.shutdown2() - server.shutdown2() + def setUp(self): + from acme.standalone import ACMEServerMixin + class _MockServer(socketserver.TCPServer, ACMEServerMixin): + def __init__(self, *args, **kwargs): + socketserver.TCPServer.__init__(self, *args, **kwargs) + ACMEServerMixin.__init__(self) + self.server = _MockServer(("", 0), socketserver.BaseRequestHandler) -class ACMEServerTest(unittest.TestCase): - """Test for acme.standalone.ACMEServer.""" + def test_serve_shutdown(self): + thread = threading.Thread(target=self.server.serve_forever2) + thread.start() + self.server.shutdown2() - def test_init(self): - from acme.standalone import ACMEServer - server = ACMEServer(("", 0), socketserver.BaseRequestHandler) - # pylint: disable=protected-access - self.assertFalse(server._stopped) + def test_shutdown2_not_running(self): + self.server.shutdown2() + self.server.shutdown2() -class ACMESimpleHTTPTLSServerTestEndToEnd(unittest.TestCase): - """End-to-end test for ACME TLS server with SimpleHTTP.""" +class DVSNIServerTest(unittest.TestCase): + """Test for acme.standalone.DVSNIServer.""" def setUp(self): self.certs = { @@ -56,46 +59,61 @@ def setUp(self): # pylint: disable=protected-access test_util.load_cert('cert.pem')._wrapped), } + from acme.standalone import DVSNIServer + self.server = DVSNIServer(("", 0), certs=self.certs) + # pylint: disable=no-member + self.thread = threading.Thread(target=self.server.handle_request) + self.thread.start() + + def tearDown(self): + self.server.shutdown2() + self.thread.join() + + def test_init(self): + # pylint: disable=protected-access + self.assertFalse(self.server._stopped) + + def test_dvsni(self): + cert = crypto_util.probe_sni( + b'localhost', *self.server.socket.getsockname()) + self.assertEqual(jose.ComparableX509(cert), + jose.ComparableX509(self.certs[b'localhost'][1])) + + +class SimpleHTTPServerTest(unittest.TestCase): + """Tests for acme.standalone.SimpleHTTPServer.""" + + def setUp(self): self.account_key = jose.JWK.load( test_util.load_vector('rsa1024_key.pem')) - - from acme.standalone import ACMETLSServer - from acme.standalone import ACMERequestHandler self.resources = set() - handler = ACMERequestHandler.partial_init( - simple_http_resources=self.resources) - self.server = ACMETLSServer(('', 0), handler, certs=self.certs) - self.server_thread = threading.Thread( - # pylint: disable=no-member - target=self.server.serve_forever2) - self.server_thread.start() + from acme.standalone import SimpleHTTPServer + self.server = SimpleHTTPServer(('', 0), resources=self.resources) + + # pylint: disable=no-member self.port = self.server.socket.getsockname()[1] + self.thread = threading.Thread(target=self.server.handle_request) + self.thread.start() def tearDown(self): self.server.shutdown2() - self.server_thread.join() + self.thread.join() def test_index(self): response = requests.get( - 'https://localhost:{0}'.format(self.port), verify=False) + 'http://localhost:{0}'.format(self.port), verify=False) self.assertEqual(response.text, 'ACME standalone client') self.assertTrue(response.ok) def test_404(self): response = requests.get( - 'https://localhost:{0}/foo'.format(self.port), verify=False) + 'http://localhost:{0}/foo'.format(self.port), verify=False) self.assertEqual(response.status_code, http_client.NOT_FOUND) - def test_dvsni(self): - cert = crypto_util.probe_sni( - b'localhost', *self.server.socket.getsockname()) - self.assertEqual(jose.ComparableX509(cert), - jose.ComparableX509(self.certs[b'localhost'][1])) - def _test_simple_http(self, add): chall = challenges.SimpleHTTP(token=(b'x' * 16)) - response = challenges.SimpleHTTPResponse(tls=True) + response = challenges.SimpleHTTPResponse(tls=False) from acme.standalone import SimpleHTTPRequestHandler resource = SimpleHTTPRequestHandler.SimpleHTTPResource( @@ -114,8 +132,8 @@ def test_simple_http_not_found(self): self.assertFalse(self._test_simple_http(add=False)) -class TestSimpleServer(unittest.TestCase): - """Tests for acme.standalone.simple_server.""" +class TestSimpleDVSNIServer(unittest.TestCase): + """Tests for acme.standalone.simple_dvsni_server.""" def setUp(self): # mirror ../examples/standalone @@ -126,9 +144,10 @@ def setUp(self): shutil.copy(test_util.vector_path('rsa512_key.pem'), os.path.join(localhost_dir, 'key.pem')) - from acme.standalone import simple_server - self.thread = threading.Thread(target=simple_server, kwargs={ - 'cli_args': ('xxx', '--port', '1234'), + from acme.standalone import simple_dvsni_server + self.port = 1234 + self.thread = threading.Thread(target=simple_dvsni_server, kwargs={ + 'cli_args': ('xxx', '--port', str(self.port)), 'forever': False, }) self.old_cwd = os.getcwd() @@ -145,12 +164,13 @@ def test_it(self): while max_attempts: max_attempts -= 1 try: - response = requests.get('https://localhost:1234', verify=False) - except requests.ConnectionError: + cert = crypto_util.probe_sni(b'localhost', b'0.0.0.0', self.port) + except errors.Error: self.assertTrue(max_attempts > 0, "Timeout!") time.sleep(1) # wait until thread starts else: - self.assertEqual(response.text, 'ACME standalone client') + self.assertEqual(jose.ComparableX509(cert), + test_util.load_cert('cert.pem')) break diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index 97964e8cce6..cb95ec408bf 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -1,7 +1,6 @@ """Standalone Authenticator.""" import argparse import collections -import functools import logging import random import socket @@ -61,25 +60,19 @@ def run(self, port, tls): if port in self._instances: return self._instances[port].server - logger.debug("Starting new server at %s (tls=%s)", port, tls) - handler = acme_standalone.ACMERequestHandler.partial_init( - self.simple_http_resources) - - if tls: - cls = functools.partial( - acme_standalone.ACMETLSServer, certs=self.certs) - else: - cls = acme_standalone.ACMEServer - + address = ("", port) try: - server = cls(("", port), handler) + if tls: + server = acme_standalone.DVSNIServer(address, self.certs) + else: + server = acme_standalone.SimpleHTTPServer( + address, self.simple_http_resources) except socket.error as error: raise errors.StandaloneBindError(error, port) # if port == 0, then random free port on OS is taken # pylint: disable=no-member host, real_port = server.socket.getsockname() - thread = threading.Thread(target=server.serve_forever2) logger.debug("Starting server at %s:%d", host, real_port) thread.start() From 454a661d444dfc3e97db826159edc2b35ee820e7 Mon Sep 17 00:00:00 2001 From: Liam Marshall Date: Fri, 9 Oct 2015 15:46:03 -0500 Subject: [PATCH 045/113] contributing.rst: fix nits pointed out by @kuba --- docs/contributing.rst | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index a76d76cd89a..b4b1619cefb 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -269,14 +269,19 @@ Please: Submitting a pull request ========================= + Steps: 1. Write your code! -2. Make sure your environment is set up properly and that you're in your virtualenv (this is a **very important** step). -3. Run ``./pep8.travis.sh`` to do a cursory check of your code style. Fix any errors. +2. Make sure your environment is set up properly and that you're in your + virtualenv. You can do this by running ``./bootstrap/dev/venv.sh``. + (this is a **very important** step) +3. Run ``./pep8.travis.sh`` to do a cursory check of your code style. + Fix any errors. 4. Run ``tox -e lint`` to check for pylint errors. Fix any errors. -5. Run ``tox`` to run the unit tests. Fix any errors. -6. Run the integration tests, see `integration`_. +5. Run ``tox`` to run the entire test suite including coverage. Fix any errors. +6. If your code touches communication with an ACME server/Boulder, you + should run the integration tests, see `integration`_. 7. Submit the PR. Updating the documentation From d1ee8311379dd1c102ed54ed1ae827b527cd255f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 9 Oct 2015 16:55:34 -0700 Subject: [PATCH 046/113] Fixed version string --- letsencrypt-compatibility-test/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-compatibility-test/setup.py b/letsencrypt-compatibility-test/setup.py index 2e70fd1d7ed..5a54239dddd 100644 --- a/letsencrypt-compatibility-test/setup.py +++ b/letsencrypt-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages -version = '0.1.0.dev0' +version = '0.0.0.dev20151008' install_requires = [ 'letsencrypt=={0}'.format(version), From ecf82c4bcf3c3e37ab8ba5746cd3902541d6a9e3 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 9 Oct 2015 16:56:28 -0700 Subject: [PATCH 047/113] Add 'strict_permissions' when creating config --- .../letsencrypt_compatibility_test/util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/letsencrypt-compatibility-test/letsencrypt_compatibility_test/util.py b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/util.py index 6181da16bb9..816f0439860 100644 --- a/letsencrypt-compatibility-test/letsencrypt_compatibility_test/util.py +++ b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/util.py @@ -25,6 +25,7 @@ def create_le_config(parent_dir): """Sets up LE dirs in parent_dir and returns the config dict""" config = copy.deepcopy(constants.CLI_DEFAULTS) + config["strict_permissions"] = False le_dir = os.path.join(parent_dir, "letsencrypt") config["config_dir"] = os.path.join(le_dir, "config") From 84418516c9a6e1ecbbc3387fc6bd3e9a8f60cc61 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Sat, 10 Oct 2015 13:31:35 -0700 Subject: [PATCH 048/113] Limit Travis runs to master and PRs. --- .travis.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.travis.yml b/.travis.yml index 46b14fe6331..3041fdd824b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,6 +21,15 @@ env: - TOXENV=lint - TOXENV=cover + +# Only build pushes to the master branch, PRs, and branches beginning with +# `test-`. This reduces the number of simultaneous Travis runs, which speeds +# turnaround time on review since there is a cap of 5 simultaneous runs. +branches: + only: + - master + - /^test-.*$/ + sudo: false # containers addons: # make sure simplehttp simple verification works (custom /etc/hosts) From f96c34546e91be25429a8a1c2f99710a9539f402 Mon Sep 17 00:00:00 2001 From: David Dworken Date: Sat, 10 Oct 2015 18:33:44 -0400 Subject: [PATCH 049/113] Fixes #902 Fix for #902 If the directory does't exist, it will create the directory before proceeding. --- letsencrypt/renewer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/letsencrypt/renewer.py b/letsencrypt/renewer.py index 1c9cddc95ab..953b372f57f 100644 --- a/letsencrypt/renewer.py +++ b/letsencrypt/renewer.py @@ -146,6 +146,10 @@ def main(config=None, args=sys.argv[1:]): # take precedence over this one. config.merge(configobj.ConfigObj(cli_config.renewer_config_file)) + # If the folder does not exist we need to create it. + if not os.path.isdir(cli_config.renewal_configs_dir): + os.makedirs(cli_config.renewal_configs_dir) + for i in os.listdir(cli_config.renewal_configs_dir): print "Processing", i if not i.endswith(".conf"): From 7a153ebf50893cc314b1536c6bb314af4ccfc817 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 11 Oct 2015 07:05:35 +0000 Subject: [PATCH 050/113] Revert "Release 0.0.0.dev20151008" This reverts commit 9e1477faa48148ad5b2c3664375a3520d589eb94. --- acme/setup.py | 2 +- letsencrypt-apache/setup.py | 2 +- letsencrypt-nginx/setup.py | 2 +- letsencrypt/__init__.py | 2 +- letshelp-letsencrypt/setup.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 36a724f97d3..6448b7fe9c2 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages -version = '0.0.0.dev20151008' +version = '0.1.0.dev0' install_requires = [ # load_pem_private/public_key (>=0.6) diff --git a/letsencrypt-apache/setup.py b/letsencrypt-apache/setup.py index ef9299b2c4f..626e700b21b 100644 --- a/letsencrypt-apache/setup.py +++ b/letsencrypt-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages -version = '0.0.0.dev20151008' +version = '0.1.0.dev0' install_requires = [ 'acme=={0}'.format(version), diff --git a/letsencrypt-nginx/setup.py b/letsencrypt-nginx/setup.py index dde7243a946..a37b8222b05 100644 --- a/letsencrypt-nginx/setup.py +++ b/letsencrypt-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages -version = '0.0.0.dev20151008' +version = '0.1.0.dev0' install_requires = [ 'acme=={0}'.format(version), diff --git a/letsencrypt/__init__.py b/letsencrypt/__init__.py index d7ec9c5e301..1155a5b0cfe 100644 --- a/letsencrypt/__init__.py +++ b/letsencrypt/__init__.py @@ -1,4 +1,4 @@ """Let's Encrypt client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.0.0.dev20151008' +__version__ = '0.1.0.dev0' diff --git a/letshelp-letsencrypt/setup.py b/letshelp-letsencrypt/setup.py index ad517c74e83..a83fc8843c9 100644 --- a/letshelp-letsencrypt/setup.py +++ b/letshelp-letsencrypt/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages -version = '0.0.0.dev20151008' +version = '0.1.0.dev0' install_requires = [ 'setuptools', # pkg_resources From 5edd809161c659d82592d53bc6e48af4f49f2e1a Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 11 Oct 2015 10:51:24 +0000 Subject: [PATCH 051/113] ApacheConfigurator.is_enabled using filecmp (fixes #838). --- letsencrypt-apache/letsencrypt_apache/configurator.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index f3d2b5f9ad6..cb81366b37f 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -1,5 +1,6 @@ """Apache Configuration based off of Augeas Configurator.""" # pylint: disable=too-many-lines +import filecmp import itertools import logging import os @@ -945,9 +946,11 @@ def is_site_enabled(self, avail_fp): """ enabled_dir = os.path.join(self.parser.root, "sites-enabled") for entry in os.listdir(enabled_dir): - if os.path.realpath(os.path.join(enabled_dir, entry)) == avail_fp: - return True - + try: + if filecmp.cmp(avail_fp, os.path.join(enabled_dir, entry)): + return True + except OSError: + pass return False def enable_site(self, vhost): From ce4120186128136376dab24715af4ed0865d56e1 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 11 Oct 2015 10:52:08 +0000 Subject: [PATCH 052/113] Require tests pass in dev release. --- tools/dev-release.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/dev-release.sh b/tools/dev-release.sh index d93a6d21fcc..cebe5001cf6 100755 --- a/tools/dev-release.sh +++ b/tools/dev-release.sh @@ -88,8 +88,7 @@ mkdir ../kgs kgs="../kgs/$version" pip freeze | tee $kgs pip install nose -# TODO: letsencrypt_apache fails due to symlink, c.f. #838 -nosetests letsencrypt $SUBPKGS || true +nosetests letsencrypt $SUBPKGS echo "New root: $root" echo "KGS is at $root/kgs" From f8daf5f09411ed9d317d310b8efc97d3f39b495e Mon Sep 17 00:00:00 2001 From: David Dworken Date: Sun, 11 Oct 2015 08:53:38 -0400 Subject: [PATCH 053/113] Fixed failing lint test and explicitly created all needed folders --- letsencrypt/renewer.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/letsencrypt/renewer.py b/letsencrypt/renewer.py index 953b372f57f..077af6a68b8 100644 --- a/letsencrypt/renewer.py +++ b/letsencrypt/renewer.py @@ -23,6 +23,8 @@ from letsencrypt import errors from letsencrypt import notify from letsencrypt import storage +from letsencrypt import constants +from letsencrypt import le_util from letsencrypt.display import util as display_util from letsencrypt.plugins import disco as plugins_disco @@ -145,10 +147,16 @@ def main(config=None, args=sys.argv[1:]): # specify a config file on the command line, which, if provided, should # take precedence over this one. config.merge(configobj.ConfigObj(cli_config.renewer_config_file)) - - # If the folder does not exist we need to create it. - if not os.path.isdir(cli_config.renewal_configs_dir): - os.makedirs(cli_config.renewal_configs_dir) + # Ensure that all of the needed folders have been created before continuing + uid = os.geteuid() + le_util.make_or_verify_dir( + cli_config.renewal_configs_dir, constants.CONFIG_DIRS_MODE, uid) + le_util.make_or_verify_dir( + cli_config.config_dir, constants.CONFIG_DIRS_MODE, uid) + le_util.make_or_verify_dir( + cli_config.work_dir, constants.CONFIG_DIRS_MODE, uid) + le_util.make_or_verify_dir( + cli_config.logs_dir, constants.CONFIG_DIRS_MODE, uid) for i in os.listdir(cli_config.renewal_configs_dir): print "Processing", i @@ -189,4 +197,4 @@ def main(config=None, args=sys.argv[1:]): cert.update_all_links_to(cert.latest_common_version()) # TODO: restart web server (invoke IInstaller.restart() method) notify.notify("Autodeployed a cert!!!", "root", "It worked!") - # TODO: explain what happened + # TODO: explain what happened \ No newline at end of file From ef9312817e230f9d579c673685acae8547d7e5f4 Mon Sep 17 00:00:00 2001 From: David Dworken Date: Sun, 11 Oct 2015 11:39:55 -0400 Subject: [PATCH 054/113] Alphabetized imports and added newline at end of file --- letsencrypt/renewer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/letsencrypt/renewer.py b/letsencrypt/renewer.py index 077af6a68b8..81a55741975 100644 --- a/letsencrypt/renewer.py +++ b/letsencrypt/renewer.py @@ -17,14 +17,14 @@ from letsencrypt import account from letsencrypt import configuration +from letsencrypt import constants from letsencrypt import cli from letsencrypt import client from letsencrypt import crypto_util from letsencrypt import errors +from letsencrypt import le_util from letsencrypt import notify from letsencrypt import storage -from letsencrypt import constants -from letsencrypt import le_util from letsencrypt.display import util as display_util from letsencrypt.plugins import disco as plugins_disco @@ -197,4 +197,4 @@ def main(config=None, args=sys.argv[1:]): cert.update_all_links_to(cert.latest_common_version()) # TODO: restart web server (invoke IInstaller.restart() method) notify.notify("Autodeployed a cert!!!", "root", "It worked!") - # TODO: explain what happened \ No newline at end of file + # TODO: explain what happened From dd8c6d6548032436008f278580db6a9c278487b4 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Sun, 11 Oct 2015 10:20:08 -0700 Subject: [PATCH 055/113] Nginx improvements Add a server_names_hash_bucket_size directive during challenges to fix an nginx crash on restart (Fixes #922). Use fullchain instead of chain (Fixes #610). Implement OCSP stapling (Fixes #937, Fixes #931). Hide Boulder output in integration tests to make them more readable. --- .../letsencrypt_apache/configurator.py | 3 +- .../configurators/apache/common.py | 5 +- .../letsencrypt_nginx/configurator.py | 30 ++++-- letsencrypt-nginx/letsencrypt_nginx/dvsni.py | 21 ++-- letsencrypt-nginx/letsencrypt_nginx/parser.py | 10 +- .../tests/configurator_test.py | 102 ++++++++++++------ .../letsencrypt_nginx/tests/dvsni_test.py | 6 +- .../letsencrypt_nginx/tests/parser_test.py | 15 +-- .../letsencrypt_nginx/tests/util.py | 15 +++ .../tests/boulder-integration.conf.sh | 1 - letsencrypt/cli.py | 9 +- letsencrypt/client.py | 9 +- letsencrypt/interfaces.py | 4 +- letsencrypt/plugins/null.py | 3 +- letsencrypt/tests/client_test.py | 12 ++- tests/boulder-start.sh | 2 +- 16 files changed, 170 insertions(+), 77 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index f3d2b5f9ad6..a3c2bd186fa 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -163,7 +163,8 @@ def prepare(self): temp_install(self.mod_ssl_conf) - def deploy_cert(self, domain, cert_path, key_path, chain_path=None): + def deploy_cert(self, domain, cert_path, key_path, + chain_path=None, fullchain_path=None): # pylint: disable=unused-argument """Deploys certificate to specified virtual host. Currently tries to find the last directives to deploy the cert in diff --git a/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/common.py b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/common.py index 0d3dbb1b560..bcc66b98a0f 100644 --- a/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/common.py +++ b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/common.py @@ -175,12 +175,13 @@ def get_testable_domain_names(self): else: return {"example.com"} - def deploy_cert(self, domain, cert_path, key_path, chain_path=None): + def deploy_cert(self, domain, cert_path, key_path, chain_path=None, + fullchain_path=None): """Installs cert""" cert_path, key_path, chain_path = self.copy_certs_and_keys( cert_path, key_path, chain_path) self._apache_configurator.deploy_cert( - domain, cert_path, key_path, chain_path) + domain, cert_path, key_path, chain_path, fullchain_path) def _is_apache_command(command): diff --git a/letsencrypt-nginx/letsencrypt_nginx/configurator.py b/letsencrypt-nginx/letsencrypt_nginx/configurator.py index a88607e58aa..ffca041ca91 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/configurator.py +++ b/letsencrypt-nginx/letsencrypt_nginx/configurator.py @@ -117,30 +117,44 @@ def prepare(self): temp_install(self.mod_ssl_conf) # Entry point in main.py for installing cert - def deploy_cert(self, domain, cert_path, key_path, chain_path=None): + def deploy_cert(self, domain, cert_path, key_path, + chain_path, fullchain_path): # pylint: disable=unused-argument """Deploys certificate to specified virtual host. .. note:: Aborts if the vhost is missing ssl_certificate or ssl_certificate_key. - .. note:: Nginx doesn't have a cert chain directive, so the last - parameter is always ignored. It expects the cert file to have - the concatenated chain. + .. note:: Nginx doesn't have a cert chain directive. + It expects the cert file to have the concatenated chain. + However, we use the chain file as input to the + ssl_trusted_certificate directive, used for verify OCSP responses. .. note:: This doesn't save the config files! """ vhost = self.choose_vhost(domain) - directives = [['ssl_certificate', cert_path], - ['ssl_certificate_key', key_path]] + cert_directives = [['ssl_certificate', fullchain_path], + ['ssl_certificate_key', key_path]] + + # OCSP stapling was introduced in Nginx 1.3.7. If we have that version + # or greater, add config settings for it. + stapling_directives = [] + if self.version >= (1, 3, 7): + stapling_directives = [ + ['ssl_trusted_certificate', chain_path], + ['ssl_stapling', 'on'], + ['ssl_stapling_verify', 'on']] try: self.parser.add_server_directives(vhost.filep, vhost.names, - directives, True) + cert_directives, True) + self.parser.add_server_directives(vhost.filep, vhost.names, + stapling_directives, False) logger.info("Deployed Certificate to VirtualHost %s for %s", vhost.filep, vhost.names) - except errors.MisconfigurationError: + except errors.MisconfigurationError, e: + logger.debug(e) logger.warn( "Cannot find a cert or key directive in %s for %s. " "VirtualHost was not modified.", vhost.filep, vhost.names) diff --git a/letsencrypt-nginx/letsencrypt_nginx/dvsni.py b/letsencrypt-nginx/letsencrypt_nginx/dvsni.py index bd9ca783f0d..f384082efe2 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/dvsni.py +++ b/letsencrypt-nginx/letsencrypt_nginx/dvsni.py @@ -90,14 +90,23 @@ def _mod_config(self, ll_addrs): # Add the 'include' statement for the challenges if it doesn't exist # already in the main config included = False - directive = ['include', self.challenge_conf] + include_directive = ['include', self.challenge_conf] root = self.configurator.parser.loc["root"] + + bucket_directive = ['server_names_hash_bucket_size', '128'] + main = self.configurator.parser.parsed[root] - for entry in main: - if entry[0] == ['http']: - body = entry[1] - if directive not in body: - body.append(directive) + for key, value in main: + if key == ['http']: + body = value + found_bucket = False + for key, value in body: # pylint: disable=unused-variable + if key == bucket_directive[0]: + found_bucket = True + if not found_bucket: + body.insert(0, bucket_directive) + if include_directive not in body: + body.insert(0, include_directive) included = True break if not included: diff --git a/letsencrypt-nginx/letsencrypt_nginx/parser.py b/letsencrypt-nginx/letsencrypt_nginx/parser.py index ef87cc6534e..fc8ed95f189 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/parser.py +++ b/letsencrypt-nginx/letsencrypt_nginx/parser.py @@ -476,8 +476,12 @@ def _add_directives(block, directives, replace=False): :param list directives: The new directives. """ - if replace: - for directive in directives: + for directive in directives: + if not replace: + # We insert new directives at the top of the block, mostly + # to work around https://trac.nginx.org/nginx/ticket/810 + block.insert(0, directive) + else: changed = False if len(directive) == 0: continue @@ -489,5 +493,3 @@ def _add_directives(block, directives, replace=False): raise errors.MisconfigurationError( 'LetsEncrypt expected directive for %s in the Nginx ' 'config but did not find it.' % directive[0]) - else: - block.extend(directives) diff --git a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py index 977a6533071..203f9920c00 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py +++ b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py @@ -63,11 +63,11 @@ def test_save(self): # pylint: disable=protected-access parsed = self.config.parser._parse_files(filep, override=True) - self.assertEqual([[['server'], [['listen', '69.50.225.155:9000'], + self.assertEqual([[['server'], [['listen', '5001 ssl'], + ['listen', '69.50.225.155:9000'], ['listen', '127.0.0.1'], ['server_name', '.example.com'], - ['server_name', 'example.*'], - ['listen', '5001 ssl']]]], + ['server_name', 'example.*']]]], parsed[0]) def test_choose_vhost(self): @@ -96,18 +96,49 @@ def test_choose_vhost(self): def test_more_info(self): self.assertTrue('nginx.conf' in self.config.more_info()) + def test_deploy_cert_stapling(self): + # Choose a version of Nginx greater than 1.3.7 so stapling code gets + # invoked. + self.config.version = (1, 9, 6) + example_conf = self.config.parser.abs_path('sites-enabled/example.com') + self.config.deploy_cert( + "www.example.com", + "example/cert.pem", + "example/key.pem", + "example/chain.pem", + "example/fullchain.pem") + self.config.save() + self.config.parser.load() + generated_conf = self.config.parser.parsed[example_conf] + + self.assertTrue(util.contains_at_depth(generated_conf, + ['ssl_stapling', 'on'], 2)) + self.assertTrue(util.contains_at_depth(generated_conf, + ['ssl_stapling_verify', 'on'], 2)) + self.assertTrue(util.contains_at_depth(generated_conf, + ['ssl_trusted_certificate', 'example/chain.pem'], 2)) + def test_deploy_cert(self): server_conf = self.config.parser.abs_path('server.conf') nginx_conf = self.config.parser.abs_path('nginx.conf') example_conf = self.config.parser.abs_path('sites-enabled/example.com') + # Choose a version of Nginx less than 1.3.7 so stapling code doesn't get + # invoked. + self.config.version = (1, 3, 1) # Get the default SSL vhost self.config.deploy_cert( "www.example.com", - "example/cert.pem", "example/key.pem") + "example/cert.pem", + "example/key.pem", + "example/chain.pem", + "example/fullchain.pem") self.config.deploy_cert( "another.alias", - "/etc/nginx/cert.pem", "/etc/nginx/key.pem") + "/etc/nginx/cert.pem", + "/etc/nginx/key.pem", + "/etc/nginx/chain.pem", + "/etc/nginx/fullchain.pem") self.config.save() self.config.parser.load() @@ -119,35 +150,34 @@ def test_deploy_cert(self): access_log = os.path.join(self.work_dir, "access.log") error_log = os.path.join(self.work_dir, "error.log") self.assertEqual([[['server'], - [['listen', '69.50.225.155:9000'], + [['include', self.config.parser.loc["ssl_options"]], + ['ssl_certificate_key', 'example/key.pem'], + ['ssl_certificate', 'example/fullchain.pem'], + ['error_log', error_log], + ['access_log', access_log], + + ['listen', '5001 ssl'], + ['listen', '69.50.225.155:9000'], ['listen', '127.0.0.1'], ['server_name', '.example.com'], - ['server_name', 'example.*'], - ['listen', '5001 ssl'], - ['access_log', access_log], - ['error_log', error_log], - ['ssl_certificate', 'example/cert.pem'], - ['ssl_certificate_key', 'example/key.pem'], - ['include', - self.config.parser.loc["ssl_options"]]]]], + ['server_name', 'example.*']]]], parsed_example_conf) self.assertEqual([['server_name', 'somename alias another.alias']], parsed_server_conf) - self.assertEqual([['server'], - [['listen', '8000'], - ['listen', 'somename:8080'], - ['include', 'server.conf'], - [['location', '/'], - [['root', 'html'], - ['index', 'index.html index.htm']]], - ['listen', '5001 ssl'], - ['access_log', access_log], - ['error_log', error_log], - ['ssl_certificate', '/etc/nginx/cert.pem'], - ['ssl_certificate_key', '/etc/nginx/key.pem'], - ['include', - self.config.parser.loc["ssl_options"]]]], - parsed_nginx_conf[-1][-1][-1]) + self.assertTrue(util.contains_at_depth(parsed_nginx_conf, + [['server'], + [['include', self.config.parser.loc["ssl_options"]], + ['ssl_certificate_key', '/etc/nginx/key.pem'], + ['ssl_certificate', '/etc/nginx/fullchain.pem'], + ['error_log', error_log], + ['access_log', access_log], + ['listen', '5001 ssl'], + ['listen', '8000'], + ['listen', 'somename:8080'], + ['include', 'server.conf'], + [['location', '/'], + [['root', 'html'], ['index', 'index.html index.htm']]]]], + 2)) def test_get_all_certs_keys(self): nginx_conf = self.config.parser.abs_path('nginx.conf') @@ -156,16 +186,22 @@ def test_get_all_certs_keys(self): # Get the default SSL vhost self.config.deploy_cert( "www.example.com", - "example/cert.pem", "example/key.pem") + "example/cert.pem", + "example/key.pem", + "example/chain.pem", + "example/fullchain.pem") self.config.deploy_cert( "another.alias", - "/etc/nginx/cert.pem", "/etc/nginx/key.pem") + "/etc/nginx/cert.pem", + "/etc/nginx/key.pem", + "/etc/nginx/chain.pem", + "/etc/nginx/fullchain.pem") self.config.save() self.config.parser.load() self.assertEqual(set([ - ('example/cert.pem', 'example/key.pem', example_conf), - ('/etc/nginx/cert.pem', '/etc/nginx/key.pem', nginx_conf), + ('example/fullchain.pem', 'example/key.pem', example_conf), + ('/etc/nginx/fullchain.pem', '/etc/nginx/key.pem', nginx_conf), ]), self.config.get_all_certs_keys()) @mock.patch("letsencrypt_nginx.configurator.dvsni.NginxDvsni.perform") diff --git a/letsencrypt-nginx/letsencrypt_nginx/tests/dvsni_test.py b/letsencrypt-nginx/letsencrypt_nginx/tests/dvsni_test.py index a09bebba215..9fc0a1ad70c 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/tests/dvsni_test.py +++ b/letsencrypt-nginx/letsencrypt_nginx/tests/dvsni_test.py @@ -85,7 +85,8 @@ def test_perform1(self, mock_save): # Make sure challenge config is included in main config http = self.sni.configurator.parser.parsed[ self.sni.configurator.parser.loc["root"]][-1] - self.assertTrue(['include', self.sni.challenge_conf] in http[1]) + self.assertTrue( + util.contains_at_depth(http, ['include', self.sni.challenge_conf], 1)) def test_perform2(self): acme_responses = [] @@ -108,7 +109,8 @@ def test_perform2(self): http = self.sni.configurator.parser.parsed[ self.sni.configurator.parser.loc["root"]][-1] self.assertTrue(['include', self.sni.challenge_conf] in http[1]) - self.assertTrue(['server_name', 'blah'] in http[1][-2][1]) + self.assertTrue( + util.contains_at_depth(http, ['server_name', 'blah'], 3)) self.assertEqual(len(sni_responses), 3) for i in xrange(3): diff --git a/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py b/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py index 0af81aefe07..a22d33e9c97 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py +++ b/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py @@ -128,18 +128,18 @@ def test_add_server_directives(self): r'~^(www\.)?(example|bar)\.']), [['foo', 'bar'], ['ssl_certificate', '/etc/ssl/cert.pem']]) - ssl_re = re.compile(r'foo bar;\n\s+ssl_certificate /etc/ssl/cert.pem') - self.assertEqual(1, len(re.findall(ssl_re, nginxparser.dumps( - nparser.parsed[nparser.abs_path('nginx.conf')])))) + ssl_re = re.compile(r'\n\s+ssl_certificate /etc/ssl/cert.pem') + dump = nginxparser.dumps(nparser.parsed[nparser.abs_path('nginx.conf')]) + self.assertEqual(1, len(re.findall(ssl_re, dump))) nparser.add_server_directives(nparser.abs_path('server.conf'), set(['alias', 'another.alias', 'somename']), [['foo', 'bar'], ['ssl_certificate', '/etc/ssl/cert2.pem']]) self.assertEqual(nparser.parsed[nparser.abs_path('server.conf')], - [['server_name', 'somename alias another.alias'], + [['ssl_certificate', '/etc/ssl/cert2.pem'], ['foo', 'bar'], - ['ssl_certificate', '/etc/ssl/cert2.pem']]) + ['server_name', 'somename alias another.alias']]) def test_add_http_directives(self): nparser = parser.NginxParser(self.config_path, self.ssl_options) @@ -148,8 +148,9 @@ def test_add_http_directives(self): [['listen', '80'], ['server_name', 'localhost']]] nparser.add_http_directives(filep, block) - self.assertEqual(nparser.parsed[filep][-1][0], ['http']) - self.assertEqual(nparser.parsed[filep][-1][1][-1], block) + root = nparser.parsed[filep] + self.assertTrue(util.contains_at_depth(root, ['http'], 1)) + self.assertTrue(util.contains_at_depth(root, block, 2)) def test_replace_server_directives(self): nparser = parser.NginxParser(self.config_path, self.ssl_options) diff --git a/letsencrypt-nginx/letsencrypt_nginx/tests/util.py b/letsencrypt-nginx/letsencrypt_nginx/tests/util.py index 36394449054..ad4dedfe720 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/tests/util.py +++ b/letsencrypt-nginx/letsencrypt_nginx/tests/util.py @@ -85,3 +85,18 @@ def traverse(tree): yield [key, values] return list(traverse(tree)) + +def contains_at_depth(haystack, needle, n): + """ + Return true if the needle is present in one of the sub-iterables in haystack + at depth n. Haystack must be an iterable. + """ + if not hasattr(haystack, '__iter__'): + return False + if n == 0: + return needle in haystack + else: + for item in haystack: + if contains_at_depth(item, needle, n - 1): + return True + return False diff --git a/letsencrypt-nginx/tests/boulder-integration.conf.sh b/letsencrypt-nginx/tests/boulder-integration.conf.sh index 006d6883695..12610d895bc 100755 --- a/letsencrypt-nginx/tests/boulder-integration.conf.sh +++ b/letsencrypt-nginx/tests/boulder-integration.conf.sh @@ -20,7 +20,6 @@ events { } http { - server_names_hash_bucket_size 2048; # Set an array of temp and cache file options that will otherwise default to # restricted locations accessible only to root. client_body_temp_path $root/client_body; diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 64cba508d48..55930e9ff6d 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -337,9 +337,9 @@ def run(args, config, plugins): # pylint: disable=too-many-branches,too-many-lo lineage = _auth_from_domains(le_client, config, domains, plugins) - # TODO: We also need to pass the fullchain (for Nginx) le_client.deploy_certificate( - domains, lineage.privkey, lineage.cert, lineage.chain) + domains, lineage.privkey, lineage.cert, + lineage.chain, lineage.fullchain) le_client.enhance_config(domains, args.redirect) if len(lineage.available_versions("cert")) == 1: @@ -392,7 +392,8 @@ def install(args, config, plugins): args, config, authenticator=None, installer=installer) assert args.cert_path is not None # required=True in the subparser le_client.deploy_certificate( - domains, args.key_path, args.cert_path, args.chain_path) + domains, args.key_path, args.cert_path, args.chain_path, + args.fullchain_path) le_client.enhance_config(domains, args.redirect) @@ -803,6 +804,8 @@ def _paths_parser(helpful): default_cp = None if verb == "auth": default_cp = flag_default("auth_chain_path") + add("paths", "--fullchain-path", default=default_cp, + help="Accompanying path to a full certificate chain (cert plus chain).") add("paths", "--chain-path", default=default_cp, help="Accompanying path to a certificate chain.") add("paths", "--config-dir", default=flag_default("config_dir"), diff --git a/letsencrypt/client.py b/letsencrypt/client.py index 7a78add38ac..123bab12180 100644 --- a/letsencrypt/client.py +++ b/letsencrypt/client.py @@ -336,7 +336,8 @@ def save_certificate(self, certr, chain_cert, cert_path, chain_path): return os.path.abspath(act_cert_path), cert_chain_abspath - def deploy_certificate(self, domains, privkey_path, cert_path, chain_path): + def deploy_certificate(self, domains, privkey_path, + cert_path, chain_path, fullchain_path): """Install certificate :param list domains: list of domains to install the certificate @@ -357,8 +358,10 @@ def deploy_certificate(self, domains, privkey_path, cert_path, chain_path): # TODO: Provide a fullchain reference for installers like # nginx that want it self.installer.deploy_cert( - dom, os.path.abspath(cert_path), - os.path.abspath(privkey_path), chain_path) + domain=dom, cert_path=os.path.abspath(cert_path), + key_path=os.path.abspath(privkey_path), + chain_path=chain_path, + fullchain_path=fullchain_path) self.installer.save("Deployed Let's Encrypt Certificate") # sites may have been enabled / final cleanup diff --git a/letsencrypt/interfaces.py b/letsencrypt/interfaces.py index 5e82d61aa90..8bf714c8859 100644 --- a/letsencrypt/interfaces.py +++ b/letsencrypt/interfaces.py @@ -241,13 +241,15 @@ def get_all_names(): """ - def deploy_cert(domain, cert_path, key_path, chain_path=None): + def deploy_cert(domain, cert_path, key_path, chain_path, fullchain_path): """Deploy certificate. :param str domain: domain to deploy certificate file :param str cert_path: absolute path to the certificate file :param str key_path: absolute path to the private key file :param str chain_path: absolute path to the certificate chain file + :param str fullchain_path: absolute path to the certificate fullchain + file (cert plus chain) :raises .PluginError: when cert cannot be deployed diff --git a/letsencrypt/plugins/null.py b/letsencrypt/plugins/null.py index 4ba6c9d64c8..632fe415a6a 100644 --- a/letsencrypt/plugins/null.py +++ b/letsencrypt/plugins/null.py @@ -30,7 +30,8 @@ def more_info(self): def get_all_names(self): return [] - def deploy_cert(self, domain, cert_path, key_path, chain_path=None): + def deploy_cert(self, domain, cert_path, key_path, + chain_path=None, fullchain_path=None): pass # pragma: no cover def enhance(self, domain, enhancement, options=None): diff --git a/letsencrypt/tests/client_test.py b/letsencrypt/tests/client_test.py index 1e63bdbb618..fddb8660733 100644 --- a/letsencrypt/tests/client_test.py +++ b/letsencrypt/tests/client_test.py @@ -177,15 +177,19 @@ def test_save_certificate(self): def test_deploy_certificate(self): self.assertRaises(errors.Error, self.client.deploy_certificate, - ["foo.bar"], "key", "cert", "chain") + ["foo.bar"], "key", "cert", "chain", "fullchain") installer = mock.MagicMock() self.client.installer = installer - self.client.deploy_certificate(["foo.bar"], "key", "cert", "chain") + self.client.deploy_certificate(["foo.bar"], "key", "cert", "chain", + "fullchain") installer.deploy_cert.assert_called_once_with( - "foo.bar", os.path.abspath("cert"), - os.path.abspath("key"), os.path.abspath("chain")) + cert_path=os.path.abspath("cert"), + chain_path=os.path.abspath("chain"), + domain='foo.bar', + fullchain_path='fullchain', + key_path=os.path.abspath("key")) self.assertEqual(installer.save.call_count, 1) installer.restart.assert_called_once_with() diff --git a/tests/boulder-start.sh b/tests/boulder-start.sh index 530f9c59884..8780cac7c1e 100755 --- a/tests/boulder-start.sh +++ b/tests/boulder-start.sh @@ -38,5 +38,5 @@ if ! go get bitbucket.org/liamstask/goose/cmd/goose ; then exit 1 fi ./test/create_db.sh -./start.py & +./start.py > /dev/null & # Hopefully start.py bootstraps before integration test is started... From 4c73db9aa1c792a831599a69fa6991a636ef501b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Sun, 11 Oct 2015 10:34:32 -0700 Subject: [PATCH 056/113] Revert "Fixed version string" This reverts commit d1ee8311379dd1c102ed54ed1ae827b527cd255f. --- letsencrypt-compatibility-test/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-compatibility-test/setup.py b/letsencrypt-compatibility-test/setup.py index 5a54239dddd..2e70fd1d7ed 100644 --- a/letsencrypt-compatibility-test/setup.py +++ b/letsencrypt-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages -version = '0.0.0.dev20151008' +version = '0.1.0.dev0' install_requires = [ 'letsencrypt=={0}'.format(version), From cd52fc02b9be44c742c88c4c2117a887c2478188 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Sun, 11 Oct 2015 11:20:21 -0700 Subject: [PATCH 057/113] Add a sleep to let Nginx finish reloading. --- letsencrypt-nginx/letsencrypt_nginx/configurator.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/letsencrypt-nginx/letsencrypt_nginx/configurator.py b/letsencrypt-nginx/letsencrypt_nginx/configurator.py index ffca041ca91..426500eb345 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/configurator.py +++ b/letsencrypt-nginx/letsencrypt_nginx/configurator.py @@ -6,6 +6,7 @@ import socket import subprocess import sys +import time import OpenSSL import zope.interface @@ -612,6 +613,10 @@ def nginx_restart(nginx_ctl, nginx_conf="/etc/nginx.conf"): except (OSError, ValueError): logger.fatal("Nginx Restart Failed - Please Check the Configuration") sys.exit(1) + # Nginx can take a moment to recognize a newly added TLS SNI servername, so sleep + # for a second. TODO: Check for expected servername and loop until it + # appears or return an error if looping too long. + time.sleep(1) return True From f0cfd69cdcd62f3c9e97f0828357a2c31104e9d4 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Sun, 11 Oct 2015 11:28:39 -0700 Subject: [PATCH 058/113] Respond to review feedback. --- letsencrypt-apache/letsencrypt_apache/configurator.py | 2 +- letsencrypt-nginx/letsencrypt_nginx/configurator.py | 8 ++++---- letsencrypt-nginx/letsencrypt_nginx/dvsni.py | 5 ++--- letsencrypt-nginx/letsencrypt_nginx/tests/util.py | 6 ++++-- tests/boulder-start.sh | 2 +- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index a3c2bd186fa..91aa02aef69 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -164,7 +164,7 @@ def prepare(self): temp_install(self.mod_ssl_conf) def deploy_cert(self, domain, cert_path, key_path, - chain_path=None, fullchain_path=None): # pylint: disable=unused-argument + chain_path=None, fullchain_path=None): # pylint: disable=unused-argument """Deploys certificate to specified virtual host. Currently tries to find the last directives to deploy the cert in diff --git a/letsencrypt-nginx/letsencrypt_nginx/configurator.py b/letsencrypt-nginx/letsencrypt_nginx/configurator.py index 426500eb345..8f7431a728e 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/configurator.py +++ b/letsencrypt-nginx/letsencrypt_nginx/configurator.py @@ -149,13 +149,13 @@ def deploy_cert(self, domain, cert_path, key_path, try: self.parser.add_server_directives(vhost.filep, vhost.names, - cert_directives, True) + cert_directives, replace=True) self.parser.add_server_directives(vhost.filep, vhost.names, - stapling_directives, False) + stapling_directives, replace=False) logger.info("Deployed Certificate to VirtualHost %s for %s", vhost.filep, vhost.names) - except errors.MisconfigurationError, e: - logger.debug(e) + except errors.MisconfigurationError as error: + logger.debug(error) logger.warn( "Cannot find a cert or key directive in %s for %s. " "VirtualHost was not modified.", vhost.filep, vhost.names) diff --git a/letsencrypt-nginx/letsencrypt_nginx/dvsni.py b/letsencrypt-nginx/letsencrypt_nginx/dvsni.py index f384082efe2..9ac2fcd7cb1 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/dvsni.py +++ b/letsencrypt-nginx/letsencrypt_nginx/dvsni.py @@ -96,11 +96,10 @@ def _mod_config(self, ll_addrs): bucket_directive = ['server_names_hash_bucket_size', '128'] main = self.configurator.parser.parsed[root] - for key, value in main: + for key, body in main: if key == ['http']: - body = value found_bucket = False - for key, value in body: # pylint: disable=unused-variable + for key, _ in body: if key == bucket_directive[0]: found_bucket = True if not found_bucket: diff --git a/letsencrypt-nginx/letsencrypt_nginx/tests/util.py b/letsencrypt-nginx/letsencrypt_nginx/tests/util.py index ad4dedfe720..9cfa6a1a11d 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/tests/util.py +++ b/letsencrypt-nginx/letsencrypt_nginx/tests/util.py @@ -86,12 +86,14 @@ def traverse(tree): return list(traverse(tree)) + def contains_at_depth(haystack, needle, n): - """ + """Is the needle in haystack at depth n? + Return true if the needle is present in one of the sub-iterables in haystack at depth n. Haystack must be an iterable. """ - if not hasattr(haystack, '__iter__'): + if not isinstance(haystack, collections.Iterable): return False if n == 0: return needle in haystack diff --git a/tests/boulder-start.sh b/tests/boulder-start.sh index 8780cac7c1e..530f9c59884 100755 --- a/tests/boulder-start.sh +++ b/tests/boulder-start.sh @@ -38,5 +38,5 @@ if ! go get bitbucket.org/liamstask/goose/cmd/goose ; then exit 1 fi ./test/create_db.sh -./start.py > /dev/null & +./start.py & # Hopefully start.py bootstraps before integration test is started... From 06c85d6b5a88be9780f7c48b3e92f6d3b628424d Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Sun, 11 Oct 2015 11:30:11 -0700 Subject: [PATCH 059/113] Fix line-wrapped function indents. --- .../configurators/apache/common.py | 2 +- letsencrypt-nginx/letsencrypt_nginx/configurator.py | 2 +- letsencrypt/plugins/null.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/common.py b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/common.py index bcc66b98a0f..5f183b6119f 100644 --- a/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/common.py +++ b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/common.py @@ -176,7 +176,7 @@ def get_testable_domain_names(self): return {"example.com"} def deploy_cert(self, domain, cert_path, key_path, chain_path=None, - fullchain_path=None): + fullchain_path=None): """Installs cert""" cert_path, key_path, chain_path = self.copy_certs_and_keys( cert_path, key_path, chain_path) diff --git a/letsencrypt-nginx/letsencrypt_nginx/configurator.py b/letsencrypt-nginx/letsencrypt_nginx/configurator.py index 8f7431a728e..d1ab8f3d13c 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/configurator.py +++ b/letsencrypt-nginx/letsencrypt_nginx/configurator.py @@ -119,7 +119,7 @@ def prepare(self): # Entry point in main.py for installing cert def deploy_cert(self, domain, cert_path, key_path, - chain_path, fullchain_path): + chain_path, fullchain_path): # pylint: disable=unused-argument """Deploys certificate to specified virtual host. diff --git a/letsencrypt/plugins/null.py b/letsencrypt/plugins/null.py index 632fe415a6a..e875376849b 100644 --- a/letsencrypt/plugins/null.py +++ b/letsencrypt/plugins/null.py @@ -31,7 +31,7 @@ def get_all_names(self): return [] def deploy_cert(self, domain, cert_path, key_path, - chain_path=None, fullchain_path=None): + chain_path=None, fullchain_path=None): pass # pragma: no cover def enhance(self, domain, enhancement, options=None): From f16489f762f3e0dd50f010850e9b61c7d01db4df Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Sun, 11 Oct 2015 12:19:39 -0700 Subject: [PATCH 060/113] Go back to hasattr and add a test. --- .../letsencrypt_nginx/tests/parser_test.py | 12 ++++++++++-- letsencrypt-nginx/letsencrypt_nginx/tests/util.py | 4 +++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py b/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py index a22d33e9c97..40ef37e2fae 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py +++ b/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py @@ -131,12 +131,14 @@ def test_add_server_directives(self): ssl_re = re.compile(r'\n\s+ssl_certificate /etc/ssl/cert.pem') dump = nginxparser.dumps(nparser.parsed[nparser.abs_path('nginx.conf')]) self.assertEqual(1, len(re.findall(ssl_re, dump))) - nparser.add_server_directives(nparser.abs_path('server.conf'), + + server_conf = nparser.abs_path('server.conf') + nparser.add_server_directives(server_conf, set(['alias', 'another.alias', 'somename']), [['foo', 'bar'], ['ssl_certificate', '/etc/ssl/cert2.pem']]) - self.assertEqual(nparser.parsed[nparser.abs_path('server.conf')], + self.assertEqual(nparser.parsed[server_conf], [['ssl_certificate', '/etc/ssl/cert2.pem'], ['foo', 'bar'], ['server_name', 'somename alias another.alias']]) @@ -152,6 +154,12 @@ def test_add_http_directives(self): self.assertTrue(util.contains_at_depth(root, ['http'], 1)) self.assertTrue(util.contains_at_depth(root, block, 2)) + # Check that our server block got inserted first among all server + # blocks. + http_block = filter(lambda x: x[0] == ['http'], root)[0][1] + server_blocks = filter(lambda x: x[0] == ['server'], http_block) + self.assertEqual(server_blocks[0], block) + def test_replace_server_directives(self): nparser = parser.NginxParser(self.config_path, self.ssl_options) target = set(['.example.com', 'example.*']) diff --git a/letsencrypt-nginx/letsencrypt_nginx/tests/util.py b/letsencrypt-nginx/letsencrypt_nginx/tests/util.py index 9cfa6a1a11d..953c5d3673e 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/tests/util.py +++ b/letsencrypt-nginx/letsencrypt_nginx/tests/util.py @@ -93,7 +93,9 @@ def contains_at_depth(haystack, needle, n): Return true if the needle is present in one of the sub-iterables in haystack at depth n. Haystack must be an iterable. """ - if not isinstance(haystack, collections.Iterable): + # Specifically use hasattr rather than isinstance(..., collections.Iterable) + # because we want to include lists but reject strings. + if not hasattr(haystack, '__iter__'): return False if n == 0: return needle in haystack From 216e589d461b33706c0014aa49b666f6e46f1c3a Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Sun, 11 Oct 2015 12:30:02 -0700 Subject: [PATCH 061/113] Replace lambda with comprehension. --- letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py b/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py index 40ef37e2fae..b28640d7f61 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py +++ b/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py @@ -156,8 +156,8 @@ def test_add_http_directives(self): # Check that our server block got inserted first among all server # blocks. - http_block = filter(lambda x: x[0] == ['http'], root)[0][1] - server_blocks = filter(lambda x: x[0] == ['server'], http_block) + http_block = [x for x in root if x[0] == ['http']][0][1] + server_blocks = [x for x in http_block if x[0] == ['server']] self.assertEqual(server_blocks[0], block) def test_replace_server_directives(self): From a809a059f0512159279b5dcbc100374393327e4b Mon Sep 17 00:00:00 2001 From: David Dworken Date: Sun, 11 Oct 2015 15:57:57 -0400 Subject: [PATCH 062/113] Don't create renewal_configs_dir and config_dir, instead just print a helpful error message and fail --- letsencrypt/renewer.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/letsencrypt/renewer.py b/letsencrypt/renewer.py index ebd9a42d176..76922e6fd73 100644 --- a/letsencrypt/renewer.py +++ b/letsencrypt/renewer.py @@ -169,14 +169,14 @@ def main(config=None, cli_args=sys.argv[1:]): config.merge(configobj.ConfigObj(cli_config.renewer_config_file)) # Ensure that all of the needed folders have been created before continuing uid = os.geteuid() - le_util.make_or_verify_dir( - cli_config.renewal_configs_dir, constants.CONFIG_DIRS_MODE, uid) - le_util.make_or_verify_dir( - cli_config.config_dir, constants.CONFIG_DIRS_MODE, uid) - le_util.make_or_verify_dir( - cli_config.work_dir, constants.CONFIG_DIRS_MODE, uid) - le_util.make_or_verify_dir( - cli_config.logs_dir, constants.CONFIG_DIRS_MODE, uid) + if (not os.path.isdir(cli_config.renewal_configs_dir) or + not os.path.isdir(cli_config.config_dir)): + print "Could not find config directory. Exiting. " + else: + le_util.make_or_verify_dir( + cli_config.work_dir, constants.CONFIG_DIRS_MODE, uid) + le_util.make_or_verify_dir( + cli_config.logs_dir, constants.CONFIG_DIRS_MODE, uid) for i in os.listdir(cli_config.renewal_configs_dir): print "Processing", i From 8ec7fdd323b6e814ae54629e507ca23f0e736beb Mon Sep 17 00:00:00 2001 From: David Dworken Date: Sun, 11 Oct 2015 16:20:18 -0400 Subject: [PATCH 063/113] Always create the folders --- letsencrypt/renewer.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/letsencrypt/renewer.py b/letsencrypt/renewer.py index 76922e6fd73..bec8684831b 100644 --- a/letsencrypt/renewer.py +++ b/letsencrypt/renewer.py @@ -172,11 +172,10 @@ def main(config=None, cli_args=sys.argv[1:]): if (not os.path.isdir(cli_config.renewal_configs_dir) or not os.path.isdir(cli_config.config_dir)): print "Could not find config directory. Exiting. " - else: - le_util.make_or_verify_dir( - cli_config.work_dir, constants.CONFIG_DIRS_MODE, uid) - le_util.make_or_verify_dir( - cli_config.logs_dir, constants.CONFIG_DIRS_MODE, uid) + le_util.make_or_verify_dir( + cli_config.work_dir, constants.CONFIG_DIRS_MODE, uid) + le_util.make_or_verify_dir( + cli_config.logs_dir, constants.CONFIG_DIRS_MODE, uid) for i in os.listdir(cli_config.renewal_configs_dir): print "Processing", i From 52f7a64b8442f099ea581b407ffaa9cae4a4cefb Mon Sep 17 00:00:00 2001 From: Brandon Kreisel Date: Sun, 11 Oct 2015 17:56:30 -0400 Subject: [PATCH 064/113] lint newline --- letsencrypt/plugins/manual.py | 1 - 1 file changed, 1 deletion(-) diff --git a/letsencrypt/plugins/manual.py b/letsencrypt/plugins/manual.py index 65d7d687687..99463c36257 100644 --- a/letsencrypt/plugins/manual.py +++ b/letsencrypt/plugins/manual.py @@ -134,7 +134,6 @@ def _perform_single(self, achall): executable = "/bin/bash" else: executable = None - try: self._httpd = subprocess.Popen( command, From 90f3b26bcff975f87a26357659543b69dad2492f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Sun, 11 Oct 2015 17:29:19 -0700 Subject: [PATCH 065/113] Fixed a2enmod --- .../configurators/apache/a2enmod.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/a2enmod.sh b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/a2enmod.sh index 0a8ade4c2a1..60a62407a38 100755 --- a/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/a2enmod.sh +++ b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/a2enmod.sh @@ -1,6 +1,6 @@ #!/bin/bash # An extremely simplified version of `a2enmod` for enabling modules in the -# httpd docker image. First argument is the server_root and the second is the +# httpd docker image. First argument is the ServerRoot and the second is the # module to be enabled. confdir=$1 @@ -8,10 +8,11 @@ module=$2 echo "LoadModule ${module}_module " \ "/usr/local/apache2/modules/mod_${module}.so" >> "${confdir}/test.conf" -available_conf=$APACHE_CONFDIR"/mods-available/${module}.conf" -enabled_dir=$APACHE_CONFDIR"/mods-enabled" -enabled_conf=$enabled_dir"/"$1".conf" -if [ -e "$available_conf" -a -d "$enabled_dir" -a ! -e "$enabled_conf" ] +availbase="/mods-available/${module}.conf" +availconf=$confdir$availbase +enabldir="$confdir/mods-enabled" +enablconf="$enabldir/${module}.conf" +if [ -e $availconf -a -d $enabldir -a ! -e $enablconf ] then - ln -s "..$available_base" $enabled_conf + ln -s "..$availbase" $enablconf fi From 9c59b5089408e9e93d5436def0f747a8202ee913 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Sun, 11 Oct 2015 17:33:00 -0700 Subject: [PATCH 066/113] Reverted incorrect change from 18adec0 --- .../letsencrypt_compatibility_test/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-compatibility-test/letsencrypt_compatibility_test/util.py b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/util.py index 816f0439860..dd4fd3b48bb 100644 --- a/letsencrypt-compatibility-test/letsencrypt_compatibility_test/util.py +++ b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/util.py @@ -40,7 +40,7 @@ def create_le_config(parent_dir): def extract_configs(configs, parent_dir): """Extracts configs to a new dir under parent_dir and returns it""" - config_dir = os.path.join(parent_dir, "renewal") + config_dir = os.path.join(parent_dir, "configs") if os.path.isdir(configs): shutil.copytree(configs, config_dir, symlinks=True) From 67ee9bf930b95c9975920a6f3079d336bbfc5520 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 12 Oct 2015 10:00:35 -0700 Subject: [PATCH 067/113] Added 'strict_permissions' to constants.py --- .../letsencrypt_compatibility_test/util.py | 1 - letsencrypt/constants.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-compatibility-test/letsencrypt_compatibility_test/util.py b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/util.py index dd4fd3b48bb..43070cf031c 100644 --- a/letsencrypt-compatibility-test/letsencrypt_compatibility_test/util.py +++ b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/util.py @@ -25,7 +25,6 @@ def create_le_config(parent_dir): """Sets up LE dirs in parent_dir and returns the config dict""" config = copy.deepcopy(constants.CLI_DEFAULTS) - config["strict_permissions"] = False le_dir = os.path.join(parent_dir, "letsencrypt") config["config_dir"] = os.path.join(le_dir, "config") diff --git a/letsencrypt/constants.py b/letsencrypt/constants.py index 762409d2517..362009ec6a2 100644 --- a/letsencrypt/constants.py +++ b/letsencrypt/constants.py @@ -27,6 +27,7 @@ auth_cert_path="./cert.pem", auth_chain_path="./chain.pem", + strict_permissions=False, ) """Defaults for CLI flags and `.IConfig` attributes.""" From 7defdb18193500cfb287bde092a089c087887b33 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 12 Oct 2015 10:08:35 -0700 Subject: [PATCH 068/113] Updated a2enmod comments --- .../configurators/apache/a2enmod.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/a2enmod.sh b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/a2enmod.sh index 60a62407a38..4da9288a22d 100755 --- a/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/a2enmod.sh +++ b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/a2enmod.sh @@ -1,7 +1,7 @@ #!/bin/bash # An extremely simplified version of `a2enmod` for enabling modules in the -# httpd docker image. First argument is the ServerRoot and the second is the -# module to be enabled. +# httpd docker image. First argument is the Apache ServerRoot which should be +# an absolute path. The second is the module to be enabled, such as `ssl`. confdir=$1 module=$2 From 589145686fd4b3f92b90357f39f05fcfa1900b5b Mon Sep 17 00:00:00 2001 From: David Dworken Date: Mon, 12 Oct 2015 15:02:07 -0400 Subject: [PATCH 069/113] Don't print error message and only call os.geteuid() once --- letsencrypt/renewer.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/letsencrypt/renewer.py b/letsencrypt/renewer.py index bec8684831b..ea1eb5ef77d 100644 --- a/letsencrypt/renewer.py +++ b/letsencrypt/renewer.py @@ -153,7 +153,8 @@ def main(config=None, cli_args=sys.argv[1:]): args = _create_parser().parse_args(cli_args) - le_util.make_or_verify_dir(args.logs_dir, 0o700, os.geteuid()) + uid = os.geteuid() + le_util.make_or_verify_dir(args.logs_dir, 0o700, uid) cli.setup_logging(args, _cli_log_handler, logfile='renewer.log') cli_config = configuration.RenewerConfiguration(args) @@ -168,10 +169,6 @@ def main(config=None, cli_args=sys.argv[1:]): # take precedence over this one. config.merge(configobj.ConfigObj(cli_config.renewer_config_file)) # Ensure that all of the needed folders have been created before continuing - uid = os.geteuid() - if (not os.path.isdir(cli_config.renewal_configs_dir) or - not os.path.isdir(cli_config.config_dir)): - print "Could not find config directory. Exiting. " le_util.make_or_verify_dir( cli_config.work_dir, constants.CONFIG_DIRS_MODE, uid) le_util.make_or_verify_dir( From 20d7576f662999c2e53a6bd48d353fcd41a8b126 Mon Sep 17 00:00:00 2001 From: David Dworken Date: Mon, 12 Oct 2015 15:14:13 -0400 Subject: [PATCH 070/113] Deleted duplicate line caused by #912 --- letsencrypt/renewer.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/letsencrypt/renewer.py b/letsencrypt/renewer.py index ea1eb5ef77d..b62e31bcead 100644 --- a/letsencrypt/renewer.py +++ b/letsencrypt/renewer.py @@ -171,8 +171,6 @@ def main(config=None, cli_args=sys.argv[1:]): # Ensure that all of the needed folders have been created before continuing le_util.make_or_verify_dir( cli_config.work_dir, constants.CONFIG_DIRS_MODE, uid) - le_util.make_or_verify_dir( - cli_config.logs_dir, constants.CONFIG_DIRS_MODE, uid) for i in os.listdir(cli_config.renewal_configs_dir): print "Processing", i From d4af07a7f86d5b6f052a184ef137a6a5a819b94f Mon Sep 17 00:00:00 2001 From: Brandon Kreisel Date: Mon, 12 Oct 2015 16:43:22 -0400 Subject: [PATCH 071/113] PEP8 Love: E126 Fix for #945 --- letsencrypt/renewer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/letsencrypt/renewer.py b/letsencrypt/renewer.py index b62e31bcead..8d8540e6f77 100644 --- a/letsencrypt/renewer.py +++ b/letsencrypt/renewer.py @@ -169,8 +169,8 @@ def main(config=None, cli_args=sys.argv[1:]): # take precedence over this one. config.merge(configobj.ConfigObj(cli_config.renewer_config_file)) # Ensure that all of the needed folders have been created before continuing - le_util.make_or_verify_dir( - cli_config.work_dir, constants.CONFIG_DIRS_MODE, uid) + le_util.make_or_verify_dir(cli_config.work_dir, + constants.CONFIG_DIRS_MODE, uid) for i in os.listdir(cli_config.renewal_configs_dir): print "Processing", i From 6ae3f4c973a5326edc696c224dff286c438c84ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Douglas=20Jose=CC=81?= Date: Mon, 12 Oct 2015 21:25:52 -0400 Subject: [PATCH 072/113] Added pip and virtualenv installation steps to Mac's bootstrap script. --- bootstrap/mac.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/bootstrap/mac.sh b/bootstrap/mac.sh index 6779188a792..06defa85320 100755 --- a/bootstrap/mac.sh +++ b/bootstrap/mac.sh @@ -6,3 +6,13 @@ fi brew install augeas brew install dialog + +if ! hash pip 2>/dev/null; then + echo "pip Not Installed\nInstalling python from Homebrew..." + brew install python +fi + +if ! hash virtualenv 2>/dev/null; then + echo "virtualenv Not Installed\nInstalling with pip" + pip install virtualenv +fi From c4042e6ce8c74e85854343b20350fb497d0232d6 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Tue, 13 Oct 2015 07:09:14 +0000 Subject: [PATCH 073/113] Busy wait loop for testing serve_forever2 This fixes race conditions, such as those in https://travis-ci.org/letsencrypt/letsencrypt/jobs/84990239: + nosetests -c /dev/null --with-cover --cover-tests --cover-package acme --cover-min-percentage=100 acme .......................................................................................................................................................................................................................................................................................................................................................Exception in thread Thread-5: Traceback (most recent call last): File "/opt/python/2.7.9/lib/python2.7/threading.py", line 810, in __bootstrap_inner self.run() File "/opt/python/2.7.9/lib/python2.7/threading.py", line 763, in run self.__target(*self.__args, **self.__kwargs) File "/opt/python/2.7.9/lib/python2.7/SocketServer.py", line 271, in handle_request timeout = self.socket.gettimeout() File "/opt/python/2.7.9/lib/python2.7/socket.py", line 224, in meth return getattr(self._sock,name)(*args) File "/opt/python/2.7.9/lib/python2.7/socket.py", line 170, in _dummy raise error(EBADF, 'Bad file descriptor') error: [Errno 9] Bad file descriptor .127.0.0.1 - - [12/Oct/2015 20:08:23] "GET /foo HTTP/1.1" 404 - .127.0.0.1 - - [12/Oct/2015 20:08:23] "GET / HTTP/1.1" 200 - .127.0.0.1 - - [12/Oct/2015 20:08:23] "GET /.well-known/acme-challenge/eHh4eHh4eHh4eHh4eHh4eA HTTP/1.1" 200 - ..... Name Stmts Miss Cover Missing ------------------------------------------------------------ acme.py 0 0 100% acme/challenges.py 215 0 100% acme/challenges_test.py 366 0 100% acme/client.py 215 0 100% acme/client_test.py 308 0 100% acme/crypto_util.py 92 0 100% acme/crypto_util_test.py 53 0 100% acme/errors.py 19 0 100% acme/errors_test.py 18 0 100% acme/fields.py 32 0 100% acme/fields_test.py 41 0 100% acme/jose.py 8 0 100% acme/jose/b64.py 15 0 100% acme/jose/b64_test.py 38 0 100% acme/jose/errors.py 12 0 100% acme/jose/errors_test.py 8 0 100% acme/jose/interfaces.py 39 0 100% acme/jose/interfaces_test.py 73 0 100% acme/jose/json_util.py 170 0 100% acme/jose/json_util_test.py 214 0 100% acme/jose/jwa.py 105 0 100% acme/jose/jwa_test.py 58 0 100% acme/jose/jwk.py 114 0 100% acme/jose/jwk_test.py 96 0 100% acme/jose/jws.py 205 0 100% acme/jose/jws_test.py 145 0 100% acme/jose/util.py 114 0 100% acme/jose/util_test.py 126 0 100% acme/jws.py 17 0 100% acme/jws_test.py 27 0 100% acme/messages.py 184 0 100% acme/messages_test.py 198 0 100% acme/other.py 21 0 100% acme/other_test.py 48 0 100% acme/standalone.py 102 1 99% 58 acme/standalone_test.py 109 0 100% acme/test_util.py 28 0 100% acme/util.py 3 0 100% acme/util_test.py 7 0 100% ------------------------------------------------------------ TOTAL 3643 1 99% nose.plugins.cover: ERROR: TOTAL Coverage did not reach minimum required: 100% --- acme/acme/standalone_test.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py index 9eb192c7489..7eda2fce3e1 100644 --- a/acme/acme/standalone_test.py +++ b/acme/acme/standalone_test.py @@ -1,6 +1,7 @@ """Tests for acme.standalone.""" import os import shutil +import socket import threading import tempfile import time @@ -40,9 +41,27 @@ def __init__(self, *args, **kwargs): ACMEServerMixin.__init__(self) self.server = _MockServer(("", 0), socketserver.BaseRequestHandler) + def _busy_wait(self): # pragma: no cover + # This function is used to avoid race coditions in tests, but + # not all of the functionality is always used, hence "no + # cover" + while True: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + # pylint: disable=no-member + sock.connect(self.server.socket.getsockname()) + except socket.error: + pass + else: + break + finally: + sock.close() + time.sleep(1) + def test_serve_shutdown(self): thread = threading.Thread(target=self.server.serve_forever2) thread.start() + self._busy_wait() self.server.shutdown2() def test_shutdown2_not_running(self): From c7732114cbedc915d14526f5c96a1a17e2becadb Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 13 Oct 2015 14:50:23 -0700 Subject: [PATCH 074/113] Only test CLI for nginx plugin if it is present - Fixes 919 --- letsencrypt/tests/cli_test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index d0fae370d5e..f690e77f968 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -75,7 +75,10 @@ def test_help(self): output.truncate(0) self.assertRaises(SystemExit, self._call_stdout, ['-h', 'nginx']) out = output.getvalue() - self.assertTrue("--nginx-ctl" in out) + from letsencrypt.plugins import disco + if "nginx" in disco.PluginsRegistry.find_all(): + # may be false while building distributions without plugins + self.assertTrue("--nginx-ctl" in out) self.assertTrue("--manual-test-mode" not in out) self.assertTrue("--checkpoints" not in out) output.truncate(0) From d5fd9986de701a2c0036a845c3443ec935a8e273 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 14 Oct 2015 09:26:59 -0700 Subject: [PATCH 075/113] Add rateLimited error type. --- acme/acme/messages.py | 1 + 1 file changed, 1 insertion(+) diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 02ae24c8f19..9d4dcbf30b9 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -24,6 +24,7 @@ class Error(jose.JSONObjectWithFields, Exception): 'connection': 'The server could not connect to the client for DV', 'dnssec': 'The server could not validate a DNSSEC signed domain', 'malformed': 'The request message was malformed', + 'rateLimited': 'There were too many requests of a given type', 'serverInternal': 'The server experienced an internal error', 'tls': 'The server experienced a TLS error during DV', 'unauthorized': 'The client lacks sufficient authorization', From 9b77c9aecb27ed95d7392cc26614e86bfa479579 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Wed, 14 Oct 2015 17:30:03 +0000 Subject: [PATCH 076/113] Uncomment simplehttp/dvsni port check --- letsencrypt/configuration.py | 10 +++++----- letsencrypt/tests/configuration_test.py | 9 ++++++++- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/letsencrypt/configuration.py b/letsencrypt/configuration.py index a2eab5ecb73..f72005233f1 100644 --- a/letsencrypt/configuration.py +++ b/letsencrypt/configuration.py @@ -7,6 +7,7 @@ from acme import challenges from letsencrypt import constants +from letsencrypt import errors from letsencrypt import interfaces @@ -36,11 +37,10 @@ class NamespaceConfig(object): def __init__(self, namespace): self.namespace = namespace - # XXX: breaks renewer in some bizarre way - #if self.simple_http_port == self.dvsni_port: - # raise errors.Error( - # "Trying to run SimpleHTTP non-TLS and DVSNI " - # "on the same port ({0})".format(self.dvsni_port)) + if self.simple_http_port == self.dvsni_port: + raise errors.Error( + "Trying to run SimpleHTTP and DVSNI " + "on the same port ({0})".format(self.dvsni_port)) def __getattr__(self, name): return getattr(self.namespace, name) diff --git a/letsencrypt/tests/configuration_test.py b/letsencrypt/tests/configuration_test.py index c1eba85703d..44bccb57754 100644 --- a/letsencrypt/tests/configuration_test.py +++ b/letsencrypt/tests/configuration_test.py @@ -4,6 +4,8 @@ import mock +from letsencrypt import errors + class NamespaceConfigTest(unittest.TestCase): """Tests for letsencrypt.configuration.NamespaceConfig.""" @@ -12,10 +14,15 @@ def setUp(self): self.namespace = mock.MagicMock( config_dir='/tmp/config', work_dir='/tmp/foo', foo='bar', server='https://acme-server.org:443/new', - dvsni_port='1234', simple_http_port=4321) + dvsni_port=1234, simple_http_port=4321) from letsencrypt.configuration import NamespaceConfig self.config = NamespaceConfig(self.namespace) + def test_init_same_ports(self): + self.namespace.dvsni_port = 4321 + from letsencrypt.configuration import NamespaceConfig + self.assertRaises(errors.Error, NamespaceConfig, self.namespace) + def test_proxy_getattr(self): self.assertEqual(self.config.foo, 'bar') self.assertEqual(self.config.work_dir, '/tmp/foo') From 3f942d742608cb97b897f0ea6167ac8fe3aa04dd Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Wed, 14 Oct 2015 18:19:10 +0000 Subject: [PATCH 077/113] Basic virtualenv bootstrap script for end users. --- bootstrap/venv.sh | 35 +++++++++++++++++++++++++++++++++++ docs/using.rst | 12 +++--------- 2 files changed, 38 insertions(+), 9 deletions(-) create mode 100755 bootstrap/venv.sh diff --git a/bootstrap/venv.sh b/bootstrap/venv.sh new file mode 100755 index 00000000000..619135d0d1a --- /dev/null +++ b/bootstrap/venv.sh @@ -0,0 +1,35 @@ +#!/bin/sh -e +# +# Installs and updates letencrypt virtualenv +# +# USAGE: source ./dev/venv.sh + + +XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share} +VENV_NAME="letsencrypt" +VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} + +# virtualenv call is not idempotent: it overwrites pip upgraded in +# later steps, causing "ImportError: cannot import name unpack_url" +if [ ! -d $VENV_PATH ] +then + virtualenv --no-site-packages --python python2 $VENV_PATH +fi + +. $VENV_PATH/bin/activate +pip install -U setuptools +pip install -U pip + +# TODO: install apache and nginx plugins by default? +# --pre is not necessary for dev releases in more recent pip versions +pip install -U letsencrypt letsencrypt-apache letsencrypt-nginx + +echo +echo "Congratulations, Let's Encrypt has been successfully installed/updated!" +echo +echo -n "Your prompt should now be prepended with ($VENV_NAME). Next " +echo -n "time, if the prompt is different, 'source' this script again " +echo -n "before running 'letsencrypt'." +echo +echo +echo "You can now run 'letsencrypt --help'." diff --git a/docs/using.rst b/docs/using.rst index 9611f37c0fd..b2d2db42ad9 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -122,15 +122,9 @@ bash``. Installation ============ -.. "pip install acme" doesn't search for "acme" in cwd, just like "pip - install -e acme" does; `-U setuptools pip` necessary for #722 - .. code-block:: shell - virtualenv --no-site-packages -p python2 venv - ./venv/bin/pip install -U setuptools - ./venv/bin/pip install -U pip - ./venv/bin/pip install -r requirements.txt acme/ . letsencrypt-apache/ letsencrypt-nginx/ + source ./bootstrap/venv.sh .. warning:: Please do **not** use ``python setup.py install``. Please do **not** attempt the installation commands as @@ -148,13 +142,13 @@ To get a new certificate run: .. code-block:: shell - sudo ./venv/bin/letsencrypt auth + (letsencrypt)$ letsencrypt auth The ``letsencrypt`` commandline tool has a builtin help: .. code-block:: shell - ./venv/bin/letsencrypt --help + (letsencrypt)$ letsencrypt --help Configuration file From 8a8dfd4bc3697e322669abc663020a1d3f5054c6 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Wed, 14 Oct 2015 18:48:43 +0000 Subject: [PATCH 078/113] More verbose tox python env tests --- tox.ini | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tox.ini b/tox.ini index b1055807793..69fce6615b3 100644 --- a/tox.ini +++ b/tox.ini @@ -12,11 +12,11 @@ envlist = py27,cover,lint commands = pip install -r requirements.txt -e acme -e .[testing] -e letsencrypt-apache -e letsencrypt-nginx -e letshelp-letsencrypt # -q does not suppress errors - python setup.py test -q - python setup.py test -q -s acme - python setup.py test -q -s letsencrypt_apache - python setup.py test -q -s letsencrypt_nginx - python setup.py test -q -s letshelp_letsencrypt + python setup.py test + python setup.py test -s acme + python setup.py test -s letsencrypt_apache + python setup.py test -s letsencrypt_nginx + python setup.py test -s letshelp_letsencrypt setenv = PYTHONPATH = {toxinidir} From 371daa42cac800c0031b251c02e8cc3e511b4b14 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Wed, 14 Oct 2015 19:16:30 +0000 Subject: [PATCH 079/113] Quickfix for boulder#985 --- tests/boulder-integration.sh | 9 +++++++-- tests/integration/_common.sh | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 1de46eda171..273fa6ef6fa 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -27,9 +27,14 @@ common() { "$@" } +# TODO: boulder#985 +common_http() { + common --dvsni-port 0 --simple-http-port 5001 "$@" +} + common --domains le1.wtf --standalone-supported-challenges dvsni auth -common --domains le2.wtf --standalone-supported-challenges simpleHttp run -common -a manual -d le.wtf auth +common_http --domains le2.wtf --standalone-supported-challenges simpleHttp run +common_http -a manual -d le.wtf auth export CSR_PATH="${root}/csr.der" KEY_PATH="${root}/key.pem" \ OPENSSL_CNF=examples/openssl.cnf diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index fd60b92588e..b4386daa887 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -16,7 +16,7 @@ letsencrypt_test () { --server "${SERVER:-http://localhost:4000/directory}" \ --no-verify-ssl \ --dvsni-port 5001 \ - --simple-http-port 5001 \ + --simple-http-porta 5002 \ --manual-test-mode \ $store_flags \ --text \ From 99a31463b0b8bf5f63a9522d133284d9a0fbc313 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Wed, 14 Oct 2015 19:23:33 +0000 Subject: [PATCH 080/113] Fix typo: porta -> port --- tests/integration/_common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index b4386daa887..ab645f6d60c 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -16,7 +16,7 @@ letsencrypt_test () { --server "${SERVER:-http://localhost:4000/directory}" \ --no-verify-ssl \ --dvsni-port 5001 \ - --simple-http-porta 5002 \ + --simple-http-port 5002 \ --manual-test-mode \ $store_flags \ --text \ From 128147af3b540979a51a4a4074b57f7cb4a065da Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 14 Oct 2015 12:27:18 -0700 Subject: [PATCH 081/113] Removed misleading renewal messages --- letsencrypt/client.py | 26 -------------------------- letsencrypt/tests/client_test.py | 31 ------------------------------- 2 files changed, 57 deletions(-) diff --git a/letsencrypt/client.py b/letsencrypt/client.py index 123bab12180..bb04f4f5a5d 100644 --- a/letsencrypt/client.py +++ b/letsencrypt/client.py @@ -5,7 +5,6 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric import rsa import OpenSSL -import zope.component from acme import client as acme_client from acme import jose @@ -19,7 +18,6 @@ from letsencrypt import crypto_util from letsencrypt import errors from letsencrypt import error_handler -from letsencrypt import interfaces from letsencrypt import le_util from letsencrypt import reverter from letsencrypt import storage @@ -258,32 +256,8 @@ def obtain_and_enroll_certificate(self, domains, plugins): OpenSSL.crypto.FILETYPE_PEM, certr.body), key.pem, crypto_util.dump_pyopenssl_chain(chain), params, config, cli_config) - self._report_renewal_status(lineage) return lineage - def _report_renewal_status(self, cert): - # pylint: disable=no-self-use - """Informs the user about automatic renewal and deployment. - - :param .RenewableCert cert: Newly issued certificate - - """ - if cert.autorenewal_is_enabled(): - if cert.autodeployment_is_enabled(): - msg = "Automatic renewal and deployment has " - else: - msg = "Automatic renewal but not automatic deployment has " - elif cert.autodeployment_is_enabled(): - msg = "Automatic deployment but not automatic renewal has " - else: - msg = "Automatic renewal and deployment has not " - - msg += ("been enabled for your certificate. These settings can be " - "configured in the directories under {0}.").format( - cert.cli_config.renewal_configs_dir) - reporter = zope.component.getUtility(interfaces.IReporter) - reporter.add_message(msg, reporter.LOW_PRIORITY) - def save_certificate(self, certr, chain_cert, cert_path, chain_path): # pylint: disable=no-self-use """Saves the certificate received from the ACME server. diff --git a/letsencrypt/tests/client_test.py b/letsencrypt/tests/client_test.py index fddb8660733..9a5a2bbe1a2 100644 --- a/letsencrypt/tests/client_test.py +++ b/letsencrypt/tests/client_test.py @@ -114,37 +114,6 @@ def test_obtain_certificate(self, mock_crypto_util): mock.sentinel.key, domains, self.config.csr_dir) self._check_obtain_certificate() - @mock.patch("letsencrypt.client.zope.component.getUtility") - def test_report_renewal_status(self, mock_zope): - # pylint: disable=protected-access - cert = mock.MagicMock() - cert.cli_config.renewal_configs_dir = "/foo/bar/baz" - - cert.autorenewal_is_enabled.return_value = True - cert.autodeployment_is_enabled.return_value = True - self.client._report_renewal_status(cert) - msg = mock_zope().add_message.call_args[0][0] - self.assertTrue("renewal and deployment has been" in msg) - self.assertTrue(cert.cli_config.renewal_configs_dir in msg) - - cert.autorenewal_is_enabled.return_value = False - self.client._report_renewal_status(cert) - msg = mock_zope().add_message.call_args[0][0] - self.assertTrue("deployment but not automatic renewal" in msg) - self.assertTrue(cert.cli_config.renewal_configs_dir in msg) - - cert.autodeployment_is_enabled.return_value = False - self.client._report_renewal_status(cert) - msg = mock_zope().add_message.call_args[0][0] - self.assertTrue("renewal and deployment has not" in msg) - self.assertTrue(cert.cli_config.renewal_configs_dir in msg) - - cert.autorenewal_is_enabled.return_value = True - self.client._report_renewal_status(cert) - msg = mock_zope().add_message.call_args[0][0] - self.assertTrue("renewal but not automatic deployment" in msg) - self.assertTrue(cert.cli_config.renewal_configs_dir in msg) - def test_save_certificate(self): certs = ["matching_cert.pem", "cert.pem", "cert-san.pem"] tmp_path = tempfile.mkdtemp() From 18ddcc72f67f07bb0cdb1bbe5d64e336769ec8ff Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Wed, 14 Oct 2015 19:40:07 +0000 Subject: [PATCH 082/113] More quickfix for boulder#985 --- tests/boulder-integration.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 273fa6ef6fa..1edae2765d7 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -28,18 +28,21 @@ common() { } # TODO: boulder#985 +common_dvsni() { + common --dvsni-port 5001 --simple-http-port 0 "$@" +} common_http() { common --dvsni-port 0 --simple-http-port 5001 "$@" } -common --domains le1.wtf --standalone-supported-challenges dvsni auth +common_dvsni --domains le1.wtf --standalone-supported-challenges dvsni auth common_http --domains le2.wtf --standalone-supported-challenges simpleHttp run common_http -a manual -d le.wtf auth export CSR_PATH="${root}/csr.der" KEY_PATH="${root}/key.pem" \ OPENSSL_CNF=examples/openssl.cnf ./examples/generate-csr.sh le3.wtf -common auth --csr "$CSR_PATH" \ +common_dvsni auth --csr "$CSR_PATH" \ --cert-path "${root}/csr/cert.pem" \ --chain-path "${root}/csr/chain.pem" openssl x509 -in "${root}/csr/0000_cert.pem" -text From 7cdcea4dd557d59dba2860cf6854a628115015ba Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 14 Oct 2015 12:43:06 -0700 Subject: [PATCH 083/113] Mock account not client --- letsencrypt/tests/account_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/letsencrypt/tests/account_test.py b/letsencrypt/tests/account_test.py index cd98e1e2073..4e2fdb1229d 100644 --- a/letsencrypt/tests/account_test.py +++ b/letsencrypt/tests/account_test.py @@ -70,12 +70,12 @@ def _call(self): from letsencrypt.account import report_new_account report_new_account(self.acc, self.config) - @mock.patch("letsencrypt.client.zope.component.queryUtility") + @mock.patch("letsencrypt.account.zope.component.queryUtility") def test_no_reporter(self, mock_zope): mock_zope.return_value = None self._call() - @mock.patch("letsencrypt.client.zope.component.queryUtility") + @mock.patch("letsencrypt.account.zope.component.queryUtility") def test_it(self, mock_zope): self._call() call_list = mock_zope().add_message.call_args_list From 4083cd42addaee2ccf115b1620ad07965ce72a61 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 14 Oct 2015 15:09:55 -0700 Subject: [PATCH 084/113] Explain email recovery better --- letsencrypt/display/ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/display/ops.py b/letsencrypt/display/ops.py index cb424a81bb7..37ce66b620b 100644 --- a/letsencrypt/display/ops.py +++ b/letsencrypt/display/ops.py @@ -124,7 +124,7 @@ def get_email(): """ while True: code, email = zope.component.getUtility(interfaces.IDisplay).input( - "Enter email address") + "Enter email address (used for urgent notices and lost key recovery)") if code == display_util.OK: if le_util.safe_email(email): From 47eaf297fba19668d7c5e483ec7abaa6085eafad Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 14 Oct 2015 15:57:09 -0700 Subject: [PATCH 085/113] Start a letsencrypt-auto script Which handles all venv-related tasks for installing from pip, and gives us auto-updating. --- letsencrypt-auto | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100755 letsencrypt-auto diff --git a/letsencrypt-auto b/letsencrypt-auto new file mode 100755 index 00000000000..82720a3fcf2 --- /dev/null +++ b/letsencrypt-auto @@ -0,0 +1,45 @@ +#!/bin/bash -e +# +# Installs and updates the letencrypt virtualenv, and runs letsencrypt +# using that virtual environment. This allows the client to function decently +# without requiring specific versions of its dependencies from the operating +# system. + +XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share} +VENV_NAME="letsencrypt" +VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} +VENV_BIN=${VENV_PATH}/bin + +for arg in "$@" ; do + if [ "$arg" = "-v" ] || [ "$arg" = "--verbose" ] ; then + VERBOSE=1 + fi +done + +# virtualenv call is not idempotent: it overwrites pip upgraded in +# later steps, causing "ImportError: cannot import name unpack_url" +if [ ! -d $VENV_PATH ] +then + echo "Creating virtual environment..." + if [ "$VERBOSE" = 1 ] ; then + virtualenv --no-site-packages --python python2 $VENV_PATH + else + virtualenv --no-site-packages --python python2 $VENV_PATH > /dev/null + fi +fi + +echo "Updating letsencrypt and virtual environment dependencies..." +if [ "$VERBOSE" = 1 ] ; then + $VENV_BIN/pip install -U setuptools + $VENV_BIN/pip install -U pip + $VENV_BIN/pip install -U letsencrypt letsencrypt-apache letsencrypt-nginx +else + $VENV_BIN/pip install -U setuptools > /dev/null + $VENV_BIN/pip install -U pip > /dev/null + $VENV_BIN/pip install -U letsencrypt letsencrypt-apache letsencrypt-nginx > /dev/null +fi + +# TODO: install apache and nginx plugins by default? +# --pre is not necessary for dev releases in more recent pip versions + +sudo $VENV_BIN/letsencrypt $@ From b85e13de3f126fca8612bf3ce01c777d1a03615e Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 14 Oct 2015 16:07:12 -0700 Subject: [PATCH 086/113] Add bootstrapping to letsencrypt-auto --- letsencrypt-auto | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/letsencrypt-auto b/letsencrypt-auto index 82720a3fcf2..77a545fc645 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -20,6 +20,34 @@ done # later steps, causing "ImportError: cannot import name unpack_url" if [ ! -d $VENV_PATH ] then + BOOTSTRAP=`dirname $0`/bootstrap + if [ ! -f $BOOTSTRAP/debian.sh ] ; then + echo "Cannot find the letsencrypt bootstrap scripts in $BOOTSTRAP" + exit 1 + fi + if [ -f /etc/debian_version ] ; then + echo "Bootstrapping dependencies for Debian-based OSes..." + sudo $BOOTSTRAP/_deb_common.sh + elif [ -f /etc/arch-release ] ; then + echo "Bootstrapping dependencies for Archlinux..." + sudo $BOOTSTRAP/archlinux.sh + elif [ -f /etc/redhat-release ] ; then + echo "Bootstrapping dependencies for RedHat-based OSes..." + sudo $BOOTSTRAP/_rpm_common.sh + elif uname | grep -iq FreeBSD ; then + echo "Bootstrapping dependencies for FreeBSD..." + sudo $BOOTSTRAP/freebsd.sh + elif uname | grep -iq Darwin ; then + echo "Bootstrapping dependencies for Mac OS X..." + sudo $BOOTSTRAP/mac.sh + else + echo "Sorry, I don't know how to bootstrap Let's Encrypt on your operating system!" + echo + echo "You will need to bootstrap, configure virtualenv, and run a pip install manually" + echo "Please see https://letsencrypt.readthedocs.org/en/latest/using.html#prerequisites" + echo "for more info" + fi + echo "Creating virtual environment..." if [ "$VERBOSE" = 1 ] ; then virtualenv --no-site-packages --python python2 $VENV_PATH From 99793c54d6a0ca7a41ca9c1ab9ec9104e82dbc44 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 14 Oct 2015 16:25:53 -0700 Subject: [PATCH 087/113] More portability --- bootstrap/mac.sh | 2 +- letsencrypt-auto | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/bootstrap/mac.sh b/bootstrap/mac.sh index 6779188a792..84e87cf5b9c 100755 --- a/bootstrap/mac.sh +++ b/bootstrap/mac.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e if ! hash brew 2>/dev/null; then echo "Homebrew Not Installed\nDownloading..." ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" diff --git a/letsencrypt-auto b/letsencrypt-auto index 77a545fc645..7f7533f3938 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -1,4 +1,4 @@ -#!/bin/bash -e +#!/bin/sh -e # # Installs and updates the letencrypt virtualenv, and runs letsencrypt # using that virtual environment. This allows the client to function decently @@ -39,7 +39,8 @@ then sudo $BOOTSTRAP/freebsd.sh elif uname | grep -iq Darwin ; then echo "Bootstrapping dependencies for Mac OS X..." - sudo $BOOTSTRAP/mac.sh + echo "WARNING: Mac support is very experimental at present..." + $BOOTSTRAP/mac.sh else echo "Sorry, I don't know how to bootstrap Let's Encrypt on your operating system!" echo From 7a1c37968cd35f9f7087cfaac24460afdf75bfeb Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 14 Oct 2015 17:21:36 -0700 Subject: [PATCH 088/113] Make the UX clearer before sudo --- letsencrypt-auto | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/letsencrypt-auto b/letsencrypt-auto index 7f7533f3938..7ee715bf425 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -68,7 +68,7 @@ else $VENV_BIN/pip install -U letsencrypt letsencrypt-apache letsencrypt-nginx > /dev/null fi -# TODO: install apache and nginx plugins by default? -# --pre is not necessary for dev releases in more recent pip versions - +# Explain what's about to happen, for the benefit of those getting sudo +# password prompts... +echo "Running with virtualenv:" sudo $VENV_BIN/letsencrypt "$@" sudo $VENV_BIN/letsencrypt $@ From 2e0dc4fc5061864f73e9742f85c5ebefd1198f5f Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 14 Oct 2015 18:25:14 -0700 Subject: [PATCH 089/113] Remove misleading Docker suggestion from the quick-install docs But keep it around in case anyone needs this? --- docs/using.rst | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/docs/using.rst b/docs/using.rst index b2d2db42ad9..3e3973d2f44 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -2,26 +2,6 @@ Using the Let's Encrypt client ============================== -Quick start -=========== - -Using Docker_ you can quickly get yourself a testing cert. From the -server that the domain your requesting a cert for resolves to, -`install Docker`_, issue the following command: - -.. code-block:: shell - - sudo docker run -it --rm -p 443:443 --name letsencrypt \ - -v "/etc/letsencrypt:/etc/letsencrypt" \ - -v "/var/lib/letsencrypt:/var/lib/letsencrypt" \ - quay.io/letsencrypt/letsencrypt:latest - -and follow the instructions. Your new cert will be available in -``/etc/letsencrypt/certs``. - -.. _Docker: https://docker.com -.. _`install Docker`: https://docs.docker.com/docker/userguide/ - Getting the code ================ @@ -174,3 +154,25 @@ By default, the following locations are searched: .. _Augeas: http://augeas.net/ .. _Virtualenv: https://virtualenv.pypa.io + +Running with Docker +=================== + +Docker_ is another way to quickly obtaintesting certs. From the +server that the domain your requesting a cert for resolves to, +`install Docker`_, issue the following command: + +.. code-block:: shell + + sudo docker auth -it --rm -p 443:443 --name letsencrypt \ + -v "/etc/letsencrypt:/etc/letsencrypt" \ + -v "/var/lib/letsencrypt:/var/lib/letsencrypt" \ + quay.io/letsencrypt/letsencrypt:latest + +and follow the instructions. Your new cert will be available in +``/etc/letsencrypt/certs``. + +.. _Docker: https://docker.com +.. _`install Docker`: https://docs.docker.com/docker/userguide/ + + From 8a5d199ddffebc0b000b26989a9451f1d12f5639 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 14 Oct 2015 18:35:34 -0700 Subject: [PATCH 090/113] Try to make the "using" instructions as simple and accurate as possible Some of the verbose stuff we had before is really of historical/developer interest only --- docs/contributing.rst | 96 +++++++++++++++++++++++++++++++++++++ docs/using.rst | 109 +++++++++++++++++++----------------------- 2 files changed, 146 insertions(+), 59 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 614f6f2aa99..ed74bbb60f7 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -296,3 +296,99 @@ commands: This should generate documentation in the ``docs/_build/html`` directory. + +Notes on OS depedencies +======================= + +OS level dependencies are managed by scripts in ``bootstrap``. Some notes +are provided here mainly for the :ref:`developers ` reference. + +In general: + +* ``sudo`` is required as a suggested way of running privileged process +* `Augeas`_ is required for the Python bindings +* ``virtualenv`` and ``pip`` are used for managing other python library + dependencies + + +Ubuntu +------ + +.. code-block:: shell + + sudo ./bootstrap/ubuntu.sh + + +Debian +------ + +.. code-block:: shell + + sudo ./bootstrap/debian.sh + +For squeeze you will need to: + +- Use ``virtualenv --no-site-packages -p python`` instead of ``-p python2``. + + +.. _`#280`: https://github.com/letsencrypt/letsencrypt/issues/280 + + +Mac OSX +------- + +.. code-block:: shell + + ./bootstrap/mac.sh + + +Fedora +------ + +.. code-block:: shell + + sudo ./bootstrap/fedora.sh + + +Centos 7 +-------- + +.. code-block:: shell + + sudo ./bootstrap/centos.sh + + +FreeBSD +------- + +.. code-block:: shell + + sudo ./bootstrap/freebsd.sh + +Bootstrap script for FreeBSD uses ``pkg`` for package installation, +i.e. it does not use ports. + +FreeBSD by default uses ``tcsh``. In order to activate virtulenv (see +below), you will need a compatbile shell, e.g. ``pkg install bash && +bash``. + + +Running with Docker +=================== + +Docker_ is another way to quickly obtaintesting certs. From the +server that the domain your requesting a cert for resolves to, +`install Docker`_, issue the following command: + +.. code-block:: shell + + sudo docker auth -it --rm -p 443:443 --name letsencrypt \ + -v "/etc/letsencrypt:/etc/letsencrypt" \ + -v "/var/lib/letsencrypt:/var/lib/letsencrypt" \ + quay.io/letsencrypt/letsencrypt:latest + +and follow the instructions. Your new cert will be available in +``/etc/letsencrypt/certs``. + +.. _Docker: https://docker.com +.. _`install Docker`: https://docs.docker.com/docker/userguide/ diff --git a/docs/using.rst b/docs/using.rst index 3e3973d2f44..1920efe3842 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -24,17 +24,62 @@ above method instead. .. _prerequisites: -Prerequisites -============= +Installation and Usage +====================== -The demo code is supported and known to work on **Ubuntu and -Debian**. Therefore, prerequisites for other platforms listed below -are provided mainly for the :ref:`developers ` reference. +To install and run the client you just need to type: + +.. code-block:: shell + + ./letsencrypt-auto + +.. warning:: Please do **not** use ``python setup.py install``. That mode of + operation might corrupt your operating system and is **not supported** + by the Let's Encrypt team! + +The ``letsencrypt`` commandline tool has a builtin help: + +.. code-block:: shell + + ./letsencrypt-auto --help + + +Configuration file +------------------ + +It is possible to specify configuration file with +``letsencrypt --config cli.ini`` (or shorter ``-c cli.ini``). For +instance, if you are a contributor, you might find the following +handy: + +.. include:: ../examples/dev-cli.ini + :code: ini + +By default, the following locations are searched: + +- ``/etc/letsencrypt/cli.ini`` +- ``$XDG_CONFIG_HOME/letsencrypt/cli.ini`` (or + ``~/.config/letsencrypt/cli.ini`` if ``$XDG_CONFIG_HOME`` is not + set). + +.. keep it up to date with constants.py + + +.. _Augeas: http://augeas.net/ +.. _Virtualenv: https://virtualenv.pypa.io + +Notes on OS depedencies +======================= + +OS level dependencies are managed by scripts in ``bootstrap``. Some notes +are provided here mainly for the :ref:`developers ` reference. In general: * ``sudo`` is required as a suggested way of running privileged process * `Augeas`_ is required for the Python bindings +* ``virtualenv`` and ``pip`` are used for managing other python library + dependencies Ubuntu @@ -99,61 +144,7 @@ below), you will need a compatbile shell, e.g. ``pkg install bash && bash``. -Installation -============ - -.. code-block:: shell - source ./bootstrap/venv.sh - -.. warning:: Please do **not** use ``python setup.py install``. Please - do **not** attempt the installation commands as - superuser/root and/or without Virtualenv_, e.g. ``sudo - python setup.py install``, ``sudo pip install``, ``sudo - ./venv/bin/...``. These modes of operation might corrupt - your operating system and are **not supported** by the - Let's Encrypt team! - - -Usage -===== - -To get a new certificate run: - -.. code-block:: shell - - (letsencrypt)$ letsencrypt auth - -The ``letsencrypt`` commandline tool has a builtin help: - -.. code-block:: shell - - (letsencrypt)$ letsencrypt --help - - -Configuration file ------------------- - -It is possible to specify configuration file with -``letsencrypt --config cli.ini`` (or shorter ``-c cli.ini``). For -instance, if you are a contributor, you might find the following -handy: - -.. include:: ../examples/dev-cli.ini - :code: ini - -By default, the following locations are searched: - -- ``/etc/letsencrypt/cli.ini`` -- ``$XDG_CONFIG_HOME/letsencrypt/cli.ini`` (or - ``~/.config/letsencrypt/cli.ini`` if ``$XDG_CONFIG_HOME`` is not - set). - -.. keep it up to date with constants.py - - -.. _Augeas: http://augeas.net/ -.. _Virtualenv: https://virtualenv.pypa.io Running with Docker =================== From 68ed333fc0ce55bfe54945a39d7f2a1190b69776 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 14 Oct 2015 21:07:40 -0700 Subject: [PATCH 091/113] Quick fix --- letsencrypt/cli.py | 6 ++++++ letsencrypt/tests/cli_test.py | 10 ++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 07ccd38fdce..92d8455382c 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -301,6 +301,12 @@ def _auth_from_domains(le_client, config, domains, plugins): raise errors.Error("Certificate could not be obtained") _report_new_cert(lineage.cert) + reporter_util = zope.component.getUtility(interfaces.IReporter) + reporter_util.add_message( + "Your certificate will expire at {0}. To obtain a new version of the " + "certificate in the future, simply run this client again.".format( + lineage.notafter().ctime()), + reporter_util.MEDIUM_PRIORITY) return lineage diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index f690e77f968..a3efd9d4078 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -128,8 +128,9 @@ def test_auth_new_request_success(self, mock_get_utility): self._auth_new_request_common(mock_client) self.assertEqual( mock_client.obtain_and_enroll_certificate.call_count, 1) - self.assertTrue( - cert_path in mock_get_utility().add_message.call_args[0][0]) + msg = mock_get_utility().add_message.call_args_list[0][0][0] + self.assertTrue(cert_path in msg) + self.assertEqual(mock_get_utility().add_message.call_count, 2) def test_auth_new_request_failure(self): mock_client = mock.MagicMock() @@ -164,8 +165,9 @@ def test_auth_renewal(self, mock_init, mock_renewal, mock_get_utility): self.assertEqual(mock_lineage.save_successor.call_count, 1) mock_lineage.update_all_links_to.assert_called_once_with( mock_lineage.latest_common_version()) - self.assertTrue( - cert_path in mock_get_utility().add_message.call_args[0][0]) + msg = mock_get_utility().add_message.call_args_list[0][0][0] + self.assertTrue(cert_path in msg) + self.assertEqual(mock_get_utility().add_message.call_count, 2) @mock.patch('letsencrypt.cli.display_ops.pick_installer') @mock.patch('letsencrypt.cli.zope.component.getUtility') From 2cd0e64537a2c774f549386640f8f9be12b1fe73 Mon Sep 17 00:00:00 2001 From: lf Date: Wed, 14 Oct 2015 22:35:52 -0600 Subject: [PATCH 092/113] Change as_string to a __str__ in nginxparser.py This change would make the RawNginxDumper more in line with other Python libraries and the standard library. --- letsencrypt-nginx/letsencrypt_nginx/nginxparser.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/letsencrypt-nginx/letsencrypt_nginx/nginxparser.py b/letsencrypt-nginx/letsencrypt_nginx/nginxparser.py index 2926a43d0c5..84b54b4dba5 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/nginxparser.py +++ b/letsencrypt-nginx/letsencrypt_nginx/nginxparser.py @@ -84,9 +84,13 @@ def __iter__(self, blocks=None, current_indent=0, spacer=' '): else: yield spacer * current_indent + key + spacer + values + ';' - def as_string(self): + def __str__(self): """Return the parsed block as a string.""" return '\n'.join(self) + '\n' + + def as_string(self): + """Return the parsed block as a string.""" + return str(self) # Shortcut functions to respect Python's serialization interface @@ -122,7 +126,7 @@ def dumps(blocks, indentation=4): :rtype: str """ - return RawNginxDumper(blocks, indentation).as_string() + return str(RawNginxDumper(blocks, indentation)) def dump(blocks, _file, indentation=4): From 87000ac5c60eda6b8f7e4c291c53bc97a006c3e7 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 14 Oct 2015 22:24:41 -0700 Subject: [PATCH 093/113] Move docker back into "using" It seems the changes removing bootstrap from "using" weren't commited earlier, either --- docs/contributing.rst | 21 ----------- docs/using.rst | 81 +------------------------------------------ 2 files changed, 1 insertion(+), 101 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index ed74bbb60f7..8661b195ed3 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -371,24 +371,3 @@ i.e. it does not use ports. FreeBSD by default uses ``tcsh``. In order to activate virtulenv (see below), you will need a compatbile shell, e.g. ``pkg install bash && bash``. - - -Running with Docker -=================== - -Docker_ is another way to quickly obtaintesting certs. From the -server that the domain your requesting a cert for resolves to, -`install Docker`_, issue the following command: - -.. code-block:: shell - - sudo docker auth -it --rm -p 443:443 --name letsencrypt \ - -v "/etc/letsencrypt:/etc/letsencrypt" \ - -v "/var/lib/letsencrypt:/var/lib/letsencrypt" \ - quay.io/letsencrypt/letsencrypt:latest - -and follow the instructions. Your new cert will be available in -``/etc/letsencrypt/certs``. - -.. _Docker: https://docker.com -.. _`install Docker`: https://docs.docker.com/docker/userguide/ diff --git a/docs/using.rst b/docs/using.rst index 1920efe3842..125aa29b30d 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -68,83 +68,6 @@ By default, the following locations are searched: .. _Augeas: http://augeas.net/ .. _Virtualenv: https://virtualenv.pypa.io -Notes on OS depedencies -======================= - -OS level dependencies are managed by scripts in ``bootstrap``. Some notes -are provided here mainly for the :ref:`developers ` reference. - -In general: - -* ``sudo`` is required as a suggested way of running privileged process -* `Augeas`_ is required for the Python bindings -* ``virtualenv`` and ``pip`` are used for managing other python library - dependencies - - -Ubuntu ------- - -.. code-block:: shell - - sudo ./bootstrap/ubuntu.sh - - -Debian ------- - -.. code-block:: shell - - sudo ./bootstrap/debian.sh - -For squeeze you will need to: - -- Use ``virtualenv --no-site-packages -p python`` instead of ``-p python2``. - - -.. _`#280`: https://github.com/letsencrypt/letsencrypt/issues/280 - - -Mac OSX -------- - -.. code-block:: shell - - ./bootstrap/mac.sh - - -Fedora ------- - -.. code-block:: shell - - sudo ./bootstrap/fedora.sh - - -Centos 7 --------- - -.. code-block:: shell - - sudo ./bootstrap/centos.sh - - -FreeBSD -------- - -.. code-block:: shell - - sudo ./bootstrap/freebsd.sh - -Bootstrap script for FreeBSD uses ``pkg`` for package installation, -i.e. it does not use ports. - -FreeBSD by default uses ``tcsh``. In order to activate virtulenv (see -below), you will need a compatbile shell, e.g. ``pkg install bash && -bash``. - - - Running with Docker =================== @@ -158,12 +81,10 @@ server that the domain your requesting a cert for resolves to, sudo docker auth -it --rm -p 443:443 --name letsencrypt \ -v "/etc/letsencrypt:/etc/letsencrypt" \ -v "/var/lib/letsencrypt:/var/lib/letsencrypt" \ - quay.io/letsencrypt/letsencrypt:latest + quay.io/letsencrypt/letsencrypt:latest auth and follow the instructions. Your new cert will be available in ``/etc/letsencrypt/certs``. .. _Docker: https://docker.com .. _`install Docker`: https://docs.docker.com/docker/userguide/ - - From ecd987b8ca9dc3c31e9702d2f871b9f840ba51ed Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 14 Oct 2015 22:30:18 -0700 Subject: [PATCH 094/113] letsencrypt-auto now knows about -vvvv --- letsencrypt-auto | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/letsencrypt-auto b/letsencrypt-auto index 7ee715bf425..5106e2faef9 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -11,7 +11,8 @@ VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} VENV_BIN=${VENV_PATH}/bin for arg in "$@" ; do - if [ "$arg" = "-v" ] || [ "$arg" = "--verbose" ] ; then + # This first clause is redundant with the third, but hedging on portability + if [ "$arg" = "-v" ] || [ "$arg" = "--verbose" ] || echo "$arg" | grep -E -- -v+ ; then VERBOSE=1 fi done From 94622f5edd136c8904a8f7aa0d78a351e4bb2ea2 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 14 Oct 2015 22:53:36 -0700 Subject: [PATCH 095/113] Date only because hard to do time with no time --- letsencrypt/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 92d8455382c..24f032ca1c8 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -303,9 +303,9 @@ def _auth_from_domains(le_client, config, domains, plugins): _report_new_cert(lineage.cert) reporter_util = zope.component.getUtility(interfaces.IReporter) reporter_util.add_message( - "Your certificate will expire at {0}. To obtain a new version of the " + "Your certificate will expire on {0}. To obtain a new version of the " "certificate in the future, simply run this client again.".format( - lineage.notafter().ctime()), + lineage.notafter().date()), reporter_util.MEDIUM_PRIORITY) return lineage From 39e489d03cd935dcbbc725ad7370441d789b4c6d Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 14 Oct 2015 23:03:18 -0700 Subject: [PATCH 096/113] Do not assume sudo is present - On Digital Ocean and perhaps other platforms, the user is root by default and sudo is uninstalled --- letsencrypt-auto | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/letsencrypt-auto b/letsencrypt-auto index 5106e2faef9..662908c4721 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -10,6 +10,12 @@ VENV_NAME="letsencrypt" VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} VENV_BIN=${VENV_PATH}/bin +if test "`id -u`" -ne "0" ; then + SUDO=sudo +else + SUDO= +fi + for arg in "$@" ; do # This first clause is redundant with the third, but hedging on portability if [ "$arg" = "-v" ] || [ "$arg" = "--verbose" ] || echo "$arg" | grep -E -- -v+ ; then @@ -28,16 +34,16 @@ then fi if [ -f /etc/debian_version ] ; then echo "Bootstrapping dependencies for Debian-based OSes..." - sudo $BOOTSTRAP/_deb_common.sh + $SUDO $BOOTSTRAP/_deb_common.sh elif [ -f /etc/arch-release ] ; then echo "Bootstrapping dependencies for Archlinux..." - sudo $BOOTSTRAP/archlinux.sh + $SUDO $BOOTSTRAP/archlinux.sh elif [ -f /etc/redhat-release ] ; then echo "Bootstrapping dependencies for RedHat-based OSes..." - sudo $BOOTSTRAP/_rpm_common.sh + $SUDO $BOOTSTRAP/_rpm_common.sh elif uname | grep -iq FreeBSD ; then echo "Bootstrapping dependencies for FreeBSD..." - sudo $BOOTSTRAP/freebsd.sh + $SUDO $BOOTSTRAP/freebsd.sh elif uname | grep -iq Darwin ; then echo "Bootstrapping dependencies for Mac OS X..." echo "WARNING: Mac support is very experimental at present..." @@ -71,5 +77,5 @@ fi # Explain what's about to happen, for the benefit of those getting sudo # password prompts... -echo "Running with virtualenv:" sudo $VENV_BIN/letsencrypt "$@" -sudo $VENV_BIN/letsencrypt $@ +echo "Running with virtualenv:" $SUDO $VENV_BIN/letsencrypt "$@" +$SUDO $VENV_BIN/letsencrypt $@ From ab036e98e08ad3c9b9f686702ab03f16c640657e Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 14 Oct 2015 23:06:20 -0700 Subject: [PATCH 097/113] Do not ship letsencrypt-nginx until it's somewhat plausibly working --- letsencrypt-auto | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/letsencrypt-auto b/letsencrypt-auto index 662908c4721..20a7a40fa0a 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -68,11 +68,13 @@ echo "Updating letsencrypt and virtual environment dependencies..." if [ "$VERBOSE" = 1 ] ; then $VENV_BIN/pip install -U setuptools $VENV_BIN/pip install -U pip - $VENV_BIN/pip install -U letsencrypt letsencrypt-apache letsencrypt-nginx + # nginx is buggy / disabled for now... + $VENV_BIN/pip install -U letsencrypt letsencrypt-apache #letsencrypt-nginx else $VENV_BIN/pip install -U setuptools > /dev/null $VENV_BIN/pip install -U pip > /dev/null - $VENV_BIN/pip install -U letsencrypt letsencrypt-apache letsencrypt-nginx > /dev/null + # nginx is buggy / disabled for now... + $VENV_BIN/pip install -U letsencrypt letsencrypt-apache > /dev/null fi # Explain what's about to happen, for the benefit of those getting sudo From 2794b762d6199dc2c3f1e63ede22eb6c7e08ffe2 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 14 Oct 2015 23:41:37 -0700 Subject: [PATCH 098/113] Add a subtle progress bar --- letsencrypt-auto | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/letsencrypt-auto b/letsencrypt-auto index 20a7a40fa0a..b43c22f879b 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -64,17 +64,23 @@ then fi fi -echo "Updating letsencrypt and virtual environment dependencies..." +echo -n "Updating letsencrypt and virtual environment dependencies..." if [ "$VERBOSE" = 1 ] ; then + echo $VENV_BIN/pip install -U setuptools $VENV_BIN/pip install -U pip # nginx is buggy / disabled for now... $VENV_BIN/pip install -U letsencrypt letsencrypt-apache #letsencrypt-nginx else $VENV_BIN/pip install -U setuptools > /dev/null + echo -n . $VENV_BIN/pip install -U pip > /dev/null + echo -n . # nginx is buggy / disabled for now... - $VENV_BIN/pip install -U letsencrypt letsencrypt-apache > /dev/null + $VENV_BIN/pip install -U letsencrypt > /dev/null + echo -n . + $VENV_BIN/pip install -U letsencrypt-apache > /dev/null + echo fi # Explain what's about to happen, for the benefit of those getting sudo From be77909ef21af57a7d10a9799bd12460770db9fb Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 14 Oct 2015 23:53:50 -0700 Subject: [PATCH 099/113] Further fixes to the docs --- docs/using.rst | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/using.rst b/docs/using.rst index 125aa29b30d..5683fba2179 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -22,8 +22,6 @@ above method instead. https://github.com/letsencrypt/letsencrypt/archive/master.zip -.. _prerequisites: - Installation and Usage ====================== @@ -33,9 +31,13 @@ To install and run the client you just need to type: ./letsencrypt-auto -.. warning:: Please do **not** use ``python setup.py install``. That mode of - operation might corrupt your operating system and is **not supported** - by the Let's Encrypt team! +(Once letsencrypt is packaged by distributions, the command will just be +``letsencrypt``. ``letsencrypt-auto`` is a wrapper which installs virtualized +dependencies and provides automated updates during the beta program) + +.. warning:: Please do **not** use ``python setup.py install`` or ``sudo pip install`. + Those mode of operation might corrupt your operating system and is + **not supported** by the Let's Encrypt team! The ``letsencrypt`` commandline tool has a builtin help: @@ -48,7 +50,7 @@ Configuration file ------------------ It is possible to specify configuration file with -``letsencrypt --config cli.ini`` (or shorter ``-c cli.ini``). For +``letsencrypt-auto --config cli.ini`` (or shorter ``-c cli.ini``). For instance, if you are a contributor, you might find the following handy: @@ -72,7 +74,7 @@ By default, the following locations are searched: Running with Docker =================== -Docker_ is another way to quickly obtaintesting certs. From the +Docker_ is another way to quickly obtain testing certs. From the server that the domain your requesting a cert for resolves to, `install Docker`_, issue the following command: From 4ef385d3ad7e174452996cb41da0c9f6c02d8ab5 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 14 Oct 2015 23:56:56 -0700 Subject: [PATCH 100/113] Protect quotes on the way to the actual command --- letsencrypt-auto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-auto b/letsencrypt-auto index b43c22f879b..72c0c8aa4d0 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -86,4 +86,4 @@ fi # Explain what's about to happen, for the benefit of those getting sudo # password prompts... echo "Running with virtualenv:" $SUDO $VENV_BIN/letsencrypt "$@" -$SUDO $VENV_BIN/letsencrypt $@ +$SUDO $VENV_BIN/letsencrypt "$@" From 0fb00ca039d390de7508f7a0cfba2f94dc9d4134 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 15 Oct 2015 00:03:49 -0700 Subject: [PATCH 101/113] Try to make the dependencies link work --- docs/contributing.rst | 2 ++ letsencrypt-auto | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 8661b195ed3..4c3d9190291 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -297,6 +297,8 @@ commands: This should generate documentation in the ``docs/_build/html`` directory. +.. _prerequisites: + Notes on OS depedencies ======================= diff --git a/letsencrypt-auto b/letsencrypt-auto index 72c0c8aa4d0..0b3d9b72d5f 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -52,7 +52,7 @@ then echo "Sorry, I don't know how to bootstrap Let's Encrypt on your operating system!" echo echo "You will need to bootstrap, configure virtualenv, and run a pip install manually" - echo "Please see https://letsencrypt.readthedocs.org/en/latest/using.html#prerequisites" + echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" echo "for more info" fi From 0fb8e3c47947e2b124734921e7b405c669703e16 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 15 Oct 2015 00:07:26 -0700 Subject: [PATCH 102/113] Move augeaus & venv references --- docs/contributing.rst | 2 ++ docs/using.rst | 4 ---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 4c3d9190291..6d0a2d4ba7f 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -312,6 +312,8 @@ In general: * ``virtualenv`` and ``pip`` are used for managing other python library dependencies +.. _Augeas: http://augeas.net/ +.. _Virtualenv: https://virtualenv.pypa.io Ubuntu ------ diff --git a/docs/using.rst b/docs/using.rst index 5683fba2179..0a781431afc 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -67,10 +67,6 @@ By default, the following locations are searched: .. keep it up to date with constants.py -.. _Augeas: http://augeas.net/ -.. _Virtualenv: https://virtualenv.pypa.io - - Running with Docker =================== From 64385cbd80cde04e9ee53fd046df5bc0ee6ef270 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 15 Oct 2015 00:09:42 -0700 Subject: [PATCH 103/113] Disable nginx in bootstrap/venv.sh too --- bootstrap/venv.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bootstrap/venv.sh b/bootstrap/venv.sh index 619135d0d1a..ce31e670352 100755 --- a/bootstrap/venv.sh +++ b/bootstrap/venv.sh @@ -20,9 +20,7 @@ fi pip install -U setuptools pip install -U pip -# TODO: install apache and nginx plugins by default? -# --pre is not necessary for dev releases in more recent pip versions -pip install -U letsencrypt letsencrypt-apache letsencrypt-nginx +pip install -U letsencrypt letsencrypt-apache # letsencrypt-nginx echo echo "Congratulations, Let's Encrypt has been successfully installed/updated!" From e7809563b16f197659b6e77e66b6a55fe8a6a470 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Thu, 15 Oct 2015 17:23:43 +0000 Subject: [PATCH 104/113] Address first batch of Seth's review comments. --- acme/acme/standalone.py | 7 ++++--- acme/acme/standalone_test.py | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index 089c2ff18c3..97e52fa9f5c 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -46,7 +46,8 @@ def server_bind(self): # pylint: disable=missing-docstring class ACMEServerMixin: # pylint: disable=old-style-class """ACME server common settings mixin.""" - server_version = "ACME standalone client" + # TODO: c.f. #858 + server_version = "ACME client standalone challenge solver" allow_reuse_address = True def __init__(self): @@ -95,7 +96,7 @@ def __init__(self, server_address, resources): class SimpleHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): """SimpleHTTP challenge handler. - Adheres to the stdlib"s `socketserver.BaseRequestHandler` interface. + Adheres to the stdlib's `socketserver.BaseRequestHandler` interface. :ivar set simple_http_resources: A set of `SimpleHTTPResource` objects. TODO: better name? @@ -119,7 +120,7 @@ def do_GET(self): # pylint: disable=invalid-name,missing-docstring def handle_index(self): """Handle index page.""" self.send_response(200) - self.send_header("Content-type", "text/html") + self.send_header("Content-Type", "text/html") self.end_headers() self.wfile.write(self.server.server_version.encode()) diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py index 7eda2fce3e1..14d212d6e9b 100644 --- a/acme/acme/standalone_test.py +++ b/acme/acme/standalone_test.py @@ -122,7 +122,8 @@ def tearDown(self): def test_index(self): response = requests.get( 'http://localhost:{0}'.format(self.port), verify=False) - self.assertEqual(response.text, 'ACME standalone client') + self.assertEqual( + response.text, 'ACME client standalone challenge solver') self.assertTrue(response.ok) def test_404(self): From 6f44bcf11795814f2d4401d76321bba1fbbf763a Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Thu, 15 Oct 2015 18:01:14 +0000 Subject: [PATCH 105/113] standalone2: move alread_listening to perform --- letsencrypt/plugins/standalone.py | 12 +++++++----- letsencrypt/plugins/standalone_test.py | 12 ++++++------ 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index cb95ec408bf..e742734a9a6 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -182,11 +182,7 @@ def more_info(self): # pylint: disable=missing-docstring return self.__doc__ def prepare(self): # pylint: disable=missing-docstring - if any(util.already_listening(port) for port in - (self.config.dvsni_port, self.config.simple_http_port)): - raise errors.MisconfigurationError( - "At least one of the (possibly) required ports is " - "already taken.") + pass def get_chall_pref(self, domain): # pylint: disable=unused-argument,missing-docstring @@ -195,6 +191,12 @@ def get_chall_pref(self, domain): return chall_pref def perform(self, achalls): # pylint: disable=missing-docstring + if any(util.already_listening(port) for port in + (self.config.dvsni_port, self.config.simple_http_port)): + raise errors.MisconfigurationError( + "At least one of the (possibly) required ports is " + "already taken.") + try: return self.perform2(achalls) except errors.StandaloneBindError as error: diff --git a/letsencrypt/plugins/standalone_test.py b/letsencrypt/plugins/standalone_test.py index e99bd473a7f..b873da6f2e4 100644 --- a/letsencrypt/plugins/standalone_test.py +++ b/letsencrypt/plugins/standalone_test.py @@ -101,16 +101,16 @@ def test_supported_challenges(self): def test_more_info(self): self.assertTrue(isinstance(self.auth.more_info(), six.string_types)) - @mock.patch("letsencrypt.plugins.standalone.util") - def test_prepare_misconfiguration(self, mock_util): - mock_util.already_listening.return_value = True - self.assertRaises(errors.MisconfigurationError, self.auth.prepare) - mock_util.already_listening.assert_called_once_with(1234) - def test_get_chall_pref(self): self.assertEqual(set(self.auth.get_chall_pref(domain=None)), set([challenges.DVSNI, challenges.SimpleHTTP])) + @mock.patch("letsencrypt.plugins.standalone.util") + def test_perform_misconfiguration(self, mock_util): + mock_util.already_listening.return_value = True + self.assertRaises(errors.MisconfigurationError, self.auth.perform, []) + mock_util.already_listening.assert_called_once_with(1234) + @mock.patch("letsencrypt.plugins.standalone.zope.component.getUtility") def test_perform(self, unused_mock_get_utility): achalls = [1, 2, 3] From ec24641511e251e9b24cb802783dbd271dfa9ec9 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Thu, 15 Oct 2015 20:31:22 +0000 Subject: [PATCH 106/113] standalone2: run(): tls -> challenge_type. --- letsencrypt/plugins/standalone.py | 16 +++++++++------- letsencrypt/plugins/standalone_test.py | 23 +++++++++++++---------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index e742734a9a6..3ad823e9c78 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -44,27 +44,29 @@ def __init__(self, certs, simple_http_resources): self.certs = certs self.simple_http_resources = simple_http_resources - def run(self, port, tls): + def run(self, port, challenge_type): """Run ACME server on specified ``port``. This method is idempotent, i.e. all calls with the same pair of - ``(port, tls)`` will reuse the same server. + ``(port, challenge_type)`` will reuse the same server. :param int port: Port to run the server on. - :param bool tls: TLS or non-TLS? + :param challenge_type: Subclass of `acme.challenges.Challenge`, + either `acme.challenge.SimpleHTTP` or `acme.challenges.DVSNI`. :returns: Server instance. :rtype: ACMEServerMixin """ + assert challenge_type in (challenges.DVSNI, challenges.SimpleHTTP) if port in self._instances: return self._instances[port].server address = ("", port) try: - if tls: + if challenge_type is challenges.DVSNI: server = acme_standalone.DVSNIServer(address, self.certs) - else: + else: # challenges.SimpleHTTP server = acme_standalone.SimpleHTTPServer( address, self.simple_http_resources) except socket.error as error: @@ -224,7 +226,7 @@ def perform2(self, achalls): for achall in achalls: if isinstance(achall, achallenges.SimpleHTTP): server = self.servers.run( - self.config.simple_http_port, tls=False) + self.config.simple_http_port, challenges.SimpleHTTP) response, validation = achall.gen_response_and_validation( tls=False) self.simple_http_resources.add( @@ -234,7 +236,7 @@ def perform2(self, achalls): cert = self.simple_http_cert domain = achall.domain else: # DVSNI - server = self.servers.run(self.config.dvsni_port, tls=True) + server = self.servers.run(self.config.dvsni_port, challenges.DVSNI) response, cert, _ = achall.gen_cert_and_response(self.key) domain = response.z_domain self.certs[domain] = (self.key, cert) diff --git a/letsencrypt/plugins/standalone_test.py b/letsencrypt/plugins/standalone_test.py index b873da6f2e4..0ccdccb1f0d 100644 --- a/letsencrypt/plugins/standalone_test.py +++ b/letsencrypt/plugins/standalone_test.py @@ -32,23 +32,23 @@ def test_init(self): self.assertTrue( self.mgr.simple_http_resources is self.simple_http_resources) - def _test_run_stop(self, tls): - server = self.mgr.run(port=0, tls=tls) + def _test_run_stop(self, challenge_type): + server = self.mgr.run(port=0, challenge_type=challenge_type) port = server.socket.getsockname()[1] # pylint: disable=no-member self.assertEqual(self.mgr.running(), {port: server}) self.mgr.stop(port=port) self.assertEqual(self.mgr.running(), {}) - def test_run_stop_tls(self): - self._test_run_stop(tls=True) + def test_run_stop_dvsni(self): + self._test_run_stop(challenges.DVSNI) - def test_run_stop_non_tls(self): - self._test_run_stop(tls=False) + def test_run_stop_simplehttp(self): + self._test_run_stop(challenges.SimpleHTTP) def test_run_idempotent(self): - server = self.mgr.run(port=0, tls=False) + server = self.mgr.run(port=0, challenge_type=challenges.SimpleHTTP) port = server.socket.getsockname()[1] # pylint: disable=no-member - server2 = self.mgr.run(port=port, tls=False) + server2 = self.mgr.run(port=port, challenge_type=challenges.SimpleHTTP) self.assertEqual(self.mgr.running(), {port: server}) self.assertTrue(server is server2) self.mgr.stop(port) @@ -59,7 +59,8 @@ def test_run_bind_error(self): some_server.bind(("", 0)) port = some_server.getsockname()[1] self.assertRaises( - errors.StandaloneBindError, self.mgr.run, port, tls=False) + errors.StandaloneBindError, self.mgr.run, port, + challenge_type=challenges.SimpleHTTP) self.assertEqual(self.mgr.running(), {}) @@ -165,7 +166,9 @@ def _run(port, tls): # pylint: disable=unused-argument self.assertTrue(isinstance(responses[1], challenges.DVSNIResponse)) self.assertEqual(self.auth.servers.run.mock_calls, [ - mock.call(4321, tls=False), mock.call(1234, tls=True)]) + mock.call(4321, challenges.SimpleHTTP), + mock.call(1234, challenges.DVSNI), + ]) self.assertEqual(self.auth.served, { "server1234": set([dvsni]), "server4321": set([simple_http]), From 50435740937ef768fc07cd6de63396e39142875c Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Thu, 15 Oct 2015 20:42:10 +0000 Subject: [PATCH 107/113] Integration tests: SimpleHTTP on port 5002. Should be pulled by @jsha in lockstep with https://github.com/letsencrypt/boulder/commit/6b8f1c27003bb43f33ffcfa9a5f63eb4154c6df9. --- tests/integration/_common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index fd60b92588e..ab645f6d60c 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -16,7 +16,7 @@ letsencrypt_test () { --server "${SERVER:-http://localhost:4000/directory}" \ --no-verify-ssl \ --dvsni-port 5001 \ - --simple-http-port 5001 \ + --simple-http-port 5002 \ --manual-test-mode \ $store_flags \ --text \ From d1b7b0553fbccfb8579615b8c38517b8596e789f Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Thu, 15 Oct 2015 20:53:47 +0000 Subject: [PATCH 108/113] Back to 5001, but with env var --- tests/integration/_common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index ab645f6d60c..418856def7d 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -16,7 +16,7 @@ letsencrypt_test () { --server "${SERVER:-http://localhost:4000/directory}" \ --no-verify-ssl \ --dvsni-port 5001 \ - --simple-http-port 5002 \ + --simple-http-port ${SIMPLE_HTTP_PORT:-5001} \ --manual-test-mode \ $store_flags \ --text \ From 22b8446efb8c8f7f4d1438e7460e3b8fa1634615 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 15 Oct 2015 14:32:52 -0700 Subject: [PATCH 109/113] Get precompiled Goose from GithHub. Rather than fetching from bitbucket and building. Bitbucket is often down, and building from scratch is slow. Github is sometimes down, but at least now we have our eggs in one basket. --- tests/boulder-start.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/boulder-start.sh b/tests/boulder-start.sh index 530f9c59884..051c832f218 100755 --- a/tests/boulder-start.sh +++ b/tests/boulder-start.sh @@ -32,11 +32,10 @@ export PATH="$GOPATH/bin:$PATH" go get -d github.com/letsencrypt/boulder/... cd $GOPATH/src/github.com/letsencrypt/boulder # goose is needed for ./test/create_db.sh -if ! go get bitbucket.org/liamstask/goose/cmd/goose ; then - echo Problems installing goose... perhaps rm -rf \$GOPATH \("$GOPATH"\) - echo and try again... - exit 1 -fi +wget https://github.com/jsha/boulder-tools/raw/master/goose.gz && \ + mkdir $GOPATH/bin && \ + zcat goose.gz > $GOPATH/bin/goose && \ + chmod +x $GOPATH/bin/goose ./test/create_db.sh ./start.py & # Hopefully start.py bootstraps before integration test is started... From 0614fba9df41df3c9b50915786e865186ad42f03 Mon Sep 17 00:00:00 2001 From: David Dworken Date: Thu, 15 Oct 2015 20:21:41 -0400 Subject: [PATCH 110/113] Make the script POSIX compliant so it works with dash (Fixes #977) --- tests/boulder-integration.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 981c8967a4a..55b42ef9a33 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -14,7 +14,7 @@ export PATH="/usr/sbin:$PATH" # /usr/sbin/nginx export GOPATH="${GOPATH:-/tmp/go}" export PATH="$GOPATH/bin:$PATH" -if [ `uname` == 'Darwin' ]; then +if [ `uname` = "Darwin" ];then readlink="greadlink" else readlink="readlink" From 2956d5d913a262cb11f53e8eda012abf6ad1958d Mon Sep 17 00:00:00 2001 From: lf Date: Thu, 15 Oct 2015 21:28:32 -0600 Subject: [PATCH 111/113] Trailing whitespace. Oops. --- letsencrypt-nginx/letsencrypt_nginx/nginxparser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-nginx/letsencrypt_nginx/nginxparser.py b/letsencrypt-nginx/letsencrypt_nginx/nginxparser.py index 84b54b4dba5..c9a24aabe10 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/nginxparser.py +++ b/letsencrypt-nginx/letsencrypt_nginx/nginxparser.py @@ -87,7 +87,7 @@ def __iter__(self, blocks=None, current_indent=0, spacer=' '): def __str__(self): """Return the parsed block as a string.""" return '\n'.join(self) + '\n' - + def as_string(self): """Return the parsed block as a string.""" return str(self) From 5c1858627b5fd1703b8bd2435d4573ea997073c4 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Fri, 16 Oct 2015 22:25:20 +0000 Subject: [PATCH 112/113] pep8 love --- letsencrypt/cli.py | 2 +- letsencrypt/client.py | 2 +- letsencrypt/plugins/null.py | 2 +- letsencrypt/tests/client_test.py | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 24f032ca1c8..477cb653f55 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -555,7 +555,7 @@ def preprocess_args(self, args): for i, token in enumerate(args): if token in VERBS: - reordered = args[:i] + args[i+1:] + [args[i]] + reordered = args[:i] + args[(i + 1):] + [args[i]] self.verb = token return reordered diff --git a/letsencrypt/client.py b/letsencrypt/client.py index bb04f4f5a5d..3e32ab01567 100644 --- a/letsencrypt/client.py +++ b/letsencrypt/client.py @@ -311,7 +311,7 @@ def save_certificate(self, certr, chain_cert, cert_path, chain_path): return os.path.abspath(act_cert_path), cert_chain_abspath def deploy_certificate(self, domains, privkey_path, - cert_path, chain_path, fullchain_path): + cert_path, chain_path, fullchain_path): """Install certificate :param list domains: list of domains to install the certificate diff --git a/letsencrypt/plugins/null.py b/letsencrypt/plugins/null.py index e875376849b..cdb96a11663 100644 --- a/letsencrypt/plugins/null.py +++ b/letsencrypt/plugins/null.py @@ -31,7 +31,7 @@ def get_all_names(self): return [] def deploy_cert(self, domain, cert_path, key_path, - chain_path=None, fullchain_path=None): + chain_path=None, fullchain_path=None): pass # pragma: no cover def enhance(self, domain, enhancement, options=None): diff --git a/letsencrypt/tests/client_test.py b/letsencrypt/tests/client_test.py index 9a5a2bbe1a2..3f7b84a6418 100644 --- a/letsencrypt/tests/client_test.py +++ b/letsencrypt/tests/client_test.py @@ -151,8 +151,8 @@ def test_deploy_certificate(self): installer = mock.MagicMock() self.client.installer = installer - self.client.deploy_certificate(["foo.bar"], "key", "cert", "chain", - "fullchain") + self.client.deploy_certificate( + ["foo.bar"], "key", "cert", "chain", "fullchain") installer.deploy_cert.assert_called_once_with( cert_path=os.path.abspath("cert"), chain_path=os.path.abspath("chain"), From 670bc1e3d3f6145e98060a069806781cc22695ba Mon Sep 17 00:00:00 2001 From: lf Date: Fri, 16 Oct 2015 19:03:21 -0600 Subject: [PATCH 113/113] Remove as_string as per request. --- letsencrypt-nginx/letsencrypt_nginx/nginxparser.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/letsencrypt-nginx/letsencrypt_nginx/nginxparser.py b/letsencrypt-nginx/letsencrypt_nginx/nginxparser.py index c9a24aabe10..cef0756d755 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/nginxparser.py +++ b/letsencrypt-nginx/letsencrypt_nginx/nginxparser.py @@ -88,10 +88,6 @@ def __str__(self): """Return the parsed block as a string.""" return '\n'.join(self) + '\n' - def as_string(self): - """Return the parsed block as a string.""" - return str(self) - # Shortcut functions to respect Python's serialization interface # (like pyyaml, picker or json)