Merge remote-tracking branch 'github/letsencrypt/master' into standal…

…one2

acme/acme/messages.py

@@ -24,6 +24,7 @@ class Error(jose.JSONObjectWithFields, Exception):
         'connection': 'The server could not connect to the client for DV',
         'dnssec': 'The server could not validate a DNSSEC signed domain',
         'malformed': 'The request message was malformed',
+        'rateLimited': 'There were too many requests of a given type',
         'serverInternal': 'The server experienced an internal error',
         'tls': 'The server experienced a TLS error during DV',
         'unauthorized': 'The client lacks sufficient authorization',

letsencrypt-apache/letsencrypt_apache/configurator.py

@@ -1,5 +1,6 @@
 """Apache Configuration based off of Augeas Configurator."""
 # pylint: disable=too-many-lines
+import filecmp
 import itertools
 import logging
 import os
@@ -163,7 +164,8 @@ def prepare(self):
 
         temp_install(self.mod_ssl_conf)
 
-    def deploy_cert(self, domain, cert_path, key_path, chain_path=None):
+    def deploy_cert(self, domain, cert_path, key_path,
+            chain_path=None, fullchain_path=None): # pylint: disable=unused-argument
         """Deploys certificate to specified virtual host.
 
         Currently tries to find the last directives to deploy the cert in
@@ -945,9 +947,11 @@ def is_site_enabled(self, avail_fp):
         """
         enabled_dir = os.path.join(self.parser.root, "sites-enabled")
         for entry in os.listdir(enabled_dir):
-            if os.path.realpath(os.path.join(enabled_dir, entry)) == avail_fp:
-                return True
-
+            try:
+                if filecmp.cmp(avail_fp, os.path.join(enabled_dir, entry)):
+                    return True
+            except OSError:
+                pass
         return False
 
     def enable_site(self, vhost):

letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/common.py

@@ -175,12 +175,13 @@ def get_testable_domain_names(self):
         else:
             return {"example.com"}
 
-    def deploy_cert(self, domain, cert_path, key_path, chain_path=None):
+    def deploy_cert(self, domain, cert_path, key_path, chain_path=None,
+            fullchain_path=None):
         """Installs cert"""
         cert_path, key_path, chain_path = self.copy_certs_and_keys(
             cert_path, key_path, chain_path)
         self._apache_configurator.deploy_cert(
-            domain, cert_path, key_path, chain_path)
+            domain, cert_path, key_path, chain_path, fullchain_path)
 
 
 def _is_apache_command(command):

letsencrypt-nginx/letsencrypt_nginx/configurator.py

@@ -6,6 +6,7 @@
 import socket
 import subprocess
 import sys
+import time
 
 import OpenSSL
 import zope.interface
@@ -117,30 +118,44 @@ def prepare(self):
         temp_install(self.mod_ssl_conf)
 
     # Entry point in main.py for installing cert
-    def deploy_cert(self, domain, cert_path, key_path, chain_path=None):
+    def deploy_cert(self, domain, cert_path, key_path,
+            chain_path, fullchain_path):
         # pylint: disable=unused-argument
         """Deploys certificate to specified virtual host.
 
         .. note:: Aborts if the vhost is missing ssl_certificate or
             ssl_certificate_key.
 
-        .. note:: Nginx doesn't have a cert chain directive, so the last
-            parameter is always ignored. It expects the cert file to have
-            the concatenated chain.
+        .. note:: Nginx doesn't have a cert chain directive.
+            It expects the cert file to have the concatenated chain.
+            However, we use the chain file as input to the
+            ssl_trusted_certificate directive, used for verify OCSP responses.
 
         .. note:: This doesn't save the config files!
 
         """
         vhost = self.choose_vhost(domain)
-        directives = [['ssl_certificate', cert_path],
-                      ['ssl_certificate_key', key_path]]
+        cert_directives = [['ssl_certificate', fullchain_path],
+                           ['ssl_certificate_key', key_path]]
+
+        # OCSP stapling was introduced in Nginx 1.3.7. If we have that version
+        # or greater, add config settings for it.
+        stapling_directives = []
+        if self.version >= (1, 3, 7):
+            stapling_directives = [
+                ['ssl_trusted_certificate', chain_path],
+                ['ssl_stapling', 'on'],
+                ['ssl_stapling_verify', 'on']]
 
         try:
             self.parser.add_server_directives(vhost.filep, vhost.names,
-                                              directives, True)
+                                              cert_directives, replace=True)
+            self.parser.add_server_directives(vhost.filep, vhost.names,
+                                              stapling_directives, replace=False)
             logger.info("Deployed Certificate to VirtualHost %s for %s",
                         vhost.filep, vhost.names)
-        except errors.MisconfigurationError:
+        except errors.MisconfigurationError as error:
+            logger.debug(error)
             logger.warn(
                 "Cannot find a cert or key directive in %s for %s. "
                 "VirtualHost was not modified.", vhost.filep, vhost.names)
@@ -598,6 +613,10 @@ def nginx_restart(nginx_ctl, nginx_conf="/etc/nginx.conf"):
     except (OSError, ValueError):
         logger.fatal("Nginx Restart Failed - Please Check the Configuration")
         sys.exit(1)
+    # Nginx can take a moment to recognize a newly added TLS SNI servername, so sleep
+    # for a second. TODO: Check for expected servername and loop until it
+    # appears or return an error if looping too long.
+    time.sleep(1)
 
     return True
 

letsencrypt-nginx/letsencrypt_nginx/dvsni.py

@@ -90,14 +90,22 @@ def _mod_config(self, ll_addrs):
         # Add the 'include' statement for the challenges if it doesn't exist
         # already in the main config
         included = False
-        directive = ['include', self.challenge_conf]
+        include_directive = ['include', self.challenge_conf]
         root = self.configurator.parser.loc["root"]
+
+        bucket_directive = ['server_names_hash_bucket_size', '128']
+
         main = self.configurator.parser.parsed[root]
-        for entry in main:
-            if entry[0] == ['http']:
-                body = entry[1]
-                if directive not in body:
-                    body.append(directive)
+        for key, body in main:
+            if key == ['http']:
+                found_bucket = False
+                for key, _ in body:
+                    if key == bucket_directive[0]:
+                        found_bucket = True
+                if not found_bucket:
+                    body.insert(0, bucket_directive)
+                if include_directive not in body:
+                    body.insert(0, include_directive)
                 included = True
                 break
         if not included:

letsencrypt-nginx/letsencrypt_nginx/parser.py

@@ -476,8 +476,12 @@ def _add_directives(block, directives, replace=False):
     :param list directives: The new directives.
 
     """
-    if replace:
-        for directive in directives:
+    for directive in directives:
+        if not replace:
+            # We insert new directives at the top of the block, mostly
+            # to work around https://trac.nginx.org/nginx/ticket/810
+            block.insert(0, directive)
+        else:
             changed = False
             if len(directive) == 0:
                 continue
@@ -489,5 +493,3 @@ def _add_directives(block, directives, replace=False):
                 raise errors.MisconfigurationError(
                     'LetsEncrypt expected directive for %s in the Nginx '
                     'config but did not find it.' % directive[0])
-    else:
-        block.extend(directives)

letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py

@@ -63,11 +63,11 @@ def test_save(self):
 
         # pylint: disable=protected-access
         parsed = self.config.parser._parse_files(filep, override=True)
-        self.assertEqual([[['server'], [['listen', '69.50.225.155:9000'],
+        self.assertEqual([[['server'], [['listen', '5001 ssl'],
+                                        ['listen', '69.50.225.155:9000'],
                                         ['listen', '127.0.0.1'],
                                         ['server_name', '.example.com'],
-                                        ['server_name', 'example.*'],
-                                        ['listen', '5001 ssl']]]],
+                                        ['server_name', 'example.*']]]],
                          parsed[0])
 
     def test_choose_vhost(self):
@@ -96,18 +96,49 @@ def test_choose_vhost(self):
     def test_more_info(self):
         self.assertTrue('nginx.conf' in self.config.more_info())
 
+    def test_deploy_cert_stapling(self):
+        # Choose a version of Nginx greater than 1.3.7 so stapling code gets
+        # invoked.
+        self.config.version = (1, 9, 6)
+        example_conf = self.config.parser.abs_path('sites-enabled/example.com')
+        self.config.deploy_cert(
+            "www.example.com",
+            "example/cert.pem",
+            "example/key.pem",
+            "example/chain.pem",
+            "example/fullchain.pem")
+        self.config.save()
+        self.config.parser.load()
+        generated_conf = self.config.parser.parsed[example_conf]
+
+        self.assertTrue(util.contains_at_depth(generated_conf,
+            ['ssl_stapling', 'on'], 2))
+        self.assertTrue(util.contains_at_depth(generated_conf,
+            ['ssl_stapling_verify', 'on'], 2))
+        self.assertTrue(util.contains_at_depth(generated_conf,
+            ['ssl_trusted_certificate', 'example/chain.pem'], 2))
+
     def test_deploy_cert(self):
         server_conf = self.config.parser.abs_path('server.conf')
         nginx_conf = self.config.parser.abs_path('nginx.conf')
         example_conf = self.config.parser.abs_path('sites-enabled/example.com')
+        # Choose a version of Nginx less than 1.3.7 so stapling code doesn't get
+        # invoked.
+        self.config.version = (1, 3, 1)
 
         # Get the default SSL vhost
         self.config.deploy_cert(
             "www.example.com",
-            "example/cert.pem", "example/key.pem")
+            "example/cert.pem",
+            "example/key.pem",
+            "example/chain.pem",
+            "example/fullchain.pem")
         self.config.deploy_cert(
             "another.alias",
-            "/etc/nginx/cert.pem", "/etc/nginx/key.pem")
+            "/etc/nginx/cert.pem",
+            "/etc/nginx/key.pem",
+            "/etc/nginx/chain.pem",
+            "/etc/nginx/fullchain.pem")
         self.config.save()
 
         self.config.parser.load()
@@ -119,35 +150,34 @@ def test_deploy_cert(self):
         access_log = os.path.join(self.work_dir, "access.log")
         error_log = os.path.join(self.work_dir, "error.log")
         self.assertEqual([[['server'],
-                           [['listen', '69.50.225.155:9000'],
+                           [['include', self.config.parser.loc["ssl_options"]],
+                            ['ssl_certificate_key', 'example/key.pem'],
+                            ['ssl_certificate', 'example/fullchain.pem'],
+                            ['error_log', error_log],
+                            ['access_log', access_log],
+
+                            ['listen', '5001 ssl'],
+                            ['listen', '69.50.225.155:9000'],
                             ['listen', '127.0.0.1'],
                             ['server_name', '.example.com'],
-                            ['server_name', 'example.*'],
-                            ['listen', '5001 ssl'],
-                            ['access_log', access_log],
-                            ['error_log', error_log],
-                            ['ssl_certificate', 'example/cert.pem'],
-                            ['ssl_certificate_key', 'example/key.pem'],
-                            ['include',
-                             self.config.parser.loc["ssl_options"]]]]],
+                            ['server_name', 'example.*']]]],
                          parsed_example_conf)
         self.assertEqual([['server_name', 'somename  alias  another.alias']],
                          parsed_server_conf)
-        self.assertEqual([['server'],
-                          [['listen', '8000'],
-                           ['listen', 'somename:8080'],
-                           ['include', 'server.conf'],
-                           [['location', '/'],
-                            [['root', 'html'],
-                             ['index', 'index.html index.htm']]],
-                           ['listen', '5001 ssl'],
-                           ['access_log', access_log],
-                           ['error_log', error_log],
-                           ['ssl_certificate', '/etc/nginx/cert.pem'],
-                           ['ssl_certificate_key', '/etc/nginx/key.pem'],
-                           ['include',
-                            self.config.parser.loc["ssl_options"]]]],
-                         parsed_nginx_conf[-1][-1][-1])
+        self.assertTrue(util.contains_at_depth(parsed_nginx_conf,
+                       [['server'],
+                        [['include', self.config.parser.loc["ssl_options"]],
+                         ['ssl_certificate_key', '/etc/nginx/key.pem'],
+                         ['ssl_certificate', '/etc/nginx/fullchain.pem'],
+                         ['error_log', error_log],
+                         ['access_log', access_log],
+                         ['listen', '5001 ssl'],
+                         ['listen', '8000'],
+                         ['listen', 'somename:8080'],
+                         ['include', 'server.conf'],
+                         [['location', '/'],
+                          [['root', 'html'], ['index', 'index.html index.htm']]]]],
+                         2))
 
     def test_get_all_certs_keys(self):
         nginx_conf = self.config.parser.abs_path('nginx.conf')
@@ -156,16 +186,22 @@ def test_get_all_certs_keys(self):
         # Get the default SSL vhost
         self.config.deploy_cert(
             "www.example.com",
-            "example/cert.pem", "example/key.pem")
+            "example/cert.pem",
+            "example/key.pem",
+            "example/chain.pem",
+            "example/fullchain.pem")
         self.config.deploy_cert(
             "another.alias",
-            "/etc/nginx/cert.pem", "/etc/nginx/key.pem")
+            "/etc/nginx/cert.pem",
+            "/etc/nginx/key.pem",
+            "/etc/nginx/chain.pem",
+            "/etc/nginx/fullchain.pem")
         self.config.save()
 
         self.config.parser.load()
         self.assertEqual(set([
-            ('example/cert.pem', 'example/key.pem', example_conf),
-            ('/etc/nginx/cert.pem', '/etc/nginx/key.pem', nginx_conf),
+            ('example/fullchain.pem', 'example/key.pem', example_conf),
+            ('/etc/nginx/fullchain.pem', '/etc/nginx/key.pem', nginx_conf),
         ]), self.config.get_all_certs_keys())
 
     @mock.patch("letsencrypt_nginx.configurator.dvsni.NginxDvsni.perform")

letsencrypt-nginx/letsencrypt_nginx/tests/dvsni_test.py

@@ -85,7 +85,8 @@ def test_perform1(self, mock_save):
         # Make sure challenge config is included in main config
         http = self.sni.configurator.parser.parsed[
             self.sni.configurator.parser.loc["root"]][-1]
-        self.assertTrue(['include', self.sni.challenge_conf] in http[1])
+        self.assertTrue(
+            util.contains_at_depth(http, ['include', self.sni.challenge_conf], 1))
 
     def test_perform2(self):
         acme_responses = []
@@ -108,7 +109,8 @@ def test_perform2(self):
         http = self.sni.configurator.parser.parsed[
             self.sni.configurator.parser.loc["root"]][-1]
         self.assertTrue(['include', self.sni.challenge_conf] in http[1])
-        self.assertTrue(['server_name', 'blah'] in http[1][-2][1])
+        self.assertTrue(
+            util.contains_at_depth(http, ['server_name', 'blah'], 3))
 
         self.assertEqual(len(sni_responses), 3)
         for i in xrange(3):

letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py

@@ -128,18 +128,20 @@ def test_add_server_directives(self):
                                            r'~^(www\.)?(example|bar)\.']),
                                       [['foo', 'bar'], ['ssl_certificate',
                                                         '/etc/ssl/cert.pem']])
-        ssl_re = re.compile(r'foo bar;\n\s+ssl_certificate /etc/ssl/cert.pem')
-        self.assertEqual(1, len(re.findall(ssl_re, nginxparser.dumps(
-            nparser.parsed[nparser.abs_path('nginx.conf')]))))
-        nparser.add_server_directives(nparser.abs_path('server.conf'),
+        ssl_re = re.compile(r'\n\s+ssl_certificate /etc/ssl/cert.pem')
+        dump = nginxparser.dumps(nparser.parsed[nparser.abs_path('nginx.conf')])
+        self.assertEqual(1, len(re.findall(ssl_re, dump)))
+
+        server_conf = nparser.abs_path('server.conf')
+        nparser.add_server_directives(server_conf,
                                       set(['alias', 'another.alias',
                                            'somename']),
                                       [['foo', 'bar'], ['ssl_certificate',
                                                         '/etc/ssl/cert2.pem']])
-        self.assertEqual(nparser.parsed[nparser.abs_path('server.conf')],
-                         [['server_name', 'somename  alias  another.alias'],
+        self.assertEqual(nparser.parsed[server_conf],
+                         [['ssl_certificate', '/etc/ssl/cert2.pem'],
                           ['foo', 'bar'],
-                          ['ssl_certificate', '/etc/ssl/cert2.pem']])
+                          ['server_name', 'somename  alias  another.alias']])
 
     def test_add_http_directives(self):
         nparser = parser.NginxParser(self.config_path, self.ssl_options)
@@ -148,8 +150,15 @@ def test_add_http_directives(self):
                  [['listen', '80'],
                   ['server_name', 'localhost']]]
         nparser.add_http_directives(filep, block)
-        self.assertEqual(nparser.parsed[filep][-1][0], ['http'])
-        self.assertEqual(nparser.parsed[filep][-1][1][-1], block)
+        root = nparser.parsed[filep]
+        self.assertTrue(util.contains_at_depth(root, ['http'], 1))
+        self.assertTrue(util.contains_at_depth(root, block, 2))
+
+        # Check that our server block got inserted first among all server
+        # blocks.
+        http_block = [x for x in root if x[0] == ['http']][0][1]
+        server_blocks = [x for x in http_block if x[0] == ['server']]
+        self.assertEqual(server_blocks[0], block)
 
     def test_replace_server_directives(self):
         nparser = parser.NginxParser(self.config_path, self.ssl_options)