From 171ee7a65467c29f161a4a39f3ebe543f5f41789 Mon Sep 17 00:00:00 2001 From: Samuel Chiang Date: Fri, 16 Feb 2024 13:27:52 -0800 Subject: [PATCH] Slight tweaks and integration CI to support Bind9 (#1423) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. This adds an integration CI dimension for Bind9 2. Resolved "cmocka unit tests" for Bind9 * Additional import in : Bind depends on some ASN1 functions, but does not directly import the corresponding header. OpenSSL imports the asn1 header file in objects.h (which Bind is pulling these symbols from), so I've added the header file reference to objects.h. * SSL_get_error error anticipation fixing: There were several failures discovered to be related this, thanks to research done in Implement SSL_MODE_AUTO_RETRY #1333. The issue was pinned down the check implemented in google/boringssl@9a38e92. This check used to exist before the final return of SSL_get_error in OpenSSL. Upstream moved this earlier in the function with google/boringssl@fcf2583. However, much of the functions guards for i < 0 checks have been removed since OpenSSL 1.1.1, so the early logic no longer applies. This check has evolved into SSL_ERROR_ZERO_RETURN in our code. Moving the check further down helps us gain better parity with OpenSSL 1.1.1. Doing so passes the bind test failures for proxystream_test, tls_test, and doh_test. This also happens to help our integration with CPython, so I've reconfigured that patch. We actually already use SSL_AUTO_RETRY by default in AWS-LC. The recent change mentioned in the point above surrounding the flag (208327e) was just to make some of the errors consistent in CPython when the flag was used. I've reverted the special behavior surrounding it since it should no longer be needed. * Assertion for SSL_set_shutdown: The assertion was added in 63006a9, where it’s stated that we didn’t want SSL_set_shutdown messing up the state machine. This assertion is causing failures in tlsdns_test for Bind9, so it appears that we'll have to remove this to gain better OpenSSL parity. 3. Patch file needed for Bind seems to be slight bug in their build configuration. This was from a fairly recent commit. We can look to contribute this sometime soon. --- .github/workflows/integrations.yml | 11 +++ include/openssl/objects.h | 1 + include/openssl/ssl.h | 10 +-- ssl/ssl_lib.cc | 22 +----- ssl/ssl_test.cc | 30 ++------ .../ci/integration/bind9_patch/bind-fix.patch | 19 +++++ .../python_patch/3.10/aws-lc-cpython.patch | 27 +++---- .../python_patch/3.11/aws-lc-cpython.patch | 23 ++---- .../python_patch/3.12/aws-lc-cpython.patch | 75 ++++++++----------- .../python_patch/main/aws-lc-cpython.patch | 23 ++---- tests/ci/integration/run_bind9_integration.sh | 73 ++++++++++++++++++ .../ci/integration/run_python_integration.sh | 2 - 12 files changed, 170 insertions(+), 146 deletions(-) create mode 100644 tests/ci/integration/bind9_patch/bind-fix.patch create mode 100755 tests/ci/integration/run_bind9_integration.sh diff --git a/.github/workflows/integrations.yml b/.github/workflows/integrations.yml index 975fa33423..87bd013f99 100644 --- a/.github/workflows/integrations.yml +++ b/.github/workflows/integrations.yml @@ -102,3 +102,14 @@ jobs: - name: Build AWS-LC, build python, run tests run: | ./tests/ci/integration/run_python_integration.sh + bind9: + runs-on: ubuntu-latest + steps: + - name: Install OS Dependencies + run: | + sudo apt-get update + sudo apt-get -y --no-install-recommends install cmake gcc ninja-build golang make python3 python3-pytest autoconf pkg-config libcmocka-dev liburcu-dev libuv1-dev libnghttp2-dev libcap-dev libprotobuf-c-dev protobuf-c-compiler libfstrm-dev libjemalloc-dev + - uses: actions/checkout@v3 + - name: Run bind9 build + run: | + ./tests/ci/integration/run_bind9_integration.sh diff --git a/include/openssl/objects.h b/include/openssl/objects.h index dd6556f264..5ebfa26f20 100644 --- a/include/openssl/objects.h +++ b/include/openssl/objects.h @@ -16,3 +16,4 @@ OpenSSL easier. */ #include "obj.h" +#include "asn1.h" diff --git a/include/openssl/ssl.h b/include/openssl/ssl.h index e9e8aea9eb..c5389f3f63 100644 --- a/include/openssl/ssl.h +++ b/include/openssl/ssl.h @@ -836,10 +836,6 @@ OPENSSL_EXPORT uint32_t SSL_get_options(const SSL *ssl); // |write|. In DTLS, it does nothing. #define SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER 0x00000002L -// SSL_MODE_AUTO_RETRY suppresses terminal errors on empty reads if the -// underlying connection state is retryable, allowing for automatic retries. -#define SSL_MODE_AUTO_RETRY 0x00000004L - // SSL_MODE_NO_AUTO_CHAIN disables automatically building a certificate chain // before sending certificates to the peer. This flag is set (and the feature // disabled) by default. @@ -5281,6 +5277,7 @@ DEFINE_STACK_OF(SSL_COMP) // The following flags do nothing and are included only to make it easier to // compile code with BoringSSL. +#define SSL_MODE_AUTO_RETRY 0 #define SSL_MODE_RELEASE_BUFFERS 0 #define SSL_MODE_SEND_CLIENTHELLO_TIME 0 #define SSL_MODE_SEND_SERVERHELLO_TIME 0 @@ -5454,9 +5451,8 @@ OPENSSL_EXPORT int SSL_state(const SSL *ssl); // receiving close_notify in |SSL_shutdown| by causing the implementation to // believe the events already happened. // -// It is an error to use |SSL_set_shutdown| to unset a bit that has already been -// set. Doing so will trigger an |assert| in debug builds and otherwise be -// ignored. +// Note: |SSL_set_shutdown| cannot be used to unset a bit that has already +// been set in AWS-LC. Doing so will be ignored. // // Use |SSL_CTX_set_quiet_shutdown| instead. OPENSSL_EXPORT void SSL_set_shutdown(SSL *ssl, int mode); diff --git a/ssl/ssl_lib.cc b/ssl/ssl_lib.cc index b3cd398b1e..24cfecea10 100644 --- a/ssl/ssl_lib.cc +++ b/ssl/ssl_lib.cc @@ -1382,23 +1382,6 @@ int SSL_get_error(const SSL *ssl, int ret_code) { return SSL_ERROR_SSL; } - if (ret_code == 0) { - if (ssl->s3->rwstate == SSL_ERROR_ZERO_RETURN) { - return SSL_ERROR_ZERO_RETURN; - } - // An EOF was observed which violates the protocol, and the underlying - // transport does not participate in the error queue. If - // |SSL_MODE_AUTO_RETRY| is unset, bubble up to the caller. - if ((ssl->ctx->mode & SSL_MODE_AUTO_RETRY) == 0) { - return SSL_ERROR_SYSCALL; - } - // If |SSL_MODE_AUTO_RETRY| is set, proceed if in a retryable state. - if (ssl->s3->rwstate != SSL_ERROR_WANT_READ - && ssl->s3->rwstate != SSL_ERROR_WANT_WRITE) { - return SSL_ERROR_SYSCALL; - } - } - switch (ssl->s3->rwstate) { case SSL_ERROR_PENDING_SESSION: case SSL_ERROR_PENDING_CERTIFICATE: @@ -1411,6 +1394,7 @@ int SSL_get_error(const SSL *ssl, int ret_code) { case SSL_ERROR_WANT_CERTIFICATE_VERIFY: case SSL_ERROR_WANT_RENEGOTIATE: case SSL_ERROR_HANDSHAKE_HINTS_READY: + case SSL_ERROR_ZERO_RETURN: return ssl->s3->rwstate; case SSL_ERROR_WANT_READ: { @@ -2611,10 +2595,6 @@ void SSL_set_quiet_shutdown(SSL *ssl, int mode) { int SSL_get_quiet_shutdown(const SSL *ssl) { return ssl->quiet_shutdown; } void SSL_set_shutdown(SSL *ssl, int mode) { - // It is an error to clear any bits that have already been set. (We can't try - // to get a second close_notify or send two.) - assert((SSL_get_shutdown(ssl) & mode) == SSL_get_shutdown(ssl)); - if (mode & SSL_RECEIVED_SHUTDOWN && ssl->s3->read_shutdown == ssl_shutdown_none) { ssl->s3->read_shutdown = ssl_shutdown_close_notify; diff --git a/ssl/ssl_test.cc b/ssl/ssl_test.cc index fae31b5c05..c390f4da2a 100644 --- a/ssl/ssl_test.cc +++ b/ssl/ssl_test.cc @@ -10458,7 +10458,9 @@ TEST(SSLTest, ErrorSyscallAfterCloseNotify) { write_failed = false; } -static void TestIntermittentEmptyRead(bool auto_retry) { +// Test that failures are supressed on (potentially) +// transient empty reads. +TEST(SSLTest, IntermittentEmptyRead) { bssl::UniquePtr client_ctx(SSL_CTX_new(TLS_method())); bssl::UniquePtr server_ctx = CreateContextWithTestCertificate(TLS_method()); @@ -10489,15 +10491,6 @@ static void TestIntermittentEmptyRead(bool auto_retry) { ASSERT_TRUE(BIO_up_ref(client_rbio.get())); SSL_set0_rbio(client.get(), rbio_empty.release()); - if (auto_retry) { - // Set flag under test - ASSERT_TRUE(SSL_CTX_set_mode(client_ctx.get(), SSL_MODE_AUTO_RETRY)); - ASSERT_TRUE(SSL_CTX_get_mode(client_ctx.get()) & SSL_MODE_AUTO_RETRY); - } else { - // |SSL_MODE_AUTO_RETRY| is off by default - ASSERT_FALSE(SSL_CTX_get_mode(client_ctx.get()) & SSL_MODE_AUTO_RETRY); - } - // Server writes some data to the client const uint8_t write_data[] = {1, 2, 3}; int ret = SSL_write(server.get(), write_data, (int) sizeof(write_data)); @@ -10507,13 +10500,9 @@ static void TestIntermittentEmptyRead(bool auto_retry) { uint8_t read_data[] = {0, 0, 0}; ret = SSL_read(client.get(), read_data, sizeof(read_data)); EXPECT_EQ(ret, 0); - if (auto_retry) { - // On empty read, client should still want a read so caller will retry - EXPECT_EQ(SSL_get_error(client.get(), ret), SSL_ERROR_WANT_READ); - } else { - // On empty read, client should error out signaling EOF - EXPECT_EQ(SSL_get_error(client.get(), ret), SSL_ERROR_SYSCALL); - } + // On empty read, client should still want a read so caller will retry. + // This would have returned |SSL_ERROR_SYSCALL| in OpenSSL 1.0.2. + EXPECT_EQ(SSL_get_error(client.get(), ret), SSL_ERROR_WANT_READ); // Reset client rbio, read should succeed SSL_set0_rbio(client.get(), client_rbio.release()); @@ -10528,13 +10517,6 @@ static void TestIntermittentEmptyRead(bool auto_retry) { EXPECT_EQ(SSL_get_error(client.get(), ret), SSL_ERROR_WANT_READ); } -// Test that |SSL_MODE_AUTO_RETRY| suppresses failure on (potentially) -// transient empty reads. -TEST(SSLTest, IntermittentEmptyRead) { - TestIntermittentEmptyRead(false); - TestIntermittentEmptyRead(true); -} - // Test that |SSL_shutdown|, when quiet shutdown is enabled, simulates receiving // a close_notify, down to |SSL_read| reporting |SSL_ERROR_ZERO_RETURN|. TEST(SSLTest, QuietShutdown) { diff --git a/tests/ci/integration/bind9_patch/bind-fix.patch b/tests/ci/integration/bind9_patch/bind-fix.patch new file mode 100644 index 0000000000..8629ad00d2 --- /dev/null +++ b/tests/ci/integration/bind9_patch/bind-fix.patch @@ -0,0 +1,19 @@ +diff --git a/tests/isc/Makefile.am b/tests/isc/Makefile.am +index 5cdd915..6ee1935 100644 +--- a/tests/isc/Makefile.am ++++ b/tests/isc/Makefile.am +@@ -115,10 +115,12 @@ proxyheader_test_SOURCES = \ + proxyheader_test_data.h + + proxystream_test_CPPFLAGS = \ +- $(AM_CPPFLAGS) ++ $(AM_CPPFLAGS) \ ++ $(OPENSSL_CFLAGS) + + proxystream_test_LDADD = \ +- $(LDADD) ++ $(LDADD) \ ++ $(OPENSSL_LIBS) + + proxystream_test_SOURCES = \ + proxystream_test.c \ diff --git a/tests/ci/integration/python_patch/3.10/aws-lc-cpython.patch b/tests/ci/integration/python_patch/3.10/aws-lc-cpython.patch index 460f9e7544..6c0e77ab8a 100644 --- a/tests/ci/integration/python_patch/3.10/aws-lc-cpython.patch +++ b/tests/ci/integration/python_patch/3.10/aws-lc-cpython.patch @@ -1,5 +1,5 @@ diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py -index 253a6c119c..2d0d10642d 100644 +index 253a6c1..2d0d106 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -1106,12 +1106,12 @@ def test_create_server_ssl_match_failed(self): @@ -20,7 +20,7 @@ index 253a6c119c..2d0d10642d 100644 # close connection diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py -index 77152cf645..be3d11b993 100644 +index 77152cf..be3d11b 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -1863,7 +1863,7 @@ def test_host_port(self): @@ -33,7 +33,7 @@ index 77152cf645..be3d11b993 100644 # just check status of PHA flag h = client.HTTPSConnection('localhost', 443) diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py -index b5c78a5d49..41235c17fc 100644 +index b5c78a5..41235c1 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -555,9 +555,10 @@ def test_ssl_raises(self): @@ -66,7 +66,7 @@ index b5c78a5d49..41235c17fc 100644 client = self.imap_class(*server.server_address, ssl_context=ssl_context) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py -index a1a581a907..c69e71159a 100644 +index a1a581a..c69e711 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -44,6 +44,7 @@ @@ -319,8 +319,8 @@ index a1a581a907..c69e71159a 100644 protocols = [ @@ -4752,6 +4776,31 @@ def test_internal_chain_server(self): self.assertEqual(res, b'\x02\n') - - + + +@unittest.skipUnless(Py_OPENSSL_IS_AWSLC, "Only test this against AWS-LC") +class TestPostHandshakeAuthAwsLc(unittest.TestCase): + def test_pha(self): @@ -350,7 +350,7 @@ index a1a581a907..c69e71159a 100644 requires_keylog = unittest.skipUnless( HAS_KEYLOG, 'test requires OpenSSL 1.1.1 with keylog callback') diff --git a/Modules/Setup b/Modules/Setup -index 87c6a152f8..3a9bc54bab 100644 +index 87c6a15..f67d7ec 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -208,8 +208,8 @@ _symtable symtablemodule.c @@ -386,7 +386,7 @@ index 87c6a152f8..3a9bc54bab 100644 # The crypt module is now disabled by default because it breaks builds # on many systems (where -lcrypt is needed), e.g. Linux (I believe). diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c -index 35addf49e9..77a12c6af5 100644 +index 35addf4..77a12c6 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -131,8 +131,12 @@ static const py_hashentry_t py_hashes[] = { @@ -403,7 +403,7 @@ index 35addf49e9..77a12c6af5 100644 }; diff --git a/Modules/_ssl.c b/Modules/_ssl.c -index 7a28f2d37f..07740af98b 100644 +index 7a28f2d..b0d2ea1 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -181,6 +181,12 @@ extern const SSL_METHOD *TLSv1_2_method(void); @@ -463,15 +463,6 @@ index 7a28f2d37f..07740af98b 100644 int err = SSL_verify_client_post_handshake(self->ssl); if (err == 0) return _setSSLError(get_state_sock(self), NULL, 0, __FILE__, __LINE__); -@@ -3186,7 +3198,7 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) - - /* Set SSL_MODE_RELEASE_BUFFERS. This potentially greatly reduces memory - usage for no cost at all. */ -- SSL_CTX_set_mode(self->ctx, SSL_MODE_RELEASE_BUFFERS); -+ SSL_CTX_set_mode(self->ctx, SSL_MODE_RELEASE_BUFFERS | SSL_MODE_AUTO_RETRY); - - #define SID_CTX "Python" - SSL_CTX_set_session_id_context(self->ctx, (const unsigned char *) SID_CTX, @@ -3199,7 +3211,7 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) X509_VERIFY_PARAM_set_flags(params, X509_V_FLAG_TRUSTED_FIRST); X509_VERIFY_PARAM_set_hostflags(params, self->hostflags); diff --git a/tests/ci/integration/python_patch/3.11/aws-lc-cpython.patch b/tests/ci/integration/python_patch/3.11/aws-lc-cpython.patch index 78f943cec3..d248433fed 100644 --- a/tests/ci/integration/python_patch/3.11/aws-lc-cpython.patch +++ b/tests/ci/integration/python_patch/3.11/aws-lc-cpython.patch @@ -1,5 +1,5 @@ diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py -index d7871d3e53..fb65ee3b8b 100644 +index d7871d3..fb65ee3 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -1103,12 +1103,12 @@ def test_create_server_ssl_match_failed(self): @@ -20,7 +20,7 @@ index d7871d3e53..fb65ee3b8b 100644 # close connection diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py -index 015a3d1e87..ec565ad181 100644 +index 015a3d1..ec565ad 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -2040,7 +2040,7 @@ def test_host_port(self): @@ -33,7 +33,7 @@ index 015a3d1e87..ec565ad181 100644 # just check status of PHA flag h = client.HTTPSConnection('localhost', 443) diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py -index bd0fc9c2da..3ab70f5c1f 100644 +index bd0fc9c..3ab70f5 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -561,9 +561,10 @@ def test_ssl_raises(self): @@ -66,7 +66,7 @@ index bd0fc9c2da..3ab70f5c1f 100644 client = self.imap_class(*server.server_address, ssl_context=ssl_context) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py -index 1f881038c3..36fa1e32e8 100644 +index 1f88103..36fa1e3 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -44,6 +44,7 @@ @@ -359,7 +359,7 @@ index 1f881038c3..36fa1e32e8 100644 requires_keylog = unittest.skipUnless( HAS_KEYLOG, 'test requires OpenSSL 1.1.1 with keylog callback') diff --git a/Modules/Setup b/Modules/Setup -index d3647ecb99..a0ff874b6d 100644 +index d3647ec..a0ff874 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -216,11 +216,11 @@ PYTHONPATH=$(COREPYTHONPATH) @@ -380,7 +380,7 @@ index d3647ecb99..a0ff874b6d 100644 # The _tkinter module. # diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c -index 57d64bd80c..1132fa520c 100644 +index 57d64bd..1132fa5 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -131,8 +131,12 @@ static const py_hashentry_t py_hashes[] = { @@ -397,7 +397,7 @@ index 57d64bd80c..1132fa520c 100644 }; diff --git a/Modules/_ssl.c b/Modules/_ssl.c -index 67ce6e97af..1132d82dd9 100644 +index 67ce6e9..6f38611 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -179,6 +179,12 @@ extern const SSL_METHOD *TLSv1_2_method(void); @@ -457,15 +457,6 @@ index 67ce6e97af..1132d82dd9 100644 int err = SSL_verify_client_post_handshake(self->ssl); if (err == 0) return _setSSLError(get_state_sock(self), NULL, 0, __FILE__, __LINE__); -@@ -3191,7 +3203,7 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) - - /* Set SSL_MODE_RELEASE_BUFFERS. This potentially greatly reduces memory - usage for no cost at all. */ -- SSL_CTX_set_mode(self->ctx, SSL_MODE_RELEASE_BUFFERS); -+ SSL_CTX_set_mode(self->ctx, SSL_MODE_RELEASE_BUFFERS | SSL_MODE_AUTO_RETRY); - - #define SID_CTX "Python" - SSL_CTX_set_session_id_context(self->ctx, (const unsigned char *) SID_CTX, @@ -3204,7 +3216,7 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) X509_VERIFY_PARAM_set_flags(params, X509_V_FLAG_TRUSTED_FIRST); X509_VERIFY_PARAM_set_hostflags(params, self->hostflags); diff --git a/tests/ci/integration/python_patch/3.12/aws-lc-cpython.patch b/tests/ci/integration/python_patch/3.12/aws-lc-cpython.patch index b10c62b765..433d708373 100644 --- a/tests/ci/integration/python_patch/3.12/aws-lc-cpython.patch +++ b/tests/ci/integration/python_patch/3.12/aws-lc-cpython.patch @@ -1,8 +1,8 @@ diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py -index b9069056c3..cffbc074a8 100644 +index ddc45fd..f982bc2 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py -@@ -1122,12 +1122,12 @@ def test_create_server_ssl_match_failed(self): +@@ -1126,12 +1126,12 @@ def test_create_server_ssl_match_failed(self): # incorrect server_hostname f_c = self.loop.create_connection(MyProto, host, port, ssl=sslcontext_client) @@ -20,10 +20,10 @@ index b9069056c3..cffbc074a8 100644 # close connection diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py -index fe8105ee2b..157c243faf 100644 +index 089bf5b..1713a5f 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py -@@ -2035,7 +2035,7 @@ def test_host_port(self): +@@ -2066,7 +2066,7 @@ def test_host_port(self): def test_tls13_pha(self): import ssl @@ -33,10 +33,10 @@ index fe8105ee2b..157c243faf 100644 # just check status of PHA flag h = client.HTTPSConnection('localhost', 443) diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py -index 60f5b671b1..56b64a6788 100644 +index def9f45..d9aa621 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py -@@ -555,9 +555,10 @@ def test_ssl_raises(self): +@@ -558,9 +558,10 @@ def test_ssl_raises(self): self.assertEqual(ssl_context.check_hostname, True) ssl_context.load_verify_locations(CAFILE) @@ -50,7 +50,7 @@ index 60f5b671b1..56b64a6788 100644 _, server = self._setup(SimpleIMAPHandler) client = self.imap_class(*server.server_address, ssl_context=ssl_context) -@@ -950,10 +951,10 @@ def test_ssl_verified(self): +@@ -954,10 +955,10 @@ def test_ssl_verified(self): ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) ssl_context.load_verify_locations(CAFILE) @@ -66,10 +66,10 @@ index 60f5b671b1..56b64a6788 100644 client = self.imap_class(*server.server_address, ssl_context=ssl_context) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py -index 6117ca3fdb..fc93c6271b 100644 +index 330515a..6745875 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py -@@ -37,6 +37,7 @@ +@@ -41,6 +41,7 @@ from ssl import TLSVersion, _TLSContentType, _TLSMessageType, _TLSAlertType Py_DEBUG_WIN32 = support.Py_DEBUG and sys.platform == 'win32' @@ -77,7 +77,7 @@ index 6117ca3fdb..fc93c6271b 100644 PROTOCOLS = sorted(ssl._PROTOCOL_NAMES) HOST = socket_helper.HOST -@@ -163,7 +164,7 @@ def is_ubuntu(): +@@ -167,7 +168,7 @@ def is_ubuntu(): except FileNotFoundError: return False @@ -86,7 +86,7 @@ index 6117ca3fdb..fc93c6271b 100644 def seclevel_workaround(*ctxs): """"Lower security level to '1' and allow all ciphers for TLS 1.0/1""" for ctx in ctxs: -@@ -540,7 +541,7 @@ def test_openssl_version(self): +@@ -544,7 +545,7 @@ def test_openssl_version(self): else: openssl_ver = f"OpenSSL {major:d}.{minor:d}.{fix:d}" self.assertTrue( @@ -95,7 +95,7 @@ index 6117ca3fdb..fc93c6271b 100644 (s, t, hex(n)) ) -@@ -1098,7 +1099,6 @@ def test_min_max_version(self): +@@ -1102,7 +1103,6 @@ def test_min_max_version(self): if has_tls_protocol(ssl.PROTOCOL_TLSv1_1): ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1_1) @@ -103,7 +103,7 @@ index 6117ca3fdb..fc93c6271b 100644 self.assertIn( ctx.minimum_version, minimum_range ) -@@ -1158,24 +1158,25 @@ def test_load_cert_chain(self): +@@ -1162,24 +1162,25 @@ def test_load_cert_chain(self): with self.assertRaises(OSError) as cm: ctx.load_cert_chain(NONEXISTINGCERT) self.assertEqual(cm.exception.errno, errno.ENOENT) @@ -135,7 +135,7 @@ index 6117ca3fdb..fc93c6271b 100644 ctx.load_cert_chain(CAFILE_CACERT, ONLYKEY) # Password protected key and cert ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD) -@@ -1243,7 +1244,7 @@ def test_load_verify_locations(self): +@@ -1247,7 +1248,7 @@ def test_load_verify_locations(self): with self.assertRaises(OSError) as cm: ctx.load_verify_locations(NONEXISTINGCERT) self.assertEqual(cm.exception.errno, errno.ENOENT) @@ -144,7 +144,7 @@ index 6117ca3fdb..fc93c6271b 100644 ctx.load_verify_locations(BADCERT) ctx.load_verify_locations(CERTFILE, CAPATH) ctx.load_verify_locations(CERTFILE, capath=BYTES_CAPATH) -@@ -1646,10 +1647,11 @@ def test_lib_reason(self): +@@ -1650,10 +1651,11 @@ def test_lib_reason(self): ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) with self.assertRaises(ssl.SSLError) as cm: ctx.load_dh_params(CERTFILE) @@ -159,7 +159,7 @@ index 6117ca3fdb..fc93c6271b 100644 def test_subclass(self): # Check that the appropriate SSLError subclass is raised -@@ -1825,7 +1827,8 @@ def test_connect_fail(self): +@@ -1833,7 +1835,8 @@ def test_connect_fail(self): s = test_wrap_socket(socket.socket(socket.AF_INET), cert_reqs=ssl.CERT_REQUIRED) self.addCleanup(s.close) @@ -169,7 +169,7 @@ index 6117ca3fdb..fc93c6271b 100644 s.connect, self.server_addr) def test_connect_ex(self): -@@ -1893,7 +1896,8 @@ def test_connect_with_context_fail(self): +@@ -1901,7 +1904,8 @@ def test_connect_with_context_fail(self): server_hostname=SIGNED_CERTFILE_HOSTNAME ) self.addCleanup(s.close) @@ -179,7 +179,7 @@ index 6117ca3fdb..fc93c6271b 100644 s.connect, self.server_addr) def test_connect_capath(self): -@@ -2110,14 +2114,14 @@ def test_bio_handshake(self): +@@ -2118,14 +2122,14 @@ def test_bio_handshake(self): self.assertIsNone(sslobj.version()) self.assertIsNone(sslobj.shared_ciphers()) self.assertRaises(ValueError, sslobj.getpeercert) @@ -196,7 +196,7 @@ index 6117ca3fdb..fc93c6271b 100644 self.assertTrue(sslobj.get_channel_binding('tls-unique')) try: self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap) -@@ -2839,11 +2843,11 @@ def test_crl_check(self): +@@ -2849,11 +2853,11 @@ def test_crl_check(self): client_context.verify_flags |= ssl.VERIFY_CRL_CHECK_LEAF server = ThreadedEchoServer(context=server_context, chatty=True) @@ -210,7 +210,7 @@ index 6117ca3fdb..fc93c6271b 100644 s.connect((HOST, server.port)) # now load a CRL file. The CRL file is signed by the CA. -@@ -2874,12 +2878,12 @@ def test_check_hostname(self): +@@ -2884,12 +2888,12 @@ def test_check_hostname(self): # incorrect hostname should raise an exception server = ThreadedEchoServer(context=server_context, chatty=True) @@ -226,7 +226,7 @@ index 6117ca3fdb..fc93c6271b 100644 s.connect((HOST, server.port)) # missing server_hostname arg should cause an exception, too -@@ -3073,7 +3077,7 @@ def test_wrong_cert_tls13(self): +@@ -3083,7 +3087,7 @@ def test_wrong_cert_tls13(self): s.connect((HOST, server.port)) with self.assertRaisesRegex( ssl.SSLError, @@ -235,7 +235,7 @@ index 6117ca3fdb..fc93c6271b 100644 ): # TLS 1.3 perform client cert exchange after handshake s.write(b'data') -@@ -3142,8 +3146,13 @@ def test_ssl_cert_verify_error(self): +@@ -3152,8 +3156,13 @@ def test_ssl_cert_verify_error(self): self.assertIsInstance(e, ssl.SSLCertVerificationError) self.assertEqual(e.verify_code, 20) self.assertEqual(e.verify_message, msg) @@ -250,7 +250,7 @@ index 6117ca3fdb..fc93c6271b 100644 def test_PROTOCOL_TLS(self): """Connecting to an SSLv23 server with various client options""" -@@ -3654,7 +3663,10 @@ def test_no_shared_ciphers(self): +@@ -3685,7 +3694,10 @@ def test_no_shared_ciphers(self): server_hostname=hostname) as s: with self.assertRaises(OSError): s.connect((HOST, server.port)) @@ -262,7 +262,7 @@ index 6117ca3fdb..fc93c6271b 100644 def test_version_basic(self): """ -@@ -3742,7 +3754,10 @@ def test_min_max_version_mismatch(self): +@@ -3773,7 +3785,10 @@ def test_min_max_version_mismatch(self): server_hostname=hostname) as s: with self.assertRaises(ssl.SSLError) as e: s.connect((HOST, server.port)) @@ -274,7 +274,7 @@ index 6117ca3fdb..fc93c6271b 100644 @requires_tls_version('SSLv3') def test_min_max_version_sslv3(self): -@@ -3784,6 +3799,9 @@ def test_tls_unique_channel_binding(self): +@@ -3815,6 +3830,9 @@ def test_tls_unique_channel_binding(self): client_context, server_context, hostname = testing_context() @@ -284,7 +284,7 @@ index 6117ca3fdb..fc93c6271b 100644 server = ThreadedEchoServer(context=server_context, chatty=True, connectionchatty=False) -@@ -3870,6 +3888,7 @@ def test_no_legacy_server_connect(self): +@@ -3901,6 +3919,7 @@ def test_no_legacy_server_connect(self): sni_name=hostname) @unittest.skipIf(Py_DEBUG_WIN32, "Avoid mixing debug/release CRT on Windows") @@ -292,7 +292,7 @@ index 6117ca3fdb..fc93c6271b 100644 def test_dh_params(self): # Check we can get a connection with ephemeral Diffie-Hellman client_context, server_context, hostname = testing_context() -@@ -3884,7 +3903,7 @@ def test_dh_params(self): +@@ -3915,7 +3934,7 @@ def test_dh_params(self): cipher = stats["cipher"][0] parts = cipher.split("-") if "ADH" not in parts and "EDH" not in parts and "DHE" not in parts: @@ -301,7 +301,7 @@ index 6117ca3fdb..fc93c6271b 100644 def test_ecdh_curve(self): # server secp384r1, client auto -@@ -4051,8 +4070,10 @@ def cb_raising(ssl_sock, server_name, initial_context): +@@ -4082,8 +4101,10 @@ def cb_raising(ssl_sock, server_name, initial_context): chatty=False, sni_name='supermessage') @@ -314,7 +314,7 @@ index 6117ca3fdb..fc93c6271b 100644 self.assertEqual(catch.unraisable.exc_type, ZeroDivisionError) def test_sni_callback_wrong_return_type(self): -@@ -4228,7 +4249,10 @@ def test_session_handling(self): +@@ -4259,7 +4280,10 @@ def test_session_handling(self): 'Session refers to a different SSLContext.') @@ -326,7 +326,7 @@ index 6117ca3fdb..fc93c6271b 100644 class TestPostHandshakeAuth(unittest.TestCase): def test_pha_setter(self): protocols = [ -@@ -4504,6 +4528,32 @@ def test_internal_chain_server(self): +@@ -4535,6 +4559,32 @@ def test_internal_chain_server(self): self.assertEqual(res, b'\x02\n') @@ -360,7 +360,7 @@ index 6117ca3fdb..fc93c6271b 100644 requires_keylog = unittest.skipUnless( HAS_KEYLOG, 'test requires OpenSSL 1.1.1 with keylog callback') diff --git a/Modules/Setup b/Modules/Setup -index a8faa1d102..8abe8fd7d2 100644 +index a8faa1d..8abe8fd 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -215,11 +215,11 @@ PYTHONPATH=$(COREPYTHONPATH) @@ -381,7 +381,7 @@ index a8faa1d102..8abe8fd7d2 100644 # The _tkinter module. # diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c -index af6d1b23d3..4b060b30a5 100644 +index af6d1b2..4b060b3 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -130,8 +130,12 @@ static const py_hashentry_t py_hashes[] = { @@ -398,7 +398,7 @@ index af6d1b23d3..4b060b30a5 100644 }; diff --git a/Modules/_ssl.c b/Modules/_ssl.c -index e939f95048..8c99e2d8a1 100644 +index b602eb0..b9f6ed5 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -182,6 +182,12 @@ extern const SSL_METHOD *TLSv1_2_method(void); @@ -458,15 +458,6 @@ index e939f95048..8c99e2d8a1 100644 int err = SSL_verify_client_post_handshake(self->ssl); if (err == 0) return _setSSLError(get_state_sock(self), NULL, 0, __FILE__, __LINE__); -@@ -3169,7 +3181,7 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) - - /* Set SSL_MODE_RELEASE_BUFFERS. This potentially greatly reduces memory - usage for no cost at all. */ -- SSL_CTX_set_mode(self->ctx, SSL_MODE_RELEASE_BUFFERS); -+ SSL_CTX_set_mode(self->ctx, SSL_MODE_RELEASE_BUFFERS | SSL_MODE_AUTO_RETRY); - - #define SID_CTX "Python" - SSL_CTX_set_session_id_context(self->ctx, (const unsigned char *) SID_CTX, @@ -3182,7 +3194,7 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) X509_VERIFY_PARAM_set_flags(params, X509_V_FLAG_TRUSTED_FIRST); X509_VERIFY_PARAM_set_hostflags(params, self->hostflags); diff --git a/tests/ci/integration/python_patch/main/aws-lc-cpython.patch b/tests/ci/integration/python_patch/main/aws-lc-cpython.patch index cce7884f25..964bf7dcf1 100644 --- a/tests/ci/integration/python_patch/main/aws-lc-cpython.patch +++ b/tests/ci/integration/python_patch/main/aws-lc-cpython.patch @@ -1,5 +1,5 @@ diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py -index b25c097573..8f07868b82 100644 +index b25c097..8f07868 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -1125,12 +1125,12 @@ def test_create_server_ssl_match_failed(self): @@ -20,7 +20,7 @@ index b25c097573..8f07868b82 100644 # close connection diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py -index 089bf5be40..1713a5fbae 100644 +index 089bf5b..1713a5f 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -2066,7 +2066,7 @@ def test_host_port(self): @@ -33,7 +33,7 @@ index 089bf5be40..1713a5fbae 100644 # just check status of PHA flag h = client.HTTPSConnection('localhost', 443) diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py -index b97474acca..f5980598c9 100644 +index b97474a..f598059 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -558,9 +558,10 @@ def test_ssl_raises(self): @@ -66,7 +66,7 @@ index b97474acca..f5980598c9 100644 client = self.imap_class(*server.server_address, ssl_context=ssl_context) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py -index 3fdfa29605..6f6fea7c36 100644 +index 3fdfa29..6f6fea7 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -41,6 +41,7 @@ @@ -359,7 +359,7 @@ index 3fdfa29605..6f6fea7c36 100644 requires_keylog = unittest.skipUnless( HAS_KEYLOG, 'test requires OpenSSL 1.1.1 with keylog callback') diff --git a/Modules/Setup b/Modules/Setup -index 8ad9a5aebb..6138085c61 100644 +index 8ad9a5a..6138085 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -208,11 +208,11 @@ PYTHONPATH=$(COREPYTHONPATH) @@ -380,7 +380,7 @@ index 8ad9a5aebb..6138085c61 100644 # The _tkinter module. # diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c -index 0e230f332f..390d6e0ed9 100644 +index 0e230f3..390d6e0 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -130,8 +130,12 @@ static const py_hashentry_t py_hashes[] = { @@ -397,7 +397,7 @@ index 0e230f332f..390d6e0ed9 100644 }; diff --git a/Modules/_ssl.c b/Modules/_ssl.c -index bc30290942..e0e59b97e5 100644 +index bc30290..0e52f96 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -187,6 +187,13 @@ extern const SSL_METHOD *TLSv1_2_method(void); @@ -458,15 +458,6 @@ index bc30290942..e0e59b97e5 100644 int err = SSL_verify_client_post_handshake(self->ssl); if (err == 0) return _setSSLError(get_state_sock(self), NULL, 0, __FILE__, __LINE__); -@@ -3203,7 +3216,7 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) - - /* Set SSL_MODE_RELEASE_BUFFERS. This potentially greatly reduces memory - usage for no cost at all. */ -- SSL_CTX_set_mode(self->ctx, SSL_MODE_RELEASE_BUFFERS); -+ SSL_CTX_set_mode(self->ctx, SSL_MODE_RELEASE_BUFFERS | SSL_MODE_AUTO_RETRY); - - params = SSL_CTX_get0_param(self->ctx); - /* Improve trust chain building when cross-signed intermediate @@ -3211,7 +3224,7 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) X509_VERIFY_PARAM_set_flags(params, X509_V_FLAG_TRUSTED_FIRST); X509_VERIFY_PARAM_set_hostflags(params, self->hostflags); diff --git a/tests/ci/integration/run_bind9_integration.sh b/tests/ci/integration/run_bind9_integration.sh new file mode 100755 index 0000000000..92ff7dfe64 --- /dev/null +++ b/tests/ci/integration/run_bind9_integration.sh @@ -0,0 +1,73 @@ +#!/bin/bash -exu +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 OR ISC + +source tests/ci/common_posix_setup.sh + +# Set up environment. + +# SYS_ROOT +# - SRC_ROOT(aws-lc) +# - SCRATCH_FOLDER +# - bind9 +# - AWS_LC_BUILD_FOLDER +# - AWS_LC_INSTALL_FOLDER +# - BIND9_BUILD_FOLDER + +# Assumes script is executed from the root of aws-lc directory +SCRATCH_FOLDER="${SRC_ROOT}/BIND9_BUILD_ROOT" +BIND9_SRC_FOLDER="${SCRATCH_FOLDER}/bind9" +BIND9_BUILD_FOLDER="${SCRATCH_FOLDER}/bind9-aws-lc" +BIND9_PATCH_FOLDER=${SRC_ROOT}/"tests/ci/integration/bind9_patch" +AWS_LC_BUILD_FOLDER="${SCRATCH_FOLDER}/aws-lc-build" +AWS_LC_INSTALL_FOLDER="${SCRATCH_FOLDER}/aws-lc-install" + +function bind9_build() { + autoreconf -fi + PKG_CONFIG_PATH="${AWS_LC_INSTALL_FOLDER}/lib/pkgconfig" ./configure \ + --with-openssl="${AWS_LC_INSTALL_FOLDER}" \ + --enable-dnstap \ + --enable-dnsrps \ + --with-cmocka \ + --with-libxml2 \ + --enable-leak-detection + make -j ${NUM_CPU_THREADS} -k all +} + +# TODO: Remove this when we make an upstream contribution. +function bind9_patch() { + for patchfile in $(find -L "${BIND9_PATCH_FOLDER}" -type f -name '*.patch'); do + echo "Apply patch $patchfile..." + patch -p1 --quiet -i "$patchfile" + done +} + +function bind9_run_tests() { + make -j ${NUM_CPU_THREADS} check +} + +mkdir -p ${SCRATCH_FOLDER} +rm -rf ${SCRATCH_FOLDER}/* +cd ${SCRATCH_FOLDER} + +git clone https://gitlab.isc.org/isc-projects/bind9.git ${BIND9_SRC_FOLDER} --depth 1 +mkdir -p ${AWS_LC_BUILD_FOLDER} ${AWS_LC_INSTALL_FOLDER} ${BIND9_BUILD_FOLDER} +ls + +aws_lc_build ${SRC_ROOT} ${AWS_LC_BUILD_FOLDER} ${AWS_LC_INSTALL_FOLDER} -DBUILD_TESTING=OFF -DBUILD_TOOL=OFF -DBUILD_SHARED_LIBS=1 +export LD_LIBRARY_PATH="${AWS_LC_INSTALL_FOLDER}/lib" + +# Build bind9 from source. +pushd ${BIND9_SRC_FOLDER} + +bind9_patch +bind9_build +bind9_run_tests + +# Iterate through all of bind's vended artifacts. +for libname in dns ns isc isccc isccfg; do + ldd "${BIND9_SRC_FOLDER}/lib/${libname}/.libs/lib${libname}.so" | grep "${AWS_LC_INSTALL_FOLDER}/lib/libcrypto.so" || exit 1 + ldd "${BIND9_SRC_FOLDER}/lib/${libname}/.libs/lib${libname}.so" | grep "${AWS_LC_INSTALL_FOLDER}/lib/libssl.so" || exit 1 +done + +popd diff --git a/tests/ci/integration/run_python_integration.sh b/tests/ci/integration/run_python_integration.sh index 168cd91403..7bdc3245df 100755 --- a/tests/ci/integration/run_python_integration.sh +++ b/tests/ci/integration/run_python_integration.sh @@ -89,8 +89,6 @@ function python_run_tests() { # authentication portion of that protocol. # - Modify the ssl module's backing C code to account for AWS-LC's divergent # function signature and return value for |sk_SSL_CIPHER_find| -# - Modify the ssl module's backing C code to set |SSL_MODE_AUTO_RETRY| in -# all calls to |SSL{_CTX}_set_mode| # # TODO: Remove these patches when we make an upstream contribution. function python_patch() {