updated documentation and added set_sign functionality

crypto/fipsmodule/rsa/internal.h

@@ -78,6 +78,7 @@ struct rsa_meth_st {
     // size returns the size of the RSA modulus in bytes.
     size_t (*size)(const RSA *rsa);
 
+    // Set via |RSA_meth_set_sign|.
     int (*sign)(int type, const uint8_t *m, unsigned int m_length,
                 uint8_t *sigret, unsigned int *siglen, const RSA *rsa);
 

crypto/fipsmodule/rsa/rsa.c

@@ -451,13 +451,15 @@ int RSA_set0_crt_params(RSA *rsa, BIGNUM *dmp1, BIGNUM *dmq1, BIGNUM *iqmp) {
 RSA_METHOD *RSA_meth_new(const char *name, int flags) {
   RSA_METHOD *meth = OPENSSL_zalloc(sizeof(*meth));
 
-  if (meth != NULL) {
-    meth->flags = flags;
-    return meth;
+  if (meth == NULL) {
+    OPENSSL_PUT_ERROR(RSA, ERR_R_MALLOC_FAILURE);
+    return NULL;
   }
 
-  OPENSSL_PUT_ERROR(RSA, ERR_R_MALLOC_FAILURE);
-  return NULL;
+  if (flags == RSA_FLAG_OPAQUE) {
+    meth->flags = flags;
+  }
+  return meth;
 }
 
 int RSA_set_method(RSA *rsa, const RSA_METHOD *meth) {
@@ -568,6 +570,18 @@ int RSA_meth_set0_app_data(RSA_METHOD *meth, void *app_data) {
   return 1;
 }
 
+int RSA_meth_set_sign(RSA_METHOD *meth, int (*sign) (int type,
+        const unsigned char *m, unsigned int m_length, unsigned char *sigret,
+        unsigned int *siglen, const RSA *rsa)) {
+  if(meth == NULL) {
+    OPENSSL_PUT_ERROR(RSA, ERR_R_PASSED_NULL_PARAMETER);
+    return 0;
+  }
+
+  meth->sign = sign;
+  return 1;
+}
+
 static int rsa_sign_raw_no_self_test(RSA *rsa, size_t *out_len, uint8_t *out,
                                      size_t max_out, const uint8_t *in,
                                      size_t in_len, int padding) {
@@ -582,6 +596,7 @@ static int rsa_sign_raw_no_self_test(RSA *rsa, size_t *out_len, uint8_t *out,
     // here.
     int ret = rsa->meth->sign_raw((int)max_out, in, out, rsa, padding);
     if(ret < 0) {
+      *out_len = 0;
       return 0;
     }
     *out_len = ret;
@@ -1011,12 +1026,12 @@ int rsa_private_transform(RSA *rsa, uint8_t *out, const uint8_t *in,
 
 int RSA_flags(const RSA *rsa) {
   SET_DIT_AUTO_DISABLE;
-  if (rsa) {
-    return rsa->flags;
+  if (rsa == NULL) {
+    OPENSSL_PUT_ERROR(RSA, ERR_R_PASSED_NULL_PARAMETER);
+    return 0;
   }
 
-  OPENSSL_PUT_ERROR(RSA, ERR_R_PASSED_NULL_PARAMETER);
-  return 0;
+  return rsa->flags;
 }
 
 void RSA_set_flags(RSA *rsa, int flags) {

crypto/fipsmodule/rsa/rsa_impl.c

@@ -422,6 +422,7 @@ int rsa_verify_raw_no_self_test(RSA *rsa, size_t *out_len, uint8_t *out,
     // here.
     int ret = rsa->meth->verify_raw((int)max_out, in, out, rsa, padding);
     if(ret < 0) {
+      *out_len = 0;
       return 0;
     }
     *out_len = ret;

crypto/rsa_extra/rsa_crypt.c

@@ -395,6 +395,7 @@ int RSA_encrypt(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out,
     // here.
     int ret = rsa->meth->encrypt((int)max_out, in, out, rsa, padding);
     if(ret < 0) {
+      *out_len = 0;
       return 0;
     }
     *out_len = ret;
@@ -566,6 +567,7 @@ int RSA_decrypt(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out,
     // OpenSSL, we initialize |out_len| based on the return value here.
     int ret = rsa->meth->decrypt((int)max_out, in, out, rsa, padding);
     if(ret < 0) {
+      *out_len = 0;
       return 0;
     }
     *out_len = ret;

include/openssl/engine.h

@@ -27,9 +27,8 @@ extern "C" {
 // be overridden via a callback. This can be used, for example, to implement an
 // RSA* that forwards operations to a hardware module.
 //
-// Methods are reference counted but |ENGINE|s are not. When creating a method,
-// you should zero the whole structure and fill in the function pointers that
-// you wish before setting it on an |ENGINE|. Any functions pointers that
+// Default Methods are zero initialized. You should set the function pointers
+// that you wish before setting it on an |ENGINE|. Any functions pointers that
 // are NULL indicate that the default behaviour should be used.
 
 

include/openssl/rsa.h

@@ -222,8 +222,9 @@ OPENSSL_EXPORT int RSA_set0_crt_params(RSA *rsa, BIGNUM *dmp1, BIGNUM *dmq1,
 OPENSSL_EXPORT const RSA_METHOD *RSA_get_default_method(void);
 
 // RSA_meth_new returns a zero-initialized |RSA_METHOD| object. It sets
-// |flags| on the object. The |name| parameter is currently ignored and
-// part of the function signature for OpenSSL compatibility.
+// |flags| on the object. Currently, only |RSA_FLAG_OPAQUE| can be set on
+// the method structure. The |name| parameter is currently ignored and
+// is part of the function signature for OpenSSL compatibility.
 OPENSSL_EXPORT RSA_METHOD *RSA_meth_new(const char *name, int flags);
 
 // RSA_set_method sets |meth| on |rsa|. Returns one on success and zero
@@ -237,8 +238,8 @@ OPENSSL_EXPORT const RSA_METHOD *RSA_get_method(const RSA *rsa);
 OPENSSL_EXPORT void RSA_meth_free(RSA_METHOD *meth);
 
 // RSA_METHOD setters
-// The following functions set the corresponding fields on |meth|. Returns one
-// on success and zero on failure.
+// The following functions set the corresponding fields on |meth|. They return
+// one on success and zero on failure.
 
 // RSA_meth_set_init sets |init| on |meth|. |init| should return one on
 // success and zero on failure.
@@ -294,10 +295,20 @@ OPENSSL_EXPORT int RSA_meth_set_pub_enc(RSA_METHOD *meth,
 // RSA_meth_set0_app_data sets |app_data| on |meth|. Although set0 functions
 // generally take ownership in AWS-LC, to maintain OpenSSL compatibility,
 // this function does not. It is the consumers responsibility to free
-// |app_data| before |meth| is freed.
+// |app_data|.
 OPENSSL_EXPORT int RSA_meth_set0_app_data(RSA_METHOD *meth, void *app_data);
 
 
+// RSA_meth_set_sign sets |sign| on |meth|. The function |sign| should return
+// one on success and zero on failure.
+OPENSSL_EXPORT int RSA_meth_set_sign(RSA_METHOD *meth,
+                                     int (*sign) (int type,
+                                     const unsigned char *m,
+                                     unsigned int m_length,
+                                     unsigned char *sigret,
+                                     unsigned int *siglen, const RSA *rsa));
+
+
 // Key generation.
 
 // RSA_generate_key_ex generates a new RSA key where the modulus has size
@@ -811,8 +822,9 @@ OPENSSL_EXPORT void *RSA_get_ex_data(const RSA *rsa, int idx);
 #define RSA_METHOD_FLAG_NO_CHECK RSA_FLAG_OPAQUE
 
 // RSAerr allows consumers to add an error for a given function |f| and reason
-// |r|. To avoid exposing internals, we ignore the |f| parameter. The |r|
-// parameter is passed into |OPENSSL_PUT_ERROR|.
+// |r|. This macro is added in for OpenSSL compatibility. To avoid exposing
+// internals, we ignore the |f| parameter. The |r| parameter is passed into
+// |OPENSSL_PUT_ERROR|.
 #define RSAerr(f,r) OPENSSL_PUT_ERROR(RSA, r);
 
 // RSA_flags returns the flags for |rsa|. These are a bitwise OR of |RSA_FLAG_*|