Fix RSA encryption/decryption with OAEP padding [#30572]

[mono.git] / mcs / class / referencesource / mscorlib / system / security / cryptography / utils.cs

diff --git a/mcs/class/referencesource/mscorlib/system/security/cryptography/utils.cs b/mcs/class/referencesource/mscorlib/system/security/cryptography/utils.cs
index 77c72021c009d631f08810f595038ab2ef33a241..0d80fa76c3e9e3a72aed9fd549be76054f123ae6 100644 (file)
--- a/mcs/class/referencesource/mscorlib/system/security/cryptography/utils.cs
+++ b/mcs/class/referencesource/mscorlib/system/security/cryptography/utils.cs
@@ -134,7 +134,7 @@ namespace System.Security.Cryptography
     }
 
     internal static class Utils {
-#if !MONO
+
 #if !FEATURE_PAL && FEATURE_CRYPTO
         [SecuritySafeCritical]
 #endif
@@ -162,10 +162,13 @@ namespace System.Security.Cryptography
                 if (!_haveDefaultRsaProviderType)
                 {
                     // The AES CSP is only supported on WinXP and higher
+#if MONO
+                    bool osSupportsAesCsp = true;
+#else
                     bool osSupportsAesCsp = Environment.OSVersion.Platform == PlatformID.Win32NT &&
                                             (Environment.OSVersion.Version.Major > 5 ||
                                             (Environment.OSVersion.Version.Major == 5 && Environment.OSVersion.Version.Minor >= 1));
-
+#endif
                     _defaultRsaProviderType = osSupportsAesCsp ? Constants.PROV_RSA_AES : Constants.PROV_RSA_FULL;
                     _haveDefaultRsaProviderType = true;
                 }
@@ -173,7 +176,7 @@ namespace System.Security.Cryptography
                 return _defaultRsaProviderType;
             }
         }
-
+#if !MONO
 #if FEATURE_CRYPTO || FEATURE_LEGACYNETCFCRYPTO
 #if !FEATURE_PAL
         [System.Security.SecurityCritical] // auto-generated
@@ -818,10 +821,14 @@ namespace System.Security.Cryptography
             return unchecked(new byte[] { (byte)(i >> 24), (byte)(i >> 16), (byte)(i >> 8), (byte)i });
         }
 
-#if !MONO
 #if FEATURE_CRYPTO || FEATURE_LEGACYNETCFCRYPTO
         [System.Security.SecurityCritical]  // auto-generated
         internal static byte[] RsaOaepEncrypt (RSA rsa, HashAlgorithm hash, PKCS1MaskGenerationMethod mgf, RandomNumberGenerator rng, byte[] data) {
+#if MONO
+            // It looks like .net managed implementation is buggy. It returns quite different
+            // result compare to old mono code, even PSLength calculation is quite different
+            return Mono.Security.Cryptography.PKCS1.Encrypt_OAEP (rsa, hash, rng, data);
+#else
             int cb = rsa.KeySize / 8;
 
             //  1. Hash the parameters to get PHash
@@ -831,7 +838,7 @@ namespace System.Security.Cryptography
             hash.ComputeHash(EmptyArray<Byte>.Value); // Use an empty octet string
 
             //  2.  Create DB object
-            byte[] DB = new byte[cb - cbHash];
+            byte[] DB = new byte[cb - cbHash + 1];
 
             //  Structure is as follows:
             //      pHash || PS || 01 || M
@@ -867,10 +874,14 @@ namespace System.Security.Cryptography
             Buffer.InternalBlockCopy(DB, 0, pad, seed.Length, DB.Length);
 
             return rsa.EncryptValue(pad);
+#endif
         }
 
         [System.Security.SecurityCritical]  // auto-generated
         internal static byte[] RsaOaepDecrypt (RSA rsa, HashAlgorithm hash, PKCS1MaskGenerationMethod mgf, byte[] encryptedData) {
+#if MONO
+            return Mono.Security.Cryptography.PKCS1.Decrypt_OAEP (rsa, hash, encryptedData);
+#else
             int cb = rsa.KeySize / 8;
 
             // 1. Decode the input data
@@ -894,11 +905,12 @@ namespace System.Security.Cryptography
             if (zeros < 0 || zeros >= cbHash)
                 throw new CryptographicException(Environment.GetResourceString("Cryptography_OAEPDecoding"));
 
+            const int leading_zeros = 0;
             byte[] seed = new byte[cbHash];
-            Buffer.InternalBlockCopy(data, 0, seed, zeros, seed.Length - zeros);
+            Buffer.InternalBlockCopy(data, leading_zeros, seed, zeros, seed.Length - zeros);
 
-            byte[] DB = new byte[data.Length - seed.Length + zeros];
-            Buffer.InternalBlockCopy(data, seed.Length - zeros, DB, 0, DB.Length);
+            byte[] DB = new byte[data.Length - seed.Length + zeros - leading_zeros];
+            Buffer.InternalBlockCopy(data, seed.Length - zeros + leading_zeros, DB, 0, DB.Length);
 
             //  4.  seedMask = MGF(maskedDB, hLen);
             byte[] mask = mgf.GenerateMask(DB, seed.Length);
@@ -946,6 +958,7 @@ namespace System.Security.Cryptography
             byte[] output = new byte[DB.Length - i];
             Buffer.InternalBlockCopy(DB, i, output, 0, output.Length);
             return output;
+ #endif
         }
 
         [System.Security.SecurityCritical]  // auto-generated
@@ -1017,7 +1030,7 @@ namespace System.Security.Cryptography
             }
             return true;
         }
-
+#if !MONO
         [System.Security.SecurityCritical]  // auto-generated
         [ResourceExposure(ResourceScope.None)]  // Creates a process resource, but it can't be scoped.
         [DllImport(JitHelpers.QCall, CharSet = CharSet.Unicode), SuppressUnmanagedCodeSecurity]
@@ -1132,7 +1145,9 @@ namespace System.Security.Cryptography
         [ResourceExposure(ResourceScope.None)]
         [MethodImplAttribute(MethodImplOptions.InternalCall)]
         internal static extern void _GenerateKey(SafeProvHandle hProv, int algid, CspProviderFlags flags, int keySize, ref SafeKeyHandle hKey);
+#endif
 #endif // FEATURE_CRYPTO
+#if !MONO
         [System.Security.SecurityCritical]  // auto-generated
         [ResourceExposure(ResourceScope.None)]
         [MethodImpl(MethodImplOptions.InternalCall)]