Search code examples
javac++openssljava-native-interfacersa

JNI OpenSSL RSA Decryption not working properly


Encryption seems to work fine, but when I start decryption I get wrong output. Keys are given in PEM and length is 2048.

Here are the methods that are called from Java

String public_key = "-----BEGIN PUBLIC KEY-----\n" +
            "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAm1AThUE8dUz6x5DeDK3J\n" +
            "SKcBbqFVtplHCdf+036+2tZ1RmHcwsKZ6AF4dtCQ/8n+2lMQgdfWSe+gKEp2lIh0\n" +
            "YQUxgQneLXGzvsEjbFVLgxSdLnEfuZrJsNC3J9LHlnvwYYvBQGaSVCx7WhvxFyDC\n" +
            "BJ7SlpjSzVY9yduxRAHKYnsFRwKgEW15N3VYbspd/LTexNHXTbuzQ968wZbWd5rX\n" +
            "qHejM9pFEQsxqBq7uIk3eFvDMZzyi47NaM9eRHX6LyDF4CtL6SL9UKRTnMCrNpDh\n" +
            "WfxaC6xbMf4tf242Bk0TGe8mK6gH5baUs4C14Tq87yUkC9IVMegNlki89nsT5uRX\n" +
            "swIDAQAB\n" +
            "-----END PUBLIC KEY-----";
    String private_key = "-----BEGIN RSA PRIVATE KEY-----\n" +
            "MIIEpQIBAAKCAQEAm1AThUE8dUz6x5DeDK3JSKcBbqFVtplHCdf+036+2tZ1RmHc\n" +
            "wsKZ6AF4dtCQ/8n+2lMQgdfWSe+gKEp2lIh0YQUxgQneLXGzvsEjbFVLgxSdLnEf\n" +
            "uZrJsNC3J9LHlnvwYYvBQGaSVCx7WhvxFyDCBJ7SlpjSzVY9yduxRAHKYnsFRwKg\n" +
            "EW15N3VYbspd/LTexNHXTbuzQ968wZbWd5rXqHejM9pFEQsxqBq7uIk3eFvDMZzy\n" +
            "i47NaM9eRHX6LyDF4CtL6SL9UKRTnMCrNpDhWfxaC6xbMf4tf242Bk0TGe8mK6gH\n" +
            "5baUs4C14Tq87yUkC9IVMegNlki89nsT5uRXswIDAQABAoIBAQCYztBl6ylwv6x9\n" +
            "bSsLjnDb6nSeRF3wqh4asUknDSz6YsY/2Uk61fxXKBs9yzbec/8rD07OcW2EkR8i\n" +
            "hSDmQts+Gb37F5phW91dcOlJTSJedYmwh9yO4JxQOwn5RIjaplZ7ouUgV8rgxmMW\n" +
            "5Sbvemtp4FmRkgrVvGROlrhyENDu0lJzPtVks8XA1Re+CrOinhTCkISChqq/sHVC\n" +
            "sJfsI6OFHzk0Oexnh0sIG5MfaMNFp4Mh38UhrXFBJ6cIBuveEGzzED+AB6O30j3k\n" +
            "XT2pQbjCJcpekfpzfcW1VRfebLIgB+2mVjSleMKgA7ImyXOg4DCJiLWLulgOKmiE\n" +
            "x7qH684BAoGBAM1onwWBR3ERui4p22dNEoZ2tOygzEJKzhjsC1tRDAyO8Gr4QWNk\n" +
            "fNt/+8+gjJkMkkoKtjaqPrOdclhdPVCyQWvJt5ZAyAv0J7HDbZU8uShAOJI4eCpi\n" +
            "G2Xf20S0v84V12MEriWKZTmXUyWWKR87XkLTcVLCyfMTqPR1gMtHfpJpAoGBAMGQ\n" +
            "0lrFeNM2xqD7fZE/465wxp+kYPf/sMNkk6IqxJljtnzjWbySDU9csSwlN03Q36vu\n" +
            "/TGFIUWx2uTvjRHGEPvOC9agEZ8p9kHWrQb1ZgICpd/yWEy9nze66OayN3JyxkVR\n" +
            "pg2R5RRfnWuI9U6CyPhuZlptqDBUHXhG2kVB3Z27AoGBALYT0CpED3zl1uBG0CqA\n" +
            "gjRZE0VRv93fi1NCIUr/y6tyJSDdELE3CQpVJ3RDf33HTAF//0bzoAL9RLeZZtma\n" +
            "OS1/sFHq+KjH80u6zO9l1UcdrkfG8JW5Q0oJpccAZakbaUJniqrSQ6pKPjTqJ2d8\n" +
            "67BW13QiIHts6O5RHiqTJFpJAoGBAJEfMgbqDJdWdv8U7mSq4NnVJaVlCWqF4hHs\n" +
            "Yx9vLyzNbHEfxxSw75ezqAWv9VG7KybtrBindnWZTcLcswhDVlJjfc6w/eU2AbIE\n" +
            "8H7KF2ukbpaDTJ5kgG25DYqAzT9aO7qW54c+/eATe6O28CuntGNF6ikcE8AAIIQf\n" +
            "ot/P7QanAoGADQ5hDdPaq+wSFhYoeh//QfXrkbIcNl18ienY5Sn4jDrNgkkxghdc\n" +
            "WV3biTqIGMf15CI+//dSShbzI7nyoYQ8bl6HsFbEnGyS95PySdF6gUhsPKG83Ih2\n" +
            "SftvM2TzsO9D8XemkOHNwayPMN6YQA3SDZTZNNv3LYD7NkqhRQI0YUo=\n" +
            "-----END RSA PRIVATE KEY-----";

    public native byte[] publicencryptRsa(String key, byte[] plainText);
    public native byte[] privatedecryptRsa(String key, byte[] encText);
byte[] plainText = "utnznxckymtyhedtumwtswlxgqkibutnznxckymtyhedtumwtswlxgqkibutnznxckymtyhedtumwtswlxgqkib1".getBytes(StandardCharsets.UTF_8);
        StringBuilder sb2 = new StringBuilder();
        for (byte b : plainText) {
            sb2.append(String.format("%02X", b));
        }
        Log.d(TAG, "onCreate: plainText = " + sb2.toString());
            //byte [] keyData = "eQg2MDbk3uUtRhMw".getBytes();
//        byte [] encryptedText = encryptAes256(keyData, plainText);
//        byte [] decryptedText = decryptAes256(keyData, encryptedText);
          byte [] encText = publicencryptRsa(public_key, plainText);
            sb2 = new StringBuilder();
            for (byte b : encText) {
                sb2.append(String.format("%02X", b));
            }
            Log.d(TAG, "onCreate: encText = " + sb2.toString());
          byte[] decBa = privatedecryptRsa(private_key, encText);
        sb2 = new StringBuilder();
        for (byte b : decBa) {
            sb2.append(String.format("%02X", b));
        }
        Log.d(TAG, "onCreate: decBa = " + sb2.toString());

Methods from .cpp file

Structures

RSA * createRSApub(std::string sKey) {
   
     RSA *rsa = nullptr;
     BIO *keybio; //  BIO_new( BIO_s_mem() );
     keybio = BIO_new_mem_buf(sKey.c_str(), sKey.length());

    if (keybio == nullptr) {
        printf( "Failed to create key BIO");
        return nullptr;
    }
    else  rsa = PEM_read_bio_RSA_PUBKEY(keybio, nullptr,nullptr, nullptr);

    BIO_free(keybio);

    return rsa;
}
RSA * createRSApriv(std::string sKey) {
    RSA *rsa = nullptr;
    BIO *keybio;
    keybio = BIO_new_mem_buf(sKey.c_str(), sKey.length());

    if (keybio == nullptr) {
        printf( "Failed to create key BIO");
        return nullptr;
    }
    else rsa = PEM_read_bio_RSAPrivateKey(keybio, nullptr,nullptr, nullptr);
  
    return rsa;
}

Encryption

Java_com_example_myapplication_MainActivity_publicencryptRsa(JNIEnv *env, jobject thiz, jstring key,
                                                       jbyteArray plain_text) {
      jboolean isCopy;

    unsigned char *key_data = (unsigned char *) (env)->GetStringUTFChars(key, &isCopy);
    std::string sKey((char*) key_data);
    int key_data_len = strlen((char*)key_data);


    int plainText_len = env->GetArrayLength(plain_text);
    jbyte* temp = env->GetByteArrayElements(plain_text,&isCopy);
    unsigned char* plainText;
    plainText = (unsigned char *) temp;


    if(nullptr == plainText)
        return nullptr;

    LOGI("The data to encrypt is : %s", plainText);

    unsigned char chEncryptedData[4098] = {};
    int iResult = RSA_public_encrypt(plainText_len, (unsigned char*)plain_text, chEncryptedData, createRSApub(sKey), RSA_PKCS1_OAEP_PADDING);

//    env->ReleaseStringUTFChars(env, inData, cData);

// If encryption fails, returns nullptr string, else returns encrypted string
    if(-1 == iResult) {
        char *chErrMsg = (char*)malloc(256);
        ERR_load_crypto_strings();
        ERR_error_string(ERR_get_error(), chErrMsg);
        LOGE("The data Encryption failed due to the reason : %s", chErrMsg);
        free(chErrMsg);
        return nullptr;
    }
//    for(int i = 0; i < 255; ++i)
//        LOGD("The Encrypted data is[%d] : %c", i, chEncryptedData[i]);

    jbyteArray EncryptedByteArray = env->NewByteArray(iResult);
    env->SetByteArrayRegion(EncryptedByteArray, 0, iResult, (jbyte *) chEncryptedData);

    return EncryptedByteArray;
}

Decryption

Java_com_example_myapplication_MainActivity_privatedecryptRsa(JNIEnv *env, jobject thiz,
                                                       jstring private_key,
                                                       jbyteArray enc_text) {
    jboolean isCopy;
    unsigned char *key_data = (unsigned char *) (env)->GetStringUTFChars(private_key, &isCopy);
    std::string sKey((char*) key_data);
    int key_data_len = strlen((char*)key_data);

    int encText_len = env->GetArrayLength(enc_text);
    jbyte* a = env->GetByteArrayElements(enc_text,&isCopy);
    const unsigned char* encText;
    encText = (unsigned char *) a;

    unsigned char chDecryptedData[4098] = {};

    int result = RSA_private_decrypt(256,encText, chDecryptedData, createRSApriv(sKey), RSA_PKCS1_OAEP_PADDING);

    if(-1 == result) {
        char *chErrMsg = (char*)malloc(256);
        ERR_load_crypto_strings();
        ERR_error_string(ERR_get_error(), chErrMsg);
        LOGE("The data Decryption failed due to the reason : %s", chErrMsg);
        free(chErrMsg);
        return nullptr;
    }

//    for(int i = 0; i < 100; ++i)
//        LOGD("The Decrypted data is[%d] : %c", i, chDecryptedData[i]);

    jbyteArray DecryptedByteArray = env->NewByteArray(result);
    env->SetByteArrayRegion(DecryptedByteArray, 0, result, (jbyte *) chDecryptedData);

    return DecryptedByteArray;
}

Here's out enter image description here

I don't know where the problem actually is, both keys are valid and RSA_private_decrypt returns correct length of plain_text, but wrong characters.


Solution

  • The problem is in this call:

        int iResult = RSA_public_encrypt(plainText_len, (unsigned char*)plain_text, chEncryptedData, createRSApub(sKey), RSA_PKCS1_OAEP_PADDING);
    

    The variable plain_text is your jbyteArray input variable. You should pass plainText instead, which is a const char *. In this case the cast masked the problem, and honestly I'm surprised it did not crash your JVM.