Search code examples
c#c++cryptographyrijndael

How to convert C++ Rijndael Cryptography to C#, when there is an error saying "Padding is invalid and cannot be removed"?


I am trying to convert c++ source to c# which encrypt and decrypt file using Rinjdael cryptography.

But c++ source has got a little bit difference from the normal en/decryptions.

And I am not really good at c++, so I am getting confused.

One of my customers' application is written in VC++, and to convert it into c# is part of my job.

And the previous c++ developer used open source code from http://www.codeproject.com/Articles/10657/A-Simple-Portable-Rinjdael-AES-Based-Stream-Cipher to manipulate en/decryption.

Here is c++ source codes.

int DCipher::DecryptFile(LPCTSTR szSrcFile, LPCTSTR szDestFile, const char* pwd, int head[19])
{
    if(CheckMemSize() != 0)
        return INSUFFICIENT_MEMORY;

    FileSize=CurPosition=0;

    _tcscpy(SrcFile, szSrcFile);
    _tcscpy(OutFile, szDestFile);
    //_tcscpy(OutFile, _T(".enc"));

    strcpy(password, pwd);

    for(int i=0; i<19; i++)
    {
        header[i] = head[i];
    }

    FILE *r, *w;
    GetFileLength();

    int nCheck = CheckIfEncrypted();

    if(nCheck != ENCRYPTED_FILE )
        return nCheck;  //either NORMAL_FILE or BAD_SIGNATURE


    if((r = _tfopen(SrcFile, _T("rb"))) == NULL)
        return ERROR_SRC_FILE;


    if((w = _tfopen(OutFile, _T("wb"))) == NULL)
    {
        fclose(r);
        return ERROR_DST_FILE;
    }

    char zzz[26];   //fixed invalid pointer - DKeesler
    fread(zzz, 25, 1, r); // Skip first 25 bytes of the file.

    int pad = header[19];
    pad *= 10;
    pad += header[20];

    // convert password to Rijndael key
    strcpy((char*)key, (const char*)CalcMD5FromString((const char*)password));

    /***************************************
    Decryption algorithm 
    ***************************************/
    int rval = NO_ERRORS_DONE;
    FileSize -= 25; 

    unsigned int BUFF_SIZE = liChunkSize;
    unsigned int WRITE_SIZE = liChunkSize;

    int nRound = FileSize / liChunkSize;
    unsigned int LAST_BLOCK = FileSize % liChunkSize;

    if(LAST_BLOCK >= 1)
        nRound++;

    const unsigned char* intext;
    unsigned char* output;

    intext = (const unsigned char*)malloc(BUFF_SIZE);
    output = (unsigned char*)malloc(BUFF_SIZE+16);

    if(intext == NULL || output == NULL)
    {
        fclose(r);
        fclose(w);
        return ALLOC_ERROR;
    }

    Rijndael rj;
    rj.init(Rijndael::CBC, Rijndael::Decrypt, key, Rijndael::Key32Bytes);

    for(int loop=1; loop <= nRound; loop++)
    {

        if(loop == nRound && LAST_BLOCK >= 1)
        {
            BUFF_SIZE = LAST_BLOCK;
            WRITE_SIZE = LAST_BLOCK - pad;
        }

        fread((void*)intext, sizeof(char), BUFF_SIZE, r); // read plaintext into intext[] buffer

        int bsize = BUFF_SIZE*8;
        int len = rj.blockDecrypt((const UINT8*)intext, bsize, (UINT8*)output);

        if(len >= 0)
        {
            fwrite((const void*)output, sizeof(char), WRITE_SIZE, w);
        }
        else
        {
            rval = READ_WRITE_ERROR;
            break;
        }
    }

    fclose(r);  //close input file
    fclose(w);  //close output file

    free((void*)intext);
    free((void*)output);

    //change these two lines if you want to leave backups or unencrypted copies...
    //that would sort of defeat the purpose of encryption in my mind, but it's your
    // app so write it like you want it.
    if(DECRYPTION_CANCEL == rval) { 
        _tremove(OutFile); 
    } 
    else { 
        //_tremove(SrcFile);    //remove input file
        //_trename(OutFile, SrcFile); //rename output file to input filename
    }

    return rval; //ZERO .. see defines for description of error codes.
}

And c# source code is from https://msdn.microsoft.com/en-us/library/system.security.cryptography.rijndael(v=vs.110).aspx.

And I changed a little bit of codes.

Here is c# codes.

public int DecryptFile(string SourceFilePath, string DestFilePath, string Password, string Signature)
{
    try
    {
        FileSize        = CurPosition = 0;

        FileInfo _fi    = new FileInfo(SourceFilePath);
        FileSize        = _fi.Length;

        // copy the signature to _header
        for(int i = 0; i < 19; i++)
        {
            _header[i] = (byte)Signature[i];
        }

        /*
            * check if the file is valid encrypted file.
            */
        int nCheck  = this.CheckIfEncrypted(SourceFilePath);
        switch (nCheck)
        {
            case ENCRYPTED_FILE:
                // The file is an encrypted file.
                break;
            case NORMAL_FILE:
                throw new ArgumentException("The file is a normal file.");
            case BAD_SIGNATURE:
                throw new ArgumentException("User signature doesn't match.");
        }

        int pad = _header[19];
        pad     *= 10;
        pad     += _header[20];

        // Rijndael session key
        byte[] session_key  = this.CalcMD5FromString(Password);

        byte[] _restFileBytes   = new byte[_fi.Length - 25];
        using (FileStream _fs = new FileStream(SourceFilePath, FileMode.Open, FileAccess.Read))
        {
            _fs.Read(_restFileBytes, 0, _restFileBytes.Length);
        }

        int rval        = NO_ERRORS_DONE;
        FileSize        -= 25; 

        int BUFF_SIZE   = liChunkSize;
        int WRITE_SIZE  = liChunkSize;

        int nRound      = (int)FileSize / liChunkSize;
        int LAST_BLOCK  = (int)FileSize % liChunkSize;

        if(LAST_BLOCK >= 1)
            nRound++;

        byte[] intext   = new byte[BUFF_SIZE];
        byte[] output   = new byte[BUFF_SIZE + 16];

        if (intext.Length == 0 || output.Length == 0)
        {
            return ALLOC_ERROR;
        }

        for (int loop = 1; loop <= nRound; loop++)
        {
            if (loop == nRound && LAST_BLOCK >= 1)
            {
                BUFF_SIZE   = LAST_BLOCK;
                WRITE_SIZE  = LAST_BLOCK - pad;
            }

            intext  = new byte[BUFF_SIZE];

            System.Buffer.BlockCopy(_restFileBytes, (loop - 1) * this.liChunkSize, intext, 0, BUFF_SIZE);

            int bsize   = BUFF_SIZE * 8;    // -> I still couldn't figure out what this bsize does on Rijndael decryption.
            using (RijndaelManaged myRijndael = new RijndaelManaged())
            {
                myRijndael.Key          = session_key;
                //myRijndael.BlockSize  = bsize;
                //myRijndael.Padding        = PaddingMode.None;
                myRijndael.GenerateIV();

                using (Rijndael rijAlg = Rijndael.Create())
                {
                    rijAlg.Key  = myRijndael.Key;
                    rijAlg.IV   = myRijndael.IV;

                    // Create a decrytor to perform the stream transform.
                    ICryptoTransform decryptor = rijAlg.CreateDecryptor(rijAlg.Key, rijAlg.IV);

                    // Create the streams used for decryption.
                    using (MemoryStream msDecrypt = new MemoryStream(intext))
                    {
                        using (CryptoStream csDecrypt = new CryptoStream(msDecrypt, decryptor, CryptoStreamMode.Read))
                        {
                            //using (StreamReader srDecrypt = new StreamReader(csDecrypt))
                            //{
                            //  // Read the decrypted bytes from the decrypting stream and place them in a string.
                            //  //string s = srDecrypt.ReadToEnd();
                            //}
                            byte[] rettt    = msDecrypt.ToArray();
                        } // --> Padding is invalid and cannot be removed error occurs here and msDecrypt byte array is just same as intext. So, it's not decrypted at all.
                    }

                }
            }
        }


        return rval;
    }
    catch
    {
        throw;
    }
}

According to Keesler(who is the writer of c++ source codes from codeproject.com), first 25 bytes are filled with user data(signature, padding and file status). So, I skipped first 25 bytes and save the rest bytes to _restFileBytes varialbes(byte array).

And Keesler has a variable called chunk size, which splits file bytes into chunk size(as long as I understand).

Anyway, I think I almost converted to c# but I still get this error message "Padding is invalid and cannot be removed" when CryptoStream disposing in c#.

Can anyone give me some guide to fix this error?


Solution

  • None should be used as padding mode. It seems like your colleague and the author of the original article made up their own padding scheme.

    Furthermore, all of the ciphertext should be streamed from the file (making sure you read all the bytes). Currently you are restarting encryption with the IV for each chunk, which is not good, the IV should only be used at the start of the ciphertext.

    Print out the key in hex for both C++ and C# and compare before you start.

    Note that the Read method differs slightly from the fread method in C++.