Decrypt in chunks a AES 128 CBC encrypted object

I have an Encrypted object in Minio, encrypted using the AES 128 bit CBC algorithm.

The object is quite large (~50 MB) so instead of loading it into the memory completely (which may cause out of memory exception), I am retrieving it in chunks of 1MB. I need to decrypt it before use.

Is it possible to decrypt the object in this way (1MB at a time, the whole object was encrypted in one go)? If yes, how can I do it? I have tried decrypting 16-byte chunks which produce the following errors:

javax.crypto.BadPaddingException: Given final block not properly padded

javax.crypto.IllegalBlockSizeException: Input length must be multiple of 16 when decrypting with padded cipher

Solution

To avoid an "out of memory error" you want to decrypt a large (encrypted) file in chunks of 1 mb size - yes it's possible with AES CBC mode.

Below you find a complete example that is generating a sample plaintext file ('plaintext.dat') with random content with the size of 50 mb + 1 byte (the + 1 byte is good to test for file sizes that are not exact multiples of 16 = AES blocksize).

In the next step this file is getting encrypted to 'ciphertext.dat' using a randomly created initialization vector and key.

The last step is the requested decryption method - it decrypts the encrypted file in chunks of 1 mb and in the lines '// obuf holds the decrypted chunk, do what you want to do with the data' and '// final data' you do have the decrypted data in the byte array obuf. For testing I'm writing the decrypted data to the file 'decryptedtext.dat' in appending mode (for that reason this file is deleted in the beginning if it exists).

To prove that decryption was successful I'm comparing the SHA256-hashes of plaintext- and decryptedtext-files.

Two notes: I'm using a 32 byte = 256 bit long key for AES CBC 256. This program has no proper exception handling and is for educational purposes only.

result:

decrypt AES CBC 256 in 1 mb chunks

file with random data created: plaintext.dat
encryption to ciphertext.dat was successfull: true

decryption in chunks of 1 mb
decrypted file written to decryptedtext.dat
plaintext equals decrytedtext file: true

code:

import javax.crypto.*;
import javax.crypto.spec.IvParameterSpec;
import javax.crypto.spec.SecretKeySpec;
import java.io.*;
import java.nio.file.Files;
import java.security.*;
import java.util.Arrays;

public class AES_CBC_chunk_decryption {
    public static void main(String[] args) throws IOException, InvalidKeyException, NoSuchAlgorithmException, NoSuchPaddingException,
            InvalidAlgorithmParameterException, BadPaddingException, IllegalBlockSizeException {
        System.out.println("https://stackoverflow.com/questions/63325528/decrypt-in-chunks-a-aes-128-cbc-encrypted-object/63325529#63325529");
        System.out.println("decrypt AES CBC 256 in 1 mb chunks");

        // setup for creation of a 50mb encrypted file
        int filesize = (50 * 1024 * 1024) + 1; // 50 mb + 1 byte = 52428801 bytes
        String filenamePlaintext = "plaintext.dat";
        String filenameCiphertext = "ciphertext.dat";
        String filenameDecryptedtext = "decryptedtext.dat";

        File file = new File("plaintext.dat");
        // fill with random bytes.
        try (FileOutputStream out = new FileOutputStream(file)) {
            byte[] bytes = new byte[filesize];
            new SecureRandom().nextBytes(bytes);
            out.write(bytes);
        }
        System.out.println("\nfile with random data created: " + filenamePlaintext);
        // delete decrypted file if it exists
        Files.deleteIfExists(new File(filenameDecryptedtext).toPath());

        // setup random key & iv
        SecureRandom secureRandom = new SecureRandom();
        byte[] iv = new byte[16];
        byte[] key = new byte[32]; // I'm using a 32 byte = 256 bit long key for aes 256
        secureRandom.nextBytes(iv);
        secureRandom.nextBytes(key);

        // encrypt complete file
        boolean resultEncryption = encryptCbcFileBufferedCipherOutputStream(filenamePlaintext, filenameCiphertext, key, iv);
        System.out.println("encryption to " + filenameCiphertext + " was successfull: " + resultEncryption);
        // encrypted file is 52428816 bytes long

        System.out.println("\ndecryption in chunks of 1 mb");
        // decryption in chunks of 1 mb
        try (FileInputStream in = new FileInputStream(filenameCiphertext)) {
            byte[] ibuf = new byte[(1024 * 1024)]; // chunks of 1 mb
            int len;
            Cipher cipher = Cipher.getInstance("AES/CBC/PKCS5PADDING");
            SecretKeySpec secretKeySpec = new SecretKeySpec(key, "AES");
            IvParameterSpec ivParameterSpec = new IvParameterSpec(iv);
            cipher.init(Cipher.DECRYPT_MODE, secretKeySpec, ivParameterSpec);
            while ((len = in.read(ibuf)) != -1) {
                byte[] obuf = cipher.update(ibuf, 0, len);
                if (obuf != null)
                    // obuf holds the decrypted chunk, do what you want to do with the data
                    // I'm writing it to a file in appending mode
                    try (FileOutputStream output = new FileOutputStream(filenameDecryptedtext, true)) {
                        output.write(obuf);
                    }
            }
            byte[] obuf = cipher.doFinal();
            if (obuf != null)
                // final data
                try (FileOutputStream output = new FileOutputStream(filenameDecryptedtext, true)) {
                    output.write(obuf);
                }
        }
        System.out.println("decrypted file written to " + filenameDecryptedtext);
        System.out.println("plaintext equals decrytedtext file: " + filecompareSha256Large(filenamePlaintext, filenameDecryptedtext));
    }


    public static boolean encryptCbcFileBufferedCipherOutputStream(String inputFilename, String outputFilename, byte[] key, byte[] iv)
            throws IOException, NoSuchPaddingException, NoSuchAlgorithmException, InvalidKeyException, InvalidAlgorithmParameterException {
        Cipher cipher = Cipher.getInstance("AES/CBC/PKCS5Padding");
        try (FileInputStream in = new FileInputStream(inputFilename);
             FileOutputStream out = new FileOutputStream(outputFilename);
             CipherOutputStream encryptedOutputStream = new CipherOutputStream(out, cipher);) {
            SecretKeySpec secretKeySpec = new SecretKeySpec(key, "AES");
            IvParameterSpec ivParameterSpec = new IvParameterSpec(iv);
            cipher.init(Cipher.ENCRYPT_MODE, secretKeySpec, ivParameterSpec);
            byte[] buffer = new byte[8096];
            int nread;
            while ((nread = in.read(buffer)) > 0) {
                encryptedOutputStream.write(buffer, 0, nread);
            }
            encryptedOutputStream.flush();
        }
        if (new File(outputFilename).exists()) {
            return true;
        } else {
            return false;
        }
    }

    public static boolean filecompareSha256Large(String filename1, String filename2) throws IOException, NoSuchAlgorithmException {
        boolean result = false;
        byte[] hash1 = generateSha256Buffered(filename1);
        byte[] hash2 = generateSha256Buffered(filename2);
        result = Arrays.equals(hash1, hash2);
        return result;
    }

    private static byte[] generateSha256Buffered(String filenameString) throws IOException, NoSuchAlgorithmException {
        // even for large files
        byte[] buffer = new byte[8192];
        int count;
        MessageDigest md = MessageDigest.getInstance("SHA-256");
        BufferedInputStream bis = new BufferedInputStream(new FileInputStream(filenameString));
        while ((count = bis.read(buffer)) > 0) {
            md.update(buffer, 0, count);
        }
        bis.close();
        return md.digest();
    }
}