In continuation to my previous pycryptodome question my requirement now got changed to support 90G of data for encryption. So I have done some design changes, de-factoring the encryption code and make them all run in the subprocess.
tar zcvf - /array22/vol4/home | openssl des3 -salt | dd of=/dev/st0
The above idea got triggered from here
Now I have 2 files:
#!/usr/bin/python
import sys, os, pwd
from Crypto.PublicKey import RSA
from Crypto.Cipher import AES, PKCS1_OAEP
from Crypto.Random import get_random_bytes
symmetric_key = get_random_bytes(16 * 2)
cipher_rsa = PKCS1_OAEP.new(RSA.import_key(open("./public.pem").read()))
enc_symmetric_key = cipher_rsa.encrypt(symmetric_key)
cipher = AES.new(symmetric_key, AES.MODE_GCM)
[sys.stdout.write(x) for x in (enc_symmetric_key, cipher.nonce,"".join(reversed(cipher.encrypt_and_digest(sys.stdin.read()))))]
#! /usr/bin/python
import os, sys, time
import tarfile, StringIO, time
from subprocess import Popen, PIPE, call
from Crypto.PublicKey import RSA
from Crypto.Cipher import AES, PKCS1_OAEP
from Crypto.Random import get_random_bytes
print "Start time %s"%time.time()
try:
p1=Popen("tar -czf - ./src", shell=True, stdout=PIPE)
p2=Popen("python ./encutil.py", shell=True, stdin=p1.stdout, stdout=PIPE)
FNULL = open(os.devnull, 'w')
p3=Popen("/bin/dd bs=10M iflag=fullblock oflag=direct,sync conv=fsync,notrunc,noerror status=progress of=./data.bin", shell=True, stdin=p2.stdout, stderr=FNULL)
p3.wait()
except Exception,e:
raise str(e)
finally:
p2.stdout.close()
p1.stdout.close()
def doRestore():
try:
privKey = RSA.import_key(open("./private.pem").read())
cipher_rsa = PKCS1_OAEP.new(privKey)
file_in = open("./data.bin", "rb")
enc_symmetric_key, nonce, tag, ciphertext = [file_in.read(x) for x in (privKey.size_in_bytes(), 16, 16, -1)]
symmetric_key = cipher_rsa.decrypt(enc_symmetric_key)
cipher = AES.new(symmetric_key, AES.MODE_GCM, nonce)
tar = tarfile.open(fileobj=StringIO.StringIO(cipher.decrypt_and_verify(ciphertext, tag)), mode='r|*')
tar.extractall(path='./dst')
except Exception,e:
print e
finally:
if file_in != None:
file_in.close()
if tar != None:
tar.close()
os.remove("./data.bin")
doRestore()
print "End time %s"%time.time()
Assume both the public and private keys are available and in place.
And, when I execute the below command after some time of execution I get the error: Ciphertext with incorrect length without any traceback:
/usr/bin/systemd-run --scope -p MemoryLimit=80G ./main.py
But it runs successful for lesser data input, like 40G of data
My system details are:
HW: HP ProLiant DL360 Gen10 with more than 500G of HDD space and 125G of RAM
OS: RHEL7.4 64-bit Kernel: 3.10.0-693.el7.x86_64
Python version: 2.7.5
Pycryptodome version: 3.7.2
If I do not control the memory resource through systemd-run then Python throws MemoryError at some point of execution and fails in the same way with "Ciphertext with incorrect length." message
Traceback (most recent call last):
File "./encutil.py", line 12, in <module>
[sys.stdout.write(x) for x in (enc_symmetric_key, cipher.nonce,"".join(reversed(cipher.encrypt_and_digest(sys.stdin.read()))))]
File "/opt/LEBackupandRestore/lib/3pp/Crypto/Cipher/_mode_gcm.py", line 547, in encrypt_and_digest
return self.encrypt(plaintext, output=output), self.digest()
File "/opt/LEBackupandRestore/lib/3pp/Crypto/Cipher/_mode_gcm.py", line 374, in encrypt
ciphertext = self._cipher.encrypt(plaintext, output=output)
File "/opt/LEBackupandRestore/lib/3pp/Crypto/Cipher/_mode_ctr.py", line 211, in encrypt
return get_raw_buffer(ciphertext)
File "/opt/LEBackupandRestore/lib/3pp/Crypto/Util/_raw_api.py", line 187, in get_raw_buffer
return buf.raw
MemoryError
Ciphertext with incorrect length.
I could not get any clue from the solution already proposed in stackoverflow
The original code design is as follows before changes:
#! /usr/bin/python
import os, pwd, sys
from subprocess import Popen, PIPE, check_call
from BackupRestoreException import BackupRestoreException, ErrorCode
from Crypto.PublicKey import RSA
from Crypto.Cipher import AES, PKCS1_OAEP
from Crypto.Random import get_random_bytes
from Crypto.Util.Padding import pad,unpad
import tarfile,StringIO,time
# Key Generation
key = RSA.generate(2048)
private_key = key.export_key()
file_out = open("private.pem", "wb")
file_out.write(private_key)
file_out.close()
public_key = key.publickey().export_key()
file_out = open("public.pem", "wb")
file_out.write(public_key)
file_out.close()
public_key = RSA.import_key(open("public.pem").read())
session_key = get_random_bytes(16)
cipher_rsa = PKCS1_OAEP.new(public_key)
enc_session_key = cipher_rsa.encrypt(session_key)
def archiveData():
cmd = ["tar", "--acls", "--selinux", "-zcPf", "-", "./src"]
return Popen(cmd,stdout=PIPE).communicate()[0]
# Encryption
cipher_aes = AES.new(session_key, AES.MODE_EAX)
ciphertext, tag = cipher_aes.encrypt_and_digest(archiveData())
file_out = open("data.bin", "wb")
[ file_out.write(x) for x in (enc_session_key, cipher_aes.nonce, tag, ciphertext) ]
file_out.close()
# Decryption
private_key = RSA.import_key(open("private.pem").read())
file_in = open("data.bin", "rb")
enc_session_key, nonce, tag, ciphertext = [ file_in.read(x) for x in (private_key.size_in_bytes(), 16, 16, -1) ]
file_in.close()
cipher_rsa = PKCS1_OAEP.new(private_key)
session_key = cipher_rsa.decrypt(enc_session_key)
cipher = AES.new(session_key, AES.MODE_EAX, nonce)
tar = tarfile.open(fileobj=StringIO.StringIO(cipher.decrypt_and_verify(ciphertext, tag)), mode='r|*')
os.chdir("/home/cfuser/target")
tar.extractall(path='.')
I have solved the problem and the below code works for larger datasize. Still any code comments / improvement ideas / suggestions are most welcome!
#! /usr/bin/python
import os, time, tarfile, io
from subprocess import Popen, PIPE, check_call
from Crypto.PublicKey import RSA
from Crypto.Cipher import AES, PKCS1_OAEP
from Crypto.Random import get_random_bytes
print "****** Start time %s" % time.time()
BLOCK_SIZE = 16
BIN_FILE = "/nfs/data.bin"
symmetric_key = get_random_bytes(BLOCK_SIZE * 2)
enc_symmetric_key = PKCS1_OAEP.new(RSA.import_key(open("./public.pem").read())).encrypt(symmetric_key)
cipher_rsa_prikey = PKCS1_OAEP.new(RSA.import_key(open("./private.pem").read()))
chunk_size = BLOCK_SIZE * 1024 * 1024 + BLOCK_SIZE
tag_size = BLOCK_SIZE
ciphertxt_size = chunk_size - tag_size
nonce_size = BLOCK_SIZE
enc_key_size = RSA.import_key(open("./private.pem").read()).size_in_bytes() # 256
def runTarCommand():
cmd = "/usr/bin/systemd-run -q --scope -p MemoryLimit=10G tar -czPf - /root/src"
return Popen(cmd, bufsize=chunk_size, shell=True, stdout=PIPE)
def doNFSBackup():
try:
p = runTarCommand()
with open(BIN_FILE,'wb') as f:
f.write(enc_symmetric_key)
while True:
dataChunk = p.stdout.read(ciphertxt_size)
if dataChunk:
cipher = AES.new(symmetric_key, AES.MODE_GCM)
f.write(cipher.nonce + b"".join(reversed(cipher.encrypt_and_digest(dataChunk))))
else:
break
except Exception,e:
print e
finally:
p.stdout.close()
def doNFSRestore():
try:
extractProc = Popen('tar -C /root/src -xzPf -', bufsize=8192, stdin=PIPE,shell=True)
file_in = open(BIN_FILE, "rb")
symmetric_key = cipher_rsa_prikey.decrypt(file_in.read(enc_key_size))
nonce = file_in.read(nonce_size)
while nonce:
ciphertxtTag = file_in.read(chunk_size)
cipher = AES.new(symmetric_key, AES.MODE_GCM, nonce)
extractProc.stdin.write(cipher.decrypt_and_verify(ciphertxtTag[BLOCK_SIZE:], ciphertxtTag[:BLOCK_SIZE]))
nonce = file_in.read(nonce_size)
except Exception,e:
print e
finally:
if file_in != None:
file_in.close()
if os.path.exists(BIN_FILE): os.remove(BIN_FILE)
def doTapeBackup():
def tarinfoFun(tar, bytsIO):
info = tarfile.TarInfo(name='test.tar')
info.size = len(bytsIO.getvalue())
info.mtime = time.time()
info.mode = 0755
tar.addfile(tarinfo=info, fileobj=bytsIO)
try:
cmd = "mt -f /dev/nst0 load; mt -f /dev/nst0 rewind; mt -f /dev/nst0 setblk %d" %(chunk_size + nonce_size)
check_call(cmd, shell=True)
p = runTarCommand()
tar = tarfile.TarFile("/dev/nst0", "w")
bytsIO = io.BytesIO()
bytsIO.write(enc_symmetric_key)
bytsIO.seek(0)
tarinfoFun(tar, bytsIO)
bytsIO.close()
bytsIO = io.BytesIO()
bytesread = 0
while True:
dataChunk = p.stdout.read(ciphertxt_size)
if not dataChunk:
if bytesread != 0:
bytsIO.seek(0)
tarinfoFun(tar, bytsIO)
bytsIO.close()
p.communicate()
break
cipher = AES.new(symmetric_key, AES.MODE_GCM)
bytsIO.write(cipher.nonce + b"".join(reversed(cipher.encrypt_and_digest(dataChunk))))
bytesread += chunk_size + nonce_size
if bytesread == (chunk_size + nonce_size) * 8:
bytsIO.seek(0)
tarinfoFun(tar, bytsIO)
bytsIO.close()
bytsIO = io.BytesIO()
bytesread = 0
except Exception,e:
print e
finally:
tar.close()
p.stdout.close()
def doTapeRestore():
try:
check_call("mt -f /dev/nst0 load; mt -f /dev/nst0 rewind", shell=True, stdout=PIPE)
p1 = Popen("tar -xPf /dev/nst0 -O", shell=True, stdout=PIPE)
p2 = Popen("tar -C /nfs -xzPf -", shell=True, stdin=PIPE)
symmetric_key = cipher_rsa_prikey.decrypt(p1.stdout.read(enc_key_size))
while True:
ciphertxtTag = p1.stdout.read(chunk_size + nonce_size)
if not ciphertxtTag:
p2.communicate()
break
nonce = ciphertxtTag[:nonce_size]
cipher = AES.new(symmetric_key, AES.MODE_GCM, nonce)
p2.stdin.write(cipher.decrypt_and_verify(ciphertxtTag[32:], ciphertxtTag[BLOCK_SIZE:32]))
except Exception,e:
print e
finally:
pass
doNFSBackup()
doNFSRestore()
doTapeBackup()
doTapeRestore()
print "****** End time %s" % time.time()