SyntaxError while trying to decrypt ciphertext (generated using openssl) with crypto.subtle.decrypt

Question

I am trying to decrypt a ciphertext (generated using openssl) with crypto.subtle.decrypt and keep getting the following error:

SyntaxError: An invalid or illegal string was specified

The same ciphertext was easily decrypted in online RSA tools and openssl. I have tried some of the examples coming up in stackoverflow like here, here and here, but even those ciphertexts are failing. I suspect I am probably missing out on some options while encrypting, but not sure which one.

I generated the keys using the following:

$ openssl genpkey -out private_key.pem -algorithm RSA -pkeyopt rsa_keygen_bits:1024
$ openssl rsa -pubout -in private_key.pem -out public_key.pem
$ cat public_key.pem
-----BEGIN PUBLIC KEY-----
MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC7P6BUs5/P9NbyztqsXvhYHFdX
sQQ5rw6R80MpH6KIgwZohpTLIuZIUjoG/9G+Artv1FHL1uvOPfqlUsj2HtVbM8Cz
E52tB/z/kIyLA6GjKYt+Ok2ZUgWMuHXFBICANaLd93Vm1RNZTcj4k5B/3UT+zZ8j
i30YJjPoLJrtx8VDTQIDAQAB
-----END PUBLIC KEY-----

I am using the below command to encrypt the plaintext

$ echo "Rohit Das" | openssl pkeyutl -encrypt -pubin -inkey public_key.pem -pkeyopt rsa_padding_mode:oaep -pkeyopt rsa_oaep_md:sha256 -pkeyopt rsa_mgf1_md:sha256 -out pass_1.enc
$ cat pass_1.enc | base64
Bubb/cepSt/BZAoPjjsRtmy0Y/50cwcP6J0GvKavFEfLYzAkMi/JbueGILIWnllnENZM0w133JoV
pPPT9B21hD2p8z2CzuOmYV4BnVV97i0oS0xawJPWdUwc6Vp+JaN9TrEX98FwCtQZHKtViyA8+Bnm
u2LBKl5N/QipKIX+Dnc=

I am using crypto.subtle in the following JS functions:

function str2ab(str) {
    const bufView = new Uint8Array(str.length);
    for (let i = 0, strLen = str.length; i < strLen; i++) {
      bufView[i] = str.charCodeAt(i);
    }
    return bufView.buffer;
}

function decrypto() {
    let pvtPem = `-----BEGIN PRIVATE KEY-----
MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBALs/oFSzn8/01vLO
2qxe+FgcV1exBDmvDpHzQykfooiDBmiGlMsi5khSOgb/0b4Cu2/UUcvW6849+qVS
yPYe1VszwLMTna0H/P+QjIsDoaMpi346TZlSBYy4dcUEgIA1ot33dWbVE1lNyPiT
kH/dRP7NnyOLfRgmM+gsmu3HxUNNAgMBAAECgYEApO8ri9BYwbWZrHCWX2Sb/gig
ysZKwYC4JckP7GZIJVS8TU/WOoQ4MZX0NPwbRPJlJeDwV8utE5K2d+9OwrRwGw7n
ZirNolvYSnBeZ8pKRwseQA2fzfCMubZcttjpyb1DfyMXwQj6oCGxuFXPLUWT8/1/
9PkPAWa8uOZhinD/fwUCQQDcxcdCn9rAFLOn1g3EJzSD56GGu6RIrkx8kdBofTYr
nwYnriiX8a//hlRg4BiOv1FdoTHFJi3QAOobX8tiUBK7AkEA2SBxS2TDOj3HJByI
32D0HY1a/O3VBEF4mkRi60HtDYRxL7fmCxebgzFytGxIQr4h+VxFqlABDoDd5vd3
cSS1lwJARm3FumhalYpFIda0f43uP+Il8mBr8U/BUMAHlz3SiSnrAb+abZaJid+l
jV4QF4HLCC6DPRyH4uJXzLHLpSpcPwJARYVVwUYqHGPbd3yLdrqcbznrgEDGi+5K
p1puMdWSCVn2w8imJ7cPXBphF9Pz7yrhxe39gGLNc89fPazO2bNfUQJBALBdr/G3
KYecLkwJeR0okQ8biSr1mAC+VhqlF9JW4pDiw4jrud6R+M4+WjU5nGjxPCfhbEHA
PCahv9z7njXL2qA=
-----END PRIVATE KEY-----`;
    const pemHeader = "-----BEGIN PRIVATE KEY-----";
    const pemFooter = "-----END PRIVATE KEY-----";
    let pvtDer = pvtPem.substring(pemHeader.length, pvtPem.length - pemFooter.length);
    let binaryDer = str2ab(window.atob(pvtDer));
    console.log("binaryDer " + binaryDer);
    window.crypto.subtle.importKey(
        "pkcs8",
        binaryDer,
        {
            name: "RSA-OAEP",
            hash: "SHA-256"
        },
        true,
        ["decrypt"]
    ).then((pvtKey) => {
        console.log("pvtKey " + pvtKey);

        let encryptedData = "Bubb/cepSt/BZAoPjjsRtmy0Y/50cwcP6J0GvKavFEfLYzAkMi/JbueGILIWnllnENZM0w133JoVpPPT9B21hD2p8z2CzuOmYV4BnVV97i0oS0xawJPWdUwc6Vp+JaN9TrEX98FwCtQZHKtViyA8+Bnmu2LBKl5N/QipKIX+Dnc=";
        window.crypto.subtle.decrypt(
            { name: "RSA-OAEP "},
            pvtKey,
            str2ab(window.atob(encryptedData))
        ).then((decryptedData) => {
            console.log("decrypted " + decryptedData + " Rohit Das");
        }).catch((err) => {console.log("decrypt error " + err);});
    }).catch((err) => {console.log("key import error " + err);});
}

Answer 1

Decryption is successful if:

• in the importKey() call "SHA-1" (instead of "SHA-256") is used. This also implies that the posted ciphertext was not actually generated with the posted OpenSSL statement (which applies SHA-256 for both digests).
• in the decrypt() call the blank is removed from the algorithm name specification ("RSA-OAEP" instead of "RSA-OAEP ")

If both are fixed, decryption works:

function str2ab(str) {
    const bufView = new Uint8Array(str.length);
    for (let i = 0, strLen = str.length; i < strLen; i++) {
      bufView[i] = str.charCodeAt(i);
    }
    return bufView.buffer;
}

function decrypto() {
    let pvtPem = `-----BEGIN PRIVATE KEY-----
MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBALs/oFSzn8/01vLO
2qxe+FgcV1exBDmvDpHzQykfooiDBmiGlMsi5khSOgb/0b4Cu2/UUcvW6849+qVS
yPYe1VszwLMTna0H/P+QjIsDoaMpi346TZlSBYy4dcUEgIA1ot33dWbVE1lNyPiT
kH/dRP7NnyOLfRgmM+gsmu3HxUNNAgMBAAECgYEApO8ri9BYwbWZrHCWX2Sb/gig
ysZKwYC4JckP7GZIJVS8TU/WOoQ4MZX0NPwbRPJlJeDwV8utE5K2d+9OwrRwGw7n
ZirNolvYSnBeZ8pKRwseQA2fzfCMubZcttjpyb1DfyMXwQj6oCGxuFXPLUWT8/1/
9PkPAWa8uOZhinD/fwUCQQDcxcdCn9rAFLOn1g3EJzSD56GGu6RIrkx8kdBofTYr
nwYnriiX8a//hlRg4BiOv1FdoTHFJi3QAOobX8tiUBK7AkEA2SBxS2TDOj3HJByI
32D0HY1a/O3VBEF4mkRi60HtDYRxL7fmCxebgzFytGxIQr4h+VxFqlABDoDd5vd3
cSS1lwJARm3FumhalYpFIda0f43uP+Il8mBr8U/BUMAHlz3SiSnrAb+abZaJid+l
jV4QF4HLCC6DPRyH4uJXzLHLpSpcPwJARYVVwUYqHGPbd3yLdrqcbznrgEDGi+5K
p1puMdWSCVn2w8imJ7cPXBphF9Pz7yrhxe39gGLNc89fPazO2bNfUQJBALBdr/G3
KYecLkwJeR0okQ8biSr1mAC+VhqlF9JW4pDiw4jrud6R+M4+WjU5nGjxPCfhbEHA
PCahv9z7njXL2qA=
-----END PRIVATE KEY-----`;
    const pemHeader = "-----BEGIN PRIVATE KEY-----";
    const pemFooter = "-----END PRIVATE KEY-----";
    let pvtDer = pvtPem.substring(pemHeader.length, pvtPem.length - pemFooter.length);
    let binaryDer = str2ab(window.atob(pvtDer));
    console.log("binaryDer " + binaryDer);
    window.crypto.subtle.importKey(
        "pkcs8",
        binaryDer,
        {
            name: "RSA-OAEP",
            hash: "SHA-1"
        },
        true,
        ["decrypt"]
    ).then((pvtKey) => {
        console.log("pvtKey " + pvtKey);

        let encryptedData = "Bubb/cepSt/BZAoPjjsRtmy0Y/50cwcP6J0GvKavFEfLYzAkMi/JbueGILIWnllnENZM0w133JoVpPPT9B21hD2p8z2CzuOmYV4BnVV97i0oS0xawJPWdUwc6Vp+JaN9TrEX98FwCtQZHKtViyA8+Bnmu2LBKl5N/QipKIX+Dnc=";
        window.crypto.subtle.decrypt(
            { name: "RSA-OAEP"},
            pvtKey,
            str2ab(window.atob(encryptedData))
        ).then((decryptedData) => {
            console.log("decrypted " + new TextDecoder().decode(decryptedData));// + " Rohit Das");
        }).catch((err) => {console.log("decrypt error " + err);});
    }).catch((err) => {console.log("key import error " + err);});
}

decrypto();

This is confirmed by decrypting with an online tool. I used https://8gwifi.org/rsafunctions.jsp with the option RSA/NONE/OAEPWithSHA1AndMGF1Padding or RSA/ECB/OAEPWithSHA-1AndMGF1Padding (for which a conversion of the key format from PKCS#8 to PKCS#1 via https://8gwifi.org/pemconvert.jsp is necessary).
The website you used did not work reliably in my tests.

---

A ciphertext generated with the posted OpenSSL statement (i.e. with SHA-256 for both digests) is e.g.:

Tu8MHqnXCrRHZjKgMJx7CF4b2r6jz1ame7+qCFJ6ZkfgzgJc1qDX5xkyN28vZMzhEiO42iFGEeUoYDA4SH+RFPoGPbM94/IFvZREgpEow0a4scMvaYfKCK76gbb5P3UbEIwcgN+QBtsHCEC/aPJmtPzEl1jUNO94fjUkjrGaKwM=

which is correctly decrypted as expected if SHA-256 is applied in the WebCrypto code:

function str2ab(str) {
    const bufView = new Uint8Array(str.length);
    for (let i = 0, strLen = str.length; i < strLen; i++) {
      bufView[i] = str.charCodeAt(i);
    }
    return bufView.buffer;
}

function decrypto() {
    let pvtPem = `-----BEGIN PRIVATE KEY-----
MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBALs/oFSzn8/01vLO
2qxe+FgcV1exBDmvDpHzQykfooiDBmiGlMsi5khSOgb/0b4Cu2/UUcvW6849+qVS
yPYe1VszwLMTna0H/P+QjIsDoaMpi346TZlSBYy4dcUEgIA1ot33dWbVE1lNyPiT
kH/dRP7NnyOLfRgmM+gsmu3HxUNNAgMBAAECgYEApO8ri9BYwbWZrHCWX2Sb/gig
ysZKwYC4JckP7GZIJVS8TU/WOoQ4MZX0NPwbRPJlJeDwV8utE5K2d+9OwrRwGw7n
ZirNolvYSnBeZ8pKRwseQA2fzfCMubZcttjpyb1DfyMXwQj6oCGxuFXPLUWT8/1/
9PkPAWa8uOZhinD/fwUCQQDcxcdCn9rAFLOn1g3EJzSD56GGu6RIrkx8kdBofTYr
nwYnriiX8a//hlRg4BiOv1FdoTHFJi3QAOobX8tiUBK7AkEA2SBxS2TDOj3HJByI
32D0HY1a/O3VBEF4mkRi60HtDYRxL7fmCxebgzFytGxIQr4h+VxFqlABDoDd5vd3
cSS1lwJARm3FumhalYpFIda0f43uP+Il8mBr8U/BUMAHlz3SiSnrAb+abZaJid+l
jV4QF4HLCC6DPRyH4uJXzLHLpSpcPwJARYVVwUYqHGPbd3yLdrqcbznrgEDGi+5K
p1puMdWSCVn2w8imJ7cPXBphF9Pz7yrhxe39gGLNc89fPazO2bNfUQJBALBdr/G3
KYecLkwJeR0okQ8biSr1mAC+VhqlF9JW4pDiw4jrud6R+M4+WjU5nGjxPCfhbEHA
PCahv9z7njXL2qA=
-----END PRIVATE KEY-----`;
    const pemHeader = "-----BEGIN PRIVATE KEY-----";
    const pemFooter = "-----END PRIVATE KEY-----";
    let pvtDer = pvtPem.substring(pemHeader.length, pvtPem.length - pemFooter.length);
    let binaryDer = str2ab(window.atob(pvtDer));
    console.log("binaryDer " + binaryDer);
    window.crypto.subtle.importKey(
        "pkcs8",
        binaryDer,
        {
            name: "RSA-OAEP",
            hash: "SHA-256"
        },
        true,
        ["decrypt"]
    ).then((pvtKey) => {
        console.log("pvtKey " + pvtKey);

        let encryptedData = "Tu8MHqnXCrRHZjKgMJx7CF4b2r6jz1ame7+qCFJ6ZkfgzgJc1qDX5xkyN28vZMzhEiO42iFGEeUoYDA4SH+RFPoGPbM94/IFvZREgpEow0a4scMvaYfKCK76gbb5P3UbEIwcgN+QBtsHCEC/aPJmtPzEl1jUNO94fjUkjrGaKwM=";
        window.crypto.subtle.decrypt(
            { name: "RSA-OAEP"},
            pvtKey,
            str2ab(window.atob(encryptedData))
        ).then((decryptedData) => {
            console.log("decrypted " + new TextDecoder().decode(decryptedData));// + " Rohit Das");
        }).catch((err) => {console.log("decrypt error " + err);});
    }).catch((err) => {console.log("key import error " + err);});
}

decrypto();