Use rsa-pss algorithm to sign data

Question

I want to use RSA-PSS algorithm to sign and verify data by using rust openssl crate. But I ran into some problems.

Here is my code:

    use openssl::sign::{Signer, Verifier, RsaPssSaltlen};
    use openssl::rsa::Rsa;
    use openssl::pkey::PKey;
    use openssl::hash::MessageDigest;
    
    // Generate a keypair
    let keypair = Rsa::generate(2048).unwrap();
    let keypair = PKey::from_rsa(keypair).unwrap();

    let data = b"hello, world!";

    // Sign the data
    let mut signer = Signer::new(MessageDigest::sha256(), &keypair).unwrap();
    // set the salt length to the digest length
    let digest_len = RsaPssSaltlen::DIGEST_LENGTH;
    signer.set_rsa_pss_saltlen(digest_len).unwrap(); // panic here

    signer.update(data).unwrap();
    let signature = signer.sign_to_vec().unwrap();

I set the salt length to the digest length, but the code throws a bug like this:

[Error { code: 67694738, library: "rsa routines", function: "pkey_rsa_ctrl", reason: "invalid pss saltlen", file: "crypto/rsa/rsa_pmeth.c", line: 438 }, Error { code: 101224595, library: "digital envelope routines", function: "EVP_PKEY_CTX_ctrl", reason: "command not supported", file: "crypto/evp/pmeth_lib.c", line: 396 }])

So the pss saltlen is invalid and I don't know how to do it properly.

I also tried to set up a saltlen myself:

let digest_len = RsaPssSaltlen::custom(32 as c_int);

which throws

[Error { code: 67694738, library: "rsa routines", function: "pkey_rsa_ctrl", reason: "invalid pss saltlen", file: "crypto/rsa/rsa_pmeth.c", line: 438 }, Error { code: 101224595, library: "digital envelope routines", function: "EVP_PKEY_CTX_ctrl", reason: "command not supported", file: "crypto/evp/pmeth_lib.c", line: 396 }]

Also an invalid pss saltlen.

Any help? Thank you!

Answer 1

Here is an example of using RSA-PSS algorithm: link.

Basically, you need to set the rsa_padding first.

let key = include_bytes!("../test/rsa.pem");
        let private_key = Rsa::private_key_from_pem(key).unwrap();
        let pkey = PKey::from_rsa(private_key).unwrap();

        let mut signer = Signer::new(MessageDigest::sha256(), &pkey).unwrap();
        signer.set_rsa_padding(Padding::PKCS1_PSS).unwrap();
        assert_eq!(signer.rsa_padding().unwrap(), Padding::PKCS1_PSS);
        signer
            .set_rsa_pss_saltlen(RsaPssSaltlen::DIGEST_LENGTH)
            .unwrap();
        signer.set_rsa_mgf1_md(MessageDigest::sha256()).unwrap();
        signer.update(&Vec::from_hex(INPUT).unwrap()).unwrap();
        let signature = signer.sign_to_vec().unwrap();

        let mut verifier = Verifier::new(MessageDigest::sha256(), &pkey).unwrap();
        verifier.set_rsa_padding(Padding::PKCS1_PSS).unwrap();
        verifier
            .set_rsa_pss_saltlen(RsaPssSaltlen::DIGEST_LENGTH)
            .unwrap();
        verifier.set_rsa_mgf1_md(MessageDigest::sha256()).unwrap();
        verifier.update(&Vec::from_hex(INPUT).unwrap()).unwrap();
        assert!(verifier.verify(&signature).unwrap());