RSA public key encryption openssl

Question

a question: Vendor says that for some encryption purpose uses PKCS#1 V2.1 OAEP with SHA-256... Is that even possible? I have checked and re-checked openssl and all they have is RSA public key encrypt with OAEP padding which is supposed to be PKCS#1 V2.1 with SHA1

So what can I do? How can I use SHA256 in RSA PUBLIC KEY encryption?

IS it even possible?

Best regards,

EDITED: ANSWER HOW TO USE RSA ENCRYPTION USING OPENSSL OAEP PADDING AND SHA256 DIGEST

#include "openssl/rsa.h"
#include <openssl/err.h>
#define RSA_F_RSA_PADDING_ADD_PKCS1_OAEP_MGF1        154

int RSA_padding_add_PKCS1_OAEP_mgf1_SHA256(unsigned char *to, int tlen,
    const unsigned char *from, int flen,
    const unsigned char *param, int plen,
    const EVP_MD *md, const EVP_MD *mgf1md)
{
    int i, emlen = tlen - 1;
    unsigned char *db, *seed;
    unsigned char *dbmask, seedmask[EVP_MAX_MD_SIZE];
    int mdlen;

    if (md == NULL)
        md = EVP_sha256(); //HERE IS THE ACTUAL USE OF SHAR256 digest!
    if (mgf1md == NULL)
        mgf1md = md;

    mdlen = EVP_MD_size(md);

    if (flen > emlen - 2 * mdlen - 1)
    {
        RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP_MGF1,
            RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE);
        return 0;
    }

    if (emlen < 2 * mdlen + 1)
    {
        RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP_MGF1, RSA_R_KEY_SIZE_TOO_SMALL);
        return 0;
    }

    to[0] = 0;
    seed = to + 1;
    db = to + mdlen + 1;

    if (!EVP_Digest((void *)param, plen, db, NULL, md, NULL))
        return 0;
    memset(db + mdlen, 0,
        emlen - flen - 2 * mdlen - 1);
    db[emlen - flen - mdlen - 1] = 0x01;
    memcpy(db + emlen - flen - mdlen, from, (unsigned int)flen);
    if (RAND_bytes(seed, mdlen) <= 0)
        return 0;
#ifdef PKCS_TESTVECT
    memcpy(seed,
        "\xaa\xfd\x12\xf6\x59\xca\xe6\x34\x89\xb4\x79\xe5\x07\x6d\xde\xc2\xf0\x6c\xb5\x8f",
        20);
#endif

    dbmask = (unsigned char*)OPENSSL_malloc(emlen - mdlen);
    if (dbmask == NULL)
    {
        RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP_MGF1, ERR_R_MALLOC_FAILURE);
        return 0;
    }

    if (PKCS1_MGF1(dbmask, emlen - mdlen, seed, mdlen, mgf1md) < 0)
        return 0;
    for (i = 0; i < emlen - mdlen; i++)
        db[i] ^= dbmask[i];

    if (PKCS1_MGF1(seedmask, mdlen, db, emlen - mdlen, mgf1md) < 0)
        return 0;
    for (i = 0; i < mdlen; i++)
        seed[i] ^= seedmask[i];

    OPENSSL_free(dbmask);
    return 1;
}

int RSA_padding_add_PKCS1_OAEP_SHA256(unsigned char *to, int tlen,
    const unsigned char *from, int flen,
    const unsigned char *param, int plen)
{
    return RSA_padding_add_PKCS1_OAEP_mgf1_SHA256(to, tlen, from, flen,
        param, plen, NULL, NULL);
}

static int RSA_eay_public_encrypt_SHA256(int flen, const unsigned char *from,
    unsigned char *to, RSA *rsa, int padding)
{
    BIGNUM *f, *ret;
    int i, j, k, num = 0, r = -1;
    unsigned char *buf = NULL;
    BN_CTX *ctx = NULL;

#ifdef OPENSSL_FIPS
    if (FIPS_selftest_failed())
    {
        FIPSerr(FIPS_F_RSA_EAY_PUBLIC_ENCRYPT, FIPS_R_FIPS_SELFTEST_FAILED);
        goto err;
    }

    if (FIPS_module_mode() && !(rsa->flags & RSA_FLAG_NON_FIPS_ALLOW)
        && (BN_num_bits(rsa->n) < OPENSSL_RSA_FIPS_MIN_MODULUS_BITS))
    {
        RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT, RSA_R_KEY_SIZE_TOO_SMALL);
        return -1;
    }
#endif

    if (BN_num_bits(rsa->n) > OPENSSL_RSA_MAX_MODULUS_BITS)
    {
        RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT, RSA_R_MODULUS_TOO_LARGE);
        return -1;
    }

    if (BN_ucmp(rsa->n, rsa->e) <= 0)
    {
        RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT, RSA_R_BAD_E_VALUE);
        return -1;
    }

    /* for large moduli, enforce exponent limit */
    if (BN_num_bits(rsa->n) > OPENSSL_RSA_SMALL_MODULUS_BITS)
    {
        if (BN_num_bits(rsa->e) > OPENSSL_RSA_MAX_PUBEXP_BITS)
        {
            RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT, RSA_R_BAD_E_VALUE);
            return -1;
        }
    }

    if ((ctx = BN_CTX_new()) == NULL) goto err;
    BN_CTX_start(ctx);
    f = BN_CTX_get(ctx);
    ret = BN_CTX_get(ctx);
    num = BN_num_bytes(rsa->n);
    buf = (unsigned char*)OPENSSL_malloc(num);
    if (!f || !ret || !buf)
    {
        RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT, ERR_R_MALLOC_FAILURE);
        goto err;
    }

    switch (padding)
    {
    case RSA_PKCS1_PADDING:
        i = RSA_padding_add_PKCS1_type_2(buf, num, from, flen);
        break;
#ifndef OPENSSL_NO_SHA
    case RSA_PKCS1_OAEP_PADDING:
        i = RSA_padding_add_PKCS1_OAEP_SHA256(buf, num, from, flen, NULL, 0);
        break;
#endif
    case RSA_SSLV23_PADDING:
        i = RSA_padding_add_SSLv23(buf, num, from, flen);
        break;
    case RSA_NO_PADDING:
        i = RSA_padding_add_none(buf, num, from, flen);
        break;
    default:
        RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT, RSA_R_UNKNOWN_PADDING_TYPE);
        goto err;
    }
    if (i <= 0) goto err;

    if (BN_bin2bn(buf, num, f) == NULL) goto err;

    if (BN_ucmp(f, rsa->n) >= 0)
    {
        /* usually the padding functions would catch this */
        RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT, RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
        goto err;
    }

    if (rsa->flags & RSA_FLAG_CACHE_PUBLIC)
        if (!BN_MONT_CTX_set_locked(&rsa->_method_mod_n, CRYPTO_LOCK_RSA, rsa->n, ctx))
            goto err;

    if (!rsa->meth->bn_mod_exp(ret, f, rsa->e, rsa->n, ctx,
        rsa->_method_mod_n)) goto err;

    /* put in leading 0 bytes if the number is less than the
    * length of the modulus */
    j = BN_num_bytes(ret);
    i = BN_bn2bin(ret, &(to[num - j]));
    for (k = 0; k<(num - i); k++)
        to[k] = 0;

    r = num;
err:
    if (ctx != NULL)
    {
        BN_CTX_end(ctx);
        BN_CTX_free(ctx);
    }
    if (buf != NULL)
    {
        OPENSSL_cleanse(buf, num);
        OPENSSL_free(buf);
    }
    return(r);
}
int RSA_public_encrypt_sha256(int flen, const unsigned char *from, unsigned char *to,
    RSA *rsa, int padding)
{
    return(RSA_eay_public_encrypt_SHA256(flen, from, to, rsa, padding));
}

Just add these few functions and call RSA_public_encrypt_sha256 instead of RSA_public_encrypt and voila you have RSA_OAEP_SHA256

Well i know this is abusing the openssl code, but this is a solution if you cannot compile openssl lib yourself, like i cannot because i received this as a part of an ARM platform

All thanks go to JARIQ in the answer below!

Thank you!

Answer 1

I am not sure about the OpenSSL API but in PKCS#11 API when you are using RSA encryption with OAEP padding you can specify message digest algorithm and also a mask generation function as you can see in my code sample (take a look at _03_EncryptAndDecryptSinglePartOaepTest() method) . It is written in C# but I believe it should be easily understandable. However I have never tried anything else than SHA1.

More information can be found in RFC 3447 and PKCS#11 specification (chapter 12.1.7 and chapter 12.1.8).

EDIT for OpenSSL:

In OpenSSL RSA encryption with public key and OAEP padding is performed in this order:

1. you need to pass RSA_PKCS1_OAEP_PADDING flag to function RSA_public_encrypt() implemented in rsa_crpt.c
2. RSA_public_encrypt() then calls function RSA_eay_public_encrypt() implemented in rsa_eay.c (unless you are using some cryptographic hardware device via ENGINE)
3. RSA_eay_public_encrypt() then calls function RSA_padding_add_PKCS1_OAEP() implemented in rsa_oaep.c

This uses SHA1 which seems to be currently the only option implemented in OpenSSL but I believe it should be possible to slightly modify code in rsa_oaep.c file to achieve what you need.