rsa encryption/decryption polarssl c++

Question

I'm using C++ library PolarSSL for RSA encryption and decryption. But I'm not able to decrypt an encrypted string unless it's an output from encryption. Following code doesn't work (it's not refactored). It encrypts text and encodes the output to the Base64 and back. Condition on strcmp works (strings are the same).

AsymetricCipher::encrypt(const std::string &pathToPublicKey, std::istream &inputData, std::ostream &encryptedData) {
    if(initServerPublicCtx(pathToPublicKey, 512)) {
        std::cout << "Encryption error: Can't load public key from file: " << pathToPublicKey << std::endl;
        return false;
    }

    entropy_context entropy;
    ctr_drbg_context ctr_drbg;
    char *pers = "rsa_encrypt";

    entropy_init(&entropy);
    if(ctr_drbg_init(&ctr_drbg, entropy_func, &entropy, (unsigned char*)pers, strlen(pers)) != 0) {
        std::cout << "Encryption error: ctr_drbg_init failed" << std::endl;
        return false;
    }

    size_t inputSize = ::getStreamSize(inputData);
    char *input = new char[inputSize];
    memset(input, 0, inputSize);
    inputData.read(input, inputSize);
    input[inputSize] = '\0';

    unsigned char *buffer = new unsigned char[ctx.len];
    memset(buffer, 0, ctx.len);
    memcpy(buffer, input, inputSize);

    // This has to be rewritten
    size_t MAX_OUTPUT_LENGTH = ctx.len;
    unsigned char *outputBuffer = new unsigned char[MAX_OUTPUT_LENGTH];
    memset(outputBuffer, 0, MAX_OUTPUT_LENGTH);

    if(rsa_pkcs1_encrypt(&ctx, ctr_drbg_random, &ctr_drbg, RSA_PUBLIC, inputSize, buffer, outputBuffer) != 0) {
        std::cout << "Encryption error: rsa_pkcs1_encrypt failed" << std::endl;
        return false;
    }

    initServerPrivateCtx("data/private.key", 512);
    size_t outputSize = 0;

    std::string copyBuffer = "";

    std::stringstream encStream;

    std::string base64 = "";

    Base64Wrapper::encode(outputBuffer, strlen((char*)outputBuffer), base64);


    for(size_t i = 0; i < base64.length();) {
        encStream << base64[i++];
    }

    unsigned char *encBuffer = new unsigned char[MAX_OUTPUT_LENGTH+10];
    memset(encBuffer, 0, MAX_OUTPUT_LENGTH);
    encStream.read((char *)encBuffer, MAX_OUTPUT_LENGTH+10);
    copyBuffer.append((char *)encBuffer);

    unsigned char *decoded = NULL;
    size_t decodedSize = 0;

    Base64Wrapper::decode(copyBuffer, &decoded, &decodedSize);
    decoded[decodedSize] = '\0';

    if(strcmp((char*)outputBuffer, (char*)decoded) != 0) {
        std::cout << "Different";
    }

    memset(buffer, 0, ctx.len);

    if(rsa_pkcs1_decrypt(&ctx, RSA_PRIVATE, &outputSize, decoded, buffer, MAX_OUTPUT_LENGTH) != 0) {
        std::cout << "Decryption error: rsa_pkcs1_decrypt failed" << std::endl;
        return false;
    }

    ::cleanMemory(outputBuffer, MAX_OUTPUT_LENGTH);
    ::cleanMemory(buffer, ctx.len);

    delete [] outputBuffer;
    delete [] buffer;
    delete [] encBuffer;
    delete [] decoded;
    //delete [] input;

    return true;
}

However, if I call rsa_pkcs1_decrypt with outputBuffer from encryption, everything works fine.

I need to encrypt text, send it and decrypt on another place in code.

Any suggestions what am I doing wrong?

Answer 1

I've finally found a solution.

The problem was strlen((char*)outputBuffer) becuase it was always 0 since output from rsa_pkcs1_encrypt starts with \0.

The right solution is

bool AsymetricCipher::encrypt(const std::string &pathToPublicKey, std::istream &inputData, std::ostream &encryptedData) {
    if(initServerPublicCtx(pathToPublicKey, 512)) {
        std::cout << "Encryption error: Can't load public key from file: " << pathToPublicKey << std::endl;
        return false;
    }

    entropy_context entropy;
    ctr_drbg_context ctr_drbg;
    char *pers = "rsa_encrypt";

    entropy_init(&entropy);
    if(ctr_drbg_init(&ctr_drbg, entropy_func, &entropy, (unsigned char*)pers, strlen(pers)) != 0) {
        std::cout << "Encryption error: ctr_drbg_init failed" << std::endl;
        return false;
    }


    size_t inputSize = ::getStreamSize(inputData);
    unsigned char *buffer = new unsigned char[inputSize];
    memset(buffer, 0, inputSize);
    inputData.read((char *)buffer, inputSize);


    size_t MAX_OUTPUT_LENGTH = ctx.len;
    unsigned char *outputBuffer = new unsigned char[MAX_OUTPUT_LENGTH];
    memset(outputBuffer, 0, MAX_OUTPUT_LENGTH);

    bool retVal = true;
    if(rsa_pkcs1_encrypt(&ctx, ctr_drbg_random, &ctr_drbg, RSA_PUBLIC, inputSize, buffer, outputBuffer) != 0) {
        std::cout << "Encryption error: rsa_pkcs1_encrypt failed" << std::endl;
        retVal = false;
    }

    if(retVal) {
        std::string base64;
        Base64Wrapper::encode(outputBuffer, MAX_OUTPUT_LENGTH, base64);
        encryptedData << base64;
        ::cleanMemory(base64);
    }

    ::cleanMemory(outputBuffer, MAX_OUTPUT_LENGTH);
    ::cleanMemory(buffer, ctx.len);

    delete [] outputBuffer;
    delete [] buffer;

    return retVal;
}

And for decryption

bool AsymetricCipher::decrypt( const std::string &pathToPrivateKey, std::istream &encryptedData, std::ostream &decryptedData ) {
    if(initServerPrivateCtx(pathToPrivateKey, 512)) {
        std::cout << "Decrypt error: Can't load private key from file: " << pathToPrivateKey << std::endl;
        return false;
    }

    size_t inputSize = ::getStreamSize(encryptedData);
    size_t outputSize = 0;

    unsigned char* buffer = NULL;
    std::string base64;
    size_t bufferSize = 0;
    encryptedData >> base64;

    Base64Wrapper::decode(base64, &buffer, &bufferSize);
    ::cleanMemory(base64);

    size_t MAX_OUTPUT_LENGTH = ctx.len;
    unsigned char *outputBuffer = new unsigned char[MAX_OUTPUT_LENGTH];

    bool retVal = true;
    if(rsa_pkcs1_decrypt(&ctx, RSA_PRIVATE, &outputSize, buffer, outputBuffer, MAX_OUTPUT_LENGTH) != 0) {
        std::cout << "Decryption error: rsa_pkcs1_decrypt failed" << std::endl;
        retVal = false;
    }

    if(retVal) {
        outputBuffer[outputSize] = '\0';
        decryptedData << outputBuffer;
    }

    ::cleanMemory(buffer, bufferSize);
    ::cleanMemory(outputBuffer, outputSize);

    delete [] outputBuffer;
    delete [] buffer;

    return retVal;
}

Answer 2

Classic memory overflow

char *input = new char[inputSize];
input[inputSize] = '\0';