C++ LZMA SDK: Uncompress function for LZMA2 compressed file

Question

I am trying to create a function that uncompresses LZAM2 compressed data. I inspired myself from this tutorial which works great for LZMA and I tried to adapt it for LZMA2. I successfully created the compression function for LZMA2, but i have no success for the uncompression one.

Here is the compression function:

static void Compress2Inc(std::vector<unsigned char> &outBuf,
                         const std::vector<unsigned char> &inBuf)
{
    CLzma2EncHandle enc = Lzma2Enc_Create(&SzAllocForLzma, &SzAllocForLzma2);
    assert(enc);

    CLzma2EncProps props;
    Lzma2EncProps_Init(&props);
    props.lzmaProps.writeEndMark = 1; // 0 or 1

    SRes res = Lzma2Enc_SetProps(enc, &props);
    assert(res == SZ_OK);

    unsigned propsSize = LZMA_PROPS_SIZE;
    outBuf.resize(propsSize);

    res = Lzma2Enc_WriteProperties(enc);
    //cout << res;
    //assert(res == SZ_OK && propsSize == LZMA_PROPS_SIZE);

    VectorInStream inStream = { &VectorInStream_Read, &inBuf, 0 };
    VectorOutStream outStream = { &VectorOutStream_Write, &outBuf };

    res = Lzma2Enc_Encode(enc,
        (ISeqOutStream*)&outStream, (ISeqInStream*)&inStream,
        0);
    assert(res == SZ_OK);

    Lzma2Enc_Destroy(enc);
}

Where:

static void *AllocForLzma2(void *, size_t size) { return BigAlloc(size); }
static void FreeForLzma2(void *, void *address) { BigFree(address); }
static ISzAlloc SzAllocForLzma2 = { AllocForLzma2, FreeForLzma2 };

static void *AllocForLzma(void *, size_t size) { return MyAlloc(size); }
static void FreeForLzma(void *, void *address) { MyFree(address); }
static ISzAlloc SzAllocForLzma = { AllocForLzma, FreeForLzma };

typedef struct
{
    ISeqInStream SeqInStream;
    const std::vector<unsigned char> *Buf;
    unsigned BufPos;
} VectorInStream;

SRes VectorInStream_Read(void *p, void *buf, size_t *size)
{
    VectorInStream *ctx = (VectorInStream*)p;
    *size = min(*size, ctx->Buf->size() - ctx->BufPos);
    if (*size)
        memcpy(buf, &(*ctx->Buf)[ctx->BufPos], *size);
    ctx->BufPos += *size;
    return SZ_OK;
}

typedef struct
{
    ISeqOutStream SeqOutStream;
    std::vector<unsigned char> *Buf;
} VectorOutStream;

size_t VectorOutStream_Write(void *p, const void *buf, size_t size)
{
    VectorOutStream *ctx = (VectorOutStream*)p;
    if (size)
    {
        unsigned oldSize = ctx->Buf->size();
        ctx->Buf->resize(oldSize + size);
        memcpy(&(*ctx->Buf)[oldSize], buf, size);
    }
    return size;
}

Here is what I have so far with the uncompression function but Lzma2Dec_DecodeToBuf function returns error code 1(SZ_ERROR_DATA) and I just couldn't find anything on the web regarding this on the web.

static void Uncompress2Inc(std::vector<unsigned char> &outBuf,
                           const std::vector<unsigned char> &inBuf)
{
    CLzma2Dec dec;
    Lzma2Dec_Construct(&dec);

    SRes res = Lzma2Dec_Allocate(&dec, outBuf.size(), &SzAllocForLzma);
    assert(res == SZ_OK);

    Lzma2Dec_Init(&dec);

    outBuf.resize(UNCOMPRESSED_SIZE);
    unsigned outPos = 0, inPos = LZMA_PROPS_SIZE;
    ELzmaStatus status;
    const unsigned BUF_SIZE = 10240;
    while (outPos < outBuf.size())
    {
        unsigned destLen = min(BUF_SIZE, outBuf.size() - outPos);
        unsigned srcLen  = min(BUF_SIZE, inBuf.size() - inPos);
        unsigned srcLenOld = srcLen, destLenOld = destLen;

        res = Lzma2Dec_DecodeToBuf(&dec,
                                   &outBuf[outPos], &destLen,
                                   &inBuf[inPos], &srcLen,
                                   (outPos + destLen == outBuf.size()) ? LZMA_FINISH_END : LZMA_FINISH_ANY,
                                   &status);

        assert(res == SZ_OK);
        inPos += srcLen;
        outPos += destLen;
        if (status == LZMA_STATUS_FINISHED_WITH_MARK)
            break;
    }

    Lzma2Dec_Free(&dec, &SzAllocForLzma);
    outBuf.resize(outPos);
}

I am using Visual Studio 2008 and LZMA SDK downloaded from here. Someone here had the exact same problem, but i couldn't make use of his code...

Did anyone ever successfully uncompressed LZMA2 compressed files using LZMA SDK?

Please help!

Answer 1

A temporary workaround would be to replace SRes res = Lzma2Dec_Allocate(&dec, outBuf.size(), &SzAllocForLzma); with SRes res = Lzma2Dec_Allocate(&dec, 8, &SzAllocForLzma); in Uncompress2Inc function where 8 is a magic number...

However this is not the right way to solve the problem...

First mistake is that Lzma2Enc_WriteProperties doesn't return a result but a property byte which will have to be used as the second parameter of the Lzma2Dec_Allocate call in the Uncompress2Inc function. As a result we replace the magic number 8 with the property byte and everything works as expected.

In order to achieve this a 5 byte header must be added to the encoded data which will be extracted in the decoding function. Here is an example that works in VS2008(not the most perfect code but it works...I will get back later, when I have time, with a better example):

void Lzma2Benchmark::compressChunk(std::vector<unsigned char> &outBuf, const std::vector<unsigned char> &inBuf)
{
    //! \todo This is a temporary workaround, size needs to be added to the 
    m_uncompressedSize = inBuf.size();

    std::cout << "Uncompressed size is: " << inBuf.size() << std::endl;

    DWORD tickCountBeforeCompression = GetTickCount();

    CLzma2EncHandle enc = Lzma2Enc_Create(&m_szAllocForLzma, &m_szAllocForLzma2);
    assert(enc);

    CLzma2EncProps props;
    Lzma2EncProps_Init(&props);
    props.lzmaProps.writeEndMark = 1; // 0 or 1
    props.lzmaProps.level = 9;
    props.lzmaProps.numThreads = 3;
    //props.numTotalThreads = 2;

    SRes res = Lzma2Enc_SetProps(enc, &props);
    assert(res == SZ_OK);

    // LZMA_PROPS_SIZE == 5 bytes
    unsigned propsSize = LZMA_PROPS_SIZE;
    outBuf.resize(propsSize);

    // I think Lzma2Enc_WriteProperties returns the encoding properties in 1 Byte
    Byte properties = Lzma2Enc_WriteProperties(enc);

    //! \todo This is a temporary workaround
    m_propByte = properties;

    //! \todo Here m_propByte and m_uncompressedSize need to be added to outBuf's 5 byte header so simply add those 2 values to outBuf and start the encoding from there.

    BenchmarkUtils::VectorInStream inStream = { &BenchmarkUtils::VectorInStream_Read, &inBuf, 0 };
    BenchmarkUtils::VectorOutStream outStream = { &BenchmarkUtils::VectorOutStream_Write, &outBuf };

    res = Lzma2Enc_Encode(enc,
                          (ISeqOutStream*)&outStream,
                          (ISeqInStream*)&inStream,
                          0);

    std::cout << "Compress time is: " << GetTickCount() - tickCountBeforeCompression << " milliseconds.\n";

    assert(res == SZ_OK);

    Lzma2Enc_Destroy(enc);

    std::cout << "Compressed size is: " << outBuf.size() << std::endl;
}

void Lzma2Benchmark::unCompressChunk(std::vector<unsigned char> &outBuf, const std::vector<unsigned char> &inBuf)
{
    DWORD tickCountBeforeUncompression = GetTickCount();

    CLzma2Dec dec;
    Lzma2Dec_Construct(&dec);

    //! \todo Heere the property size and the uncompressed size need to be extracted from inBuf, which is the compressed data.

    // The second parameter is a temporary workaround.
    SRes res = Lzma2Dec_Allocate(&dec, m_propByte/*8*/, &m_szAllocForLzma);
    assert(res == SZ_OK);

    Lzma2Dec_Init(&dec);

    outBuf.resize(m_uncompressedSize);
    unsigned outPos = 0, inPos = LZMA_PROPS_SIZE;
    ELzmaStatus status;
    const unsigned BUF_SIZE = 10240;

    while(outPos < outBuf.size())
    {
        SizeT destLen = std::min(BUF_SIZE, outBuf.size() - outPos);
        SizeT srcLen  = std::min(BUF_SIZE, inBuf.size() - inPos);
        SizeT srcLenOld = srcLen, destLenOld = destLen;

        res = Lzma2Dec_DecodeToBuf(&dec,
                                   &outBuf[outPos],
                                   &destLen,
                                   &inBuf[inPos],
                                   &srcLen,
                                   (outPos + destLen == outBuf.size()) ? LZMA_FINISH_END : LZMA_FINISH_ANY,
                                   &status);

        assert(res == SZ_OK);
        inPos += srcLen;
        outPos += destLen;

        if(status == LZMA_STATUS_FINISHED_WITH_MARK)
        {
            break;
        }
    }

    Lzma2Dec_Free(&dec, &m_szAllocForLzma);

    outBuf.resize(outPos);

    std::cout << "Uncompress time is: " << GetTickCount() - tickCountBeforeUncompression << " milliseconds.\n";
}