FFmpeg + OpenAL - playback streaming sound from video won't work

Question

I am decoding an OGG video (theora & vorbis as codecs) and want to show it on the screen (using Ogre 3D) while playing its sound. I can decode the image stream just fine and the video plays perfectly with the correct frame rate, etc.

However, I cannot get the sound to play at all with OpenAL.

Edit: I managed to make the playing sound resemble the actual audio in the video at least somewhat. Updated sample code.

Edit 2: I was able to get "almost" correct sound now. I had to set OpenAL to use AL_FORMAT_STEREO_FLOAT32 (after initializing the extension) instead of just STEREO16. Now the sound is "only" extremely high pitched and stuttering, but at the correct speed.

Here is how I decode audio packets (in a background thread, the equivalent works just fine for the image stream of the video file):

//------------------------------------------------------------------------------
int decodeAudioPacket(  AVPacket& p_packet, AVCodecContext* p_audioCodecContext, AVFrame* p_frame,
                        FFmpegVideoPlayer* p_player, VideoInfo& p_videoInfo)
{
    // Decode audio frame
    int got_frame = 0;
    int decoded = avcodec_decode_audio4(p_audioCodecContext, p_frame, &got_frame, &p_packet);
    if (decoded < 0) 
    {
        p_videoInfo.error = "Error decoding audio frame.";
        return decoded;
    }

    // Frame is complete, store it in audio frame queue
    if (got_frame)
    {
        int bufferSize = av_samples_get_buffer_size(NULL, p_audioCodecContext->channels, p_frame->nb_samples, 
                                                    p_audioCodecContext->sample_fmt, 0);

        int64_t duration = p_frame->pkt_duration;
        int64_t dts = p_frame->pkt_dts;

        if (staticOgreLog)
        {
            staticOgreLog->logMessage("Audio frame bufferSize / duration / dts: " 
                    + boost::lexical_cast<std::string>(bufferSize) + " / "
                    + boost::lexical_cast<std::string>(duration) + " / "
                    + boost::lexical_cast<std::string>(dts), Ogre::LML_NORMAL);
        }

        // Create the audio frame
        AudioFrame* frame = new AudioFrame();
        frame->dataSize = bufferSize;
        frame->data = new uint8_t[bufferSize];
        if (p_frame->channels == 2)
        {
            memcpy(frame->data, p_frame->data[0], bufferSize >> 1);
            memcpy(frame->data + (bufferSize >> 1), p_frame->data[1], bufferSize >> 1);
        }
        else
        {
            memcpy(frame->data, p_frame->data, bufferSize);
        }
        double timeBase = ((double)p_audioCodecContext->time_base.num) / (double)p_audioCodecContext->time_base.den;
        frame->lifeTime = duration * timeBase;

        p_player->addAudioFrame(frame);
    }

    return decoded;
}

So, as you can see, I decode the frame, memcpy it to my own struct, AudioFrame. Now, when the sound is played, I use these audio frame like this:

    int numBuffers = 4;
    ALuint buffers[4];
    alGenBuffers(numBuffers, buffers);
    ALenum success = alGetError();
    if(success != AL_NO_ERROR)
    {
        CONSOLE_LOG("Error on alGenBuffers : " + Ogre::StringConverter::toString(success) + alGetString(success));
        return;
    }

    // Fill a number of data buffers with audio from the stream
    std::vector<AudioFrame*> audioBuffers;
    std::vector<unsigned int> audioBufferSizes;
    unsigned int numReturned = FFMPEG_PLAYER->getDecodedAudioFrames(numBuffers, audioBuffers, audioBufferSizes);

    // Assign the data buffers to the OpenAL buffers
    for (unsigned int i = 0; i < numReturned; ++i)
    {
        alBufferData(buffers[i], _streamingFormat, audioBuffers[i]->data, audioBufferSizes[i], _streamingFrequency);

        success = alGetError();
        if(success != AL_NO_ERROR)
        {
            CONSOLE_LOG("Error on alBufferData : " + Ogre::StringConverter::toString(success) + alGetString(success)
                            + " size: " + Ogre::StringConverter::toString(audioBufferSizes[i]));
            return;
        }
    }

    // Queue the buffers into OpenAL
    alSourceQueueBuffers(_source, numReturned, buffers);
    success = alGetError();
    if(success != AL_NO_ERROR)
    {
        CONSOLE_LOG("Error queuing streaming buffers: " + Ogre::StringConverter::toString(success) + alGetString(success));
        return;
    }
}

alSourcePlay(_source);

The format and frequency I give to OpenAL are AL_FORMAT_STEREO_FLOAT32 (it is a stereo sound stream, and I did initialize the FLOAT32 extension) and 48000 (which is the sample rate of the AVCodecContext of the audio stream).

And during playback, I do the following to refill OpenAL's buffers:

ALint numBuffersProcessed;

// Check if OpenAL is done with any of the queued buffers
alGetSourcei(_source, AL_BUFFERS_PROCESSED, &numBuffersProcessed);
if(numBuffersProcessed <= 0)
    return;

// Fill a number of data buffers with audio from the stream
std::vector<AudiFrame*> audioBuffers;
std::vector<unsigned int> audioBufferSizes;
unsigned int numFilled = FFMPEG_PLAYER->getDecodedAudioFrames(numBuffersProcessed, audioBuffers, audioBufferSizes);

// Assign the data buffers to the OpenAL buffers
ALuint buffer;
for (unsigned int i = 0; i < numFilled; ++i)
{
    // Pop the oldest queued buffer from the source, 
    // fill it with the new data, then re-queue it
    alSourceUnqueueBuffers(_source, 1, &buffer);

    ALenum success = alGetError();
    if(success != AL_NO_ERROR)
    {
        CONSOLE_LOG("Error Unqueuing streaming buffers: " + Ogre::StringConverter::toString(success));
        return;
    }

    alBufferData(buffer, _streamingFormat, audioBuffers[i]->data, audioBufferSizes[i], _streamingFrequency);

    success = alGetError();
    if(success != AL_NO_ERROR)
    {
        CONSOLE_LOG("Error on re- alBufferData: " + Ogre::StringConverter::toString(success));
        return;
    }

    alSourceQueueBuffers(_source, 1, &buffer);

    success = alGetError();
    if(success != AL_NO_ERROR)
    {
        CONSOLE_LOG("Error re-queuing streaming buffers: " + Ogre::StringConverter::toString(success) + " "
                    + alGetString(success));
        return;
    }
}

// Make sure the source is still playing, 
// and restart it if needed.
ALint playStatus;
alGetSourcei(_source, AL_SOURCE_STATE, &playStatus);
if(playStatus != AL_PLAYING)
    alSourcePlay(_source);

As you can see, I do quite heavy error checking. But I do not get any errors, neither from OpenAL nor from FFmpeg. Edit: What I hear somewhat resembles the actual audio from the video, but VERY high pitched and stuttering VERY much. Also, it seems to be playing on top of TV noise. Very strange. Plus, it is playing much slower than the correct audio would. Edit: 2 After using AL_FORMAT_STEREO_FLOAT32, the sound plays at the correct speed, but is still very high pitched and stuttering (though less than before).

The video itself is not broken, it can be played fine on any player. OpenAL can also play *.way files just fine in the same application, so it is also working.

Any ideas what could be wrong here or how to do this correctly?

My only guess is that somehow, FFmpeg's decode function does not produce data OpenGL can read. But this is as far as the FFmpeg decode example goes, so I don't know what's missing. As I understand it, the decode_audio4 function decodes the frame to raw data. And OpenAL should be able to work with RAW data (or rather, doesn't work with anything else).

Answer 1

So, I finally figured out how to do it. Gee, what a mess. It was a hint from a user on the libav-users mailing list that put me on the correct path.

Here are my mistakes:

1. Using the wrong format in the alBufferData function. I used AL_FORMAT_STEREO16 (as that is what every single streaming example with OpenAL uses). I should have used AL_FORMAT_STEREO_FLOAT32, as the video I stream is Ogg and vorbis is stored in floating points. And using swr_convert to convert from AV_SAMPLE_FMT_FLTP to AV_SAMPLE_FMT_S16 just crashes. No idea why.

2. Not using swr_convert to convert the decoded audio frame to the target format. After I was trying to use swr_convert to convert from FLTP to S16, and it would simply crash without a reason given, I assumed it was broken. But after figuring out my first mistake, I tried again, converting from FLTP to FLT (non-planar) and then it worked! So OpenAL uses interleaved format, not planar. Good to know.

So here is the decodeAudioPacket function that is working for me with Ogg video, vorbis audio stream:

int decodeAudioPacket(  AVPacket& p_packet, AVCodecContext* p_audioCodecContext, AVFrame* p_frame,
                        SwrContext* p_swrContext, uint8_t** p_destBuffer, int p_destLinesize,
                        FFmpegVideoPlayer* p_player, VideoInfo& p_videoInfo)
{
    // Decode audio frame
    int got_frame = 0;
    int decoded = avcodec_decode_audio4(p_audioCodecContext, p_frame, &got_frame, &p_packet);
    if (decoded < 0) 
    {
        p_videoInfo.error = "Error decoding audio frame.";
        return decoded;
    }

    if(decoded <= p_packet.size)
    {
        /* Move the unread data to the front and clear the end bits */
        int remaining = p_packet.size - decoded;
        memmove(p_packet.data, &p_packet.data[decoded], remaining);
        av_shrink_packet(&p_packet, remaining);
    }

    // Frame is complete, store it in audio frame queue
    if (got_frame)
    {
        int outputSamples = swr_convert(p_swrContext, 
                                        p_destBuffer, p_destLinesize, 
                                        (const uint8_t**)p_frame->extended_data, p_frame->nb_samples);

        int bufferSize = av_get_bytes_per_sample(AV_SAMPLE_FMT_FLT) * p_videoInfo.audioNumChannels
                            * outputSamples;

        int64_t duration = p_frame->pkt_duration;
        int64_t dts = p_frame->pkt_dts;

        if (staticOgreLog)
        {
            staticOgreLog->logMessage("Audio frame bufferSize / duration / dts: " 
                    + boost::lexical_cast<std::string>(bufferSize) + " / "
                    + boost::lexical_cast<std::string>(duration) + " / "
                    + boost::lexical_cast<std::string>(dts), Ogre::LML_NORMAL);
        }

        // Create the audio frame
        AudioFrame* frame = new AudioFrame();
        frame->dataSize = bufferSize;
        frame->data = new uint8_t[bufferSize];
        memcpy(frame->data, p_destBuffer[0], bufferSize);
        double timeBase = ((double)p_audioCodecContext->time_base.num) / (double)p_audioCodecContext->time_base.den;
        frame->lifeTime = duration * timeBase;

        p_player->addAudioFrame(frame);
    }

    return decoded;
}

And here is how I initialize the context and the destination buffer:

// Initialize SWR context
SwrContext* swrContext = swr_alloc_set_opts(NULL, 
            audioCodecContext->channel_layout, AV_SAMPLE_FMT_FLT, audioCodecContext->sample_rate,
            audioCodecContext->channel_layout, audioCodecContext->sample_fmt, audioCodecContext->sample_rate, 
            0, NULL);
int result = swr_init(swrContext);

// Create destination sample buffer
uint8_t** destBuffer = NULL;
int destBufferLinesize;
av_samples_alloc_array_and_samples( &destBuffer,
                                    &destBufferLinesize,
                                    videoInfo.audioNumChannels,
                                    2048,
                                    AV_SAMPLE_FMT_FLT,
                                    0);