Android MediaCodec dropping frames

Question

I'm doing live video processing via OpenGL, MediaCodec and MediaMuxer.

The output video seems to have frames dropped. The video seems to be running at 1 or 2 FPS, even though the app is running at 15 FPS. I've debugged the output of the encoder, and no frames are being dropped. What is going on?

I have added the core code below.

public class VideoSavingController
{
    // Static Variables
    private static final String MIME_TYPE = "video/avc";        
    private static final int FRAME_RATE = 15;                   
    private static final int IFRAME_INTERVAL = 1;               
    private static final int TIMEOUT_USEC = 10000;          

    private static final int BIT_RATE = 16 * 1000 * 1000;

    // Member Variables
    private boolean mIsRecordingStarted = false;
    private boolean mIsStartRequsted    = false;
    private boolean mIsStopRequested    = false;

    private MediaCodec mEncoder;
    private MediaCodec.BufferInfo mBufferInfo;

    private MediaMuxer mMuxer;
    private int mTrackIndex;
    private boolean mMuxerStarted = false;

    private VideoSavingSurface mInputSurface;

    private String mOutputPath;

    private long mStartTime;

    // Constructor
    public VideoSavingController(){}

    // Controls
    public void requestStartRecording()
    {
        mIsStartRequsted = true;
    }
    public void updateStartRecording()
    {
        if (mIsStartRequsted)
        {
            startRecording();
            mIsStartRequsted = false;
            mStartTime = SnapDat.camera().mCamera.timestamp();
        }
    }
    private void startRecording()
    {
        if (mIsRecordingStarted)
            return;
        mIsRecordingStarted = true;

        prepareEncoder();
    }
    public void recordFrameStep1()
    {
        if (!mIsRecordingStarted)
            return;

        mInputSurface.makeCurrent();

        drainEncoder(false);
    }
    /**
     * Draw the Image Between These Steps
     * Share texture between contexts by passing the GLSurfaceView's EGLContext as eglCreateContext()'s share_context argument
     * */
    public void recordFrameStep2()
    {
        if (!mIsRecordingStarted)
            return;

        // Set the presentation time stamp from the SurfaceTexture's time stamp.  This
        // will be used by MediaMuxer to set the PTS in the video.
        long time = SnapDat.camera().mCamera.timestamp() - mStartTime;
        mInputSurface.setPresentationTime( time );


        // Submit it to the encoder.  The eglSwapBuffers call will block if the input
        // is full, which would be bad if it stayed full until we dequeued an output
        // buffer (which we can't do, since we're stuck here).  So long as we fully drain
        // the encoder before supplying additional input, the system guarantees that we
        // can supply another frame without blocking.
        mInputSurface.swapBuffers();
    }
    public void requestStopRecording()
    {
        mIsStopRequested = true;
    }
    public void updateStopRecording()
    {
        if (mIsStopRequested)
        {
            mIsStopRequested = false;
            stopRecording();
        }
    }
    private void stopRecording()
    {
        if (!mIsRecordingStarted)
            return;
        mIsRecordingStarted = false;

        drainEncoder(true);
        releaseEncoder();

        // Notify Video File Added
        File videoFile = new File(mOutputPath);
        UtilityVideo.addVideo(videoFile, SnapDat.currentActivity());
    }
    public boolean isRecording()
    {
        return mIsRecordingStarted;
    }

    // Encoder
    private void prepareEncoder()
    {
        // Determine Size
        Size previewSize = xxxx
        int maxSize = Math.max(previewSize.width, previewSize.height);
        int width  = (640 * previewSize.width ) / maxSize;
        int height = (640 * previewSize.height) / maxSize;

        if ( !xxxx.isLandscape() )
        {
            int oldWidth = width;
            width = height;
            height = oldWidth;
        }

        // Force Factor of 16 Size
        width  = (width  / 16) * 16;
        height = (height / 16) * 16;

        mBufferInfo = new MediaCodec.BufferInfo();

        MediaFormat format = MediaFormat.createVideoFormat(MIME_TYPE, width, height);
        format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
        format.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE);
        format.setInteger(MediaFormat.KEY_FRAME_RATE, FRAME_RATE);
        format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, IFRAME_INTERVAL);

        mEncoder = MediaCodec.createEncoderByType(MIME_TYPE);
        mEncoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
        mInputSurface = new VideoSavingSurface( mEncoder.createInputSurface() );
        mEncoder.start();

        // Output filename
        mOutputPath = VideoSaver.getVideoPath();

        // Create a MediaMuxer.  We can't add the video track and start() the muxer here,
        // because our MediaFormat doesn't have the Magic Goodies.  These can only be
        // obtained from the encoder after it has started processing data.
        //
        // We're not actually interested in multiplexing audio.  We just want to convert
        // the raw H.264 elementary stream we get from MediaCodec into a .mp4 file.
        try
        {
            mMuxer = new MediaMuxer(mOutputPath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
        } 
        catch (IOException ioe)
        {
            throw new RuntimeException("MediaMuxer creation failed", ioe);
        }

        mTrackIndex = -1;
        mMuxerStarted = false;
    }
    private void releaseEncoder()
    {
        if (mEncoder != null)
        {
            mEncoder.stop();
            mEncoder.release();
            mEncoder = null;
        }
        if (mInputSurface != null) 
        {
            mInputSurface.release();
            mInputSurface = null;
        }
        if (mMuxer != null) 
        {
            mMuxer.stop();
            mMuxer.release();
            mMuxer = null;
        }
    }
    private void drainEncoder(boolean endOfStream)
    {
        if (endOfStream)
            mEncoder.signalEndOfInputStream();  

        ByteBuffer[] encoderOutputBuffers = mEncoder.getOutputBuffers();
        while (true) 
        {
            int encoderStatus = mEncoder.dequeueOutputBuffer(mBufferInfo, TIMEOUT_USEC);
            if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) 
            {
                break;
            } 
            else if (encoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) 
            {
                // not expected for an encoder
                encoderOutputBuffers = mEncoder.getOutputBuffers();
            } 
            else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) 
            {
                // should happen before receiving buffers, and should only happen once
                if (mMuxerStarted) 
                    throw new RuntimeException("format changed twice");
                MediaFormat newFormat = mEncoder.getOutputFormat();

                // now that we have the Magic Goodies, start the muxer
                mTrackIndex = mMuxer.addTrack(newFormat);
                mMuxer.start();
                mMuxerStarted = true;
            } 
            else if (encoderStatus < 0) 
            {
                // Unexpected status, ignore it
            } 
            else 
            {
                ByteBuffer encodedData = encoderOutputBuffers[encoderStatus];
                if (encodedData == null) 
                    throw new RuntimeException("encoderOutputBuffer " + encoderStatus + " was null");

                if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0)
                    mBufferInfo.size = 0;

                if (mBufferInfo.size != 0)
                {
                    if (!mMuxerStarted) 
                        throw new RuntimeException("muxer hasn't started");

                    // adjust the ByteBuffer values to match BufferInfo (not needed?)
                    encodedData.position(mBufferInfo.offset);
                    encodedData.limit(mBufferInfo.offset + mBufferInfo.size);

                    mMuxer.writeSampleData(mTrackIndex, encodedData, mBufferInfo);
                }

                mEncoder.releaseOutputBuffer(encoderStatus, false);

                if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0)
                    break;      // out of while
            }
        }
    }
}

The code which drives this is given below:

    OpenGLState oldState = OpenGLState.createCurrent();

    mSaveVideo.updateStartRecording();
    if (mSaveVideo.isRecording())
    {
        mSaveVideo.recordFrameStep1();

        // Draws Image here

        mSaveVideo.recordFrameStep2();
    }
    mSaveVideo.updateStopRecording();

    oldState.makeCurrent();

Answer 1

This appears to be a bug in the driver when shared contexts are used.

This post has the details. In short, one of the contexts isn't noticing that the texture contents have changed, so it keeps rendering the previous data. You can work around the problem by binding to texture 0 and then back to the actual texture ID.