How to draw Optical flow images from ocl::PyrLKOpticalFlow::dense()

Question 1

A well establische method used in the optical flow community is to display a motion vector field as a color coded image as you can see at one of the various data sets. E.g MPI dataset or the Middlebury dataset.

Therefor you estimate the length and the angle of your motion vector. And use a HSV to RGB colorspace transformation (see OpenCV cvtColor function) to create your color coded image. Use the angle of your motion vector as H (Hue) - channel and the normalized length as the S (Saturation) - channel and set V (Value) to 1. The the color of your image will show you the direction of your motion and the saturation the length ( speed ).

The code will should like this ( Note if use_value == true, the Saturation will be set to 1 and the Value channel is related to the motion vector length):

    void FlowToRGB(const cv::Mat & inpFlow,
                cv::Mat & rgbFlow,
                const float & max_size ,
                bool use_value)
{
    if(inpFlow.empty()) return;
    if( inpFlow.depth() != CV_32F)
        throw(std::exception("FlowToRGB: error inpFlow wrong data type ( has be CV_32FC2"));
    const float grad2deg = (float)(90/3.141);
    cv::Mat pol(inpFlow.size(), CV_32FC2);

    float mean_val = 0, min_val = 1000, max_val = 0;
    float _dx, _dy;

    for(int r = 0; r < inpFlow.rows; r++)
    {
        for(int c = 0; c < inpFlow.cols; c++)
        {
            cv::Point2f polar = cvmath::toPolar(inpFlow.at<cv::Point2f>(r,c));
            polar.y *= grad2deg;
            mean_val +=polar.x;
            max_val = MAX(max_val, polar.x);
            min_val = MIN(min_val, polar.x);
            pol.at<cv::Point2f>(r,c) = cv::Point2f(polar.y,polar.x);
        }
    }
    mean_val /= inpFlow.size().area();
    float scale = max_val - min_val;
    float shift = -min_val;//-mean_val + scale;
    scale = 255.f/scale;
    if( max_size > 0)
    {
        scale = 255.f/max_size;
        shift = 0;
    }

    //calculate the angle, motion value 
    cv::Mat hsv(inpFlow.size(), CV_8UC3);
    uchar * ptrHSV = hsv.ptr<uchar>();
    int idx_val  = (use_value) ? 2:1;
    int idx_sat  = (use_value) ? 1:2;


    for(int r = 0; r < inpFlow.rows; r++, ptrHSV += hsv.step1())
    {
        uchar * _ptrHSV = ptrHSV;
        for(int c = 0; c < inpFlow.cols; c++, _ptrHSV+=3)
        {
            cv::Point2f vpol = pol.at<cv::Point2f>(r,c);

            _ptrHSV[0] = cv::saturate_cast<uchar>(vpol.x);
            _ptrHSV[idx_val] = cv::saturate_cast<uchar>( (vpol.y + shift) * scale);  
            _ptrHSV[idx_sat] = 255;
        }
    }   
    cv::Mat rgbFlow32F;
    cv::cvtColor(hsv, rgbFlow32F, CV_HSV2BGR);
    rgbFlow32F.convertTo(rgbFlow, CV_8UC3);}
 }

Question 2

Python

Please refer to opt_flow.py#draw_flow

def draw_flow(img, flow, step=16):
    h, w = img.shape[:2]
    y, x = np.mgrid[step/2:h:step, step/2:w:step].reshape(2,-1).astype(int)
    fx, fy = flow[y,x].T
    lines = np.vstack([x, y, x+fx, y+fy]).T.reshape(-1, 2, 2)
    lines = np.int32(lines + 0.5)
    vis = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
    cv2.polylines(vis, lines, 0, (0, 255, 0))
    for (x1, y1), (x2, y2) in lines:
        cv2.circle(vis, (x1, y1), 1, (0, 255, 0), -1)
    return vis

C++

Please can refer to tvl1_optical_flow.cpp#drawOpticalFlow

static void drawOpticalFlow(const Mat_<Point2f>& flow, Mat& dst, float maxmotion = -1)
{
    dst.create(flow.size(), CV_8UC3);
    dst.setTo(Scalar::all(0));

    // determine motion range:
    float maxrad = maxmotion;

    if (maxmotion <= 0)
    {
        maxrad = 1;
        for (int y = 0; y < flow.rows; ++y)
        {
            for (int x = 0; x < flow.cols; ++x)
            {
                Point2f u = flow(y, x);

                if (!isFlowCorrect(u))
                    continue;

                maxrad = max(maxrad, sqrt(u.x * u.x + u.y * u.y));
            }
        }
    }

    for (int y = 0; y < flow.rows; ++y)
    {
        for (int x = 0; x < flow.cols; ++x)
        {
            Point2f u = flow(y, x);

            if (isFlowCorrect(u))
                dst.at<Vec3b>(y, x) = computeColor(u.x / maxrad, u.y / maxrad);
        }
    }
}

Question 3

I did something like this in my code, a while ago:

        calcOpticalFlowPyrLK(frame_prec,frame_cur,v_corners_prec[i],corners_cur,status, err);


        for(int j=0; j<corners_cur.size(); j++){
                if(status[j]){

                    line(frame_cur,v_corners_prec[i][j],corners_cur[j],colors[i]);
                }
        }

Basically I draw a line between the points tracked by the OF in this iteration and the previous ones, this draws the optical flow lines which represent the flow on the image.

Hope this helps..