calcOpticalFlowPyrLK performance worse at high FPS

StackOverflow https://stackoverflow.com/questions/22204315

Question

I am using cv::calcOpticalFlowPyrLK to calculate the optical flow from one frame to another in a video sequence. I noticed that the tracking is less accurate in high fps compared to low fps.

The source fps is 30fps, and I found if I down sample it by 8, the tracking is much more accurate than using all the frames.

The frame size is 360*480 and the search window size is 21*21

Any help is appreciated!

Was it helpful?

Solution 2

Actually, I found out my problem is a float to int conversion bug in my code.

In my code I was looping through all the frames and did a conversion from optical flow tracked points to IOS points(CGPoint) and back again. During this process I accidentally converted from float to int (I used cv::Point instead of cv::Point2f).

The performance got worse at high fps because the error accumulated a lot more times, since the tracking is called a lot more times.

OTHER TIPS

the main cpp

#include <iostream>
#include <queue>
#include <opencv2/opencv.hpp>
#include "corner_tracker.h"
using namespace std;
using namespace cv;

int main(int argc, char** argv) {
  if (argc != 2) {
    cout << "usage: " << argv[0] << " <video path>" << endl;
    exit(1);
  }
  int frame_lag = 4;
  string video_filepath(argv[1]);
  VideoCapture vidcap(video_filepath);
  Mat ref_frame, curr_frame, prev_frame;
  queue<Mat> frame_buffer;
  vector<Point2f> tracked_corners;
  vector<Point2f> optical_flow;
  CornerTrackerParameterBlock param;
  CornerTracker corner_tracker(param);
  Mat mask;
  while (true){
    vidcap >> ref_frame;
    if (ref_frame.empty()) break;
    cvtColor(ref_frame, curr_frame, CV_BGR2GRAY);

    Mat tmp_frame;
    curr_frame.copyTo(tmp_frame);
    frame_buffer.push(tmp_frame);
    if ((int)frame_buffer.size() < frame_lag+1 ) {
      continue;
    }
    prev_frame = frame_buffer.front();
    frame_buffer.pop();
    corner_tracker.TrackCorners(prev_frame, curr_frame, mask, 100, tracked_corners, optical_flow);
    for (int i = 0; i < (int)tracked_corners.size(); i++) {
      //because optical flow is calculated between current frame and the frame_lag frame before it
      //the actual value of the optical flow vector has to be normalized
      Point2f normalized_optical_flow = optical_flow[i]*(1.0/(double)frame_lag);
      line(ref_frame, tracked_corners[i], tracked_corners[i] + normalized_optical_flow, Scalar(0,255,0));
      circle(ref_frame, tracked_corners[i], 2, Scalar(0,0,255));
    }
    imshow("window",ref_frame);
    if((char)waitKey(30) == 27) {
      break;
    }
  }
  return 0;
}

corner tracker header file

#ifndef CORNER_TRACKER_H_
#define CORNER_TRACKER_H_
#include <opencv2/core/core.hpp>

struct CornerTrackerParameterBlock {
  double lkt_max_bidirectioal_error;
  int lkt_maxlevel;
  int lkt_winsize;
  int feature_blocksize;
  double feature_k;
  double feature_mindist;
  double feature_quality_level;

//default constructor
CornerTrackerParameterBlock(void) :
  lkt_max_bidirectioal_error(2.0),
  lkt_maxlevel(3),
  lkt_winsize(16),
  feature_blocksize(3),
  feature_k(0.04),
  feature_mindist(5.0),
  feature_quality_level(0.01)
  {}
};

class CornerTracker {
public:
  CornerTracker(const CornerTrackerParameterBlock& param);
  void TrackCorners(const cv::Mat& prev_frame, const cv::Mat& curr_frame, const cv::Mat& mask, int max_corners, std::vector<cv::Point2f>& tracked_corners, std::vector<cv::Point2f>& optical_flow_vectors) const;
private:
  void AddAdditionalCorners(const cv::Mat& curr_frame, const cv::Mat& mask, int max_corners, std::vector<cv::Point2f>& tracked_corners) const;
  CornerTrackerParameterBlock m_param;
};

#endif //CORNER_TRACKER_H_

corner tracker cpp file

#include <iostream>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/video/tracking.hpp>
#include "corner_tracker.h"

using namespace std;
using namespace cv;

CornerTracker::CornerTracker(const CornerTrackerParameterBlock& param) :
  m_param(param)
{}

void CornerTracker::AddAdditionalCorners(const cv::Mat& curr_frame, const cv::Mat& mask, int max_corners, std::vector<cv::Point2f>& tracked_corners) const {
  //detect additional features
  int additional_corners = max_corners - tracked_corners.size();
  if (additional_corners <= 0) return;
  //generate mask
  Mat tmp_mask;
  if (mask.rows != curr_frame.rows || mask.cols != curr_frame.cols || mask.type() != CV_8U) {
    tmp_mask.create(curr_frame.rows, curr_frame.cols, CV_8U);
    tmp_mask = Scalar::all(255);
  }
  else {
    mask.copyTo(tmp_mask);
  }
  //mask out current points
  for (const Point2f& p : tracked_corners) {
    circle(tmp_mask, p, m_param.feature_mindist, Scalar::all(0), -1); //filled black circle
  }
  vector<Point2f> corners;
  goodFeaturesToTrack(curr_frame, corners, additional_corners, m_param.feature_quality_level, m_param.feature_mindist, tmp_mask, m_param.feature_blocksize, true, m_param.feature_k );
  for (const Point2f& p : corners) {
    tracked_corners.push_back(p);
  }
}

void CornerTracker::TrackCorners(const cv::Mat& prev_frame, const cv::Mat& curr_frame, const cv::Mat& mask, int max_corners, std::vector<cv::Point2f>& tracked_corners, std::vector<cv::Point2f>& optical_flow_vectors) const {
  AddAdditionalCorners(curr_frame, mask, max_corners, tracked_corners);
  vector<Point2f> prev_corners(tracked_corners);
  vector<Point2f> next_corners(tracked_corners);
  //optical flow corner tracking
  vector<uchar> status1,status2;
  vector<float> error1,error2;
  calcOpticalFlowPyrLK(curr_frame, prev_frame, tracked_corners, prev_corners, status1, error1, Size(m_param.lkt_winsize,m_param.lkt_winsize), m_param.lkt_maxlevel, TermCriteria(TermCriteria::COUNT+TermCriteria::EPS, 30, 0.01), OPTFLOW_USE_INITIAL_FLOW);
  calcOpticalFlowPyrLK(prev_frame, curr_frame, prev_corners, next_corners, status2, error2, Size(m_param.lkt_winsize,m_param.lkt_winsize), m_param.lkt_maxlevel, TermCriteria(TermCriteria::COUNT+TermCriteria::EPS, 30, 0.01), OPTFLOW_USE_INITIAL_FLOW);
  //check tracked corner quality
  vector<Point2f> temp_corners;
  optical_flow_vectors.clear();
  for (unsigned int i = 0; i < tracked_corners.size(); i++) {
    if (status1[i] == 0 || status2[i] == 0) {
      continue;
    }
    float bidirectional_error = norm(next_corners[i] - tracked_corners[i]);
    //bidirectional error check
    if (bidirectional_error > m_param.lkt_max_bidirectioal_error) {
      continue;
    }
    optical_flow_vectors.push_back(tracked_corners[i] - prev_corners[i]);
    temp_corners.push_back(tracked_corners[i]);
  }
  tracked_corners.swap(temp_corners);
}

This could also happen if the quality of the video changes. At lower FPS but the same kBPS, some types of video encoder (eg h.264) have more bits to encode each frame, leading to higher quality.

Licensed under: CC-BY-SA with attribution
Not affiliated with StackOverflow
scroll top