Video Stabilization using goodFeaturesToTrack and calcOpticalFlowPyrLK

edit

Hi all,

I'm currently trying to achieve video stabilization using goodFeaturesToTrack and calcOpticalFlowPyrLK.

I get the good features and calculate the good optical flow. I then get the image trajectory and smooth out the image trajectory and apply it to the video frames.

This all seems to be working correct, but the problem I'm facing is that the processed frames contain large black portions. How can I get the frames centered in the video without seeing the large black areas? Any help or suggestions will really be appreciated.

Below is the code I'm currently using:

#include <opencv2/opencv.hpp>
#include <iostream>
#include <cassert>
#include <cmath>
#include <fstream>
#include <ctime>

using namespace std;
using namespace cv;

const int SMOOTHING_RADIUS = 30; // In frames. The larger the more stable the video, but less reactive to sudden panning
const int HORIZONTAL_BORDER_CROP = 60; // In pixels. Crops the border to reduce the black borders from stabilisation being too noticeable.

struct TransformParam {
    TransformParam() {}
    TransformParam(double _dx, double _dy, double _da)
    {
        dx = _dx;
        dy = _dy;
        da = _da;
    }

    double dx;
    double dy;
    double da; // angle
};

struct Trajectory {
    Trajectory() {}
    Trajectory(double _x, double _y, double _a)
    {
        x = _x;
        y = _y;
        a = _a;
    }

    double x;
    double y;
    double a; // angle
};

void stabilize(const vector<Mat> &images, vector<Mat> &resultImages) {
  vector<TransformParam> prev_to_cur_transform;
  Mat last_T;

  for (int i = 1; i < images.size(); i++) {
    Mat prev = images[i-1];
    Mat cur = images[i];

    // vector from prev to cur
    vector<Point2f> prevCorner, curCorner;
    vector<Point2f> prevCorner2, curCorner2;
    vector<uchar> status;
    vector<float> error;

    Mat curGrey;
    Mat prevGrey;

    cvtColor(cur, curGrey, COLOR_BGR2GRAY);
    cvtColor(prev, prevGrey, COLOR_BGR2GRAY);

    size_t totalEntries = curGrey.cols * curGrey.rows;

    goodFeaturesToTrack(prevGrey, prevCorner, totalEntries, 0.01, 1);
    calcOpticalFlowPyrLK(prevGrey, curGrey, prevCorner, curCorner, status, error);

    // weed out bad matches
    for (int i = 0; i < status.size(); i++) {
      if (status[i]) {
        prevCorner2.push_back(prevCorner[i]);
        curCorner2.push_back(curCorner[i]);
      }
    }

    // translation + rotation only
    // false = rigidTransform, no scaling/shearing
    Mat T = estimateRigidTransform(prevCorner2, curCorner2, false);

    // in rare cases no valid transform is found, use the last known good transform
    if (T.data == NULL) {
      last_T.copyTo(T);
    }

    T.copyTo(last_T);

    // decompose T
    double dx = T.at<double>(0, 2);
    double dy = T.at<double>(1, 2);
    double da = atan2(T.at<double>(1, 0), T.at<double>(0, 0));

    prev_to_cur_transform.push_back(TransformParam(dx, dy, da));
    cout << "i: " << i << " " << "dx: " << dx << " " << "dy: " << dy << " " << "da: " << da << endl;
    cout << endl;

    curGrey.copyTo(prevGrey);
    cout << "Image: " << i << "/" << images.size() << " - good optical flow: " << prevCorner2.size() << endl;
  }

  // Step 2 - Accumulate transformations to get image trajectory
  // Accumulated frame to frame transformation
  double a = 0;
  double x = 0;
  double y = 0;

  vector<Trajectory> trajectory; // trajectory at all frames

  for (int i = 0; i < prev_to_cur_transform.size(); i++) {
    x += prev_to_cur_transform[i].dx;
    y += prev_to_cur_transform[i].dy;
    a += prev_to_cur_transform[i].da;

    trajectory.push_back(Trajectory(x, y, a));
    cout << "Trajectory " << (i+1) << ":" << " " << "x: " << x << " " << "y: " << y << " " << "a: " << a << endl;
  }

  // Step 3 - Smooth out trajectory using an averaging window
  vector<Trajectory> smoothed_trajectory; // trajectory at all frames

  for (int i = 0; i < trajectory.size(); i++) {
    double sum_x = 0;
    double sum_y ...