martisan's profile - activity

Hi all,

I'm currently trying to achieve video stabilization using goodFeaturesToTrack and calcOpticalFlowPyrLK.

I get the good features and calculate the good optical flow. I then get the image trajectory and smooth out the image trajectory and apply it to the video frames.

This all seems to be working correct, but the problem I'm facing is that the processed frames contain large black portions. How can I get the frames centered in the video without seeing the large black areas? Any help or suggestions will really be appreciated.

Below is the code I'm currently using:

#include <opencv2/opencv.hpp>
#include <iostream>
#include <cassert>
#include <cmath>
#include <fstream>
#include <ctime>

using namespace std;
using namespace cv;

const int SMOOTHING_RADIUS = 30; // In frames. The larger the more stable the video, but less reactive to sudden panning
const int HORIZONTAL_BORDER_CROP = 60; // In pixels. Crops the border to reduce the black borders from stabilisation being too noticeable.

struct TransformParam {
    TransformParam() {}
    TransformParam(double _dx, double _dy, double _da)
    {
        dx = _dx;
        dy = _dy;
        da = _da;
    }

    double dx;
    double dy;
    double da; // angle
};

struct Trajectory {
    Trajectory() {}
    Trajectory(double _x, double _y, double _a)
    {
        x = _x;
        y = _y;
        a = _a;
    }

    double x;
    double y;
    double a; // angle
};

void stabilize(const vector<Mat> &images, vector<Mat> &resultImages) {
  vector<TransformParam> prev_to_cur_transform;
  Mat last_T;

  for (int i = 1; i < images.size(); i++) {
    Mat prev = images[i-1];
    Mat cur = images[i];

    // vector from prev to cur
    vector<Point2f> prevCorner, curCorner;
    vector<Point2f> prevCorner2, curCorner2;
    vector<uchar> status;
    vector<float> error;

    Mat curGrey;
    Mat prevGrey;

    cvtColor(cur, curGrey, COLOR_BGR2GRAY);
    cvtColor(prev, prevGrey, COLOR_BGR2GRAY);

    size_t totalEntries = curGrey.cols * curGrey.rows;

    goodFeaturesToTrack(prevGrey, prevCorner, totalEntries, 0.01, 1);
    calcOpticalFlowPyrLK(prevGrey, curGrey, prevCorner, curCorner, status, error);

    // weed out bad matches
    for (int i = 0; i < status.size(); i++) {
      if (status[i]) {
        prevCorner2.push_back(prevCorner[i]);
        curCorner2.push_back(curCorner[i]);
      }
    }

    // translation + rotation only
    // false = rigidTransform, no scaling/shearing
    Mat T = estimateRigidTransform(prevCorner2, curCorner2, false);

    // in rare cases no valid transform is found, use the last known good transform
    if (T.data == NULL) {
      last_T.copyTo(T);
    }

    T.copyTo(last_T);

    // decompose T
    double dx = T.at<double>(0, 2);
    double dy = T.at<double>(1, 2);
    double da = atan2(T.at<double>(1, 0), T.at<double>(0, 0));

    prev_to_cur_transform.push_back(TransformParam(dx, dy, da));
    cout << "i: " << i << " " << "dx: " << dx << " " << "dy: " << dy << " " << "da: " << da << endl;
    cout << endl;

    curGrey.copyTo(prevGrey);
    cout << "Image: " << i << "/" << images.size() << " - good optical flow: " << prevCorner2.size() << endl;
  }

  // Step 2 - Accumulate transformations to get image trajectory
  // Accumulated frame to frame transformation
  double a = 0;
  double x = 0;
  double y = 0;

  vector<Trajectory> trajectory; // trajectory at all frames

  for (int i = 0; i < prev_to_cur_transform.size(); i++) {
    x += prev_to_cur_transform[i].dx;
    y += prev_to_cur_transform[i].dy;
    a += prev_to_cur_transform[i].da;

    trajectory.push_back(Trajectory(x, y, a));
    cout << "Trajectory " << (i+1) << ":" << " " << "x: " << x << " " << "y: " << y << " " << "a: " << a << endl;
  }

  // Step 3 - Smooth out trajectory using an averaging window
  vector<Trajectory> smoothed_trajectory; // trajectory at all frames

  for (int i = 0; i < trajectory.size(); i++) {
    double sum_x = 0;
    double sum_y ...

Hi all,

I hope someone could please help me understand why processing would take so long on an iPhone 6. I'm currently trying to do video stabilization using the following:

+ (void)stabilizeVideoFileAtURL:(NSURL *)inputURL writeToURL:(NSURL *)outputURL {
  dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0), ^{
    String inputFile = *new String(inputURL.path.UTF8String);
    cout << "Input: " << inputFile << endl;

    String outputFile = *new String(outputURL.path.UTF8String);
    cout << "Output: " << outputFile << endl;

    VideoCapture cap(inputFile);
    assert(cap.isOpened());

    Mat cur, cur_grey, cur_orig;
    Mat prev, prev_grey, prev_orig;

    cap >> prev;
    cvtColor(prev, prev_grey, COLOR_BGR2GRAY);

    // Step 1 - Get previous to current frame transformation (dx, dy, da) for all frames
    vector<TransformParam> prev_to_cur_transform;

    int frames = 1;
    int max_frames = cap.get(CV_CAP_PROP_FRAME_COUNT);
    cout << "Max Frames: " << max_frames << endl;

    Mat last_T;

    while (true) {
      cap >> cur;

      if (cur.data == NULL) {
        cout << "Current data is NULL, breaking out...1" << endl;
        break;
      }

      cvtColor(cur, cur_grey, COLOR_BGR2GRAY);

      // Vector from prev to cur
      vector<Point2f> prev_corner, cur_corner;
      vector<Point2f> prev_corner2, cur_corner2;
      vector<uchar> status;
      vector<float> error;

      goodFeaturesToTrack(prev_grey, prev_corner, 200, 0.01, 30);
      calcOpticalFlowPyrLK(prev_grey, cur_grey, prev_corner, cur_corner, status, error);

      // Weed out bad matches
      for (size_t i = 0; i < status.size(); i++) {
        if (status[i]) {
          prev_corner2.push_back(prev_corner[i]);
          cur_corner2.push_back(cur_corner[i]);
        }
      }

      // Translation + Rotation only
      Mat T = estimateRigidTransform(prev_corner, cur_corner, false);

      if (T.data == NULL) {
        cout << "No Transform was found" << endl;
        last_T.copyTo(T);
      }

      T.copyTo(last_T);

      // Decompose T
      double dx = T.at<double>(0,2);
      double dy = T.at<double>(1,2);
      double da = atan2(T.at<double>(1,0), T.at<double>(0,0));

      prev_to_cur_transform.push_back(TransformParam(dx, dy, da));

      cur.copyTo(prev);
      cur_grey.copyTo(prev_grey);

      frames++;
    }

    // Step 2 - Accumulate the transformations to get the image trajectory
    // Accumulated frame to frame transform
    double x = 0;
    double y = 0;
    double a = 0;

    vector<Trajectory> trajectory; // Trajectory at all frames

    for (size_t i = 0; i < prev_to_cur_transform.size(); i++) {
      x += prev_to_cur_transform[i].dx;
      y += prev_to_cur_transform[i].dy;
      a += prev_to_cur_transform[i].da;

      trajectory.push_back(Trajectory(x, y, a));
    }

    // Step 3 - Smooth out the trajectory using an averaging window
    vector<Trajectory> smoothed_trajectory; // Trajectory at all frames

    for (size_t i = 0; i < trajectory.size(); i++) {
      double sum_x = 0;
      double sum_y = 0;
      double sum_a = 0;
      int count = 0;

      for (int j = -SMOOTHING_RADIUS; j <= SMOOTHING_RADIUS; j++) {
        if (i + j >= 0 && i + j < trajectory.size()) {
          sum_x += trajectory[i+j].x;
          sum_y += trajectory[i+j].y;
          sum_a += trajectory[i+j].a;

          count++;
        }
      }

      double avg_x = sum_x / count;
      double avg_y = sum_y / count;
      double avg_a = sum_a / count;

      smoothed_trajectory.push_back(Trajectory(avg_x, avg_y, avg_a));
    }

    // Step 4 - Generate new set of previous to current transform, such that the trajectory ends up being the same as the smoothed trajectory
    vector<TransformParam> new_prev_to_cur_transform;

    // Accumulated frame to frame transform
    x = 0;
    y = 0;
    a = 0;

    for (size_t i = 0; i < prev_to_cur_transform.size(); i++) {
      x += prev_to_cur_transform[i].dx;
      y += prev_to_cur_transform[i].dy;
      a += prev_to_cur_transform[i].da;

      // Target - Current
      double diff_x = smoothed_trajectory[i].x - x;
      double diff_y = smoothed_trajectory[i].y - y;
      double diff_a = smoothed_trajectory[i].a - a;

      double dx = prev_to_cur_transform[i].dx + diff_x;
      double dy = prev_to_cur_transform[i].dy + diff_y;
      double da = prev_to_cur_transform[i].da + diff_a;

      new_prev_to_cur_transform ...