Having Trouble Calibrating Stereo Fisheye/Undistorting Image

edit

Hello everyone,

I've been struggling with figuring out how to properly calibrate and undistort a stereo image taken from two identical fisheye cameras. I think the issue has something to do with my program for calibration since when I run undistortImage() with the matrices obtained from the calibration program I get an image that is a small spec in the upper left corner and the rest of the image is black. Here is the code for obtaining the calibration matrices:

#include <iostream>
#include <vector>
#include <string>

#include <opencv2/core.hpp>
#include <opencv2/videoio.hpp>
#include <opencv2/calib3d.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/highgui.hpp>

#define NUM_CAL 1
#define CHESS_ROW 9
#define CHESS_COL 6
#define SQUARE_SIZE_CM 0.023

using namespace std;
using namespace cv;

uint_fast8_t i;
Mat frame0[NUM_CAL], frame1[NUM_CAL];
vector<vector<Point2f>> imagePoints0, imagePoints1;
vector<Point2f> chessCorners0[NUM_CAL], chessCorners1[NUM_CAL];
vector<vector<Point2d>> leftPoints, rightPoints;
vector<vector<Point3d>> objectPoints;
Size chessSize(CHESS_COL, CHESS_ROW);


int main(int argc, char **argv)
{
    // Create matrix representing the chessboard corners
    vector<Point3d> obj;
    for(i = 0; i < CHESS_ROW; ++i )
    {
        for(uint_fast8_t j = 0; j < CHESS_COL; ++j )
        {
            obj.push_back(Point3d(double((float)j * SQUARE_SIZE_CM), double( (float)i * SQUARE_SIZE_CM ), 0));
        }
    }

    // Turn on cameras
    VideoCapture cam0(0), cam1(2);
    cam0.open(0,cv::CAP_V4L2);
    cam1.open(2,cv::CAP_V4L2);
    if(cam0.isOpened() == false) puts("No left camera found.  Please check connection \n");
    if(cam1.isOpened() == false) puts("No right camera found.  Please check connection \n");

    // Get calibration images and store them to RAM
    for(i = 0; i < NUM_CAL; ++i)
    {
        puts("Press enter to take a calibration picture \n");
        cin.get();
        cam0 >> frame0[i];
        cam1 >> frame1[i];
    }

    // Further refine corner points and store in intermediate matrix
    for(i = 0; i < NUM_CAL; ++i)
    {
        bool found1, found2;
        found1 = findChessboardCorners(frame0[i], chessSize, chessCorners0[i], CALIB_CB_ADAPTIVE_THRESH + CALIB_CB_NORMALIZE_IMAGE + CALIB_CB_FILTER_QUADS);
        found2 = findChessboardCorners(frame1[i], chessSize, chessCorners1[i], CALIB_CB_ADAPTIVE_THRESH + CALIB_CB_NORMALIZE_IMAGE + CALIB_CB_FILTER_QUADS);
        if((found1 == false) || (found2 == false)) puts("Not all corners found \n");
        cvtColor(frame0[i], frame0[i], CV_BGR2GRAY);
        cvtColor(frame1[i], frame1[i], CV_BGR2GRAY);
        cornerSubPix(frame0[i], chessCorners0[i], Size(1, 1), Size(-1, -1), TermCriteria(TermCriteria::COUNT+TermCriteria::EPS, 30, 0.01));
        cornerSubPix(frame1[i], chessCorners1[i], Size(1, 1), Size(-1, -1), TermCriteria(TermCriteria::COUNT+TermCriteria::EPS, 30, 0.01));
        drawChessboardCorners(frame0[i], chessSize, chessCorners0[i], found1);
        drawChessboardCorners(frame1[i], chessSize, chessCorners1[i], found2);
        // namedWindow("Image", WINDOW_AUTOSIZE);
        // imshow("Image", obj);
        // waitKey(0);
        imagePoints0.push_back(chessCorners0[i]);
        imagePoints1.push_back(chessCorners1[i]);
        objectPoints.push_back(obj);
    }

    // Finalize creation of the left and right calibration matrices
    // for(i = 0; i < imagePoints1.size(); ++i)
    for(i = 0; i < NUM_CAL; ++i)
    {
        long unsigned int j;
        vector<Point2d> v0, v1;
        for(j = 0; j < imagePoints1[i].size(); ++j)
        {
            v0.push_back(Point2d((double)imagePoints0[i][j].x, (double)imagePoints0[i][j].y));
            v1.push_back(Point2d((double)imagePoints1[i][j].x, (double)imagePoints1[i][j].y));
        }
        leftPoints.push_back(v0);
        rightPoints.push_back(v1);
    }

    // Begin actual calibration!
    printf("Calculating... \n");
    // Size newSize(0,0);
    // Matx33d K1, K2, R;
    // Vec3d T;
    // Vec4d D1, D2;
    Mat K1, K2, R, T, D1, D2 ...