I am trying to stereo calibrate the camera pair 
.
I created a set of stereo images using the chessboard image on my home TV. The images can be found at https://drive.google.com/drive/folders/19iIDvAjQXD5v6lC7f-SGh6s1yLElQDAV?usp=sharing
The code for detecting the intrinsic and extrinsic parameters looks as follows:
def calibrate_stereo_pair(image_folder : str,
coefficients_folder: str,
dist_coefficient_pattern = 'dist_c%s.npy',
camera_matrix_pattern = 'cammat_%s.npy',
win_names : List[str] = ['left', 'right'],
nrows = 9,
ncols = 6) -> bool:
N = len(win_names)
if N != 2:
raise NotImplementedError("")
criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)
objp = np.zeros((nrows * ncols, 3), np.float32)
objp[:, :2] = np.mgrid[0:nrows, 0:ncols].T.reshape(-1, 2)
objpoints = [] # 3d point in real world space
imgpoints = []
for i in range(N):
imgpoints.append([])
index = 1
while True:
filenames = [os.path.join(image_folder , win_names[i] + str(index) + '.png') for i in range(N)]
if not all(os.path.exists(fname) for fname in filenames):
if index == 1:
print('No images detected.')
return False
else:
break
frames = [cv2.imread(fname, 0) for fname in filenames]
found = [cv2.findChessboardCorners(gray, (nrows, ncols), None) for gray in frames]
if not all(q[0] for q in found):
print ('WARNING: Failed to detect chessboard cornesr in the {0}th set. Skipping...'.format(index))
continue
objpoints.append(objp)
for i in range(N):
cv2.cornerSubPix(frames[i], found[i][1], (11, 11), (-1, -1), criteria)
imgpoints[i].append(found[i][1])
index += 1
if len(objpoints) == 0:
print('No images found')
return False
retval1, cammat1, K1, L11, L12 = cv2.calibrateCamera(objpoints, imgpoints[0], frames[0].shape[::-1], None, None)
retval2, cammat2, K2, L21, L22 = cv2.calibrateCamera(objpoints, imgpoints[1], frames[1].shape[::-1], None, None)
#TODO: resize images to the minimal w, length
retval, cameraMatrix1, distCoeffs1, cameraMatrix2, distCoeffs2, R, T, E, F = cv2.stereoCalibrate(objpoints,imgpoints[0], imgpoints[1],cammat1 , K1, cammat2, K2, frames[0].shape[::-1], flags = cv2.CALIB_FIX_FOCAL_LENGTH | cv2.CALIB_FIX_S1_S2_S3_S4 | cv2.CALIB_FIX_K1 | cv2.CALIB_FIX_K2 | cv2.CALIB_FIX_K3 | cv2.CALIB_FIX_K4 | cv2.CALIB_FIX_K5 | cv2.CALIB_FIX_PRINCIPAL_POINT | cv2.CALIB_FIX_INTRINSIC | cv2.CALIB_RATIONAL_MODEL, criteria=criteria)
np.save(os.path.join(coefficients_folder, dist_coefficient_pattern % win_names[0]), distCoeffs1)
np.save(os.path.join(coefficients_folder, dist_coefficient_pattern % win_names[1]), distCoeffs2)
np.save(os.path.join(coefficients_folder, camera_matrix_pattern % win_names[0]), cameraMatrix1)
np.save(os.path.join(coefficients_folder, camera_matrix_pattern % win_names[1]), cameraMatrix2)
np.save(os.path.join(coefficients_folder, 'R.npy'), R)
np.save(os.path.join(coefficients_folder, 'T.npy'), T)
np.save(os.path.join(coefficients_folder, 'E.npy'), E)
np.save(os.path.join(coefficients_folder, 'F.npy'), F)
return True
Initially I used the function stereoCalibrate without separately calibrating each camera, but experiments show that this approach produces slightly better results.
Next I used the following code for further rectification and block matching:
cv2.namedWindow('left', cv2.WINDOW_NORMAL)
cv2.namedWindow('right', cv2.WINDOW_NORMAL)
#uses np.load for loading the coefficients saved in the previous step
cam_mat1, cam_mat2, dist1, dist2, R, T, E, F = camera_calibration.load_stereo_pair([1,2],config.coefficient_folder)
caps = [cv2.VideoCapture(i) for i in [1,2]]
frames = [ cv2.cvtColor(cap.read()[1],cv2.COLOR_BGR2GRAY) for cap in caps]
[cap.release() for cap in caps]
h, w = frames[0].shape[0:2]
R1, R2, P1,P2, Q, validPixROI1, validPixROI2 = cv2.stereoRectify(cam_mat1, dist1, cam_mat2,dist2,(w,h),R,T, flags=cv2.CALIB_ZERO_DISPARITY)
m11, m12 = cv2.initUndistortRectifyMap(cam_mat1, dist1, R1, P1, (w,h), cv2.CV_32FC1)
m21, m22 = cv2.initUndistortRectifyMap(cam_mat2, dist2, R2,P2, (w,h), cv2.CV_32FC1)
rectIm1 = cv2.remap(frames[0], m11, m12, cv2.INTER_LINEAR)
rectIm2 = cv2.remap(frames[1], m21, m22, cv2.INTER_LINEAR)
sbm = cv2.StereoBM_create(numDisparities=16,blockSize=5)
sbm.setROI1(validPixROI1)
sbm.setROI2(validPixROI2)
disp = sbm.compute(rectIm1,rectIm2)
Although I learnt the procedure from the opencv examples, I have been getting strange results. First of all the rectified images look strange. For example while the left image looks like:
the rectified image looks more distorted:
Notice the wall edge in the left part of the image.
Next, I tried to sift match the images and this is the result. I would expect the matching lines to be horizontal, but they have some angles. Is this OK?
Finally, the disparity map (normalized, etc..) looks very noisy. What am I doing wrong?
Thanks in advance.
