Something is wrong with the calibration

edit

I am trying to stereo calibrate the camera pair .

I created a set of stereo images using the chessboard image on my home TV. The images can be found at https://drive.google.com/drive/folder...

The code for detecting the intrinsic and extrinsic parameters looks as follows:

def calibrate_stereo_pair(image_folder : str,
                      coefficients_folder: str,
                      dist_coefficient_pattern = 'dist_c%s.npy',
                      camera_matrix_pattern = 'cammat_%s.npy',
                      win_names : List[str] = ['left', 'right'],
                      nrows = 9,
                      ncols = 6) -> bool:

    N = len(win_names)
    if N != 2:
        raise NotImplementedError("")


    criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)
    objp = np.zeros((nrows * ncols, 3), np.float32)
    objp[:, :2] = np.mgrid[0:nrows, 0:ncols].T.reshape(-1, 2)

    objpoints = []  # 3d point in real world space
    imgpoints = []
    for i in range(N):
        imgpoints.append([])
    index = 1
    while True:
        filenames = [os.path.join(image_folder , win_names[i] + str(index) + '.png') for i in range(N)]
        if not all(os.path.exists(fname) for fname in filenames):
            if index == 1:
                print('No images detected.')
                return False
            else:
                break

        frames = [cv2.imread(fname, 0) for fname in filenames]
        found = [cv2.findChessboardCorners(gray, (nrows, ncols), None) for gray in frames]
        if not all(q[0] for q in found):
            print ('WARNING: Failed to detect chessboard cornesr in the {0}th set. Skipping...'.format(index))
            continue

        objpoints.append(objp)
        for i in range(N):
            cv2.cornerSubPix(frames[i], found[i][1], (11, 11), (-1, -1), criteria)
            imgpoints[i].append(found[i][1])
        index += 1

    if len(objpoints) == 0:
        print('No images found')
        return False

    retval1, cammat1, K1, L11, L12 = cv2.calibrateCamera(objpoints, imgpoints[0], frames[0].shape[::-1], None, None)
    retval2, cammat2, K2, L21, L22 = cv2.calibrateCamera(objpoints, imgpoints[1], frames[1].shape[::-1], None, None)

    #TODO: resize images to the minimal w, length
    retval, cameraMatrix1, distCoeffs1, cameraMatrix2, distCoeffs2, R, T, E, F = cv2.stereoCalibrate(objpoints,imgpoints[0], imgpoints[1],cammat1 , K1, cammat2, K2, frames[0].shape[::-1], flags = cv2.CALIB_FIX_FOCAL_LENGTH  | cv2.CALIB_FIX_S1_S2_S3_S4 | cv2.CALIB_FIX_K1 | cv2.CALIB_FIX_K2 | cv2.CALIB_FIX_K3 | cv2.CALIB_FIX_K4 | cv2.CALIB_FIX_K5 | cv2.CALIB_FIX_PRINCIPAL_POINT | cv2.CALIB_FIX_INTRINSIC  | cv2.CALIB_RATIONAL_MODEL, criteria=criteria)


    np.save(os.path.join(coefficients_folder, dist_coefficient_pattern % win_names[0]), distCoeffs1)
    np.save(os.path.join(coefficients_folder, dist_coefficient_pattern % win_names[1]), distCoeffs2)

    np.save(os.path.join(coefficients_folder, camera_matrix_pattern % win_names[0]), cameraMatrix1)
    np.save(os.path.join(coefficients_folder, camera_matrix_pattern % win_names[1]), cameraMatrix2)

    np.save(os.path.join(coefficients_folder, 'R.npy'), R)
    np.save(os.path.join(coefficients_folder, 'T.npy'), T)
    np.save(os.path.join(coefficients_folder, 'E.npy'), E)
    np.save(os.path.join(coefficients_folder, 'F.npy'), F)
    return True

Initially I used the function stereoCalibrate without separately calibrating each camera, but experiments show that this approach produces slightly better results.
Next I used the following code for further rectification and block matching:

cv2.namedWindow('left', cv2.WINDOW_NORMAL)
cv2.namedWindow('right', cv2.WINDOW_NORMAL)

#uses np.load for loading the coefficients saved in the previous step
cam_mat1, cam_mat2, dist1, dist2, R, T, E, F = camera_calibration.load_stereo_pair([1,2],config.coefficient_folder)


caps = [cv2.VideoCapture(i) for i in [1,2]]
frames = [ cv2.cvtColor(cap.read()[1],cv2.COLOR_BGR2GRAY) for ...