Using fisheye correction program works over webcam, but not rtsp stream?

edit

So, I'm trying to make a program using openCV to make a fisheye camera have a regular image in real time from an rtsp stream, I started with this code: http://aishack.in/tutorials/calibrati... and am now trying to adapt it so that it will work with an IP camera that I get the video from using rtsp. It works with a webcam the way it says it's supposed to in the link, however, when I put in an rtsp stream, It seems to just freeze on the first frame and not be taking video at all.

// ConsoleApplication2.cpp : Defines the entry point for the console application.

#include <opencv2\videoio.hpp>
#include <opencv2\highgui.hpp>
#include <opencv\cv.hpp>
#include <opencv\cv.h>
#include <iostream>
#include <stdio.h>
#include <chrono>
#include <thread>


using namespace cv;
using namespace std;

int main (){
int numBoards = 0;
int numCornersHor;
int numCornersVer;

printf("Enter number of corners along width: ");
scanf_s("%d", &numCornersHor);

printf("Enter number of corners along height: ");
scanf_s("%d", &numCornersVer);

printf("Enter number of boards: ");
scanf_s("%d", &numBoards);

int numSquares = numCornersHor * numCornersVer;
Size board_sz = Size(numCornersHor, numCornersVer);

VideoCapture capture = VideoCapture("rtsp://172.16.127.27:554/mpeg4"); //Works just fine if value is set to 0 which grabs webcam by default.

vector<vector<Point3f>> object_points;
vector<vector<Point2f>> image_points;

vector<Point2f> corners;
int successes = 0;

Mat image;
Mat gray_image;
capture >> image;

vector<Point3f> obj;
for (int j = 0; j < numSquares; j++)
    obj.push_back(Point3f(j / numCornersHor, j%numCornersHor, 0.0f));

while (successes < numBoards) {
    this_thread::sleep_for(chrono::milliseconds(100)); // Tried this to alleviate it, but it did nothing, still happens with or without this.
    cvtColor(image, gray_image, CV_BGR2GRAY);

    bool found = findChessboardCorners(image, board_sz, corners, CV_CALIB_CB_ADAPTIVE_THRESH | CV_CALIB_CB_FILTER_QUADS);

    if (found) {
        cornerSubPix(gray_image, corners, Size(11,11), Size(-1, -1), TermCriteria(CV_TERMCRIT_EPS | CV_TERMCRIT_ITER, 30, 0.1));
        drawChessboardCorners(gray_image, board_sz, corners, found);
    }

    imshow("win1", image);
    imshow("win2", gray_image);

    capture >> image;
    int key = waitKey(1);

    if (key == 27)

        return 0;

    if (key == ' ' && found != 0){
        image_points.push_back(corners);
        object_points.push_back(obj);

        printf("Snap stored!");

        successes++;

        if (successes >= numBoards)
            break;
    }
}

Mat intrinsic = Mat(3, 3, CV_32FC1);
Mat distCoeffs;
vector<Mat> rvecs;
vector<Mat> tvecs;

intrinsic.ptr<float>(0)[0] = 1;
intrinsic.ptr<float>(1)[1] = 1;

calibrateCamera(object_points, image_points, image.size(), intrinsic, distCoeffs, rvecs, tvecs);

Mat imageUndistorted;
while (1) {
    capture >> image;
    undistort(image, imageUndistorted, intrinsic, distCoeffs);

    imshow("win1", image);
    imshow("win2", imageUndistorted);
    waitKey(1);
}

capture.release();

return 0;
}

I'm still rather new to OpenCV, so I'm open to any and all suggestions and I'm still trying to fully learn what is going on here, eventually, I want this to translate into just having it take in the video and then spitting it back out modified correctly without the fisheye effect, just having to take this one step at a time since this is proving to be incredibly difficult which is odd because there are cameras that do it by default in their firmware, but those cameras are not always an option for who I'm working for.