Lucas Kanade Optical Flow Tracking Problem

edit

I have been trying to do some homography estimation between different frames in a video using Lucas Kanade Optical Flow Tracking (yes, I have already taken a look at the opencv sample). I have written up some code and tested it to see if I could start out by just tracking points in some videos I took. In every video, the points start out fine, and are tracked well for a few frames. Then, all of a sudden, the following happens:

This happens about 10 frames in after the points seem to be tracked just fine. Similar results occur in all of the other videos I have tested. Why is this happening and how can I fix it?

Update #1

Here is a code snippet that may help in solving the issue (ignore the formatting errors that occurred while posting):

def findNthFeatures(prevImg, prevPnts, nxtImg):

nxtDescriptors = []
prevGrey = None
nxtGrey = None
nxtPnts = prevPnts[:]

prevGrey = cv2.cvtColor(prevImg, cv2.COLOR_BGR2GRAY)
nxtGrey = cv2.cvtColor(nxtImg, cv2.COLOR_BGR2GRAY)

lucasKanadeParams = dict( winSize = (19,19), maxLevel = 10, criteria = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10, 0.03))

nxtPnts, status, err = cv2.calcOpticalFlowPyrLK(prevGrey, nxtGrey, prevPnts, None, **lucasKanadeParams)


goodNew = nxtPnts[status==1]
return goodNew

def stitchRow(videoName): color = np.random.randint(0,255,(100,3))
lastFrame = None currentFrame = None lastKeypoints = None currentKeypoints = None lastDescriptors = None currentDescriptors = None firstImage = True

feature_params = dict( maxCorners = 100,
                        qualityLevel = 0.1,
                        minDistance = 8,
                        blockSize = 15)

frameCount = 0

Homographies = []

cv2.namedWindow('display', cv2.WINDOW_NORMAL) 
cap = cv2.VideoCapture(videoName)
flags, frame = cap.read()

while flags:
if firstImage:                                                
    firstImage = False
        lastFrame = frame[:,:].copy()
        lastGray = cv2.cvtColor(lastFrame, cv2.COLOR_BGR2GRAY)
    lastKeypoints = cv2.goodFeaturesToTrack(lastGray, mask = None, **feature_params)
        flags, frame = cap.read()
        frameCount += 1
    else:
        mask = np.zeros_like(lastFrame)           
        currentFrame = frame[:,:].copy()
        frameCount += 1
        #if(frameCount % 3 == 0):

        cv2.imshow('display', currentFrame)
    currentKeypoints = findNthFeatures(lastFrame, lastKeypoints, currentFrame)
        #for i,(new,old) in enumerate(zip(currentKeypoints, lastKeypoints)):
         #   a, b = new.ravel()
          #  c, d = old.ravel()
           # mask = cv2.line(mask, (a,b), (c,d), color[i].tolist(), 2)
           # frame = cv2.circle(frame, (a,b), 5, color[i].tolist(), -1)
        #img = cv2.add(frame,mask)

        cv2.imshow('display', img)
        cv2.waitKey(0)
        for i in range(0, len(lastKeypoints)):
            lastKeypoints[i] = tuple(lastKeypoints[i])
            print lastKeypoints[i]
            cv2.waitKey(0)
        homographyMatrix = cv2.findHomography(lastKeypoints, currentKeypoints)
    Homographies.append(homographyMatrix)   
    lastFrame = currentFrame
    lastDescriptors = currentDescriptors
    lastKeypoints = currentKeypoints

        flags, frame = cap.read()