# Lucas Kanade Optical Flow Tracking Problem

I have been trying to do some homography estimation between different frames in a video using Lucas Kanade Optical Flow Tracking (yes, I have already taken a look at the opencv sample). I have written up some code and tested it to see if I could start out by just tracking points in some videos I took. In every video, the points start out fine, and are tracked well for a few frames. Then, all of a sudden, the following happens:

This happens about 10 frames in after the points seem to be tracked just fine. Similar results occur in all of the other videos I have tested. Why is this happening and how can I fix it?

Update #1

Here is a code snippet that may help in solving the issue (ignore the formatting errors that occurred while posting):

def findNthFeatures(prevImg, prevPnts, nxtImg):

nxtDescriptors = []
prevGrey = None
nxtGrey = None
nxtPnts = prevPnts[:]

prevGrey = cv2.cvtColor(prevImg, cv2.COLOR_BGR2GRAY)
nxtGrey = cv2.cvtColor(nxtImg, cv2.COLOR_BGR2GRAY)

lucasKanadeParams = dict( winSize = (19,19), maxLevel = 10, criteria = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10, 0.03))

goodNew = nxtPnts[status==1]
return goodNew


def stitchRow(videoName): color = np.random.randint(0,255,(100,3))
lastFrame = None currentFrame = None lastKeypoints = None currentKeypoints = None lastDescriptors = None currentDescriptors = None firstImage = True

feature_params = dict( maxCorners = 100,
qualityLevel = 0.1,
minDistance = 8,
blockSize = 15)

frameCount = 0

Homographies = []

cv2.namedWindow('display', cv2.WINDOW_NORMAL)
cap = cv2.VideoCapture(videoName)

while flags:
if firstImage:
firstImage = False
lastFrame = frame[:,:].copy()
lastGray = cv2.cvtColor(lastFrame, cv2.COLOR_BGR2GRAY)
lastKeypoints = cv2.goodFeaturesToTrack(lastGray, mask = None, **feature_params)
frameCount += 1
else:
currentFrame = frame[:,:].copy()
frameCount += 1
#if(frameCount % 3 == 0):

cv2.imshow('display', currentFrame)
currentKeypoints = findNthFeatures(lastFrame, lastKeypoints, currentFrame)
#for i,(new,old) in enumerate(zip(currentKeypoints, lastKeypoints)):
#   a, b = new.ravel()
#  c, d = old.ravel()
# frame = cv2.circle(frame, (a,b), 5, color[i].tolist(), -1)

cv2.imshow('display', img)
cv2.waitKey(0)
for i in range(0, len(lastKeypoints)):
lastKeypoints[i] = tuple(lastKeypoints[i])
print lastKeypoints[i]
cv2.waitKey(0)
homographyMatrix = cv2.findHomography(lastKeypoints, currentKeypoints)
Homographies.append(homographyMatrix)
lastFrame = currentFrame
lastDescriptors = currentDescriptors
lastKeypoints = currentKeypoints