Revision history [back]

Inspired by the Mathworks Tutorial for Augmented Reality I wanted to create a similar Application for Android, where the Recognition and Tracking are implemented.

After some research I've seen, that the mentioned pointTracker from Mathworks uses the KLT-Algorithm which is also implemented in OpenCV with calcOpticalFlowPyrLK I've implemented the algorithm, which takes the last frame, where I recognized my Points and try to estimate their new position in the current frame with this method:

int BruteForceMatcher::trackWithOpticalFlow(cv::Mat prevImg, cv::Mat nextImg, std::vector<cv::Point2f> &srcPoints, std::vector<cv::Point2f> &srcCorners){

std::vector<cv::Point2f> estPoints;
std::vector<cv::Point2f> estCorners;
std::vector<cv::Point2f> goodPoints;
std::vector<cv::Point2f> leftsrc;
std::vector<uchar> status;
std::vector<float> error;

if(srcPoints.size() > 0) {

    cv::calcOpticalFlowPyrLK(prevImg, nextImg, srcPoints, estPoints, status, error);

    for (int i = 0; i < estPoints.size(); i++) {
        if (error[i] < 20.f) {
            //LOGW("ERROR : %f\n", error[i]);
            goodPoints.push_back(estPoints[i]);
            leftsrc.push_back(srcPoints[i]);
        }
    }

    //LOGD("Left Points (est/src): %i, %i", goodPoints.size(), leftsrc.size());

    if(goodPoints.size() <= 0){
        //LOGD("No good Points calculated");
        return 0;
    }
    cv::Mat f = cv::findHomography(leftsrc, goodPoints);

    if(cv::countNonZero(f) < 1){
        //LOGD("Homography Matrix is empty!");
        return 0;
    }

    cv::perspectiveTransform(srcCorners, estCorners, f);

    srcCorners.swap(estCorners);
    srcPoints.swap(goodPoints);
    status.clear();
    error.clear();

    return srcPoints.size();
}

return 0;

}

And the Method which will be called through a JNICALL:

std::vector<cv::Point2f> findBruteForceMatches(cv::Mat img){

int matches = 0;
std::vector<cv::Point2f> ransacs;
BruteForceMatcher *bruteForceMatcher = new BruteForceMatcher();
double tf = cv::getTickFrequency();


if(trackKLT){
    LOGD("TRACK WITH KLT");

    double kltTime = (double) cv::getTickCount();

    matches = bruteForceMatcher->trackWithOpticalFlow(prevImg, img, srcPoints, scene_corners);

    kltTime = (double) cv::getTickCount() - kltTime;
    LOGD("KLT Track Time: %f\n", kltTime*1000./tf);

    if(matches > 3){
        trackKLT = true;
        prevImg = img;
        delete bruteForceMatcher;
        return scene_corners;
    }else{

        trackKLT = false;
        prevImg.release();
        srcPoints.clear();
        scene_corners.clear();
        delete bruteForceMatcher;
        return scene_corners;

    }
} else{
    LOGD("RECOGNIZE OBJECT");
    [... FAST/SIFT Recognition! ...]
}
}

Unfortunately this method runs only at 200 ms (~5 Fps) which is to slow for my Application. Is there any other similar algorithm, which could track a couple of points in a Image? Or is there a way, to speed up my algorithm?

In a paper I read, that they use a cross correlation tracking algorithm, is there sth. like that in openCV?

Some specs:

Phone: Nexus 5x (6.0.1)

OpenCV C++ Native Library Vers. (Master-Branch downloaded 26.09.2016)

Android SDK 21 - 24

Inspired by the Mathworks Tutorial for Augmented Reality I wanted to create a similar Application for Android, where the Recognition and Tracking are implemented.

After some research I've seen, that the mentioned pointTracker from Mathworks uses the KLT-Algorithm which is also implemented in OpenCV with calcOpticalFlowPyrLK I've implemented the algorithm, which takes the last frame, where I recognized my Points and try to estimate their new position in the current frame with this method:

int BruteForceMatcher::trackWithOpticalFlow(cv::Mat prevImg, cv::Mat nextImg, std::vector<cv::Point2f> &srcPoints, std::vector<cv::Point2f> &srcCorners){

std::vector<cv::Point2f> estPoints;
std::vector<cv::Point2f> estCorners;
std::vector<cv::Point2f> goodPoints;
std::vector<cv::Point2f> leftsrc;
std::vector<uchar> status;
std::vector<float> error;

if(srcPoints.size() > 0) {

    cv::Mat smallerPrev;
    cv::Mat smallerNext;

    cv::resize(prevImg, smallerPrev, cv::Size(prevImg.cols/2, prevImg.rows/2));
    cv::resize(nextImg, smallerNext, cv::Size(nextImg.cols/2, nextImg.rows/2));

    for(int i = 0; i < srcPoints.size(); i++)
        srcPoints[i] *= 0.5f;

    cv::calcOpticalFlowPyrLK(prevImg, nextImg, srcPoints, estPoints, status, error);

    for (int i = 0; i < estPoints.size(); i++) {
        if (error[i] < 20.f) {
            //LOGW("ERROR : %f\n", error[i]);
            goodPoints.push_back(estPoints[i]);
            leftsrc.push_back(srcPoints[i]);
goodPoints.push_back(estPoints[i] *= 2);
            leftsrc.push_back(srcPoints[i] *= 2);
        }
    }

    //LOGD("Left Points (est/src): %i, %i", goodPoints.size(), leftsrc.size());

    if(goodPoints.size() <= 0){
        //LOGD("No good Points calculated");
        return 0;
    }
    cv::Mat f = cv::findHomography(leftsrc, goodPoints);

    if(cv::countNonZero(f) < 1){
        //LOGD("Homography Matrix is empty!");
        return 0;
    }

    cv::perspectiveTransform(srcCorners, estCorners, f);

    srcCorners.swap(estCorners);
    srcPoints.swap(goodPoints);
    status.clear();
    error.clear();

    return srcPoints.size();
}

return 0;

}

And the Method which will be called through a JNICALL:

std::vector<cv::Point2f> findBruteForceMatches(cv::Mat img){

int matches = 0;
std::vector<cv::Point2f> ransacs;
BruteForceMatcher *bruteForceMatcher = new BruteForceMatcher();
double tf = cv::getTickFrequency();


if(trackKLT){
    LOGD("TRACK WITH KLT");

    double kltTime = (double) cv::getTickCount();

    matches = bruteForceMatcher->trackWithOpticalFlow(prevImg, img, srcPoints, scene_corners);

    kltTime = (double) cv::getTickCount() - kltTime;
    LOGD("KLT Track Time: %f\n", kltTime*1000./tf);

    if(matches > 3){
        trackKLT = true;
        prevImg = img;
        delete bruteForceMatcher;
        return scene_corners;
    }else{

        trackKLT = false;
        prevImg.release();
        srcPoints.clear();
        scene_corners.clear();
        delete bruteForceMatcher;
        return scene_corners;

    }
} else{
    LOGD("RECOGNIZE OBJECT");
    [... FAST/SIFT Recognition! ...]
}
}

Unfortunately this method runs only at 200 ms (~5 Fps) which is to slow for my Application. Is there any other similar algorithm, which could track a couple of points in a Image? Or is there a way, to speed up my algorithm?

In a paper I read, that they use a cross correlation tracking algorithm, is there sth. like that in openCV?

Some specs:

Phone: Nexus 5x (6.0.1)

OpenCV C++ Native Library Vers. (Master-Branch downloaded 26.09.2016)

Android SDK 21 - 24

EDIT: Added Beraks suggestion of downscale/upscale.

Inspired by the Mathworks Tutorial for Augmented Reality I wanted to create a similar Application for Android, where the Recognition and Tracking are implemented.

After some research I've seen, that the mentioned pointTracker from Mathworks uses the KLT-Algorithm which is also implemented in OpenCV with calcOpticalFlowPyrLK I've implemented the algorithm, which takes the last frame, where I recognized my Points and try to estimate their new position in the current frame with this method:

int BruteForceMatcher::trackWithOpticalFlow(cv::Mat prevImg, cv::Mat nextImg, std::vector<cv::Point2f> &srcPoints, std::vector<cv::Point2f> &srcCorners){

std::vector<cv::Point2f> estPoints;
std::vector<cv::Point2f> estCorners;
std::vector<cv::Point2f> goodPoints;
std::vector<cv::Point2f> leftsrc;
std::vector<uchar> status;
std::vector<float> error;

if(srcPoints.size() > 0) {

    //Added with Edit:
    //downscaling
    cv::Mat smallerPrev;
    cv::Mat smallerNext;

    cv::resize(prevImg, smallerPrev, cv::Size(prevImg.cols/2, prevImg.rows/2));
    cv::resize(nextImg, smallerNext, cv::Size(nextImg.cols/2, nextImg.rows/2));

    for(int i = 0; i < srcPoints.size(); i++)
        srcPoints[i] *= 0.5f;

    cv::calcOpticalFlowPyrLK(prevImg, nextImg, srcPoints, estPoints, status, error);

    for (int i = 0; i < estPoints.size(); i++) {
        if (error[i] < 20.f) {
            //LOGW("ERROR : %f\n", error[i]);
            //upscaling of the Points
            goodPoints.push_back(estPoints[i] *= 2);
            leftsrc.push_back(srcPoints[i] *= 2);
        }
    }

    //LOGD("Left Points (est/src): %i, %i", goodPoints.size(), leftsrc.size());

    if(goodPoints.size() <= 0){
        //LOGD("No good Points calculated");
        return 0;
    }
    cv::Mat f = cv::findHomography(leftsrc, goodPoints);

    if(cv::countNonZero(f) < 1){
        //LOGD("Homography Matrix is empty!");
        return 0;
    }

    cv::perspectiveTransform(srcCorners, estCorners, f);

    srcCorners.swap(estCorners);
    srcPoints.swap(goodPoints);
    status.clear();
    error.clear();

    return srcPoints.size();
}

return 0;

}

And the Method which will be called through a JNICALL:

std::vector<cv::Point2f> findBruteForceMatches(cv::Mat img){

int matches = 0;
std::vector<cv::Point2f> ransacs;
BruteForceMatcher *bruteForceMatcher = new BruteForceMatcher();
double tf = cv::getTickFrequency();


if(trackKLT){
    LOGD("TRACK WITH KLT");

    double kltTime = (double) cv::getTickCount();

    matches = bruteForceMatcher->trackWithOpticalFlow(prevImg, img, srcPoints, scene_corners);

    kltTime = (double) cv::getTickCount() - kltTime;
    LOGD("KLT Track Time: %f\n", kltTime*1000./tf);

    if(matches > 3){
        trackKLT = true;
        prevImg = img;
        delete bruteForceMatcher;
        return scene_corners;
    }else{

        trackKLT = false;
        prevImg.release();
        srcPoints.clear();
        scene_corners.clear();
        delete bruteForceMatcher;
        return scene_corners;

    }
} else{
    LOGD("RECOGNIZE OBJECT");
    [... FAST/SIFT Recognition! ...]
}
}

Unfortunately this method runs only at 200 ms (~5 Fps) which is to slow for my Application. Is there any other similar algorithm, which could track a couple of points in a Image? Or is there a way, to speed up my algorithm?

In a paper I read, that they use a cross correlation tracking algorithm, is there sth. like that in openCV?

Some specs:

Phone: Nexus 5x (6.0.1)

OpenCV C++ Native Library Vers. (Master-Branch downloaded 26.09.2016)

Android SDK 21 - 24

EDIT: Added Beraks suggestion of downscale/upscale.

Inspired by the Mathworks Tutorial for Augmented Reality I wanted to create a similar Application for Android, where the Recognition and Tracking are implemented.

After some research I've seen, that the mentioned pointTracker from Mathworks uses the KLT-Algorithm which is also implemented in OpenCV with calcOpticalFlowPyrLK I've implemented the algorithm, which takes the last frame, where I recognized my Points and try to estimate their new position in the current frame with this method:

int BruteForceMatcher::trackWithOpticalFlow(cv::Mat prevImg, cv::Mat nextImg, std::vector<cv::Point2f> &srcPoints, std::vector<cv::Point2f> &srcCorners){

std::vector<cv::Point2f> estPoints;
std::vector<cv::Point2f> estCorners;
std::vector<cv::Point2f> goodPoints;
std::vector<cv::Point2f> leftsrc;
std::vector<uchar> status;
std::vector<float> error;

if(srcPoints.size() > 0) {

    //Added with Edit:
    //downscaling
    cv::Mat smallerPrev;
    cv::Mat smallerNext;

    cv::resize(prevImg, smallerPrev, cv::Size(prevImg.cols/2, prevImg.rows/2));
    cv::resize(nextImg, smallerNext, cv::Size(nextImg.cols/2, nextImg.rows/2));

    for(int i = 0; i < srcPoints.size(); i++)
        srcPoints[i] *= 0.5f;

    cv::calcOpticalFlowPyrLK(prevImg, nextImg, srcPoints, estPoints, status, error);

    for (int i = 0; i < estPoints.size(); i++) {
        if (error[i] < 20.f) {
            //LOGW("ERROR : %f\n", error[i]);
            //upscaling of the Points
            goodPoints.push_back(estPoints[i] *= 2);
            leftsrc.push_back(srcPoints[i] *= 2);
        }
    }

    //LOGD("Left Points (est/src): %i, %i", goodPoints.size(), leftsrc.size());

    if(goodPoints.size() <= 0){
        //LOGD("No good Points calculated");
        return 0;
    }
    cv::Mat f = cv::findHomography(leftsrc, goodPoints);

    if(cv::countNonZero(f) < 1){
        //LOGD("Homography Matrix is empty!");
        return 0;
    }

    cv::perspectiveTransform(srcCorners, estCorners, f);

    srcCorners.swap(estCorners);
    srcPoints.swap(goodPoints);
    status.clear();
    error.clear();

    return srcPoints.size();
}

return 0;

}

And the Method which will be called through a JNICALL:

std::vector<cv::Point2f> findBruteForceMatches(cv::Mat img){

int matches = 0;
std::vector<cv::Point2f> ransacs;
BruteForceMatcher *bruteForceMatcher = new BruteForceMatcher();
double tf = cv::getTickFrequency();


if(trackKLT){
    LOGD("TRACK WITH KLT");

    double kltTime = (double) cv::getTickCount();

    matches = bruteForceMatcher->trackWithOpticalFlow(prevImg, img, srcPoints, scene_corners);

    kltTime = (double) cv::getTickCount() - kltTime;
    LOGD("KLT Track Time: %f\n", kltTime*1000./tf);

    if(matches > 3){
        trackKLT = true;
        prevImg = img;
        delete bruteForceMatcher;
        return scene_corners;
    }else{

        trackKLT = false;
        prevImg.release();
        srcPoints.clear();
        scene_corners.clear();
        delete bruteForceMatcher;
        return scene_corners;

    }
} else{
    LOGD("RECOGNIZE OBJECT");
    [... FAST/SIFT Recognition! ...]
double bfMatchTime = (double) cv::getTickCount();

    matches = bruteForceMatcher->findMatchesBF(img, features2d, descriptors, scene_corners, ransacs);

    bfMatchTime = (double) cv::getTickCount() - bfMatchTime;
    LOGD("BruteForceMatch Time: %f\n", bfMatchTime*1000./tf);
    if(matches > 3){

        trackKLT = true;
        prevImg = img;
        srcPoints = ransacs;
        delete bruteForceMatcher;
        return scene_corners;

    }else{

        scene_corners.clear();
        ransacs.clear();
        delete bruteForceMatcher;
        return scene_corners;

    }
}
}

Unfortunately this method runs only at 200 ms (~5 Fps) which is to slow for my Application. Is there any other similar algorithm, which could track a couple of points in a Image? Or is there a way, to speed up my algorithm?

In a paper I read, that they use a cross correlation tracking algorithm, is there sth. like that in openCV?

Some specs:

Phone: Nexus 5x (6.0.1)

OpenCV C++ Native Library Vers. (Master-Branch downloaded 26.09.2016)

Android SDK 21 - 24

EDIT: Added Beraks suggestion of downscale/upscale.downscale/upscale. EDIT: Added additional Changes, Inspired by Tetragram.

Inspired by the Mathworks Tutorial for Augmented Reality I wanted to create a similar Application for Android, where the Recognition and Tracking are implemented.

After some research I've seen, that the mentioned pointTracker from Mathworks uses the KLT-Algorithm which is also implemented in OpenCV with calcOpticalFlowPyrLK I've implemented the algorithm, which takes the last frame, where I recognized my Points and try to estimate their new position in the current frame with this method:

int BruteForceMatcher::trackWithOpticalFlow(cv::Mat prevImg, cv::Mat nextImg, BruteForceMatcher::trackWithOpticalFlow(std::vector<cv::Mat> prevPyr, std::vector<cv::Mat> nextPyr, std::vector<cv::Point2f> &srcPoints, std::vector<cv::Point2f> &srcCorners){

std::vector<cv::Point2f> estPoints;
std::vector<cv::Point2f> estCorners;
std::vector<cv::Point2f> goodPoints;
std::vector<cv::Point2f> leftsrc;
std::vector<uchar> status;
std::vector<float> error;

if(srcPoints.size() > 0) {

    //Added with Edit:
    //downscaling
    cv::Mat smallerPrev;
    cv::Mat smallerNext;

    cv::resize(prevImg, smallerPrev, cv::Size(prevImg.cols/2, prevImg.rows/2));
    cv::resize(nextImg, smallerNext, cv::Size(nextImg.cols/2, nextImg.rows/2));

    for(int i = 0; i < srcPoints.size(); i++)
        srcPoints[i] *= 0.5f;

    cv::calcOpticalFlowPyrLK(prevImg, nextImg, cv::calcOpticalFlowPyrLK(prevPyr, nextPyr, srcPoints, estPoints, status, error);

    for (int i = 0; i < estPoints.size(); i++) {
        if (error[i] < 20.f) {
            //LOGW("ERROR : %f\n", error[i]);
            //upscaling of the Points
            goodPoints.push_back(estPoints[i] *= 2);
4);
            leftsrc.push_back(srcPoints[i] *= 2);
4);
        }
    }

    //LOGD("Left Points (est/src): %i, %i", goodPoints.size(), leftsrc.size());

    if(goodPoints.size() <= 0){
        //LOGD("No good Points calculated");
        return 0;
    }
    cv::Mat f = cv::findHomography(leftsrc, goodPoints);

    if(cv::countNonZero(f) < 1){
        //LOGD("Homography Matrix is empty!");
        return 0;
    }

    cv::perspectiveTransform(srcCorners, estCorners, f);

    srcCorners.swap(estCorners);
    srcPoints.swap(goodPoints);
    status.clear();
    error.clear();

    return srcPoints.size();
}

return 0;

}

And the Method which will be called through a JNICALL:

std::vector<cv::Point2f> findBruteForceMatches(cv::Mat img){

int matches = 0;
std::vector<cv::Point2f> ransacs;
BruteForceMatcher *bruteForceMatcher = new BruteForceMatcher();
double tf = cv::getTickFrequency();


if(trackKLT){
    LOGD("TRACK WITH KLT");

    std::vector<cv::Mat> nextPyr;
    cv::resize(img, img, cv::Size(img.cols/4, img.rows/4));
    cv::buildOpticalFlowPyramid(img, nextPyr, cv::Size(8,8), 3);        

    double kltTime = (double) cv::getTickCount();

    matches = bruteForceMatcher->trackWithOpticalFlow(prevImg, img, bruteForceMatcher->trackWithOpticalFlow(prevPyr, nextPyr, srcPoints, scene_corners);

    kltTime = (double) cv::getTickCount() - kltTime;
    LOGD("KLT Track Time: %f\n", kltTime*1000./tf);

    if(matches > 3){
10){
        trackKLT = true;
        prevImg = img;
prevPyr.swap(currPyr);
        delete bruteForceMatcher;
        return scene_corners;
    }else{

        trackKLT = false;
        prevImg.release();
prevPyr.clear();
        srcPoints.clear();
        scene_corners.clear();
        delete bruteForceMatcher;
        return scene_corners;

    }
} else{
    LOGD("RECOGNIZE OBJECT");
    double bfMatchTime = (double) cv::getTickCount();

    matches = bruteForceMatcher->findMatchesBF(img, features2d, descriptors, scene_corners, ransacs);

    bfMatchTime = (double) cv::getTickCount() - bfMatchTime;
    LOGD("BruteForceMatch Time: %f\n", bfMatchTime*1000./tf);
    if(matches > 3){

        trackKLT = true;
        prevImg = img;
        srcPoints = ransacs;
cv::resize(img, img, cv::Size(img.cols/4, img.rows/4));
        cv::buildOpticalFlowPyramid(img, prevPyr, cv::Size(8,8), 3);

        for(int i = 0; i < ransacs.size(); i++){
            ransacs[i] *= 0.25;
        }
        srcPoints.swap(ransacs);
        delete bruteForceMatcher;
        return scene_corners;

    }else{
         img.release();
        scene_corners.clear();
        ransacs.clear();
        delete bruteForceMatcher;
        return scene_corners;

    }
}
}

Unfortunately this method runs only at 200 ms (~5 Fps) which is to slow for my Application. Is there any other similar algorithm, which could track a couple of points in a Image? Or is there a way, to speed up my algorithm?

In a paper I read, that they use a cross correlation tracking algorithm, is there sth. like that in openCV?

Some specs:

specs: Phone: Nexus 5x (6.0.1)