Revision history [back]

ArUco marker translation deviation for some video frames

We are using ArUco markers for pose estimation and are encountering an issue.

The detected pose translation will sometimes be significantly off without any visible disturbance in the source image.

This is a follow-up to an issue that occurs when corner refinement is turned off. http://answers.opencv.org/question/145829/aruco-marker-z-axis-mirrored/ The issue seems to be is less frequent than the one without corner refinement, but it still very desirable to work with the cleanest possible data before any filtering.

Is it possible to fix this issue with ArUco parameters or otherwise?

Below is a test program that takes a file name as argument and two example images to demonstrate the issue. The two images are two consecutive frames of a video and have very little differences.

The program output is:

$ ./arucobug frame104.png 
R: [1.59948, -2.23327, 0.380623]  T: [22.319, -4.29597, 94.5112]
$ ./arucobug frame103.png 
R: [1.5954, -2.22185, 0.386173]  T: [24.8896, -4.84338, 105.443]

So the marker is detected about 10cm (intrinsics are in cm) further away than it should be.

#include "opencv2/opencv.hpp"
#include "opencv2/aruco.hpp"


int main(int argc, char *argv[])
{
    if (argc != 2)
        return 1;


    cv::Mat frame = cv::imread(argv[1]);
    cv::Mat debugFrame = frame.clone();

    // Intrinsic camera parameters.
    cv::Mat camMatrix = cv::Mat::eye(3, 3, CV_64F);
    cv::Mat distortionCoeffs = cv::Mat::zeros(8, 1, CV_64F);
    camMatrix.at<double>(0, 0) = 2.3396381685789738e+03;
    camMatrix.at<double>(0, 2) = 960.;
    camMatrix.at<double>(1, 1) = 2.3396381685789738e+03;
    camMatrix.at<double>(1, 2) = 540.;
    distortionCoeffs.at<double>(0, 0) = -1.0982746232841779e-01;
    distortionCoeffs.at<double>(1, 0) = 2.2689585715220828e-01;
    distortionCoeffs.at<double>(2, 0) = 0.;
    distortionCoeffs.at<double>(3, 0) = 0.;
    distortionCoeffs.at<double>(4, 0) = -2.2112148171171589e-01;

    cv::Ptr<cv::aruco::Dictionary> dict = cv::aruco::getPredefinedDictionary(cv::aruco::DICT_4X4_50);
    cv::Ptr<cv::aruco::DetectorParameters> params = cv::aruco::DetectorParameters::create();
    std::vector<int> markerIds;
    std::vector<std::vector<cv::Point2f>> markerCorners;
    std::vector<cv::Vec3d> rotationVec, translationVec;


    params->cornerRefinementMethod = cv::aruco::CORNER_REFINE_CONTOUR;
    cv::aruco::detectMarkers(frame, dict, markerCorners, markerIds, params);
    auto numMarkers = markerIds.size();

    if (numMarkers > 0)
    {
        cv::aruco::drawDetectedMarkers(debugFrame, markerCorners, markerIds);

        cv::aruco::estimatePoseSingleMarkers(markerCorners, 5.0, camMatrix, distortionCoeffs, rotationVec, translationVec);

        for (int i = 0; i < numMarkers; i++)
        {
            // Official aruco axis drawing.
            cv::aruco::drawAxis(debugFrame, camMatrix, distortionCoeffs, rotationVec[i], translationVec[i], 0.5 * 5.0);

            std::cout << "R: " << rotationVec[i] << "  T: " << translationVec[i] << std::endl;


            // Project marker origin.
            std::vector< cv::Point3f > axisPoints;
            axisPoints.push_back(cv::Point3f(0, 0, 0));
            std::vector< cv::Point2f > imagePoints;
            cv::projectPoints(axisPoints, rotationVec[i], translationVec[i], camMatrix, distortionCoeffs, imagePoints);

            // Convert rotation vector to rotation matrix.
            cv::Mat rotMat;
            cv::Rodrigues(rotationVec[i], rotMat);

            // Check for proper rotation matrix.
            assert(cv::determinant(rotMat) > 0.99 && cv::determinant(rotMat) < 1.01);

            // Manually rotate the z-axis vector.
            cv::Mat testVec(cv::Point3f(0, 0, 10.0f));
            testVec.convertTo(testVec, CV_64FC1);
            cv::Mat testTransformed = rotMat * testVec;

            // Project the rotated z-axis vector.
            std::vector<cv::Point3f> pts;
            pts.push_back(cv::Point3f(testTransformed));
            std::vector<cv::Point2f> pts2d;
            cv::projectPoints(pts, cv::Mat(cv::Point3f(0, 0, 0)), translationVec[i], camMatrix, distortionCoeffs, pts2d);

            // Draw the projected z-axis.
            cv::line(debugFrame, imagePoints[0], pts2d[0], 4);


            // Project cube.
            float length = 10.0f;
            std::vector<cv::Point3f> testObj3d;
            testObj3d.push_back(cv::Point3f(0, 0, 0));
            testObj3d.push_back(cv::Point3f(length, 0, 0));
            testObj3d.push_back(cv::Point3f(0, length, 0));
            testObj3d.push_back(cv::Point3f(length, length, 0));
            testObj3d.push_back(cv::Point3f(0, 0, length));
            testObj3d.push_back(cv::Point3f(length, 0, length));
            testObj3d.push_back(cv::Point3f(0, length, length));
            testObj3d.push_back(cv::Point3f(length, length, length));
            std::vector<cv::Point2f> testObj2d;
            cv::projectPoints(testObj3d, rotationVec[i], translationVec[i], camMatrix, distortionCoeffs, testObj2d);

            cv::line(debugFrame, testObj2d[0], testObj2d[1], 4);
            cv::line(debugFrame, testObj2d[0], testObj2d[2], 4);
            cv::line(debugFrame, testObj2d[1], testObj2d[3], 4);
            cv::line(debugFrame, testObj2d[2], testObj2d[3], 4);
            cv::line(debugFrame, testObj2d[0], testObj2d[4], 4);
            cv::line(debugFrame, testObj2d[1], testObj2d[5], 4);
            cv::line(debugFrame, testObj2d[2], testObj2d[6], 4);
            cv::line(debugFrame, testObj2d[3], testObj2d[7], 4);
            cv::line(debugFrame, testObj2d[4], testObj2d[5], 4);
            cv::line(debugFrame, testObj2d[4], testObj2d[6], 4);
            cv::line(debugFrame, testObj2d[5], testObj2d[7], 4);
            cv::line(debugFrame, testObj2d[6], testObj2d[7], 4);
        }
    }

    cv::imshow("debug", debugFrame);
    cv::waitKey(0);
}

Source image with marker working:

good_src

Source image with marker flipped:

smaller_src

Debug image marker working:

good_dbg

Debug image marker flipped:

smaller_dbg