Head pose estimation fails with specific image sizes
I want to find angles of rotation of the head using opencv
and dlib
. So, I tried to use this code from the tutorial:
cv::Mat im = imread("img.jpg");
matrix<bgr_pixel> dlibImage;
assign_image(dlibImage, cv_image<bgr_pixel>(im));
auto face = detector(dlibImage)[0];
auto shape = sp(dlibImage, face);
// 2D image points.
std::vector<cv::Point2d> image_points;
image_points.push_back(cv::Point2d(shape.part(30).x(), shape.part(30).y())); // Nose tip
image_points.push_back(cv::Point2d(shape.part(8).x(), shape.part(8).y())); // Chin
image_points.push_back(cv::Point2d(shape.part(36).x(), shape.part(36).y())); // Left eye left corner
image_points.push_back(cv::Point2d(shape.part(45).x(), shape.part(45).y())); // Right eye right corner
image_points.push_back(cv::Point2d(shape.part(48).x(), shape.part(48).y())); // Left Mouth corner
image_points.push_back(cv::Point2d(shape.part(54).x(), shape.part(54).y())); // Right mouth corner
// 3D model points.
std::vector<cv::Point3d> model_points;
model_points.push_back(cv::Point3d(0.0f, 0.0f, 0.0f)); // Nose tip
model_points.push_back(cv::Point3d(0.0f, -330.0f, -65.0f)); // Chin
model_points.push_back(cv::Point3d(-225.0f, 170.0f, -135.0f)); // Left eye left corner
model_points.push_back(cv::Point3d(225.0f, 170.0f, -135.0f)); // Right eye right corner
model_points.push_back(cv::Point3d(-150.0f, -150.0f, -125.0f)); // Left Mouth corner
model_points.push_back(cv::Point3d(150.0f, -150.0f, -125.0f)); // Right mouth corner
// Camera internals
double focal_length = im.cols; // Approximate focal length.
Point2d center = cv::Point2d(im.cols/2,im.rows/2);
cv::Mat camera_matrix = (cv::Mat_<double>(3,3) << focal_length, 0, center.x, 0 , focal_length, center.y, 0, 0, 1);
cv::Mat dist_coeffs = cv::Mat::zeros(4,1,cv::DataType<double>::type); // Assuming no lens distortion
cout << "Camera Matrix " << endl << camera_matrix << endl ;
// Output rotation and translation
cv::Mat rotation_vector; // Rotation in axis-angle form
cv::Mat translation_vector;
// Solve for pose
cv::solvePnP(model_points, image_points, camera_matrix, dist_coeffs, rotation_vector, translation_vector);
// Project a 3D point (0, 0, 1000.0) onto the image plane.
// We use this to draw a line sticking out of the nose
std::vector<Point3d> nose_end_point3D;
std::vector<Point2d> nose_end_point2D;
nose_end_point3D.push_back(Point3d(0,0,1000.0));
projectPoints(nose_end_point3D, rotation_vector, translation_vector, camera_matrix, dist_coeffs, nose_end_point2D);
for(int i=0; i < image_points.size(); i++)
{
circle(im, image_points[i], 3, Scalar(0,0,255), -1);
}
cv::line(im,image_points[0], nose_end_point2D[0], cv::Scalar(255,0,0), 2);
cout << "Rotation Vector " << endl << rotation_vector << endl;
cout << "Translation Vector" << endl << translation_vector << endl;
cout << nose_end_point2D << endl;
// Display image.
cv::imshow("Output", im);
cv::waitKey(0);
But, unfortunately, I get completely different results depending on the size of the same image!
If I use this img.jpg which has size 299x299 px(many sizes are ok, but we take the nearest), then all ok and I get right result:
Output:
Rotation Vector
[-0,04450161828760668;
-2,133664002574712;
-0,2208024002827168]
But if I use this img.jpg which has size 298x298 px, then I get absolutely wrong result:
Output:
Rotation Vector
[-2,999117288644056;
0,0777816930911016;
-0,7573144061217354]
I also understood that it ...
people here will only be able to reproduce it, if you throw out the dlib parts, and replace them with a fixed set of 2d points (like you did with the 3d ones)
@berak of course, please check UPD section of the question.
Are you interested only with rotation angles without detecting face direction, or do you want the side towards the face is directed as well?