This forum is disabled, please visit https://forum.opencv.org
 | 1 | initial version |
i still think, you want a head pose estimation here. the gist is: you can use solvePnP to get an rvec and a tvec, containing rotation / translation wrt. the camera.
for this, you will need 2d landmarks (e.g from dlib, or from opencv's landmark models) and corresponding 3d points. i'll try with FacemarkLBF here:
// create and load the facemarks model
cv::Ptr<cv::face::Facemark> facemark;
facemark = cv::face::createFacemarkLBF();
facemark->loadModel(landmarksModel);
// load the 68 3d points from file (see below !)
std::vector<cv::Point3d> pts3d;
cv::FileStorage fs2("points3d.yml",0);
fs2["points"] >> pts3d;
fs2.release();
then for each image, detect a face, then get the current landmarks:
std::vector<cv::Rect> rects;
face_cascade.detectMultiScale(gray_img, faces, 1.4, 2, cv::CASCADE_SCALE_IMAGE, cv::Size(30, 30));
std::vector<cv::Rect> faces(1,rects[0]);
std::vector< std::vector<cv::Point2f> > shapes;
facemark->fit(I,faces,shapes)
std::vector<cv::Point2d> &pts2d;
for(size_t k=0; k<shapes[0].size(); k++)
pts2d.push_back(shapes[0][k]);
now we can apply solvePnP:
// camMatrix based on img size
cv::Mat rvec,tvec;
cv::Mat camMatrix;
int max_d = std::max(s.width,s.height);
camMatrix = (cv::Mat_<double>(3,3) <<
max_d, 0, s.width/2.0,
0, max_d, s.height/2.0,
0, 0, 1.0);
// 2d -> 3d correspondence
cv::solvePnP(pts3d, pts2d, camMatrix, cv::Mat(1,4,CV_64F,0.0), rvec, tvec, false, cv::SOLVEPNP_EPNP);
and, for your convenience, here's points3d.yml:
%YAML:1.0
---
points: [ -7.4077796936035156e+001, 4.5610500335693359e+001,
1.7611330032348633e+001, -7.6078399658203125e+001,
2.4455335617065430e+001, 1.4652364253997803e+000,
-7.0680282592773438e+001, 3.8770267963409424e+000,
1.6104341506958008e+001, -6.9542381286621094e+001,
-1.8663349151611328e+001, -5.0522098541259766e+000,
-6.0891132354736328e+001, -3.7201663970947266e+001,
-4.9577393531799316e+000, -4.7551403045654297e+001,
-5.1671474456787109e+001, 2.1515935897827148e+001,
-3.3833507537841797e+001, -6.4209655761718750e+001,
3.3763854980468750e+001, -1.8493196487426758e+001,
-7.5527656555175781e+001, 4.2787197113037109e+001,
-2.5200850963592529e+000, -7.7372253417968750e+001,
4.5473331451416016e+001, 1.3970505714416504e+001,
-7.3213897705078125e+001, 6.4313529968261719e+001,
3.0962856292724609e+001, -6.6279350280761719e+001,
3.5533737182617188e+001, 4.6108547210693359e+001,
-5.3055961608886719e+001, 1.9751256942749023e+001,
5.1060363769531250e+001, -3.2454692840576172e+001,
6.6386039733886719e+001, 5.8377300262451172e+001,
-1.4232730865478516e+001, 5.9445739746093750e+001,
6.3227752685546875e+001, 4.3665418624877930e+000,
4.6228359222412109e+001, 7.0979812622070312e+001,
2.6926740646362305e+001, 4.6090355515480042e-001,
6.5315643310546875e+001, 4.6220058441162109e+001,
4.7723823547363281e+001, -5.8375602722167969e+001,
5.7522071838378906e+001, 7.4911415100097656e+001,
-4.7820030212402344e+001, 6.2965705871582031e+001,
9.6279922485351562e+001, -3.4796894073486328e+001,
6.4059089660644531e+001, 1.0583456420898438e+002,
-2.3467020034790039e+001, 6.1613960266113281e+001,
1.1014395904541016e+002, -1.2404515266418457e+001,
5.8514854431152344e+001, 1.1110581970214844e+002,
9.6587600708007812e+000, 5.9617969512939453e+001,
1.1002123260498047e+002, 2.0711694717407227e+001,
6.3654747009277344e+001, 1.0981579589843750e+002,
3.2074752807617188e+001, 6.5145515441894531e+001,
1.0512077331542969e+002, 4.5245258331298828e+001,
6.3173934936523438e+001, 9.4144226074218750e+001,
5.4559543609619141e+001, 5.6469257354736328e+001,
7.4634750366210938e+001, -1.3849945068359375e+000,
4.5427635192871094e+001, 1.1209272003173828e+002,
-1.3849945068359375e+000, 3.3059230804443359e+001,
1.2122914123535156e+002, -1.3849945068359375e+000,
2.0191362380981445e+001, 1.2911428833007812e+002,
-1.3849945068359375e+000, 7.7342357635498047e+000,
1.3653234863281250e+002, -1.4418341636657715e+001,
4.0165048837661743e-001, 1.1071662902832031e+002,
-8.3029985427856445e+000, -1.3892960548400879e+000,
1.1774624633789062e+002, -2.3706998825073242e+000,
-3.3219952583312988e+000, 1.1901679229736328e+002,
4.5612163543701172e+000, -2.4215652942657471e+000,
1.1639292907714844e+002, 1.0586941719055176e+001,
-5.3226190805435181e-001, 1.1313079071044922e+002,
-4.0067256927490234e+001, 4.5844459533691406e+001,
8.9595939636230469e+001, -3.2541095733642578e+001,
4.9783782958984375e+001, 9.3007537841796875e+001,
-2.5366222381591797e+001, 4.8885505676269531e+001,
9.0976623535156250e+001, -1.9025533676147461e+001,
4.3527683258056641e+001, 9.3429924011230469e+001,
-2.5210683822631836e+001, 4.3476085662841797e+001,
9.4305854797363281e+001, -3.3464397430419922e+001,
4.3444763183593750e+001, 9.4837562561035156e+001,
1.4225140571594238e+001, 4.3614593505859375e+001,
9.1954528808593750e+001, 2.2479167938232422e+001,
4.7674880981445312e+001, 9.3482368469238281e+001,
2.9758741378784180e+001, 4.7693107604980469e+001,
9.3210464477539062e+001, 3.6196487426757812e+001,
4.4752899169921875e+001, 9.0352371215820312e+001,
2.9635911941528320e+001, 4.2387844085693359e+001,
9.5226150512695312e+001, 2.2422723770141602e+001,
4.2418308258056641e+001, 9.4690505981445312e+001,
-2.4785892486572266e+001, -2.3192493438720703e+001,
1.0339839935302734e+002, -1.7388151168823242e+001,
-1.8571313858032227e+001, 1.1188236999511719e+002,
-9.2983198165893555e+000, -1.6239971160888672e+001,
1.1731330871582031e+002, -3.3596007823944092e+000,
-1.8158702850341797e+001, 1.1823022460937500e+002,
4.5456194877624512e+000, -1.7206506729125977e+001,
1.1767270660400391e+002, 1.1584952354431152e+001,
-2.0486402511596680e+001, 1.1311756896972656e+002,
1.9875576019287109e+001, -2.5036510467529297e+001,
1.0587088012695312e+002, 1.1517845153808594e+001,
-3.0236152648925781e+001, 1.1565891265869141e+002,
4.5666022300720215e+000, -3.2193908691406250e+001,
1.1595104217529297e+002, -3.3861181735992432e+000,
-3.2590847015380859e+001, 1.1170287322998047e+002,
-1.0372335433959961e+001, -3.1501707077026367e+001,
1.1267192840576172e+002, -1.7461778640747070e+001,
-2.8766559600830078e+001, 1.0961690521240234e+002,
-2.0606306076049805e+001, -2.3995977401733398e+001,
1.0624508666992188e+002, -1.0359879493713379e+001,
-2.1467332839965820e+001, 1.1335329437255859e+002,
-3.3716876506805420e+000, -2.3318824768066406e+001,
1.1525502777099609e+002, 4.5927386283874512e+000,
-2.2431402206420898e+001, 1.1380648040771484e+002,
1.5679312705993652e+001, -2.4708747863769531e+001,
1.1011718750000000e+002, 4.5823898315429688e+000,
-2.3363874435424805e+001, 1.1465561676025391e+002,
-3.3716876506805420e+000, -2.3318824768066406e+001,
1.1525502777099609e+002, -1.0346333503723145e+001,
-2.2410360336303711e+001, 1.1409424591064453e+002
| | 2 | No.2 Revision |
i still think, you want a head pose estimation here. the gist is: you can use solvePnP to get an rvec and a tvec, containing rotation / translation wrt. the camera.
for this, you will need 2d landmarks (e.g from dlib, or from opencv's landmark models) and corresponding 3d points. i'll try with FacemarkLBF here:
// create and load the facemarks model
cv::Ptr<cv::face::Facemark> facemark;
facemark = cv::face::createFacemarkLBF();
facemark->loadModel(landmarksModel);
// load the 68 3d points from file (see below !)
std::vector<cv::Point3d> pts3d;
cv::FileStorage fs2("points3d.yml",0);
fs2["points"] >> pts3d;
fs2.release();
then for each image, detect a face, then get the current landmarks:
std::vector<cv::Rect> rects;
face_cascade.detectMultiScale(gray_img, faces, 1.4, 2, cv::CASCADE_SCALE_IMAGE, cv::Size(30, 30));
std::vector<cv::Rect> faces(1,rects[0]);
std::vector< std::vector<cv::Point2f> > shapes;
facemark->fit(I,faces,shapes)
facemark->fit(gray_img,faces,shapes)
std::vector<cv::Point2d> &pts2d;
for(size_t k=0; k<shapes[0].size(); k++)
pts2d.push_back(shapes[0][k]);
now we can apply solvePnP:
// if you did not calibrate it, use a camMatrix based on img size
size:
cv::Mat rvec,tvec;
cv::Mat camMatrix;
int max_d = std::max(s.width,s.height);
camMatrix = (cv::Mat_<double>(3,3) <<
max_d, 0, s.width/2.0,
0, max_d, s.height/2.0,
0, 0, 1.0);
// 2d -> 3d correspondence
cv::solvePnP(pts3d, pts2d, camMatrix, cv::Mat(1,4,CV_64F,0.0), rvec, tvec, false, cv::SOLVEPNP_EPNP);
and, for your convenience, here's points3d.yml:
%YAML:1.0
---
points: [ -7.4077796936035156e+001, 4.5610500335693359e+001,
1.7611330032348633e+001, -7.6078399658203125e+001,
2.4455335617065430e+001, 1.4652364253997803e+000,
-7.0680282592773438e+001, 3.8770267963409424e+000,
1.6104341506958008e+001, -6.9542381286621094e+001,
-1.8663349151611328e+001, -5.0522098541259766e+000,
-6.0891132354736328e+001, -3.7201663970947266e+001,
-4.9577393531799316e+000, -4.7551403045654297e+001,
-5.1671474456787109e+001, 2.1515935897827148e+001,
-3.3833507537841797e+001, -6.4209655761718750e+001,
3.3763854980468750e+001, -1.8493196487426758e+001,
-7.5527656555175781e+001, 4.2787197113037109e+001,
-2.5200850963592529e+000, -7.7372253417968750e+001,
4.5473331451416016e+001, 1.3970505714416504e+001,
-7.3213897705078125e+001, 6.4313529968261719e+001,
3.0962856292724609e+001, -6.6279350280761719e+001,
3.5533737182617188e+001, 4.6108547210693359e+001,
-5.3055961608886719e+001, 1.9751256942749023e+001,
5.1060363769531250e+001, -3.2454692840576172e+001,
6.6386039733886719e+001, 5.8377300262451172e+001,
-1.4232730865478516e+001, 5.9445739746093750e+001,
6.3227752685546875e+001, 4.3665418624877930e+000,
4.6228359222412109e+001, 7.0979812622070312e+001,
2.6926740646362305e+001, 4.6090355515480042e-001,
6.5315643310546875e+001, 4.6220058441162109e+001,
4.7723823547363281e+001, -5.8375602722167969e+001,
5.7522071838378906e+001, 7.4911415100097656e+001,
-4.7820030212402344e+001, 6.2965705871582031e+001,
9.6279922485351562e+001, -3.4796894073486328e+001,
6.4059089660644531e+001, 1.0583456420898438e+002,
-2.3467020034790039e+001, 6.1613960266113281e+001,
1.1014395904541016e+002, -1.2404515266418457e+001,
5.8514854431152344e+001, 1.1110581970214844e+002,
9.6587600708007812e+000, 5.9617969512939453e+001,
1.1002123260498047e+002, 2.0711694717407227e+001,
6.3654747009277344e+001, 1.0981579589843750e+002,
3.2074752807617188e+001, 6.5145515441894531e+001,
1.0512077331542969e+002, 4.5245258331298828e+001,
6.3173934936523438e+001, 9.4144226074218750e+001,
5.4559543609619141e+001, 5.6469257354736328e+001,
7.4634750366210938e+001, -1.3849945068359375e+000,
4.5427635192871094e+001, 1.1209272003173828e+002,
-1.3849945068359375e+000, 3.3059230804443359e+001,
1.2122914123535156e+002, -1.3849945068359375e+000,
2.0191362380981445e+001, 1.2911428833007812e+002,
-1.3849945068359375e+000, 7.7342357635498047e+000,
1.3653234863281250e+002, -1.4418341636657715e+001,
4.0165048837661743e-001, 1.1071662902832031e+002,
-8.3029985427856445e+000, -1.3892960548400879e+000,
1.1774624633789062e+002, -2.3706998825073242e+000,
-3.3219952583312988e+000, 1.1901679229736328e+002,
4.5612163543701172e+000, -2.4215652942657471e+000,
1.1639292907714844e+002, 1.0586941719055176e+001,
-5.3226190805435181e-001, 1.1313079071044922e+002,
-4.0067256927490234e+001, 4.5844459533691406e+001,
8.9595939636230469e+001, -3.2541095733642578e+001,
4.9783782958984375e+001, 9.3007537841796875e+001,
-2.5366222381591797e+001, 4.8885505676269531e+001,
9.0976623535156250e+001, -1.9025533676147461e+001,
4.3527683258056641e+001, 9.3429924011230469e+001,
-2.5210683822631836e+001, 4.3476085662841797e+001,
9.4305854797363281e+001, -3.3464397430419922e+001,
4.3444763183593750e+001, 9.4837562561035156e+001,
1.4225140571594238e+001, 4.3614593505859375e+001,
9.1954528808593750e+001, 2.2479167938232422e+001,
4.7674880981445312e+001, 9.3482368469238281e+001,
2.9758741378784180e+001, 4.7693107604980469e+001,
9.3210464477539062e+001, 3.6196487426757812e+001,
4.4752899169921875e+001, 9.0352371215820312e+001,
2.9635911941528320e+001, 4.2387844085693359e+001,
9.5226150512695312e+001, 2.2422723770141602e+001,
4.2418308258056641e+001, 9.4690505981445312e+001,
-2.4785892486572266e+001, -2.3192493438720703e+001,
1.0339839935302734e+002, -1.7388151168823242e+001,
-1.8571313858032227e+001, 1.1188236999511719e+002,
-9.2983198165893555e+000, -1.6239971160888672e+001,
1.1731330871582031e+002, -3.3596007823944092e+000,
-1.8158702850341797e+001, 1.1823022460937500e+002,
4.5456194877624512e+000, -1.7206506729125977e+001,
1.1767270660400391e+002, 1.1584952354431152e+001,
-2.0486402511596680e+001, 1.1311756896972656e+002,
1.9875576019287109e+001, -2.5036510467529297e+001,
1.0587088012695312e+002, 1.1517845153808594e+001,
-3.0236152648925781e+001, 1.1565891265869141e+002,
4.5666022300720215e+000, -3.2193908691406250e+001,
1.1595104217529297e+002, -3.3861181735992432e+000,
-3.2590847015380859e+001, 1.1170287322998047e+002,
-1.0372335433959961e+001, -3.1501707077026367e+001,
1.1267192840576172e+002, -1.7461778640747070e+001,
-2.8766559600830078e+001, 1.0961690521240234e+002,
-2.0606306076049805e+001, -2.3995977401733398e+001,
1.0624508666992188e+002, -1.0359879493713379e+001,
-2.1467332839965820e+001, 1.1335329437255859e+002,
-3.3716876506805420e+000, -2.3318824768066406e+001,
1.1525502777099609e+002, 4.5927386283874512e+000,
-2.2431402206420898e+001, 1.1380648040771484e+002,
1.5679312705993652e+001, -2.4708747863769531e+001,
1.1011718750000000e+002, 4.5823898315429688e+000,
-2.3363874435424805e+001, 1.1465561676025391e+002,
-3.3716876506805420e+000, -2.3318824768066406e+001,
1.1525502777099609e+002, -1.0346333503723145e+001,
-2.2410360336303711e+001, 1.1409424591064453e+002
| | 3 | No.3 Revision |
i still think, you want a head pose estimation here. the gist is: you can use solvePnP to get an rvec and a tvec, containing rotation / translation wrt. the camera.
for this, you will need 2d landmarks (e.g from dlib, or from opencv's landmark models) and corresponding 3d points. i'll try with FacemarkLBF here:
// create and load the facemarks model
cv::Ptr<cv::face::Facemark> facemark;
facemark = cv::face::createFacemarkLBF();
facemark->loadModel(landmarksModel);
// load the 68 3d points from file (see below !)
std::vector<cv::Point3d> pts3d;
cv::FileStorage fs2("points3d.yml",0);
fs2["points"] >> pts3d;
fs2.release();
then for each image, detect a face, then get the current landmarks:
std::vector<cv::Rect> rects;
face_cascade.detectMultiScale(gray_img, faces, 1.4, 2, cv::CASCADE_SCALE_IMAGE, cv::Size(30, 30));
std::vector<cv::Rect> faces(1,rects[0]);
std::vector< std::vector<cv::Point2f> > shapes;
facemark->fit(gray_img,faces,shapes)
std::vector<cv::Point2d> &pts2d;
for(size_t k=0; k<shapes[0].size(); k++)
pts2d.push_back(shapes[0][k]);
now we can apply solvePnP:
// if you did not calibrate it, use a camMatrix based on img size:
cv::Mat rvec,tvec;
cv::Mat camMatrix;
int max_d = std::max(s.width,s.height);
camMatrix = (cv::Mat_<double>(3,3) <<
max_d, 0, s.width/2.0,
0, max_d, s.height/2.0,
0, 0, 1.0);
// 2d -> 3d correspondence
cv::solvePnP(pts3d, pts2d, camMatrix, cv::Mat(1,4,CV_64F,0.0), rvec, tvec, false, cv::SOLVEPNP_EPNP);
and, for your convenience, here's points3d.yml:
%YAML:1.0
---
points: [ -7.4077796936035156e+001, 4.5610500335693359e+001,
1.7611330032348633e+001, -7.6078399658203125e+001,
2.4455335617065430e+001, 1.4652364253997803e+000,
-7.0680282592773438e+001, 3.8770267963409424e+000,
1.6104341506958008e+001, -6.9542381286621094e+001,
-1.8663349151611328e+001, -5.0522098541259766e+000,
-6.0891132354736328e+001, -3.7201663970947266e+001,
-4.9577393531799316e+000, -4.7551403045654297e+001,
-5.1671474456787109e+001, 2.1515935897827148e+001,
-3.3833507537841797e+001, -6.4209655761718750e+001,
3.3763854980468750e+001, -1.8493196487426758e+001,
-7.5527656555175781e+001, 4.2787197113037109e+001,
-2.5200850963592529e+000, -7.7372253417968750e+001,
4.5473331451416016e+001, 1.3970505714416504e+001,
-7.3213897705078125e+001, 6.4313529968261719e+001,
3.0962856292724609e+001, -6.6279350280761719e+001,
3.5533737182617188e+001, 4.6108547210693359e+001,
-5.3055961608886719e+001, 1.9751256942749023e+001,
5.1060363769531250e+001, -3.2454692840576172e+001,
6.6386039733886719e+001, 5.8377300262451172e+001,
-1.4232730865478516e+001, 5.9445739746093750e+001,
6.3227752685546875e+001, 4.3665418624877930e+000,
4.6228359222412109e+001, 7.0979812622070312e+001,
2.6926740646362305e+001, 4.6090355515480042e-001,
6.5315643310546875e+001, 4.6220058441162109e+001,
4.7723823547363281e+001, -5.8375602722167969e+001,
5.7522071838378906e+001, 7.4911415100097656e+001,
-4.7820030212402344e+001, 6.2965705871582031e+001,
9.6279922485351562e+001, -3.4796894073486328e+001,
6.4059089660644531e+001, 1.0583456420898438e+002,
-2.3467020034790039e+001, 6.1613960266113281e+001,
1.1014395904541016e+002, -1.2404515266418457e+001,
5.8514854431152344e+001, 1.1110581970214844e+002,
9.6587600708007812e+000, 5.9617969512939453e+001,
1.1002123260498047e+002, 2.0711694717407227e+001,
6.3654747009277344e+001, 1.0981579589843750e+002,
3.2074752807617188e+001, 6.5145515441894531e+001,
1.0512077331542969e+002, 4.5245258331298828e+001,
6.3173934936523438e+001, 9.4144226074218750e+001,
5.4559543609619141e+001, 5.6469257354736328e+001,
7.4634750366210938e+001, -1.3849945068359375e+000,
4.5427635192871094e+001, 1.1209272003173828e+002,
-1.3849945068359375e+000, 3.3059230804443359e+001,
1.2122914123535156e+002, -1.3849945068359375e+000,
2.0191362380981445e+001, 1.2911428833007812e+002,
-1.3849945068359375e+000, 7.7342357635498047e+000,
1.3653234863281250e+002, -1.4418341636657715e+001,
4.0165048837661743e-001, 1.1071662902832031e+002,
-8.3029985427856445e+000, -1.3892960548400879e+000,
1.1774624633789062e+002, -2.3706998825073242e+000,
-3.3219952583312988e+000, 1.1901679229736328e+002,
4.5612163543701172e+000, -2.4215652942657471e+000,
1.1639292907714844e+002, 1.0586941719055176e+001,
-5.3226190805435181e-001, 1.1313079071044922e+002,
-4.0067256927490234e+001, 4.5844459533691406e+001,
8.9595939636230469e+001, -3.2541095733642578e+001,
4.9783782958984375e+001, 9.3007537841796875e+001,
-2.5366222381591797e+001, 4.8885505676269531e+001,
9.0976623535156250e+001, -1.9025533676147461e+001,
4.3527683258056641e+001, 9.3429924011230469e+001,
-2.5210683822631836e+001, 4.3476085662841797e+001,
9.4305854797363281e+001, -3.3464397430419922e+001,
4.3444763183593750e+001, 9.4837562561035156e+001,
1.4225140571594238e+001, 4.3614593505859375e+001,
9.1954528808593750e+001, 2.2479167938232422e+001,
4.7674880981445312e+001, 9.3482368469238281e+001,
2.9758741378784180e+001, 4.7693107604980469e+001,
9.3210464477539062e+001, 3.6196487426757812e+001,
4.4752899169921875e+001, 9.0352371215820312e+001,
2.9635911941528320e+001, 4.2387844085693359e+001,
9.5226150512695312e+001, 2.2422723770141602e+001,
4.2418308258056641e+001, 9.4690505981445312e+001,
-2.4785892486572266e+001, -2.3192493438720703e+001,
1.0339839935302734e+002, -1.7388151168823242e+001,
-1.8571313858032227e+001, 1.1188236999511719e+002,
-9.2983198165893555e+000, -1.6239971160888672e+001,
1.1731330871582031e+002, -3.3596007823944092e+000,
-1.8158702850341797e+001, 1.1823022460937500e+002,
4.5456194877624512e+000, -1.7206506729125977e+001,
1.1767270660400391e+002, 1.1584952354431152e+001,
-2.0486402511596680e+001, 1.1311756896972656e+002,
1.9875576019287109e+001, -2.5036510467529297e+001,
1.0587088012695312e+002, 1.1517845153808594e+001,
-3.0236152648925781e+001, 1.1565891265869141e+002,
4.5666022300720215e+000, -3.2193908691406250e+001,
1.1595104217529297e+002, -3.3861181735992432e+000,
-3.2590847015380859e+001, 1.1170287322998047e+002,
-1.0372335433959961e+001, -3.1501707077026367e+001,
1.1267192840576172e+002, -1.7461778640747070e+001,
-2.8766559600830078e+001, 1.0961690521240234e+002,
-2.0606306076049805e+001, -2.3995977401733398e+001,
1.0624508666992188e+002, -1.0359879493713379e+001,
-2.1467332839965820e+001, 1.1335329437255859e+002,
-3.3716876506805420e+000, -2.3318824768066406e+001,
1.1525502777099609e+002, 4.5927386283874512e+000,
-2.2431402206420898e+001, 1.1380648040771484e+002,
1.5679312705993652e+001, -2.4708747863769531e+001,
1.1011718750000000e+002, 4.5823898315429688e+000,
-2.3363874435424805e+001, 1.1465561676025391e+002,
-3.3716876506805420e+000, -2.3318824768066406e+001,
1.1525502777099609e+002, -1.0346333503723145e+001,
-2.2410360336303711e+001, 1.1409424591064453e+002
| | 4 | No.4 Revision |
i still think, you want a head pose estimation here. the gist is: you can use solvePnP to get an rvec and a tvec, containing rotation / translation wrt. the camera.
for this, you will need 2d landmarks (e.g from dlib, openpose, or from opencv's landmark models) and corresponding 3d points. i'll try with FacemarkLBF here:
// create and load the facemarks model
cv::Ptr<cv::face::Facemark> facemark;
facemark = cv::face::createFacemarkLBF();
facemark->loadModel(landmarksModel);
// load the 68 3d points from file (see below !)
std::vector<cv::Point3d> pts3d;
cv::FileStorage fs2("points3d.yml",0);
fs2["points"] >> pts3d;
fs2.release();
then for each image, detect a face, then get the current landmarks:
std::vector<cv::Rect> rects;
face_cascade.detectMultiScale(gray_img, faces, 1.4, 2, cv::CASCADE_SCALE_IMAGE, cv::Size(30, 30));
std::vector<cv::Rect> faces(1,rects[0]);
std::vector< std::vector<cv::Point2f> > shapes;
facemark->fit(gray_img,faces,shapes)
std::vector<cv::Point2d> &pts2d;
for(size_t k=0; k<shapes[0].size(); k++)
pts2d.push_back(shapes[0][k]);
now we can apply solvePnP:
// if you did not calibrate it, use a camMatrix based on img size:
cv::Mat rvec,tvec;
cv::Mat camMatrix;
int max_d = std::max(s.width,s.height);
camMatrix = (cv::Mat_<double>(3,3) <<
max_d, 0, s.width/2.0,
0, max_d, s.height/2.0,
0, 0, 1.0);
// 2d -> 3d correspondence
cv::solvePnP(pts3d, pts2d, camMatrix, cv::Mat(1,4,CV_64F,0.0), rvec, tvec, false, cv::SOLVEPNP_EPNP);
and, for your convenience, here's points3d.yml:
%YAML:1.0
---
points: [ -7.4077796936035156e+001, 4.5610500335693359e+001,
1.7611330032348633e+001, -7.6078399658203125e+001,
2.4455335617065430e+001, 1.4652364253997803e+000,
-7.0680282592773438e+001, 3.8770267963409424e+000,
1.6104341506958008e+001, -6.9542381286621094e+001,
-1.8663349151611328e+001, -5.0522098541259766e+000,
-6.0891132354736328e+001, -3.7201663970947266e+001,
-4.9577393531799316e+000, -4.7551403045654297e+001,
-5.1671474456787109e+001, 2.1515935897827148e+001,
-3.3833507537841797e+001, -6.4209655761718750e+001,
3.3763854980468750e+001, -1.8493196487426758e+001,
-7.5527656555175781e+001, 4.2787197113037109e+001,
-2.5200850963592529e+000, -7.7372253417968750e+001,
4.5473331451416016e+001, 1.3970505714416504e+001,
-7.3213897705078125e+001, 6.4313529968261719e+001,
3.0962856292724609e+001, -6.6279350280761719e+001,
3.5533737182617188e+001, 4.6108547210693359e+001,
-5.3055961608886719e+001, 1.9751256942749023e+001,
5.1060363769531250e+001, -3.2454692840576172e+001,
6.6386039733886719e+001, 5.8377300262451172e+001,
-1.4232730865478516e+001, 5.9445739746093750e+001,
6.3227752685546875e+001, 4.3665418624877930e+000,
4.6228359222412109e+001, 7.0979812622070312e+001,
2.6926740646362305e+001, 4.6090355515480042e-001,
6.5315643310546875e+001, 4.6220058441162109e+001,
4.7723823547363281e+001, -5.8375602722167969e+001,
5.7522071838378906e+001, 7.4911415100097656e+001,
-4.7820030212402344e+001, 6.2965705871582031e+001,
9.6279922485351562e+001, -3.4796894073486328e+001,
6.4059089660644531e+001, 1.0583456420898438e+002,
-2.3467020034790039e+001, 6.1613960266113281e+001,
1.1014395904541016e+002, -1.2404515266418457e+001,
5.8514854431152344e+001, 1.1110581970214844e+002,
9.6587600708007812e+000, 5.9617969512939453e+001,
1.1002123260498047e+002, 2.0711694717407227e+001,
6.3654747009277344e+001, 1.0981579589843750e+002,
3.2074752807617188e+001, 6.5145515441894531e+001,
1.0512077331542969e+002, 4.5245258331298828e+001,
6.3173934936523438e+001, 9.4144226074218750e+001,
5.4559543609619141e+001, 5.6469257354736328e+001,
7.4634750366210938e+001, -1.3849945068359375e+000,
4.5427635192871094e+001, 1.1209272003173828e+002,
-1.3849945068359375e+000, 3.3059230804443359e+001,
1.2122914123535156e+002, -1.3849945068359375e+000,
2.0191362380981445e+001, 1.2911428833007812e+002,
-1.3849945068359375e+000, 7.7342357635498047e+000,
1.3653234863281250e+002, -1.4418341636657715e+001,
4.0165048837661743e-001, 1.1071662902832031e+002,
-8.3029985427856445e+000, -1.3892960548400879e+000,
1.1774624633789062e+002, -2.3706998825073242e+000,
-3.3219952583312988e+000, 1.1901679229736328e+002,
4.5612163543701172e+000, -2.4215652942657471e+000,
1.1639292907714844e+002, 1.0586941719055176e+001,
-5.3226190805435181e-001, 1.1313079071044922e+002,
-4.0067256927490234e+001, 4.5844459533691406e+001,
8.9595939636230469e+001, -3.2541095733642578e+001,
4.9783782958984375e+001, 9.3007537841796875e+001,
-2.5366222381591797e+001, 4.8885505676269531e+001,
9.0976623535156250e+001, -1.9025533676147461e+001,
4.3527683258056641e+001, 9.3429924011230469e+001,
-2.5210683822631836e+001, 4.3476085662841797e+001,
9.4305854797363281e+001, -3.3464397430419922e+001,
4.3444763183593750e+001, 9.4837562561035156e+001,
1.4225140571594238e+001, 4.3614593505859375e+001,
9.1954528808593750e+001, 2.2479167938232422e+001,
4.7674880981445312e+001, 9.3482368469238281e+001,
2.9758741378784180e+001, 4.7693107604980469e+001,
9.3210464477539062e+001, 3.6196487426757812e+001,
4.4752899169921875e+001, 9.0352371215820312e+001,
2.9635911941528320e+001, 4.2387844085693359e+001,
9.5226150512695312e+001, 2.2422723770141602e+001,
4.2418308258056641e+001, 9.4690505981445312e+001,
-2.4785892486572266e+001, -2.3192493438720703e+001,
1.0339839935302734e+002, -1.7388151168823242e+001,
-1.8571313858032227e+001, 1.1188236999511719e+002,
-9.2983198165893555e+000, -1.6239971160888672e+001,
1.1731330871582031e+002, -3.3596007823944092e+000,
-1.8158702850341797e+001, 1.1823022460937500e+002,
4.5456194877624512e+000, -1.7206506729125977e+001,
1.1767270660400391e+002, 1.1584952354431152e+001,
-2.0486402511596680e+001, 1.1311756896972656e+002,
1.9875576019287109e+001, -2.5036510467529297e+001,
1.0587088012695312e+002, 1.1517845153808594e+001,
-3.0236152648925781e+001, 1.1565891265869141e+002,
4.5666022300720215e+000, -3.2193908691406250e+001,
1.1595104217529297e+002, -3.3861181735992432e+000,
-3.2590847015380859e+001, 1.1170287322998047e+002,
-1.0372335433959961e+001, -3.1501707077026367e+001,
1.1267192840576172e+002, -1.7461778640747070e+001,
-2.8766559600830078e+001, 1.0961690521240234e+002,
-2.0606306076049805e+001, -2.3995977401733398e+001,
1.0624508666992188e+002, -1.0359879493713379e+001,
-2.1467332839965820e+001, 1.1335329437255859e+002,
-3.3716876506805420e+000, -2.3318824768066406e+001,
1.1525502777099609e+002, 4.5927386283874512e+000,
-2.2431402206420898e+001, 1.1380648040771484e+002,
1.5679312705993652e+001, -2.4708747863769531e+001,
1.1011718750000000e+002, 4.5823898315429688e+000,
-2.3363874435424805e+001, 1.1465561676025391e+002,
-3.3716876506805420e+000, -2.3318824768066406e+001,
1.1525502777099609e+002, -1.0346333503723145e+001,
-2.2410360336303711e+001, 1.1409424591064453e+002
| | 5 | No.5 Revision |
i still think, you want a head pose estimation here. the gist is: you can use solvePnP to get an rvec and a tvec, containing rotation / translation wrt. the camera.
for this, you will need 2d landmarks (e.g from dlib, openpose, or from opencv's landmark models) and corresponding 3d points. i'll try with FacemarkLBF FacemarkLBF here:
// create and load the facemarks model
cv::Ptr<cv::face::Facemark> facemark;
facemark = cv::face::createFacemarkLBF();
facemark->loadModel(landmarksModel);
// load the 68 3d points from file (see below !)
std::vector<cv::Point3d> pts3d;
cv::FileStorage fs2("points3d.yml",0);
fs2["points"] >> pts3d;
fs2.release();
then for each image, detect a face, then get the current landmarks:
std::vector<cv::Rect> rects;
face_cascade.detectMultiScale(gray_img, faces, 1.4, 2, cv::CASCADE_SCALE_IMAGE, cv::Size(30, 30));
std::vector<cv::Rect> faces(1,rects[0]);
std::vector< std::vector<cv::Point2f> > shapes;
facemark->fit(gray_img,faces,shapes)
std::vector<cv::Point2d> &pts2d;
for(size_t k=0; k<shapes[0].size(); k++)
pts2d.push_back(shapes[0][k]);
now we can apply solvePnP:
// if you did not calibrate it, use a camMatrix based on img size:
cv::Mat rvec,tvec;
cv::Mat camMatrix;
int max_d = std::max(s.width,s.height);
camMatrix = (cv::Mat_<double>(3,3) <<
max_d, 0, s.width/2.0,
0, max_d, s.height/2.0,
0, 0, 1.0);
// 2d -> 3d correspondence
cv::solvePnP(pts3d, pts2d, camMatrix, cv::Mat(1,4,CV_64F,0.0), rvec, tvec, false, cv::SOLVEPNP_EPNP);
and, for your convenience, here's points3d.yml:
%YAML:1.0
---
points: [ -7.4077796936035156e+001, 4.5610500335693359e+001,
1.7611330032348633e+001, -7.6078399658203125e+001,
2.4455335617065430e+001, 1.4652364253997803e+000,
-7.0680282592773438e+001, 3.8770267963409424e+000,
1.6104341506958008e+001, -6.9542381286621094e+001,
-1.8663349151611328e+001, -5.0522098541259766e+000,
-6.0891132354736328e+001, -3.7201663970947266e+001,
-4.9577393531799316e+000, -4.7551403045654297e+001,
-5.1671474456787109e+001, 2.1515935897827148e+001,
-3.3833507537841797e+001, -6.4209655761718750e+001,
3.3763854980468750e+001, -1.8493196487426758e+001,
-7.5527656555175781e+001, 4.2787197113037109e+001,
-2.5200850963592529e+000, -7.7372253417968750e+001,
4.5473331451416016e+001, 1.3970505714416504e+001,
-7.3213897705078125e+001, 6.4313529968261719e+001,
3.0962856292724609e+001, -6.6279350280761719e+001,
3.5533737182617188e+001, 4.6108547210693359e+001,
-5.3055961608886719e+001, 1.9751256942749023e+001,
5.1060363769531250e+001, -3.2454692840576172e+001,
6.6386039733886719e+001, 5.8377300262451172e+001,
-1.4232730865478516e+001, 5.9445739746093750e+001,
6.3227752685546875e+001, 4.3665418624877930e+000,
4.6228359222412109e+001, 7.0979812622070312e+001,
2.6926740646362305e+001, 4.6090355515480042e-001,
6.5315643310546875e+001, 4.6220058441162109e+001,
4.7723823547363281e+001, -5.8375602722167969e+001,
5.7522071838378906e+001, 7.4911415100097656e+001,
-4.7820030212402344e+001, 6.2965705871582031e+001,
9.6279922485351562e+001, -3.4796894073486328e+001,
6.4059089660644531e+001, 1.0583456420898438e+002,
-2.3467020034790039e+001, 6.1613960266113281e+001,
1.1014395904541016e+002, -1.2404515266418457e+001,
5.8514854431152344e+001, 1.1110581970214844e+002,
9.6587600708007812e+000, 5.9617969512939453e+001,
1.1002123260498047e+002, 2.0711694717407227e+001,
6.3654747009277344e+001, 1.0981579589843750e+002,
3.2074752807617188e+001, 6.5145515441894531e+001,
1.0512077331542969e+002, 4.5245258331298828e+001,
6.3173934936523438e+001, 9.4144226074218750e+001,
5.4559543609619141e+001, 5.6469257354736328e+001,
7.4634750366210938e+001, -1.3849945068359375e+000,
4.5427635192871094e+001, 1.1209272003173828e+002,
-1.3849945068359375e+000, 3.3059230804443359e+001,
1.2122914123535156e+002, -1.3849945068359375e+000,
2.0191362380981445e+001, 1.2911428833007812e+002,
-1.3849945068359375e+000, 7.7342357635498047e+000,
1.3653234863281250e+002, -1.4418341636657715e+001,
4.0165048837661743e-001, 1.1071662902832031e+002,
-8.3029985427856445e+000, -1.3892960548400879e+000,
1.1774624633789062e+002, -2.3706998825073242e+000,
-3.3219952583312988e+000, 1.1901679229736328e+002,
4.5612163543701172e+000, -2.4215652942657471e+000,
1.1639292907714844e+002, 1.0586941719055176e+001,
-5.3226190805435181e-001, 1.1313079071044922e+002,
-4.0067256927490234e+001, 4.5844459533691406e+001,
8.9595939636230469e+001, -3.2541095733642578e+001,
4.9783782958984375e+001, 9.3007537841796875e+001,
-2.5366222381591797e+001, 4.8885505676269531e+001,
9.0976623535156250e+001, -1.9025533676147461e+001,
4.3527683258056641e+001, 9.3429924011230469e+001,
-2.5210683822631836e+001, 4.3476085662841797e+001,
9.4305854797363281e+001, -3.3464397430419922e+001,
4.3444763183593750e+001, 9.4837562561035156e+001,
1.4225140571594238e+001, 4.3614593505859375e+001,
9.1954528808593750e+001, 2.2479167938232422e+001,
4.7674880981445312e+001, 9.3482368469238281e+001,
2.9758741378784180e+001, 4.7693107604980469e+001,
9.3210464477539062e+001, 3.6196487426757812e+001,
4.4752899169921875e+001, 9.0352371215820312e+001,
2.9635911941528320e+001, 4.2387844085693359e+001,
9.5226150512695312e+001, 2.2422723770141602e+001,
4.2418308258056641e+001, 9.4690505981445312e+001,
-2.4785892486572266e+001, -2.3192493438720703e+001,
1.0339839935302734e+002, -1.7388151168823242e+001,
-1.8571313858032227e+001, 1.1188236999511719e+002,
-9.2983198165893555e+000, -1.6239971160888672e+001,
1.1731330871582031e+002, -3.3596007823944092e+000,
-1.8158702850341797e+001, 1.1823022460937500e+002,
4.5456194877624512e+000, -1.7206506729125977e+001,
1.1767270660400391e+002, 1.1584952354431152e+001,
-2.0486402511596680e+001, 1.1311756896972656e+002,
1.9875576019287109e+001, -2.5036510467529297e+001,
1.0587088012695312e+002, 1.1517845153808594e+001,
-3.0236152648925781e+001, 1.1565891265869141e+002,
4.5666022300720215e+000, -3.2193908691406250e+001,
1.1595104217529297e+002, -3.3861181735992432e+000,
-3.2590847015380859e+001, 1.1170287322998047e+002,
-1.0372335433959961e+001, -3.1501707077026367e+001,
1.1267192840576172e+002, -1.7461778640747070e+001,
-2.8766559600830078e+001, 1.0961690521240234e+002,
-2.0606306076049805e+001, -2.3995977401733398e+001,
1.0624508666992188e+002, -1.0359879493713379e+001,
-2.1467332839965820e+001, 1.1335329437255859e+002,
-3.3716876506805420e+000, -2.3318824768066406e+001,
1.1525502777099609e+002, 4.5927386283874512e+000,
-2.2431402206420898e+001, 1.1380648040771484e+002,
1.5679312705993652e+001, -2.4708747863769531e+001,
1.1011718750000000e+002, 4.5823898315429688e+000,
-2.3363874435424805e+001, 1.1465561676025391e+002,
-3.3716876506805420e+000, -2.3318824768066406e+001,
1.1525502777099609e+002, -1.0346333503723145e+001,
-2.2410360336303711e+001, 1.1409424591064453e+002