This forum is disabled, please visit https://forum.opencv.org
 | 1 | initial version |
The methods in the mapping3d module are (at the moment) just a multi-camera/multi-view triangulation method, and one to match a 3d model to observations. It does not do the image processing to find the points in multiple images. It does not have any real intelligence.
You give it a set of image points (one per image), and the position and orientation of the cameras that took the images, and it will give you the best-estimate 3d location of that point. In order to get enough information, there must be either multiple cameras, or a moving camera, or both. One image is not enough. Multiple images from a stationary camera is not enough.
It will not figure out where the cameras are based on the images. You have to provide that information. Either a tracked camera, known scene points through solvePnP, or a Structure from Motion (SfM) algorithm.
The example you have been looking at are one scene I captured with a tracked camera. The poses.xml is valid for that image set only. If you want to use your own images, you need to either modify the poses.xml, or more usefully, just read that information from however you're already storing and pass it to the mapping3d:: functions in a manner similar to the example.
As for how to use Viz, it can read .ply files, and the tutorial HERE shows how to do that and draw camera axes.
Example code including VIZ: Sorry it's not as clean as the example, I just pulled this from history.
FileStorage fs;
fs.open("D:/Data/Vive/Poses.xml", FileStorage::FORMAT_AUTO);
Mat cameraMatrix;
fs["Camera_Matrix"] >> cameraMatrix;
cameraMatrix.convertTo(cameraMatrix, CV_64F);
double fovx, fovy, fl, ar;
Point2d pp;
calibrationMatrixValues( cameraMatrix, Size( 612, 460 ), 1, 1, fovx, fovy, fl, pp, ar );
cout << fovx << " " << fovy << "\n";
calibrationMatrixValues( cameraMatrix, Size( 612, 460 ), 100, 100, fovx, fovy, fl, pp, ar );
cout << fovx << " " << fovy << "\n";
int numFrames;
fs["nr_of_frames"] >> numFrames;
viz::Viz3d myWindow("Coordinate Frame");
myWindow.showWidget("Coordinate Widget", viz::WCoordinateSystem());
Affine3f cam_pose;
vector<Mat> imgBuffer, rBuffer, tBuffer;
vector<Mat> keyFramePyr, dstFramePyr;
vector<Point2f> keyFramePts, dstFramePts;
vector<vector<Point2f>> trackingPts;
for (int i = 1, j = 0; i < numFrames; i+=10, ++j)
{
std::stringstream str1, str2;
str1 << "D:/Data/Vive/" << i << ".png";
str2 << "Pose_Matrix_" << i;
cout << "Reading Frame " << i << "\n";
imgBuffer.push_back(imread(str1.str()));
Mat pose, rvec;
fs[str2.str()] >> pose;
Mat ident;
ident.create(3, 3, pose.type());
setIdentity(ident, -1);
ident.at<float>(0, 0) = 1;
pose(Rect(0, 0, 3, 3)) = (ident * pose(Rect(0, 0, 3, 3)).t()).t();
Rodrigues(pose(Rect(0, 0, 3, 3)), rvec);
rBuffer.push_back(rvec.clone());
tBuffer.push_back(pose(Rect(3, 0, 1, 3)).clone());
Vec3f rVec3f, tVec3f;
rVec3f = Vec3f(rBuffer[j].at<float>(0), rBuffer[j].at<float>(1), rBuffer[j].at<float>(2));
tVec3f = Vec3f(tBuffer[j].at<float>(0), tBuffer[j].at<float>(1), tBuffer[j].at<float>(2));
cam_pose = Affine3f(rVec3f, tVec3f);
viz::WCameraPosition cpw(0.25); // Coordinate axes
viz::WCameraPosition cpw_frustum(Vec2f(2.0944, 1.57422), 0.25); // Camera frustum
std::stringstream str3;
str3 << "CPW_" << i;
myWindow.showWidget(str3.str(), cpw, cam_pose);
str3 << "_FRUSTUM";
myWindow.showWidget(str3.str(), cpw_frustum, cam_pose);
Mat R;
Rodrigues(rBuffer[j], R);
R = R.t();
tBuffer[j] = (-R * tBuffer[j]);
Rodrigues(R, rBuffer[j]);
tBuffer[j].convertTo(tBuffer[j], CV_64F);
rBuffer[j].convertTo(rBuffer[j], CV_64F);
}
Mat state, PCL(1, 3, CV_64F), cov;
PCL.setTo(0);
for (int i = 0; i < imgBuffer.size(); i += 1)
{
trackingPts.clear();
keyFramePts.clear();
dstFramePts.clear();
Ptr<ORB> pORB = ORB::create(10000, 1.2, 1);
vector<KeyPoint> kps;
pORB->detect(imgBuffer[i], kps);
vector<Mat> localt;
vector<Mat> localr;
localt.push_back(tBuffer[i]);
localr.push_back(rBuffer[i]);
for (KeyPoint& p : kps)
{
keyFramePts.push_back(p.pt);
trackingPts.push_back(vector<Point2f>(1, p.pt));
}
buildOpticalFlowPyramid(imgBuffer[i], keyFramePyr, Size(11, 11), 3);
for (int j = 0; j < imgBuffer.size(); j++)
{
if (j != i)
{
localt.push_back(tBuffer[j]);
localr.push_back(rBuffer[j]);
Mat status, err;
buildOpticalFlowPyramid(imgBuffer[j], dstFramePyr, Size(11, 11), 3);
if (dstFramePts.size() == keyFramePts.size())
cv::calcOpticalFlowPyrLK(keyFramePyr, dstFramePyr, keyFramePts, dstFramePts, status, err, Size(11, 11), 3, TermCriteria(TermCriteria::COUNT + TermCriteria::EPS, 30, 0.01), OPTFLOW_USE_INITIAL_FLOW);
else
cv::calcOpticalFlowPyrLK(keyFramePyr, dstFramePyr, keyFramePts, dstFramePts, status, err, Size(11, 11), 3);
for (int idx = 0, idx2 = 0; idx < keyFramePts.size(); ++idx, ++idx2)
{
if (status.at<uchar>(idx2) == 1)
trackingPts[idx].push_back(dstFramePts[idx2]);
else
{
trackingPts.erase(trackingPts.begin() + idx);
keyFramePts.erase(keyFramePts.begin() + idx);
idx--;
}
}
}
}
auto start = high_resolution_clock::now();
for (int idx = 0; idx < trackingPts.size(); ++idx)
{
mapping3d::calcPosition(localt, localr, trackingPts[idx], cameraMatrix, noArray(), state, cov);
//std::cout << cov << "\n\n";
double min, max;
minMaxIdx(cov, &min, &max);
if(MAX(abs(max), abs(min)) < 0.003)
vconcat(PCL, state.t(), PCL);
else
{
trackingPts.erase(trackingPts.begin() + idx);
keyFramePts.erase(keyFramePts.begin() + idx);
idx--;
}
}
auto stop = high_resolution_clock::now();
cout << duration_cast<nanoseconds>( stop - start ).count() / 1.0e9 << " seconds for "<<trackingPts.size()<<" points\n";
}
PCL = PCL.reshape(3, PCL.rows);
cout << PCL << "\n";
Affine3f cloud_pose_global;
viz::WCloud cloud_widget(PCL, viz::Color::green());
myWindow.showWidget("points", cloud_widget, cloud_pose_global);
myWindow.setViewerPose(cam_pose);
myWindow.spin();