I think everything is in doc here (see detailed description)

n-dimensional dense array class

The class Mat represents an
n-dimensional dense numerical
single-channel or multi-channel array.
It can be used to store real or
complex-valued vectors and matrices,
grayscale or color images, voxel
volumes, vector fields, point clouds,
tensors, histograms (though, very
high-dimensional histograms may be
better stored in a SparseMat ). The
data layout of the array M is defined
by the array M.step[], so that the
address of element (i0,...,iM.dims−1),
where 0≤ik<m.size[k], is="" computed="" as:<="" p="">

addr(Mi0,...,iM.dims−1)=M.data+M.step[0]∗i0+M.step[1]∗i1+...+M.step[M.dims−1]∗iM.dims−1

In case of a 2-dimensional array, the
above formula is reduced to:

addr(Mi,j)=M.data+M.step[0]∗i+M.step[1]∗j

Note that M.step[i] >= M.step[i+1] (in
fact, M.step[i] >=
M.step[i+1]*M.size[i+1] ). This means
that 2-dimensional matrices are stored
row-by-row, 3-dimensional matrices are
stored plane-by-plane, and so on.
M.step[M.dims-1] is minimal and always
equal to the element size M.elemSize()
.

Now if you want to use calhist for 3d histogram (calcbackproject for example with dim=3 it is a 3d histogram) you can do like this :

```
Mat m = imread("f:/lib/opencv/samples/data/baboon.jpg", CV_LOAD_IMAGE_ANYCOLOR);
Mat mask(m.size(), CV_8UC1, Scalar::all(0));
Rect rZone(Point(220, 330), Point(260, 360));
mask(rZone) = 255;
float hRange[] = { 0, 255 };
const float* etendu[] = { hRange, hRange,hRange };
int hBins = 30;
int tailleHist[] = { hBins, hBins+5, hBins+10 };
int canaux[] = { 2, 1,0 };
double hMin, hMax;
cv::Mat histogramme;
cv::Mat resultMap;
cv::Mat backproj1d;
for (int dim = 1; dim <= 3; dim++)
{
cv::calcHist(&m, 1, canaux, mask, histogramme, dim, tailleHist, etendu, true, false);
minMaxIdx(histogramme, &hMin, &hMax);
cout << "Hist extremum = " << hMin << "\t" << hMax << "\n";
cout << "Pixel in Mat = " <<histogramme.total()<<endl;
for (int j=0;j<dim;j++)
{
cout <<"dimension " << j <<" : " << tailleHist[j]<<endl;
if (j<dim-1)
cout << "offset in dimension "<<j<<" -> "<<histogramme.step[j]<<endl;
}
normalize(histogramme, histogramme, 0, 255, cv::NormTypes::NORM_MINMAX, -1, cv::Mat());
minMaxIdx(histogramme, &hMin, &hMax);
cout << "after normalize Hist extremum = " << hMin << "\t" << hMax << "\n";
cv::calcBackProject(&m, 1, canaux, histogramme, backproj1d, etendu, 1, true);
applyColorMap(backproj1d, resultMap, cv::COLORMAP_JET);
imshow(format("backproject %dD", dim), resultMap);
waitKey();
}
```