Kmeans support needed

edit

1. the labels are an integer Mat, with 1 lookup entry per pixel
2. 3 floats per row, 1 row per pixel
3. just try. below, 10 iterations seemed to be enough

assuming, you start with an bgr, uchar image:

//
// step 1: preprocess the data:
//
Mat data;
img.convertTo(data,CV_32F); // float
// each pixel should be a seperate row of 3 numbers, like 
// b g r
// b g r
// ...
data = data.reshape(1,data.total()); // to single channel, 3 cols
cerr << data.type() << " " << data.size() << endl;
// 5 [3 x 168750]

//
// step 2: now we can apply kmeans, and get 8 clusters (of 3 numbers).
//   for each original pixel (row), there will be an index in labels 
//   (lookup for the resp. cluster center)
//

Mat labels, centers;
cv::kmeans(data, 8, labels, cv::TermCriteria(CV_TERMCRIT_ITER, 10, 1.0), 3, cv::KMEANS_PP_CENTERS, centers);

cerr << labels.type() << " " << labels.size() << endl;
cerr << centers.type() << " " << centers.size() << endl;
cerr << centers << endl;
// 4 [1 x 168750]
// 5 [3 x 8]
// [106.11583, 154.31699, 166.2114;
// 56.6311, 128.48317, 81.664391;
//  55.669727, 43.721508, 207.35027;
//  76.740356, 181.49567, 115.45416;
//  159.78735, 177.61528, 180.26251;
//  27.584431, 46.930195, 61.983833;
//  54.100143, 77.063377, 104.66669;
//  82.145332, 115.53667, 132.77074]    

//
// step 3: replace the original pixels with the cluster centers:
//

// move both centers and data from 3 cols to 3 channels
// (makes it easier to copy later)
centers = centers.reshape(3,centers.rows);
data = data.reshape(3,data.rows);

// copy loop. lookup the center from the labels, then replace Vec3f pixel
Vec3f *p = data.ptr<Vec3f>();
for (size_t i=0; i<data.rows; i++) {
   int center_id = labels.at<int>(i);
   p[i] = centers.at<Vec3f>(center_id);
}

//
// step 4: back to original shape & type:
//
result = data.reshape(3, img.rows);
result.convertTo(result, CV_8U);