Revision history [back]

Here's the whole code :

#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <iostream>
#include <opencv2\opencv.hpp>
#include <vector>
#include <cmath>
#include <opencv2/features2d/features2d.hpp>
#include <opencv\highgui.h>
#include <opencv2\imgproc\imgproc_c.h>
#include <stdarg.h>

using namespace cv;
using namespace std;
/////////////////////////////////////////////////////////////
/// Global Variables
int filterKernelSize = 15; // size of median filter, need to be odd number
int resultDigit; // the outcome of the recognition algorithm
Mat src; Mat srcSm;
/// Function headers
int getContourAndHull(cv::Mat);
vector<int> elimNeighborHulls(vector<int>, vector<Point>); // to remove neighbor hulls
vector<int> filterHulls(vector<int>, vector<Point>, RotatedRect); // to remove hulls below a height
vector<int> filterHulls2(vector<int>, vector<Point>, vector<Point>, RotatedRect); // to further remove hulls around palm
vector<Point> filterDefects(vector<Point>, RotatedRect); // to remove defects below a height
void findConvexityDefects(vector<Point>&, vector<int>&, vector<Point>&);
void display(char*, cv::Mat);
/////////////////////////////////////////////////////////////
// void cvShowManyImages(char* title, int nArgs, ...);
int main(int argc, char** argv)
{
    int i, j;
    int n;
    double handDepth = 24;
    double maxVal1 = 0, minVal = 0;
    unsigned char data = 90;
    int lastNum = 0;
    //Mat image;
    Mat num1, num2, num3, num4, num5, num0;

    const char* inFileSrc1 = "C:/Users/HDR/Documents/Visual Studio 2010/Projects/openCV_tutorial/1.png";
    const char* inFileSrc2 = "C:/Users/HDR/Documents/Visual Studio 2010/Projects/openCV_tutorial/2.png";
    const char* inFileSrc3 = "C:/Users/HDR/Documents/Visual Studio 2010/Projects/openCV_tutorial/3.png";
    const char* inFileSrc4 = "C:/Users/HDR/Documents/Visual Studio 2010/Projects/openCV_tutorial/4.png";
    const char* inFileSrc5 = "C:/Users/HDR/Documents/Visual Studio 2010/Projects/openCV_tutorial/5.png";
    const char* inFileSrc0 = "C:/Users/HDR/Documents/Visual Studio 2010/Projects/openCV_tutorial/0.png";

    /*const char* inFileSrc1 = "C:/Users/Project Student/Desktop/Bonie/Preprocessing/images/chandrakalha1.jpg";
    const char* inFileSrc2 = "C:/Users/Project Student/Desktop/Bonie/Preprocessing/images/chandrakalha2.jpg";
    const char* inFileSrc3 = "C:/Users/Project Student/Desktop/Bonie/Preprocessing/images/chandrakalha3.jpg";
    const char* inFileSrc4 = "C:/Users/Project Student/Desktop/Bonie/Preprocessing/images/chandrakalha4.jpg";
    const char* inFileSrc5 = "C:/Users/Project Student/Desktop/Bonie/Preprocessing/images/chandrakalha5.jpg";
    const char* inFileSrc0 = "C:/Users/Project Student/Desktop/Bonie/Preprocessing/images/chandrakalha6.jpg";*/

    num1 = imread(inFileSrc1, CV_LOAD_IMAGE_GRAYSCALE);
    num2 = imread(inFileSrc2, CV_LOAD_IMAGE_GRAYSCALE);
    num3 = imread(inFileSrc3, CV_LOAD_IMAGE_GRAYSCALE);
    num4 = imread(inFileSrc4, CV_LOAD_IMAGE_GRAYSCALE);
    num5 = imread(inFileSrc5, CV_LOAD_IMAGE_GRAYSCALE);
    num0 = imread(inFileSrc0, CV_LOAD_IMAGE_GRAYSCALE);


    VideoCapture capture("pathaka.mp4");

    if (!capture.isOpened())
    {
        cout << "Can not open a capture object." << endl;
        return -1;
    }
    capture.set(CV_CAP_OPENNI_IMAGE_GENERATOR_OUTPUT_MODE, CV_CAP_OPENNI_VGA_30HZ);
    cout << "\nDepth generator output mode:" << endl <<
        "FRAME_WIDTH " << capture.get(CV_CAP_PROP_FRAME_WIDTH) << endl <<
        "FRAME_HEIGHT " << capture.get(CV_CAP_PROP_FRAME_HEIGHT) << endl <<
        "FRAME_MAX_DEPTH " << capture.get(CV_CAP_PROP_OPENNI_FRAME_MAX_DEPTH) << " mm"
        << endl <<
        "FPS " << capture.get(CV_CAP_PROP_FPS) << endl;
    for (;;)
    {
        Mat image, image1;

        //dataptr=image.data;

        //capture.retrieve(image,CV_16UC1);
        if (!capture.grab())
        {
            cout << "Can not grab images." << endl;
            return -1;
        }
        else
        {
            if (capture.retrieve(image, CV_CAP_OPENNI_DEPTH_MAP))
            {
                // imshow("original", image);
                //printf("image captured\n");
                const float scaleFactor = 0.3f;
                image.convertTo(image, CV_8UC1, scaleFactor);
                image = 255 - image;
                //printf("image optimized\n");

                n = image.cols;

                    uchar* dataptr = image.data;
                //dataptr=image.data;
                //printf("start shadow elimination\n");
                for (i = 0; i <= image.rows; i++)
                {
                    for (j = 0; j <= image.cols; j++)
                    {
                        if (dataptr[n*i + j] == 255)
                            dataptr[n*i + j] = 0;
                    };
                };
                image1 = image.clone();
                //printf("shadow elimination done\n");
                //printf("start to calculate maxVal\n");
                minMaxLoc(image, &minVal, &maxVal1, NULL, NULL);
                //printf("the maxVal is %f\n",maxVal1);
                //printf("the interger of maxVal is %u\n",(unsigned int)maxVal1);
                //printf("the hand Depth is %u\n",handDepth);
                uchar* dataptr1 = image1.data;

                for (i = 0; i<image1.rows; i++)
                {
                    for (j = 0; j<image1.cols; j++)
                    {
                        //data=dataptr1[n*i+j];
                        /*if((unsigned int)dataptr1[n*i+j]<maxVal1-handDepth)
                        dataptr1[n*i+j]=0;
                        else if(maxVal1!=255||maxVal1!=0)
                        dataptr1[n*i+j]=255;*/
                        if (maxVal1<130 && maxVal1>24)
                            if ((unsigned int)dataptr1[n*i + j]<maxVal1 - handDepth)
                                dataptr1[n*i + j] = 0;
                            else
                                dataptr1[n*i + j] = 255;
                        else
                            dataptr1[n*i + j] = 0;
                    };
                };
                //printf("processing finished\n");

                imshow("depth map", image1);
                cvMoveWindow("depth map", 50, 100);
                //printf("image showed\n");
                ///////////////////////////////////////////////////////////////////////////
                medianBlur(image1, srcSm, filterKernelSize); // medianBlur smoothing filter
                // processing to get result digit
                resultDigit = getContourAndHull(srcSm);
                if (resultDigit != lastNum)
                    if (resultDigit == 0)
                    {
                    imshow("number", num0);
                    cvMoveWindow("number", 380, 625);
                    }
                    else if (resultDigit == 1)
                    {
                        imshow("number", num1);
                            cvMoveWindow("number", 380, 625);
                    }
                    else if (resultDigit == 2)
                    {
                        imshow("number", num2);
                        cvMoveWindow("number", 380, 625);
                    }
                    else if (resultDigit == 3)
                    {
                        imshow("number", num3);
                        cvMoveWindow("number", 380, 625);
                    }
                    else if (resultDigit == 4)
                    {
                        imshow("number", num4);
                        cvMoveWindow("number", 380, 625);
                    }
                    else if (resultDigit == 5)
                    {
                        imshow("number", num5);
                        cvMoveWindow("number", 380, 625);
                    }
                    lastNum = resultDigit;
                    cout << "This is: " << resultDigit << endl;
                    //////////////////////////////////////////////////////////////////////////

            }
        }
        // capture>>image;
        // namedWindow( "kinect", CV_WINDOW_AUTOSIZE );
        // imshow( "kinect", image );
        if (waitKey(30) >= 0)
            break;
    }
    waitKey(0);
    return 0;
}

int getContourAndHull(cv::Mat image) {
    // declarations
    Mat imageContour = image.clone(); // make a copy to work on
    vector< vector<Point> > contours; // all contours of image
    vector<Point> biggestContour; // the outermost contour
    vector<Point> approxContour; // to obtain polygon
    vector<int> hull; // convex points
    vector<int> filteredHulls; // filtered convex points
    vector<Point> defects; // concave points
    vector<Point> filteredDefects; // filtered concave points
    vector< Vec4i > hierarchy; // store informmation about contour
    double tmpContourArea1 = 0;
    double tmpContourArea2 = 0;
    double approxPolyDist = 15; // parameter to determine accuracy of appoximating polygon, the higher the less accurate
        int r = 5; // radius of point for whatever being drawn
    RotatedRect minRect; // to roughly find the center of palm...
    Scalar color(rand() & 255, rand() & 255, rand() & 255); // color of line when plotting (if it's on colored image) 

        bool zeroOrOne = true; // initialized to zero = true, turn to one if false
    // obtain contour
        cv::cvtColor(imageContour, imageContour, CV_RGB2GRAY);

        cout << "b4 contour" << endl;
    findContours(imageContour, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE,
        Point(0, 0));
    // get the biggest (outmermost) contour
    vector< vector<Point> >::iterator iter;
    iter = contours.begin();
    while (iter != contours.end()){
        tmpContourArea1 = contourArea(*iter, false);
        if (tmpContourArea1 > tmpContourArea2) {
            tmpContourArea2 = tmpContourArea1;
            biggestContour = *iter;
        }
        iter++;
    }
    // quit method if no contour found
    if (biggestContour.empty() == true) {
        return -1;
    }
    /// draw contours
    drawContours(imageContour, contours, 0, color, 1, 8, hierarchy);
    /// get approximation polygon
    //double approxAcc = arcLength(biggestContour, false) / 40; // picked @ perimeter * 1/400
    approxPolyDP(biggestContour, approxContour, approxPolyDist, false); // somewhere between 5-30 is good for apporixmation acc
        biggestContour = approxContour;
    /// find center of palm by the min enclosing rectangle
    minRect = minAreaRect(biggestContour);
    /// find convex and concave points
    convexHull(biggestContour, hull, false, false); /// get convex hull
    findConvexityDefects(biggestContour, hull, defects); /// get convexity defects
    /// filter convex/concave points
    filteredDefects = defects; // assign in case no filtering
    filteredDefects = filterDefects(defects, minRect);
    filteredHulls = hull;
    filteredHulls = filterHulls(hull, biggestContour, minRect);
    filteredHulls = elimNeighborHulls(filteredHulls, biggestContour);
    filteredHulls = filterHulls2(filteredHulls, filteredDefects, biggestContour, minRect);

    /// draw polygon
    // fillConvexPoly(imagePolygon, biggestContour, color, 8,0);
    /// draw enclosing rectangle and center
    // ellipse(imageContour, minRect,color,1,8);
    circle(imageContour, minRect.center, 2, color, 5, 8, 0);
    for (unsigned int i = 0; i < filteredDefects.size(); i++)
    {
        circle(imageContour, filteredDefects[i], r, color, 2, 8, 0);
    }
    for (unsigned int i = 0; i<filteredHulls.size(); i++) {
        circle(imageContour, biggestContour[filteredHulls[i]], r, color, 1, 8, 0);
    }

        /// plot
        display("Filtered convex and concave points and center", imageContour);
    cvMoveWindow("Filtered convex and concave points and center", 710, 100);
    // cout << "Unfiltered defects: " << defects.size() << endl;
    // cout << "Unfiltered hull size: " << hull.size() << endl;
    // cout << "FilteredHull size: " << filteredHulls.size() << endl;
    // cout << "FilteredDefect size: " << filteredDefects.size() << endl;
    // determine resulting number of digits, if no convex defect found, use convex hull to determine if it's 0 or 1
    if (filteredDefects.size() > 0) {
        return filteredHulls.size(); // it's not 0 nor 1
    }
    else { // given no concave points detected, figure out if it's zero or one

        float palmRadius;
        if (minRect.size.height <= minRect.size.width) {
            palmRadius = (minRect.size.height) / 2; // the normal case
        }
        else {
            palmRadius = (minRect.size.width) / 2;
        }
        for (unsigned int i = 0; i<filteredHulls.size(); i++) {
            if (biggestContour[filteredHulls[i]].y < (minRect.center.y - (palmRadius * 2))) { //multiply by two with trial and error
                    zeroOrOne = false; // this is one
            }
        }
        if (zeroOrOne == true){
            return 0;
        }
        else if (zeroOrOne == false) {
            return 1;
        }
        else { // no digit assigned
            return -1;
        }

    }
}

void findConvexityDefects(vector<Point>& contour, vector<int>& hull, vector<Point>& convexDefects){
    if (hull.size() > 0 && contour.size() > 0){
        CvSeq* contourPoints;
        CvSeq* defects;
        CvMemStorage* storage;
        CvMemStorage* strDefects;
        CvMemStorage* contourStr;
        CvConvexityDefect *defectArray = 0;

        strDefects = cvCreateMemStorage();
        defects = cvCreateSeq(CV_SEQ_KIND_GENERIC | CV_32SC2, sizeof(CvSeq), sizeof(CvPoint), strDefects);

        // transform our vector<Point> into a CvSeq* object of CvPoint.
        contourStr = cvCreateMemStorage();
        contourPoints = cvCreateSeq(CV_SEQ_KIND_GENERIC | CV_32SC2, sizeof(CvSeq), sizeof(CvPoint),contourStr);
        for (int i = 0; i<(int)contour.size(); i++) {
            CvPoint cp = { contour[i].x, contour[i].y };
            cvSeqPush(contourPoints, &cp);
        }
        // do the same thing with the hull index
        int count = (int)hull.size();
        int* hullK = (int*)malloc(count*sizeof(int));
        for (int i = 0; i<count; i++){hullK[i] = hull.at(i); }
        CvMat hullMat = cvMat(1, count, CV_32SC1, hullK);

        // calculate convexity defects
        storage = cvCreateMemStorage(0);
        defects = cvConvexityDefects(contourPoints, &hullMat, storage);
        defectArray = (CvConvexityDefect*)malloc(sizeof(CvConvexityDefect)*defects->total);
        cvCvtSeqToArray(defects, defectArray, CV_WHOLE_SEQ);

        // store defects points in the convexDefects parameter.
        for (int i = 0; i<defects->total; i++){
            CvPoint ptf;
            ptf.x = defectArray[i].depth_point->x;
            ptf.y = defectArray[i].depth_point->y;
            convexDefects.push_back(ptf);
        }

        // release memory
        cvReleaseMemStorage(&contourStr);
        cvReleaseMemStorage(&strDefects);
        cvReleaseMemStorage(&storage);


    }
}
void display(char* window, cv::Mat image)
{
    namedWindow(window, CV_WINDOW_AUTOSIZE);
    imshow(window, image);
}
vector<int> elimNeighborHulls(vector<int> inputIndex, vector<Point> inputPoints) {
    vector<int> tempfilteredHulls;
    float distance;
    float distThreshold = 20;
    if (inputIndex.size() == 0) {
        return inputIndex; // it's empty
    }
    if (inputIndex.size() == 1) {
        return inputIndex; // only one hull
    }
    for (unsigned int i = 0; i<inputIndex.size() - 1; i++) { // eliminate points that are close 

            distance = sqrt((float)pow((float)inputPoints[inputIndex[i]].x - inputPoints[inputIndex[i + 1]].x, 2)
            + pow((float)inputPoints[inputIndex[i]].y - inputPoints[inputIndex[i + 1]].y, 2));
        if (distance > distThreshold) { // set distance threshold to be 10
            tempfilteredHulls.push_back(inputIndex[i]);
        }
    }
    // get take of the last one, compare it with the first one
    distance = sqrt((float)pow((float)inputPoints[inputIndex[0]].x - inputPoints[inputIndex[inputIndex.size() -
        1]].x, 2) + pow((float)inputPoints[inputIndex[0]].y - inputPoints[inputIndex[inputIndex.size() - 1]].y, 2));
    if (distance > distThreshold) { // set distance threshold to be 10
        tempfilteredHulls.push_back(inputIndex[inputIndex.size() - 1]);
    }
    else if (inputIndex.size() == 2) { // the case when there are only two pts and they are together
        tempfilteredHulls.push_back(inputIndex[0]);
    }
    return tempfilteredHulls;
}
vector<int> filterHulls(vector<int> inputIndex, vector<Point> inputPoints, RotatedRect rect) {
    vector<int> tempFilteredHulls;
    float distThres = 20;
    for (unsigned int i = 0; i < inputIndex.size(); i++) {
        if (inputPoints[inputIndex[i]].y < (rect.center.y + distThres)) { // 10 being threshold height difference
                tempFilteredHulls.push_back(inputIndex[i]);
        }
    }
    return tempFilteredHulls;
}
vector<int> filterHulls2(vector<int> inputIndex, vector<Point> inputDefects, vector<Point> inputPoints,
    RotatedRect rect) {
    if (inputIndex.size() > 2 && inputDefects.size() > 1) {
        return inputIndex;
    }
    // only do filtering if there are less than 3 convex points
    vector<int> tempFilteredHulls;
    float palmRadius;
    if (rect.size.height <= rect.size.width) {
        palmRadius = (rect.size.height) / 2; // the normal case
    }
    else {
        palmRadius = (rect.size.width) / 2;
    }
    // for now ignore angle or rotation
    for (unsigned int i = 0; i < inputIndex.size(); i++) {
        if (inputPoints[inputIndex[i]].y < (rect.center.y - palmRadius)) { // 10 being threshold height difference
                tempFilteredHulls.push_back(inputIndex[i]);
        }
    }
    return tempFilteredHulls;
}

vector<Point> filterDefects(vector<Point> inputDefects, RotatedRect rect) {
    vector<Point> tempFilteredDefects;
    for (unsigned int i = 0; i <inputDefects.size(); i++) {
        if (inputDefects[i].y < (rect.center.y + 10)) {
            tempFilteredDefects.push_back(inputDefects[i]);
        }
    }
    return tempFilteredDefects;
}