Revision history [back]

You can build your own opencv_traincascade-generated classifier tester. Here's a code sample that processes samples created with opencv_createsamples. A true positive is obtained when the ratio between the intersection of the two rectangles (classified rectangle and test rectangle, i.e. where the object actually is in the image) and the test rectangle areas is greater than 0.5. This is of course arbitrary.

#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/objdetect/objdetect.hpp>
#include <iostream>
#include <stdio.h>
#include <time.h>

using namespace cv;
using namespace std;

// signatures
vector<std::string> split(const std::string &s, char delim);
void split(const std::string &s, char delim, std::vector<std::string> &elems);

string cascadeClassifierName = "cascade.xml";
CascadeClassifier cascadeClassifier;
string windows_name = "Detection";
String folder = "images";
int truePositives = 0;
int falseNegatives = 0;
int falsePositives = 0;
int repeatedTruePositives = 0; // Targets detected more than once. Actually part of false positives. Counted separately so as to give a clue to what's wrong with FPs.
int trueNegatives = 0;

int populationSize = 0;
int positives = 0;
int negatives = 0;

int main(int argc, char** argv)
{
    clock_t timeStart = clock();
    // Error handling
    if (!cascadeClassifier.load(cascadeClassifierName))
    {
        cout << "Error loading cascade classifier" << endl;
        return -1;
    }

    vector<cv::String> filenames;
    glob(folder, filenames);
    populationSize = filenames.size();
    for (size_t i = 0; i < populationSize; i++)
    {
        // Each file contains AT MOST ONE target and contains x, y, width and height of the surrounding rectangle test_rect
        // (all set to zero in case it does not contain it)
        Mat frame = imread(filenames[i]);
        vector<std::string>  dimensions = split(filenames[i], '_');
        int x = std::stoi(dimensions[1]);
        int y = std::stoi(dimensions[2]);
        int width = std::stoi(dimensions[3]);
        int height = std::stoi(dimensions[4]);
        Rect test_rect = Rect(x, y, width, height);
        double test_area = test_rect.area();
        int targetsInImage = (width + height > 0) ? 1 : 0;
        if (targetsInImage >= 1) { positives++; }
        else { negatives++; }

        // Detect objects and store them in objectsDetected. Also draw rectangle around true object
        Mat frame_gray;
        cvtColor(frame, frame_gray, CV_BGR2GRAY);
        equalizeHist(frame_gray, frame_gray);
        std::vector<Rect>* objectsDetected = new vector<Rect>;
        cascadeClassifier.detectMultiScale(frame_gray, *objectsDetected, 1.4, 2, 0 | CV_HAAR_SCALE_IMAGE, Size(80, 80));
        // rectangle(frame, test_rect, Scalar(0, 255, 0), 1, 8, 0);

        // Determine how many detected rectangles intersect test rectangle and how many don't
        int goodIntersectionCount = 0;
        size_t badIntersectionCount = 0;
        if (targetsInImage > 0)
        {
            for (size_t i = 0; i < (*objectsDetected).size(); i++)
            {
                Rect class_rect = (*objectsDetected)[i];
                Rect intersect = class_rect & test_rect;
                if (intersect.area() / test_area > 0.5) goodIntersectionCount++;
                else badIntersectionCount++;
                // rectangle(frame, class_rect, Scalar(255, 0, 0), 1, 8, 0);
            }
        }
        else
        {
            badIntersectionCount += (*objectsDetected).size();
        }
        // Translate these results into TP, TN, FP, FN and other stats
        truePositives += goodIntersectionCount >= targetsInImage ? targetsInImage : goodIntersectionCount; // num(good detected rects) >= num(actual targets) ? TP = num(actual targets), else TP = num(good detected rects)
        trueNegatives += (targetsInImage == 0 && (*objectsDetected).size() == 0) ? 1 : 0; // only add by one if image does not contain target object
        falseNegatives += std::max(targetsInImage - goodIntersectionCount, 0); // num(actual targets) - num(good detected rectangles) -> FN
        falsePositives += badIntersectionCount; // Detected rectangles out of test region -> FPs
        repeatedTruePositives += std::max(goodIntersectionCount - targetsInImage, 0);
        /*while (true)
        {
        imshow(windows_name, frame);
        int c = waitKey(5000);
        if ((char)c == 'q') { break; }
        }*/
    }
    cout << "POPULATION SIZE: " << populationSize << endl;
    cout << "POSITIVES:       " << positives << endl;
    cout << "NEGATIVES:       " << negatives << endl;
    cout << "======RESULTS======" << '\r' << endl;
    cout << "True Positives   " << truePositives << endl;
    cout << "True Negatives   " << trueNegatives << endl;
    cout << "False Positives  " << falsePositives + repeatedTruePositives << " actual: " << falsePositives << " repeated true pos: " << repeatedTruePositives << endl;
    cout << "False Negatives  " << falseNegatives << endl;
    cout << "=======STATS======" << endl;
    if (positives + negatives != 0) cout << "Accuracy         " << (double)(truePositives + trueNegatives) / (double)(positives + negatives) << endl;
    if (truePositives + falsePositives != 0) cout << "Precision        " << (double)(truePositives) / (double)(truePositives + falsePositives) << endl;
    if (truePositives + falseNegatives != 0) cout << "Recall           " << (double)(truePositives) / (double)(truePositives + falseNegatives) << endl;
    cout << "Time taken:      " << (double)(clock() - timeStart) / CLOCKS_PER_SEC << "s" << endl;
    return 0;
}

std::vector<std::string> split(const std::string &s, char delim)
{
    std::vector<std::string> elems;
    split(s, delim, elems);
    return elems;
}

void split(const std::string &s, char delim, std::vector<std::string> &elems)
{
    std::stringstream ss;
    ss.str(s);
    std::string item;
    while (std::getline(ss, item, delim))
    {
        elems.push_back(item);
    }
}