Android+CV4.3.0: Why did NDK run 50% slower to detect the faces??

asked 2020-05-12 03:54:42 -0600

HomanH gravatar image

Kotlin: 17fps

fun drawFaceRectangle() {
    faceRects = MatOfRect()

    // image width & height
    val scrW = grayMat.width().toDouble()
    val scrH = grayMat.height().toDouble()

    val scale = getRatio(Size(scrW, scrH), 480) + 1
    val neighbor = 3

    rotateImage()

    faceDetector!!.detectMultiScale(
        grayMat, // image
        faceRects, // array
        scale, // scale
        neighbor, // min neighbors
        0, // flags,
        Size(30.0, 30.0), // min size
        Size(scrW, scrH) // max size
    )

    for (rect in faceRects!!.toArray()) {
        val x = rect.x.toDouble()
        val y = rect.y.toDouble()
        val rw = rect.width.toDouble() // rectangle width
        val rh = rect.height.toDouble() // rectangle height

        val w = x + rw
        val h = y + rh

        when (screenRotation) {
            0-> {
                rectFace(y, x, h, w, RED)
                drawDot(y, x, GREEN)
            }
            90-> {
                rectFace(x, y, w, h, RED)
                drawDot(x, y, GREEN)
            }
            180-> {
                // fix height
                val yFix = scrW - y
                val hFix = yFix - rh

                rectFace(yFix, x, hFix, w, YELLOW)
                drawDot(yFix, x, BLUE)
            }
            270-> {
                // fix height
                val yFix = scrH - y
                val hFix = yFix - rh

                rectFace(x, yFix, w, hFix, YELLOW)
                drawDot(x, yFix, BLUE)
            }
        }
    }
}

ndk: 9fps

void drawFaceRectangle(
    Mat &rgba,
    Mat &gray,
    String path,
    double ratio,
    int rotation) {
// width and height of frame
float scrW = (float)rgba.size().width;
float scrH = (float)rgba.size().height;

vector<cv::Rect> faces;
double scale = 1.0+ratio;
int neighbor = 3;

// fix orientation so it can be detected
rotateGray(gray, rotation);

/*
 * void detectMultiScale (
 * InputArray image,
 * std::vector< Rect > &objects,
 * double scaleFactor=1.1,
 * int minNeighbors=3,
 * int flags=0,
 * Size minSize=Size(),
 * Size maxSize=Size())
Detects objects of different sizes in the input image.
 The detected objects are returned as a list of rectangles.
 */
faceDetector.detectMultiScale(
        gray, // image
        faces, // array
        scale, // scale
        neighbor, // min neighbors
        0, // flags,
        Size(30, 30), // min size
        Size((int)scrW, (int)scrH) // max size
);

for (int i=0; i<faces.size(); i++) {
    Rect rect = faces[i];

    float x = (float)rect.x;
    float y = (float)rect.y;        
    float rw = (float)rect.width;
    float rh = (float)rect.height;

    float w = x + rw;
    float h = y + rh;
    float yFix, hFix;

    // draw rectangle
    switch (rotation) {
        case 0:
            rectFace(rgba, y, x, h, w, RED);
            drawDot(rgba, y, x, GREEN);
            break;

        case 90:
            rectFace(rgba, x, y, w, h, RED);
            drawDot(rgba, x, y, GREEN);
            break;

        case 180:
            // fix height
            yFix = scrW - y;
            hFix = yFix - rh;
            rectFace(rgba, yFix, x, hFix, w, YELLOW);
            drawDot(rgba, yFix, x, BLUE);
            break;

        case 270:
            // fix height
            yFix = scrH - y;
            hFix = yFix - rh;
            rectFace(rgba, x, yFix, w, hFix, YELLOW);
            drawDot(rgba, x, yFix, BLUE);
            break;

        default:
            string msg = "Error: wrong rotation data -- " +
                         to_string(rotation);
            lge(msg.c_str());
            break;
    }
}

}

You see, they are same function. The timing are much different.

    override fun onCameraFrame(inputFrame: CvCameraViewFrame?): Mat {
    if (pCounter == 1000) {
        evalPerfomance()
        finish()
    }
    // start time
    frameStart = System.currentTimeMillis()

    imageMat = inputFrame!!.rgba()

    if (!javaEnabled) {
        // C++
        /*
        if (!OpenCvNativeCall().convertGray(
                imageMat.nativeObjAddr,
                grayMat.nativeObjAddr
            )
        ) {
            throw CvException("Data is corrupted!")
        }
        */

        if (!OpenCvNativeCall().faceDetection(
                imageMat.nativeObjAddr,
                480,
                screenRotation,
                faceModel!!.absolutePath)) {
            longMsg(this, "Failed to load Face Detector!!!")
        }
    } else {
        grayMat = inputFrame!!.gray()

        // detect face rectangle
        drawFaceRectangle()
    }

    postText("$pCounter", 100.0, 100.0, YELLOW)

    // end time
    frameEnd = System.currentTimeMillis()
    frameRecord[pCounter] = frameEnd ...
(more)
edit retag flag offensive close merge delete

Comments

Have you compiled it with the RELEASE flag instead of the DEBUG flag?

HYPEREGO gravatar imageHYPEREGO ( 2020-05-12 05:04:25 -0600 )edit

I add

        externalNativeBuild {
        def opencvsdk = 'D:/Android/OpenCV-Android-Lib/OpenCV-android-sdk-430/sdk'
        cmake {
            // Sets a flag to enable format macro constants for the C compiler.
            cFlags "-D__STDC_FORMAT_MACROS"
            // Sets optional flags for the C++ compiler.
            cppFlags "-std=c++14 -frtti -fexceptions"
            // Passes optional arguments to CMake.
            arguments "-DANDROID_TOOLCHAIN=clang",
                    "-DANDROID_STL=c++_shared",
                    "-DANDROID_PLATFORM=android-21",
                    "DCMAKE_BUILD_TYPE=Release", <== I added
                    "-DOpenCV_DIR=" + opencvsdk
        }

No improvement.

HomanH gravatar imageHomanH ( 2020-05-12 12:47:59 -0600 )edit

Compare #1 issue: Kotlin(19-21Fps) : imageMat = inputFrame!!.gray() NDK(15Fps): cvtColor(mRgb, mRgb, CV_RGBA2GRAY); 4 to 6 Fps difference

HomanH gravatar imageHomanH ( 2020-05-12 12:50:41 -0600 )edit

The change could be explained if Kotlin was using the input frame from camera (in YUV_420_888 format), while C converted the frame to RGBA and then extracted the gray from it.

alexcohn gravatar imagealexcohn ( 2020-07-15 02:30:36 -0600 )edit