KevinWorkman's profile - activity

I'm using Android4OpenCV to do some live image processing, and I'd like to use the smallest resolution the camera can offer. The default resolution is the largest the camera can offer.

I'm looking at the 3rd example, which allows the user to change resolutions via a menu. I'd like to modify that example to change the resolution at startup instead of requiring the user go through the menu. To do that, I simply add two lines to the otherwise empty onCameraViewStarted() function:

public void onCameraViewStarted(int width, int height) {
    android.hardware.Camera.Size res = mOpenCvCameraView.getResolutionList().get(mOpenCvCameraView.getResolutionList().size()-1);
    mOpenCvCameraView.setResolution(res);
}

And the thing is, this works perfectly fine on my Galaxy Nexus, running Android 4.2.2. The app starts up, and the resolution is set correctly.

However, when I run the exact same app on a Nexus 7 tablet, running Android 5.1, the app hangs on the call to setResolution(). Actually it works okay one time, but then hangs the second time you try to run it- even if you completely exit the app, remove it from the running apps, or restart the device. Other users are reporting the same error as well, so it's not just the Nexus 7 device- in fact, my Galaxy Nexus seems to be the only device where this works.

Specifically, the application goes into the setResolution() function, which then calls org.opencv.android.JavaCameraView.disconnectCamera(), which looks like this:

(Note: this code is internal to the OpenCV4Android library, this is not my code)

protected void disconnectCamera() {
    /* 1. We need to stop thread which updating the frames
     * 2. Stop camera and release it
     */
    Log.d(TAG, "Disconnecting from camera");
    try {
        mStopThread = true;
        Log.d(TAG, "Notify thread");
        synchronized (this) {
            this.notify();
        }
        Log.d(TAG, "Wating for thread");
        if (mThread != null)
            mThread.join();
    } catch (InterruptedException e) {
        e.printStackTrace();
    } finally {
        mThread =  null;
    }

    /* Now release camera */
    releaseCamera();
}

Looking at the logs, I can see that the thread gets stuck on the synchronized(this) line. The only other thing that synchronizes on that Object is the inner JavaCameraView.CameraWorker class, which is the mThread variable in the above code, started by the JavaCameraView class:

(Note: this code is internal to the OpenCV4Android library, this is not my code)

private class CameraWorker implements Runnable {

    public void run() {
        do {
            synchronized (JavaCameraView.this) {
                try {
                    JavaCameraView.this.wait();
                } catch (InterruptedException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                }
            }

            if (!mStopThread) {
                if (!mFrameChain[mChainIdx].empty())
                    deliverAndDrawFrame(mCameraFrame[mChainIdx]);
                mChainIdx = 1 - mChainIdx;
            }
        } while (!mStopThread);
        Log.d(TAG, "Finish processing thread");
    }
}

I've tried futzing with that code, changing the notify() to notifyAll(), and maintaining a List of CameraWorker threads and joining each one. But no matter what, the app still hangs at the disconnectCamera() call.

My questions are:

• How can I modify the third OpenCV4Android example so that its resolution is set at startup?

• What is causing the app to hang?

• Why does this work on some devices but not others?

I've also posted ... (more)

@HaDang pointed me to these links:

http://www.walking-productions.com/no...

https://code.google.com/p/javacv/sour...

That example uses a Java wrapper of FFMPEG to do the video recording. This project is a pretty useful starting point for anybody wanting to do the same: https://github.com/vanevery/JavaCV_0....

I took that above project and hammered it into my example. It's very messy, but it works:

package com.example.videotest;


import java.io.File;
import java.io.IOException;
import java.nio.ShortBuffer;
import java.util.List;

import org.opencv.android.BaseLoaderCallback;
import org.opencv.android.LoaderCallbackInterface;
import org.opencv.android.OpenCVLoader;
import org.opencv.android.CameraBridgeViewBase.CvCameraViewFrame;
import org.opencv.android.CameraBridgeViewBase.CvCameraViewListener2;
import org.opencv.core.Mat;
import org.opencv.core.Scalar;
import org.opencv.imgproc.Imgproc;

import com.googlecode.javacv.FFmpegFrameRecorder;
import com.googlecode.javacv.FrameRecorder.Exception;
import com.googlecode.javacv.cpp.opencv_core.IplImage;

import android.app.Activity;
import android.media.AudioFormat;
import android.media.AudioRecord;
import android.media.MediaRecorder;
import android.media.MediaScannerConnection;
import android.os.Bundle;
import android.os.Environment;
import android.util.Log;
import android.view.SurfaceView;
import android.view.View;
import android.view.WindowManager;
import android.widget.Toast;

public class MainActivity extends Activity implements CvCameraViewListener2{

    private CameraView cameraView;
    private Mat edgesMat;
    private final Scalar greenScalar = new Scalar(0,255,0);
    private int resolutionIndex = 0;

    private IplImage videoImage = null;

    boolean recording = false;
    private volatile FFmpegFrameRecorder recorder;

    private int sampleAudioRateInHz = 44100;
    private int imageWidth = 320;
    private int imageHeight = 240;
    private int frameRate = 30;

    private Thread audioThread;
    volatile boolean runAudioThread = true;
    private AudioRecord audioRecord;
    private AudioRecordRunnable audioRecordRunnable;

    private String ffmpeg_link;

    long startTime = 0;

    private String LOG_TAG = "VideoTest";

    private BaseLoaderCallback mLoaderCallback = new BaseLoaderCallback(this) {
        @Override
        public void onManagerConnected(int status) {
            switch (status) {
            case LoaderCallbackInterface.SUCCESS:
                Log.i("VideoTest", "OpenCV loaded successfully");
                cameraView.enableView();
                break;
            default:
                super.onManagerConnected(status);
                break;
            }
        }
    };


    public void onCreate(Bundle savedInstanceState) {

        super.onCreate(savedInstanceState);
        getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);

        setContentView(R.layout.activity_main);

        cameraView = (CameraView) findViewById(R.id.cameraView);
        cameraView.setVisibility(SurfaceView.VISIBLE);
        cameraView.setCvCameraViewListener(this);
    }

    private void initRecorder() {
        Log.w(LOG_TAG,"initRecorder");

        int depth = com.googlecode.javacv.cpp.opencv_core.IPL_DEPTH_8U;
        int channels = 4;

        // if (yuvIplimage == null) {
        // Recreated after frame size is set in surface change method
        videoImage = IplImage.create(imageWidth, imageHeight, depth, channels);
        //yuvIplimage = IplImage.create(imageWidth, imageHeight, IPL_DEPTH_32S, 2);

        Log.v(LOG_TAG, "IplImage.create");
        // }

        File videoFile = new File(getExternalFilesDir(null), "VideoTest/images/video.mp4");
        boolean mk = videoFile.getParentFile().mkdirs();
        Log.v(LOG_TAG, "Mkdir: " + mk);

        boolean del = videoFile.delete();
        Log.v(LOG_TAG, "del: " + del);

        try {
            boolean created = videoFile.createNewFile();
            Log.v(LOG_TAG, "Created: " + created);
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }

        ffmpeg_link = videoFile.getAbsolutePath();
        recorder = new FFmpegFrameRecorder(ffmpeg_link, imageWidth, imageHeight, 1);
        Log.v(LOG_TAG, "FFmpegFrameRecorder: " + ffmpeg_link + " imageWidth: " + imageWidth + " imageHeight " + imageHeight);

        recorder.setFormat("mp4");
        Log.v(LOG_TAG, "recorder.setFormat(\"mp4\")");

        recorder.setSampleRate(sampleAudioRateInHz);
        Log.v(LOG_TAG, "recorder.setSampleRate(sampleAudioRateInHz)");

        // re-set in the surface changed method as well
        recorder.setFrameRate(frameRate);
        Log.v(LOG_TAG, "recorder.setFrameRate(frameRate)");

        // Create audio recording thread
        audioRecordRunnable = new AudioRecordRunnable();
        audioThread = new Thread(audioRecordRunnable);
    }

    @Override
    public void ...

I originally asked this question on StackOverflow here, but I haven't received any replies or answers.

My goal is to do a couple things:

1. Use OpenCV and the JavaCameraView to process frames from the phone's camera feed
2. Enable recording of that processed video as it happens

I have both of them working, but the way I had to implement number 2 is ridiculous:

1. For each frame, write the processed Mat as an image file.
2. When the recording stops, use JCodec's Android library to stitch them together into a video file.

That works, but it comes with a ton of drawbacks: the framerate drops unbearably low during a recording, and the stitching step takes about half a second per frame, and runs out of memory for videos more than a couple seconds long- and that's after I lower my camera's resolution to make sure the images are as small as possible. Even then, the video framerate is way out of whack with reality, and the video looks insanely sped up.

This seems ridiculous for a lot of reasons, so my question is: is there a better way to do this?

Here's a little example if anybody wants to run it. This requires the OpenCV Android project available here, and the JCodec Andrpod project available here.

Manifest.xml:

<manifest xmlns:android="http://schemas.android.com/apk/res/android" package="com.example.videotest" android:versioncode="1" android:versionname="1.0" &gt;<="" p="">

<uses-sdk
    android:minSdkVersion="8"
    android:targetSdkVersion="22"
/>

<application
    android:allowBackup="true"
    android:icon="@drawable/ic_launcher"
    android:label="@string/app_name"
    android:theme="@android:style/Theme.NoTitleBar.Fullscreen" >

    <activity
        android:name=".MainActivity"
        android:screenOrientation="landscape"
        android:configChanges="orientation|keyboardHidden|screenSize"
        android:label="@string/app_name" >
        <intent-filter>
            <action android:name="android.intent.action.MAIN" />
            <category android:name="android.intent.category.LAUNCHER" />
        </intent-filter>
    </activity>

</application>

<uses-permission android:name="android.permission.CAMERA"/>
<uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />

</manifest>

MainActivity:

package com.example.videotest;

import java.io.File;
import java.util.List;

import org.opencv.android.BaseLoaderCallback;
import org.opencv.android.LoaderCallbackInterface;
import org.opencv.android.OpenCVLoader;
import org.opencv.android.CameraBridgeViewBase.CvCameraViewFrame;
import org.opencv.android.CameraBridgeViewBase.CvCameraViewListener2;
import org.opencv.core.Mat;
import org.opencv.core.Scalar;
import org.opencv.imgproc.Imgproc;

import android.app.Activity;
import android.media.MediaScannerConnection;
import android.os.Bundle;
import android.os.Environment;
import android.util.Log;
import android.view.SurfaceView;
import android.view.View;
import android.view.WindowManager;
import android.widget.Toast;

public class MainActivity extends Activity implements CvCameraViewListener2{

    private CameraView cameraView;
    private Mat edgesMat;
    private final Scalar greenScalar = new Scalar(0,255,0);
    private int resolutionIndex = 0;
    private MatVideoWriter matVideoWriter = new MatVideoWriter();


    private BaseLoaderCallback mLoaderCallback = new BaseLoaderCallback(this) {
        @Override
        public void onManagerConnected(int status) {
            switch (status) {
            case LoaderCallbackInterface.SUCCESS:
            {
                Log.i("VideoTest", "OpenCV loaded successfully");

                cameraView.enableView();

            } break;
            default:
            {
                super.onManagerConnected(status);
            } break;
            }
        }
    };


    public void onCreate(Bundle savedInstanceState) {

        super.onCreate(savedInstanceState);
        getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);

        setContentView(R.layout.activity_main);

        cameraView = (CameraView) findViewById(R.id.cameraView);
        cameraView.setVisibility(SurfaceView.VISIBLE);
        cameraView.setCvCameraViewListener ...