erohan's profile - activity

I understood that you want to indicate the areas on frame which have motion. You may use motion history approach or contour approach for this work.

Here is a motion history approach used sample method (optimized through forking the OpenCV sample - update_mhi) written with Java. You may call this method in a loop through passing the retrieved video frame's Mat reference to img parameter of method. // Fields... Mat motion, mhi, orient, mask, segmask;

private void update_mhi(Mat img, Mat dst, int diff_threshold) {
    if (videoSignalOkay) {
        double timestamp = (System.nanoTime() - startTime) / 1e9;
        int idx1 = last, idx2;
        Mat silh = Mat.zeros(size, CvType.CV_32FC1);
        cvtColor(img, buf[last], COLOR_BGR2GRAY);
        double angle, count;

        idx2 = (last + 1) % Constants.fps; // index of (last - (N-1))th frame
        last = idx2;

        silh = buf[idx2];
        if (silh == null || silh.empty()) {
            silh = Mat.zeros(size, CvType.CV_32FC1);
        }
        absdiff(buf[idx1], buf[idx2], silh);

        threshold(silh, silh, diff_threshold, 1, THRESH_BINARY);

        updateMotionHistory(silh, mhi, timestamp, Constants.mhiDuration);

        mhi.convertTo(mask, mask.type(), 255.0 / Constants.mhiDuration, (Constants.mhiDuration - timestamp) * 255.0 / (Constants.mhiDuration));
        dst.setTo(new Scalar(0));
        List<Mat> list = new ArrayList<Mat>(3);
        list.add(mask);
        list.add(Mat.zeros(mask.size(), mask.type()));
        list.add(Mat.zeros(mask.size(), mask.type()));

        merge(list, dst);

        calcMotionGradient(mhi, mask, orient, Constants.maxTimeDelta, Constants.minTimeDelta, 3);

        MatOfRect roi = new MatOfRect();
        segmentMotion(mhi, segmask, roi, timestamp, Constants.maxTimeDelta);

        int total = roi.toArray().length;
        Rect[] rois = roi.toArray();
        Rect comp_rect;
        Scalar color;

        for (int i = -1; i < total; i++) {
            if (i < 0) {
                comp_rect = new Rect(0, 0, videoWidth, videoHeight);
                color = new Scalar(255, 255, 255);
                magnitude = 100;
            } else {
                comp_rect = rois[i];
                if (comp_rect.width >= videoWidth/2 || comp_rect.height >= videoHeight/2 || 
                    comp_rect.width < Constants.recfactorx || comp_rect.height < Constants.recfactory || 
                    comp_rect.width + comp_rect.height < (Constants.recfactorx*Constants.recfactory)) // reject very small things
                    continue;
                color = new Scalar(0, 0, 255);
                magnitude = 30;
            }

            Mat silhROI = silh.submat(comp_rect);
            Mat mhiROI = mhi.submat(comp_rect);
            Mat orientROI = orient.submat(comp_rect);
            Mat maskROI = mask.submat(comp_rect);

            angle = calcGlobalOrientation(orientROI, maskROI, mhiROI, timestamp, Constants.mhiDuration);
            angle = 360.0 - angle;
            count = Core.norm(silhROI, NORM_L1);

            silhROI.release();
            mhiROI.release();
            orientROI.release();
            maskROI.release();
            if (count < comp_rect.height * comp_rect.width * Constants.pixelFactor ||
                comp_rect.width == videoWidth || comp_rect.height == videoHeight) {
                continue;
            } else {
                isAnyTrack = true;
            }
            Point center = new Point((comp_rect.x + comp_rect.width / 2), (comp_rect.y + comp_rect.height / 2));
            // Optimizer part! Compare the tracked thing in previous list to control empty area movement detection...
            /*if (isAnyTrack) {
                TrackingPojo trackPojo = new TrackingPojo(comp_rect, center, null); 
                if (notNeededTracking(trackPojo)) {
                    isAnyTrack = false;
                    continue;
                } else {
                    trackList.add(trackPojo);
                    // Show the warning icon...
                    ControlManager.getTheInstance().toggleAlert(true);
                }
            }*/
            circle(img, center, (int) Math.round(magnitude * 1.2), color, 3, LINE_AA, 0);
            Point thePoint = new Point(
                    Math.round(center.x - magnitude * Math.sin(angle * Math.PI / 180)),
                    Math.round(center.y + magnitude * Math.cos(angle * Math.PI / 180)));
            Point thePoint2 = new Point(thePoint.x, thePoint.y - (comp_rect.height / 2 + 10));
            Point thePoint3 = new Point(thePoint2.x, thePoint2.y - 15);
            Core.putText(img, "(" + center.x + ", " + center.y + ")", thePoint, 16, 0.50, new Scalar(255, 0, 0 ...

I tried this CPP code and adopt it to Java side as below. I changed some of the lines because no need of them. So, the last state of it with Java goes here. I used *.mp4 file but i resize the frame before processing because more than a 640x480 resolution, processing is too slow. I prefer to dynamic resize the frame to 320x240 or max. 640x480 before processing :

// Fields...

 Mat fg;     
 Mat blurred;
 Mat thresholded;
 Mat gray;
 Mat blob;
 Mat bgmodel;                                            

 BackgroundSubtractorMOG2 bgs;      

 List<MatOfPoint> contours = new ArrayList<MatOfPoint>();            

 CascadeClassifier body;

// Initializer part (you may put it to a private method and call it from constructor)...

     bgs = new BackgroundSubtractorMOG2();
     bgs.setInt("nmixtures", 3);
     bgs.setInt("history", 1000);
     bgs.setInt("varThresholdGen", 15);
     //bgs.setBool("bShadowDetection", true);   // This feautee does not exist on OpenCV-2.4.11
     bgs.setInt("nShadowDetection",0);                               
     bgs.setDouble("fTau", 0.5);                                         

     body=new CascadeClassifier("mapbase/hogcascade_pedestrians.xml"); // Standart sample pedestrian trained data file
      if(body.empty())  
      {  
           System.out.println("--(!)Error loading hogcascade file\n");  
            return;  
      }  
      else  
      {  
                 System.out.println("Pedestrian classifier loaded up");  
      }

// Frame reader part...

VideoCapture cam = new VideoCapture("D:\\Sample_Videos\\pedestrian-much.mp4"); // int 0 : Webcam, String : rtsp://IP:PORT/PATH for ip camera etc.
        if(cam.isOpened())  
          {  
           Thread.sleep(500); /// This one-time delay allows the Webcam to initialize itself  
           while( true )  
           {  
             cam.read(cam_image);  
             if(!cam_image.empty())  
              {            
                   // Thread.sleep(50); // If too much computation, you may give a time to CPU        
                   Imgproc.resize(cam_image, resizeImage, new Size(320, 240));
                   //Apply the classifier to the captured image  
                   renderImage=faceDetector.detect(resizeImage);
                   //Display the image  
                   BufferedImage img = facePanel.matToBufferedImage(renderImage);
                   facePanel.repaint();
                   renderImage.release();
              }  
              else  
              {   
                   System.out.println(" --(!) No captured frame from video source!");
                   break;   
              }  
             }  
            }
           cam.release(); //release the webcam   }

// Detect method which is called in while read frame loop... public Mat detect(Mat inputframe){

        Imgproc.resize(webcam_image, resizeImage, new Size(320, 240));
        Mat mRgba=new Mat();
        inputframe.copyTo(mRgba); // The frame read from video source (camera, *.mp4, *.avi file etc)
        blurred = new Mat();
        fg = blurred.clone();
        thresholded = fg.clone();
        blob = thresholded.clone();
        gray = blob.clone();

        Imgproc.cvtColor(mRgba,gray,Imgproc.COLOR_BGR2GRAY);  // Before processing we convert to gray
        Imgproc.GaussianBlur(gray,blurred,new Size(3,3),0,0,Imgproc.BORDER_DEFAULT); // Avoid from unnecessary pixels

             bgs.apply(blurred,fg); // Extract background from scene                             

            Imgproc.threshold(fg,thresholded,70.0f,255,Imgproc.THRESH_BINARY); // Apply max and min threshold

        Mat elementCLOSE = new Mat(5,5,CvType.CV_8U,new Scalar(255,255,255));
        Imgproc.morphologyEx(thresholded,thresholded,Imgproc.MORPH_CLOSE,elementCLOSE); // Do a body figure analysis (for object scene change optimization)

        Mat v = new Mat();
        Imgproc.findContours(thresholded,contours,v, Imgproc.RETR_EXTERNAL,Imgproc.CHAIN_APPROX_SIMPLE); // Find the changes of scene (the moving parts)

        Imgproc.equalizeHist(gray, gray); // Balance the contrast to gain proper result

        int cmin = 500; 
        int cmax = 80000;
        MatOfRect rects = new MatOfRect();
        if (contours != null && !contours.isEmpty()) {
            //if (contours.size() >= 50) { // You may use this control and a threshold value if too much contour you have
                //Imgproc.drawContours(mRgba,contours,-1,new Scalar(0, 0, 255),2); // If you want you ...

Hi,

I think your code is right. Sample Java code is as below. I also tried on lots of environments like you. With Gstream ugly & good codecs on Ubuntu 12.04, H.264 streaming is working with a latency without any frame corruption.

The code is like that :

public static void main(String[] args) {

    ImageIO.setUseCache(false);
    JFrame jframe = new JFrame("ETS New Algorithm - Suzuki85");
    jframe.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
    JLabel vidpanel = new JLabel();
    jframe.setContentPane(vidpanel);
    jframe.setSize(1280, 720);
    jframe.setVisible(true);

    Mat frame = new Mat();
    VideoCapture camera = new VideoCapture("rtsp://192.168.1.229:554/av0_1");
    while (true) {
        if (camera.read(frame)) {
                ImageIcon image = new ImageIcon(Mat2bufferedImage(frame));
                vidpanel.setIcon(image);
                vidpanel.repaint();
            }
    }
}

public static BufferedImage Mat2bufferedImage(Mat image) {
    MatOfByte bytemat = new MatOfByte();
    Highgui.imencode(".jpg", image, bytemat);
    byte[] bytes = bytemat.toArray();
    InputStream in = new ByteArrayInputStream(bytes);
    BufferedImage img = null;
    try {
        img = ImageIO.read(in);
    } catch (IOException e) {
        e.printStackTrace();
    } finally {
        bytemat.release();
        bytes = null;
    }
    return img;
}

But if you need to process data (for example; motion tracking, histogram equalization etc. i mean Highgui procs) you need the Mat reference and this reference can not be gotten without FFMPEG. And OpenCV which is built from source with FFMPEG ability uses CPU more than 100% on Linux.So, you can not run properly on Linux. (Some folks said that they were succedded here and was me..:) but FFMPEG Needed not GStreamer but i couldn't get a stable environment)

So, i choosed to go on Win7 64 bit platform wt. OpenCV 2.4.11. Currently i can get MJPEG streming wt. 1920x1080 resolution at 16384 bitrate from my ip camera. Its quality is near to H.264 but data cost is twice of H.264. (H.264 = 500K, MJPEG = 800K) I also downloaded source code of OpenCV2.4.11 and open the project in Visual Studio .NET. I succedded to compile it and currently i am investigating the source code at CPP side if i will be able to find anything, i may write to here.

I saw that lots of codes exist at CPP side about comparing the FFMPEG versions. Sample like that :

And lastly, you are right that MPEG4 or H.264 encoded files on disk are played without any error.

IP Camera H.264 640x480 RTSP Sample on Win7 64 bit and OpenCV2.4.11 (wt. Rendering errors)

H.264 Encoded MP4 File 640x480 Sample on Win7 64 bit and OpenCV2.4.11 (wt. out errors)

I guess the problem is related to network side. If it was related to FFMPEG, the H.264 encoded file also rendered wt. errors. GStreamer on Linux can render it wt. out errors with a latency. So, I think if grapb - retrieve cylec can wait some time about being sure that the frame is gotten properly than there will be no problem. I mean i can accept 2-3 seconds delay if Mat reference will be able to be given after reading the H..264 encoded video data.

See you on ... (more)