Revision history [back]

Thread scheme is too poor because grab and process aren't sync. Depends on grab/process timing you will lost some frames or process same frame twice. You could implements some sync using mutex||semaphores||std::atomic<bool> but it's would be better if you use a thread safe circular buffer with copy constructor, in this case 1 producer 1 consumer: Grab/Push and Process/Pop.

A simple implementation could be done use std:queue but isn't memory effective because you will create a new Mat for each grab (even if STL and Mat optimization will recycle the memory). look at this simple example and try moving the window "Image main"

#include <thread>
#include <mutex>
#include <queue>
#include <atomic>

using namespace std;
using namespace cv;

std::queue<cv::Mat> buffer;
std::mutex mtxCam;
std::atomic<bool> grabOn; // this is lock free

void GrabTask(VideoCapture *cap)
{
    cv::Mat tmp;
    *cap >> tmp;   // grab a frame for get size and type

    // we wont lost the 1st frame
    mtxCam.lock();
    buffer.push(tmp);
    mtxCam.unlock();

    while (grabOn.load()==true)
    {
        mtxCam.lock();

        // AVOID below because buffer.push will not copy the frame but create just a ref
        // than the frame it will be overwritten on next grab
        //*cap >> tmp; buffer.push(tmp);

        // this will create a new Mat for each grab
        buffer.push(cv::Mat(tmp.size(), tmp.type()));    // size and type must never change

        *cap >> buffer.back();        // grab directly into the buffer
        buffer.back().copyTo(tmp);    // just for imshow 
        mtxCam.unlock();

        imshow("Image thread", tmp);
        waitKey(1);    //just for imshow
    }
}
void ProcessFrame(const Mat &src)
{
    imshow("Image main", src);

}
int MainProc() {
    Mat frame;
    VideoCapture cap;
    cap.open(0);
    if (!cap.isOpened()) // check if we succeeded
        return -1;

    grabOn.store(true);                // set the grabbing control variable
    thread t(GrabTask, &cap);          // start the grabbing task
    int bufSize;
    while (true) {

        mtxCam.lock();                // lock memory for exclusive access
        bufSize = buffer.size();      // check how many frames are waiting 
        if (bufSize > 0)              // if some 
        {
            frame = buffer.front();    // get the oldest grabbed frame (queue=FIFO)
            // frame is a reference of a cv::Mat created with buffer.push(...
            // it should survive after pop because of cv::Mat memory reference counting
            // in case take a copy of it:
            // frame.copyTo(image);
            buffer.pop();            // release the queue item
        }
        mtxCam.unlock();            // unlock the memory

        if (bufSize > 0)            // if a new frame is available
        {
            ProcessFrame(frame);    // process it
            bufSize--;
        }

        // if bufSize is increasing means that process time is too slow regards to grab time 
        // may be you will have out of memory soon
        cout << endl << "frame to process:" << bufSize;

        if (waitKey(1) >= 0)        // press any key to terminate
        {
            grabOn.store(false);    // stop the grab loop 
            t.join();               // wait for the grab loop

            cout << endl << "Flushing buffer of:" << bufSize << " frames...";
            while (!buffer.empty())    // flushing the buffer
            {
                frame = buffer.front();    
                buffer.pop();
                ProcessFrame(frame);
            }
            cout << "done"<<endl;
            break; // exit from process loop
        }
    }
    cout << endl << "Press Enter to terminate"; cin.get();
    return 0;
}

Thread scheme is too poor because grab and process aren't sync. Depends on grab/process timing you will lost some frames or process same frame twice. You could implements some sync using mutex||semaphores||std::atomic<bool> but it's would be better if you use a thread safe circular buffer with copy constructor, in this case 1 producer 1 consumer: Grab/Push and Process/Pop.

A simple implementation could be done use std:queue but isn't memory effective because you will create a new Mat for each grab (even if STL and Mat optimization will recycle the memory). look at this simple example and try moving the window "Image main"

#include <thread>
#include <mutex>
#include <queue>
#include <atomic>

using namespace std;
using namespace cv;

std::queue<cv::Mat> buffer;
std::mutex mtxCam;
std::atomic<bool> grabOn; // this is lock free

void GrabTask(VideoCapture *cap)
{
    cv::Mat tmp;
    *cap >> tmp;   // grab a frame for get size and type

    // we wont lost the 1st frame
    mtxCam.lock();
    buffer.push(tmp);
    mtxCam.unlock();

    while (grabOn.load()==true)
    {
        mtxCam.lock();

        // AVOID below because buffer.push will not copy the frame but create just a ref
        // than the frame it will be overwritten on next grab
        //*cap >> tmp; buffer.push(tmp);

        // this will create a new Mat for each grab
        buffer.push(cv::Mat(tmp.size(), tmp.type()));    // size and type must never change

        *cap >> buffer.back();        // grab directly into the buffer
        buffer.back().copyTo(tmp);    // just for imshow 
        mtxCam.unlock();

        putText(tmp, "THREAD FRAME", Point(10, 10), CV_FONT_HERSHEY_PLAIN, 1, cv::Scalar(0, 255, 0));
        imshow("Image thread", tmp);
        waitKey(1);    //just for imshow
    }
}
void ProcessFrame(const Mat &src)
{
    if(src.empty()) return;
    putText(src, "PROC FRAME", Point(10, 10), CV_FONT_HERSHEY_PLAIN, 1, cv::Scalar(0, 255, 0));
    imshow("Image main", src);
 }
int MainProc() {
    Mat frame;
    VideoCapture cap;
    cap.open(0);
    if (!cap.isOpened()) // check if we succeeded
        return -1;

    grabOn.store(true);                // set the grabbing control variable
    thread t(GrabTask, &cap);          // start the grabbing task
    int bufSize;
    while (true) {

        mtxCam.lock();                // lock memory for exclusive access
        bufSize = buffer.size();      // check how many frames are waiting 
        if (bufSize > 0)              // if some 
        {
            frame = buffer.front();    // get the oldest grabbed frame (queue=FIFO)
            // frame is a // reference of a cv::Mat created with buffer.push(...
            // it to buffer.front() should survive be valid after 
            // pop because of cv::Mat memory reference counting
            // in case take but it's content can change after unlock, better to keep a copy of it:
            // frame.copyTo(image);
copy
            // an alternative is to unlock after processing (this will lock grabbing)
            buffer.front().copyTo(frame);   // get the oldest grabbed frame (queue=FIFO)
            buffer.pop();            // release the queue item
        }
        mtxCam.unlock();            // unlock the memory

        if (bufSize > 0)            // if a new frame is available
        {
            ProcessFrame(frame);    // process it
            bufSize--;
        }

        // if bufSize is increasing means that process time is too slow regards to grab time 
        // may be you will have out of memory soon
        cout << endl << "frame to process:" << bufSize;

        if (waitKey(1) >= 0)        // press any key to terminate
        {
            grabOn.store(false);    // stop the grab loop 
            t.join();               // wait for the grab loop

            cout << endl << "Flushing buffer of:" << bufSize << " frames...";
            while (!buffer.empty())    // flushing the buffer
            {
                frame = buffer.front();    
                buffer.pop();
                ProcessFrame(frame);
            }
            cout << "done"<<endl;
            break; // exit from process loop
        }
    }
    cout << endl << "Press Enter to terminate"; cin.get();
    return 0;
}

Thread scheme is too poor because grab and process aren't sync. Depends on grab/process timing you will lost some frames or process same frame twice. You could implements some sync using mutex||semaphores||std::atomic<bool> but it's would be better if you use a thread safe circular buffer with copy constructor, in this case 1 producer 1 consumer: Grab/Push and Process/Pop.

A simple implementation could be done use std:queue but isn't memory effective because you will create a new Mat for each grab (even if STL and Mat optimization will recycle the memory). look at this simple example and try moving the window "Image main"

EDIT: keep a copy of grabbed frame is safer

#include <thread>
#include <mutex>
#include <queue>
#include <atomic>

using namespace std;
using namespace cv;

std::queue<cv::Mat> buffer;
std::mutex mtxCam;
std::atomic<bool> grabOn; // this is lock free

void GrabTask(VideoCapture GrabTask(cv::VideoCapture *cap)
{
    cv::Mat tmp;
    *cap >> tmp;   // grab  //catch a frame for get size and type

    // we wont lost cv::VideoWriter vw;
    OpenVideo(vw, tmp.size(), "../img/file2.avi");

    // keep the 1st frame
    mtxCam.lock();
    buffer.push(tmp);
buffer.push(cv::Mat(tmp.size(), tmp.type()));
    tmp.copyTo(buffer.back());
    mtxCam.unlock();
     while (grabOn.load()==true)
(grabOn.load() == true) //this is lock free
    {
        mtxCam.lock();

        // AVOID below because buffer.push //grab will not copy the frame but create just a ref
        // than the frame it will wait for cam FPS
        //keep grab out of lock so that 
        //idle time can be overwritten on next grab
        //*cap used by other threads
        *cap >> tmp; buffer.push(tmp);

//this will wait for cam FPS

        if (tmp.empty()) continue;

        //get lock only when we have a frame
        mtxCam.lock();
        // buffer.push_back(tmp) stores item by reference than avoid
        // this will create a new Mat cv::Mat for each grab
        buffer.push(cv::Mat(tmp.size(), tmp.type()));    // size and type must never change

        *cap >> buffer.back();        // grab directly into the buffer
        buffer.back().copyTo(tmp);    // just for imshow 
tmp.type()));
        tmp.copyTo(buffer.back());
        mtxCam.unlock();

        putText(tmp, bool show = true;
        if (show)
        {
            // SHOW THE FRAME
            int font = CV_FONT_HERSHEY_PLAIN;
            cv::putText(tmp, "THREAD FRAME", Point(10, cv::Point(10, 10), CV_FONT_HERSHEY_PLAIN, font, 1, cv::Scalar(0, 255, 0));
         imshow("Image thread", tmp);
        waitKey(1);     cv::waitKey(1);    //just for imshow
        }
        vw << tmp; //save frame to avi
    }
}
 void ProcessFrame(const Mat &src)
{
    if(src.empty()) return;
    putText(src, "PROC FRAME", Point(10, 10), CV_FONT_HERSHEY_PLAIN, 1, cv::Scalar(0, 255, 0));
    imshow("Image main", src);
}
int MainProc() {
    Mat frame;
    VideoCapture cap;
    cap.open(0);
    if (!cap.isOpened()) // check if we succeeded
        return -1;

    grabOn.store(true);                // set the grabbing control variable
    thread t(GrabTask, &cap);          // start the grabbing task
    int bufSize;
    while (true) {

        mtxCam.lock();                // lock memory for exclusive access
        bufSize = buffer.size();      // check how many frames are waiting 
        if (bufSize > 0)              // if some 
        {
            // reference to buffer.front() should be valid after 
            // pop because of cv::Mat memory reference counting
            // but it's content can change after unlock, better to keep a copy
            // an alternative is to unlock after processing (this will lock grabbing)
            buffer.front().copyTo(frame);   // get the oldest grabbed frame (queue=FIFO)
            buffer.pop();            // release the queue item
        }
        mtxCam.unlock();            // unlock the memory

        if (bufSize > 0)            // if a new frame is available
        {
            ProcessFrame(frame);    // process it
            bufSize--;
        }

        // if bufSize is increasing means that process time is too slow regards to grab time 
        // may be you will have out of memory soon
        cout << endl << "frame to process:" << bufSize;

        if (waitKey(1) >= 0)        // press any key to terminate
        {
            grabOn.store(false);    // stop the grab loop 
            t.join();               // wait for the grab loop

            cout << endl << "Flushing buffer of:" << bufSize << " frames...";
            while (!buffer.empty())    // flushing the buffer
            {
                frame = buffer.front();    
                buffer.pop();
                ProcessFrame(frame);
            }
            cout << "done"<<endl;
            break; // exit from process loop
        }
    }
    cout << endl << "Press Enter to terminate"; cin.get();
    return 0;
}

Thread scheme is too poor because grab and process aren't sync. Depends on grab/process timing you will lost some frames or process same frame twice. You could implements some sync using mutex||semaphores||std::atomic<bool> but it's would be better if you use a thread safe circular buffer with copy constructor, in this case 1 producer 1 consumer: Grab/Push and Process/Pop.

A simple implementation could be done use std:queue but isn't memory effective because you will create a new Mat for each grab (even if STL and Mat optimization will recycle the memory). look at this simple example and try moving the window "Image main"

EDIT: keep a copy of grabbed frame is safer

safer. EDIT2: move grab out of lockinglocking. EDIT3: removed useless VideoWriter. Added counter for cv::Mat memory allocation/recycling

#include <thread>
#include <mutex>
#include <queue>
#include <atomic>

using namespace std;
using namespace cv;

std::queue<cv::Mat> std::queue<Mat> buffer;
std::mutex mtxCam;
std::atomic<bool> grabOn; // this //this is lock free

void GrabTask(cv::VideoCapture GrabThread(VideoCapture *cap)
{
    Mat tmp;

    //To know how many memory blocks will be allocated to store frames in the queue.
    //Even if you grab N frames and create N x Mat in the queue
    //only few real memory blocks will be allocated
    //thanks to std::queue and cv::Mat tmp;
    *cap >> tmp;    //catch a frame for get size and type

    cv::VideoWriter vw;
    OpenVideo(vw, tmp.size(), "../img/file2.avi");

    // keep the 1st frame
    mtxCam.lock();
    buffer.push(cv::Mat(tmp.size(), tmp.type()));
    tmp.copyTo(buffer.back());
    mtxCam.unlock();
memory recycling
    std::map<unsigned char*, int> matMemoryCounter;
    uchar * frameMemoryAddr;

    while (grabOn.load() == true) //this is lock free
    {
        //grab will wait for cam FPS
        //keep grab out of lock so that 
        //idle time can be used by other threads
        *cap >> tmp; //this will wait for cam FPS

        if (tmp.empty()) continue;

        //get lock only when we have a frame
        mtxCam.lock();
        // buffer.push_back(tmp) //buffer.push(tmp) stores item by reference than avoid
        // this //this will create a new cv::Mat for each grab
        buffer.push(cv::Mat(tmp.size(), buffer.push(Mat(tmp.size(), tmp.type()));
        tmp.copyTo(buffer.back());
        frameMemoryAddr = buffer.front().data;
        mtxCam.unlock();
        //count how many times this memory block has been used
        matMemoryCounter[frameMemoryAddr]++; 

        bool show = true;
        if (show)
        {
            // SHOW THE FRAME
            int font = CV_FONT_HERSHEY_PLAIN;
            cv::putText(tmp, putText(tmp, "THREAD FRAME", cv::Point(10, Point(10, 10), font, 1, cv::Scalar(0, Scalar(0, 255, 0));
            imshow("Image thread", tmp);
            cv::waitKey(1); waitKey(1);    //just for imshow
        }
        vw }
    std::cout << tmp; //save frame to avi
    }
std::endl << "Number of Mat in memory: " << matMemoryCounter.size();
}

void ProcessFrame(const Mat &src)
{
    if(src.empty()) return;
    putText(src, "PROC FRAME", Point(10, 10), CV_FONT_HERSHEY_PLAIN, 1, cv::Scalar(0, Scalar(0, 255, 0));
    imshow("Image main", src);
}
int MainProc() {
    Mat frame;
    VideoCapture cap;
    cap.open(0);
    if (!cap.isOpened()) // check //check if we succeeded
        return -1;

    grabOn.store(true);                // set //set the grabbing control variable
    thread t(GrabTask, t(GrabThread, &cap);          // start //start the grabbing task
thread
    int bufSize;
    while (true) (true)
    {
         mtxCam.lock();                // lock //lock memory for exclusive access
        bufSize = buffer.size();      // check //check how many frames are waiting 
        if (bufSize > 0)              // if //if some 
        {
            // reference //reference to buffer.front() should be valid after 
            // pop //pop because of cv::Mat Mat memory reference counting
            // but it's //but if content can change after unlock, unlock is better to keep a copy
            // an //an alternative is to unlock after processing (this will lock grabbing)
            buffer.front().copyTo(frame);   // get //get the oldest grabbed frame (queue=FIFO)
            buffer.pop();            // release //release the queue item
        }
        mtxCam.unlock();            // unlock //unlock the memory

        if (bufSize > 0)            // if //if a new frame is available
        {
            ProcessFrame(frame);    // process //process it
            bufSize--;
        }

        // if //if bufSize is increasing means that process time is too slow regards to grab time 
        // may //may be you will have out of memory soon
        cout << endl << "frame to process:" << bufSize;

        if (waitKey(1) >= 0)        // press //press any key to terminate
        {
            grabOn.store(false);    // stop //stop the grab loop 
            t.join();               // wait //wait for the grab loop

            cout << endl << "Flushing buffer of:" << bufSize << " frames...";
            while (!buffer.empty())    // flushing //flushing the buffer
            {
                frame = buffer.front();    
                ProcessFrame(frame);
                buffer.pop();
                ProcessFrame(frame);
            }
            cout << "done"<<endl;
            break; // exit //exit from process loop
        }
    }
    cout << endl << "Press Enter to terminate"; cin.get();
    return 0;
}