How to detect the direction of moving objects?

edit

Hi all, This is my work to detect any moving object on the screen:

#include"stdafx.h"
#include<vector>
#include<iostream>
#include<opencv2/opencv.hpp>
#include<opencv2/core/core.hpp>
#include<opencv2/imgproc/imgproc.hpp>
#include<opencv2/highgui/highgui.hpp>


int main(int argc, char *argv[])
{
    cv::Mat frame;                                              
    cv::Mat fg;     
    cv::Mat blurred;
    cv::Mat thresholded;
    cv::Mat gray;
    cv::Mat blob;
    cv::Mat bgmodel;                                            
    cv::namedWindow("Frame");   
    cv::namedWindow("Background Model");
    cv::namedWindow("Blob");
    cv::VideoCapture cap(0);    
cv::BackgroundSubtractorMOG2 bgs;                           

    bgs.nmixtures = 3;
    bgs.history = 1000;
    bgs.varThresholdGen = 15;
    bgs.bShadowDetection = true;                            
    bgs.nShadowDetection = 0;                               
    bgs.fTau = 0.5;                                         

std::vector<std::vector<cv::Point>> contours;               

for(;;)
{
    cap >> frame;                                           

    cv::GaussianBlur(frame,blurred,cv::Size(3,3),0,0,cv::BORDER_DEFAULT);

    bgs.operator()(blurred,fg);                         
    bgs.getBackgroundImage(bgmodel);                                

    cv::erode(fg,fg,cv::Mat(),cv::Point(-1,-1),1);                         
    cv::dilate(fg,fg,cv::Mat(),cv::Point(-1,-1),3);       

    cv::threshold(fg,thresholded,70.0f,255,CV_THRESH_BINARY);

    cv::findContours(thresholded,contours,CV_RETR_EXTERNAL,CV_CHAIN_APPROX_SIMPLE);
    cv::cvtColor(thresholded,blob,CV_GRAY2RGB);
    cv::drawContours(blob,contours,-1,cv::Scalar(255,255,255),CV_FILLED,8);

    cv::cvtColor(frame,gray,CV_RGB2GRAY);
    cv::equalizeHist(gray, gray);

    int cmin = 20; 
    int cmax = 1000;
    std::vector<cv::Rect> rects;
    std::vector<std::vector<cv::Point>>::iterator itc=contours.begin();
    while (itc!=contours.end()) {   
        if (itc->size() > cmin && itc->size() < cmax){ 

                    std::vector<cv::Point> pts = *itc;
                    cv::Mat pointsMatrix = cv::Mat(pts);
                    cv::Scalar color( 0, 255, 0 );

                    cv::Rect r0= cv::boundingRect(pointsMatrix);
                    cv::rectangle(frame,r0,color,2);                    

                    //DETECT THE DIRECTION OF MOVING OBJECTS HERE!

                    ++itc;}
         else{++itc;}
    }

    cv::imshow("Frame",frame);
    cv::imshow("Background Model",bgmodel);
    cv::imshow("Blob",blob);
    if(cv::waitKey(30) >= 0) break;
}
    return 0;
}

And now I want to add some code to detect the direction of moving object on

//DETECT THE DIRECTION OF MOVING OBJECTS HERE!

for example when a person walking from left to right it will show a blue rectangle, when a person walking from right to left it will show a yellow rectangle, and when a person stop walking it overwrite the blue/yellow rectangle with red rectangle. Last if that person start to walking again, it will overwrite the red rectangle with blue/yellow rectangle depend on the direction. So how to do that? Any tutorial of it?

I'll appreciate any help here, thanks! :)