Revision history [back]

I want to fuse two videos and record fused video. Two videos have different frames per second, one video has 25 fps and another video has 10 fps and different widths and heights. And I have function, where an input is two Mat objects and an output is fused image. My problem is what sometimes my function returns totally black image, through if I save two image which produce totally black image and call this functon out of while(1) loop, I get correct(not black) image. here's my main() function:

    //TWO VIDEOS, different fps and lenghts
    VideoCapture video1("test1.mp4"); 
    VideoCapture video2("test2.mp4");

    bool v1end = false;
    bool v2end = false;
    bool even = true;
    int frames_counter = 0;

    float width = video2.get(CAP_PROP_FRAME_WIDTH);  
    float height = video2.get(CAP_PROP_FRAME_HEIGHT); 
    Mat frame1;  
    Mat frame2;

    VideoWriter video("outcpp.avi",VideoWriter::fourcc('M','J','P','G'),24, Size(width, height));

    while(1){
    //here's logic for manipulating videos with different fps
    if (!v1end) {
        if (frames_counter == 0 ){
            video1 >> frame1;
            cvtColor(frame1, frame1, COLOR_RGB2GRAY); 
        }

        if (frame1.empty()){
            v1end = true;
            video1.release();
            destroyWindow( "Frame1" );
        }
        else { 
            resize(frame1, frame1, cv::Size(width, height), 0, 0 , cv::INTER_NEAREST);
            imshow( "Frame1", frame1);
        }
    }
    if (!v2end) {
        frames_counter++;
        if (frames_counter == 3 && even == false){
            frames_counter = 0;
            even = true;
        }
        if (frames_counter == 2 && even == true){
            frames_counter = 0;
            even = false;
        }
        video2 >> frame2;  
        //cvtColor(frame, frame, CV_RGB2GRAY); 
        if (frame2.empty()){
            video2.release();
            destroyWindow( "Frame2" );
            v2end = true;
        }
        else{ 
            imshow( "Frame2", frame2);
        }
    }

    if (!v1end){
        cvtColor(frame2, frame2, COLOR_RGB2GRAY );
        //HERE IS  A FUNCTION CALL 
        Mat temp = fusion(frame1, frame2);   
        cvtColor(temp, temp, COLOR_GRAY2RGB);
        imshow( "Frame ", temp); 
        //recording a fused image
        video.write(temp);
    }
    else {
    break;
    }


    char c = (char)waitKey(1);
    if( c == 27 )     
    break;
    }

and here's my function for fusing images

Mat fusion(Mat img1, Mat img2){ 
    //Here I check for input images, img1 and img2 always are correct pictures, so, pixel values are correct
    imshow("img1",img1);
    imshow("img2",img2);

    //Logic for fusion algorithm
    int all_pixels = img1.cols*img1.rows;     
    Mat_<float> samples;
    int sum1 = 0;
    int sum2 = 0;
    samples.create(all_pixels, 2);
    int counter = 0;
    for(int i = 0; i < img1.rows; i++){
        for(int j = 0; j < img1.cols; j++){
            samples.at<float>(0,counter) = (img1.at<uchar>(i,j));
            samples.at<float>(1,counter) = (img1.at<uchar>(i,j));
            counter++;
        }
    }

    Mat cov, mu;
    cv::calcCovarMatrix(samples, cov, mu, COVAR_NORMAL | COVAR_ROWS);

    cov = cov / (samples.rows - 1);
    // If fused image is black, here is usually all are "nans"     
    cout << "cov: " << endl;
    cout << cov << endl;

    /* cout << "mu: " << endl;
    cout << mu << endl;*/

    Mat eigenvalues, eigenvectors;
    cv::eigen(cov, eigenvalues, eigenvectors);

    /* cout << "eigenvalues: " << endl;
    cout << eigenvalues << endl;*/

    /* cout << "eigenvectors: " << endl;
    cout << eigenvectors << endl;*/
    float av1 = mu.at<double>(0);
    float av2 = mu.at<double>(1);
    Mat finalImg;
    finalImg.create(img1.rows, img1.cols, CV_8UC1);

    float p1 = eigenvectors.at<double>(0,0);
    float p2 = eigenvectors.at<double>(0,1);

    for(int i = 0; i < img1.rows; i++){
        for(int j = 0; j < img1.cols; j++){
            float tmp = (((int)img1.at<uchar>(i,j) - av1)*p1 + ((int)img2.at<uchar>(i,j) - av2) * p2);
            tmp = cvRound(tmp);
            finalImg.at<uchar>(i,j) = tmp;
            if (tmp > 255){
                float k = tmp - 255;
                finalImg.at<uchar>(i,j) = 255 - k;
            }
            if (tmp < 0 ){
                finalImg.at<uchar>(i,j) = -tmp ;
            }
        }
    }
    return finalImg;
}

So, my problem is what extremely often this function works wrong, and I have no idea why. If I save pictures (img1) and (img2) from fusion() which produce black picture on file system and then call my function on these saved pictures, I get right fused picture. I tested this code on different videos with different fps. And yet this code works wrong. And even sometimes on same videos it gets me different output. I mean sometimes it gets me whole fused correct video, sometimes it gets me video which consists of black pictures. I don't know where is problem in this code. Please help me.