Read h264 RTP stream with libvlc and process it with OpenCV

edit

Hi everybody !

I want to read stream from ipCamera in h264 1920*1072 and process it using Opencv.

I tried the easy way with videoCapture("rtp://myadress") but the image is very bad and i have error like : "error while decoding : MB...".

So I want to use libVlc to read the stream and access the data with openCV. I find this post and I use his code : http://answers.opencv.org/question/56... I change the size of the data for matching mine 1920*1072. But I don't really understand this code. But i have always the same console logs :

• [rawvideo @ 0x7fe04c2cd440] AVFrame.format is not set
• [rawvideo @ 0x7fe04c2cd440]AVFrame.width or height is not set
• MediaPlayerPositionChanged
• MediaPlayerTimeChanged 0 ms

And i have in my file log:

• rtp debug: ignoring late packet (sequence: 27130)
• rtp debug: ignoring late packet (sequence: 27131)
• packetizer_h264 warning: waiting for SPS/PPS
• rtp debug: ignoring late packet (sequence: 27132) and no image is shown.

using namespace cv;
    using namespace std;

struct VideoDataStruct {
  int param;
};

int done = 0;
libvlc_media_player_t *mp;
unsigned int videoBufferSize = 0;
uint8_t *videoBuffer = 0;

 void cbVideoPrerender(void *p_video_data, uint8_t **pp_pixel_buffer, int size) {
// Locking
  if (size > videoBufferSize || !videoBuffer)
  {
    printf("Reallocate raw video buffer\n");
    free(videoBuffer);
    videoBuffer = (uint8_t *) malloc(size);
    videoBufferSize = size;
  }

   *pp_pixel_buffer = videoBuffer;
}

 void cbVideoPostrender(void *p_video_data, uint8_t *p_pixel_buffer, int width, int height, int pixel_pitch, int size, int64_t pts) {
     // Unlocking
     //CloseHandle(hMutex);
 }

 static void handleEvent(const libvlc_event_t* pEvt, void* pUserData) {
   libvlc_time_t time;
   switch(pEvt->type)
   {
      case libvlc_MediaPlayerTimeChanged:
          time = libvlc_media_player_get_time(mp);
          printf("MediaPlayerTimeChanged %lld ms\n", (long long)time);
          break;
      case libvlc_MediaPlayerEndReached:
          printf ("MediaPlayerEndReached\n");
          done = 1;
        break;
       default:
          printf("%s\n", libvlc_event_type_name(pEvt->type));
    }
}

int main(int argc, char* argv[]) {

// VLC pointers
libvlc_instance_t *inst;
libvlc_media_t *m;
void *pUserData = 0;
VideoDataStruct dataStruct;

// VLC options
char smem_options[1000];
// RV24
sprintf(smem_options
    , "#transcode{vcodec=RV24}:smem{"
     "video-prerender-callback=%lld,"
     "video-postrender-callback=%lld,"
     "video-data=%lld,"
     "no-time-sync},"
    , (long long int)(intptr_t)(void*)&cbVideoPrerender
    , (long long int)(intptr_t)(void*)&cbVideoPostrender
    , (long long int)(intptr_t)(void*)&dataStruct
);

const char * const vlc_args[] = {
          "-I", "dummy",            // Don't use any interface
          "--ignore-config",        // Don't use VLC's config
          "--extraintf=logger",     // Log anything
          "--verbose=2",            // Be verbose
          "--sout", smem_options    // Stream to memory
           };

// We launch VLC
inst = libvlc_new(sizeof(vlc_args) / sizeof(vlc_args[0]), vlc_args);

/* Create a new item */
//m = libvlc_media_new_path(inst, "outputDiff.avi");
    m= libvlc_media_new_location(inst, "rtp://myadress:myport");
/* Create a media player playing environement */
mp = libvlc_media_player_new_from_media (m);

libvlc_event_manager_t* eventManager = libvlc_media_player_event_manager(mp);
libvlc_event_attach(eventManager, libvlc_MediaPlayerTimeChanged, handleEvent, pUserData);
libvlc_event_attach(eventManager, libvlc_MediaPlayerEndReached, handleEvent, pUserData);
libvlc_event_attach(eventManager, libvlc_MediaPlayerPositionChanged, handleEvent, pUserData);

libvlc_video_set_format(mp, "RV24", VIDEOWIDTH, VIDEOHEIGHT, VIDEOWIDTH* 3 );

/* play the media_player */
libvlc_media_player_play (mp);

 while(1)
 {
     if(videoBuffer)            // Check for invalid input
     {
         // CV_8UC3 = 8 bits, 3 chanels
         Mat img = Mat(Size(VIDEOWIDTH, VIDEOHEIGHT), CV_8UC3, videoBuffer);
         cvtColor(img, img, CV_RGB2BGR);
         namedWindow("Display window", WINDOW_AUTOSIZE);    // Create a window for display.
         imshow("Display window", img);     // Show our image inside it.
     }
 }
 libvlc_release (inst);
}
 
Thank you guys.
Bye