将opencv处理添加到gstreamer应用程序

Question

我正在尝试执行以下操作:使用gstreamer接收视频流并使用opencv进行处理.我找到了一些解决方案,其中一个是将视频写入(来自gstreamer)fifo,然后使用opencv读取它.(OPTION3这里是MJPEG流媒体和解码).问题是我无法打开管道.cvCreateFileCapture永远不会返回.这是我写的部分代码:

if(mkfifo("fifo.avi", S_IRUSR| S_IWUSR) == -1)
{
    cout<<"Cant create fifo"<<endl;
    cout<<errno<<endl;
}

loop = g_main_loop_new(NULL, false);

fsink = gst_element_factory_make("filesink", "fsink");
g_object_set(G_OBJECT(fsink), "location", "fifo.avi", NULL);

playbin = gst_element_factory_make("playbin2", "play");    
g_object_set(G_OBJECT(playbin), "uri", uri.c_str(), NULL);
g_object_set(G_OBJECT(playbin), "video-sink", fsink, NULL);

bus = gst_pipeline_get_bus(GST_PIPELINE(playbin));
gst_bus_add_signal_watch(bus);

g_signal_connect(bus, "message::buffering", G_CALLBACK(&set_playbin_state), playbin);
gst_object_unref(bus);

cvNamedWindow("output", CV_WINDOW_AUTOSIZE);
capture = cvCreateFileCapture("fifo.avi");

程序堆叠在最后一行.PS:我正在使用opencv 2.3.1.

Answer 1

所以.经过一段时间的搜索,我找到了一个解决方案,其中包括从缓冲区中检索数据.因此,我们的想法是创建playbin并将appsink设置为"视频接收器".这是代码示例:

cout<<"Creating appsink"<<endl;
appsink = gst_element_factory_make("appsink", "asink");
gst_app_sink_set_emit_signals((GstAppSink*)appsink, true);
gst_app_sink_set_drop((GstAppSink*)appsink, true);
gst_app_sink_set_max_buffers((GstAppSink*)appsink, 1);

//creating and initialising pipeline

g_object_set(G_OBJECT(playbin), "video-sink", appsink, NULL);

g_signal_connect(appsink, "new-buffer", G_CALLBACK(DisplayFrame), (gpointer) mark);

//callback function looks like this

gboolean Core::DisplayFrame(GstAppSink *fks, gpointer mark)
{
static bool init = false;
static IplImage *frame;
GstBuffer* buf;
Mark* mk = (Mark*) mark;

if(!init)
{
    init = true;
    frame = cvCreateImage(cvSize(mk->frame_w, mk->frame_h), IPL_DEPTH_8U, 1);
}
buf = gst_app_sink_pull_buffer(fks);
frame->imageData = (char*)GST_BUFFER_DATA(buf);

ProcessFrame(frame);
gst_buffer_unref(buf);
return true;
}

这很有效.PS.关于这种方法有很多信息,但我花了很多时间来搜索它.所以我决定在这里发布它,以便提供至少一些搜索关键字.

UPDATE.关于连接gstreamer和opencv的更多信息.这是关于现在将缓冲区转换为iplimage.首先,我们需要接收rgb缓冲区,以使转换尽可能简单.为了做到这一点,我们将使用连接到ffmpegcolorspace的appsink替换appsink

cout<<"Creating appsink"<<endl;
appsink = gst_element_factory_make("appsink", "asink");
gst_app_sink_set_emit_signals((GstAppSink*)appsink, true);
gst_app_sink_set_drop((GstAppSink*)appsink, true);
gst_app_sink_set_max_buffers((GstAppSink*)appsink, 1);
csp = gst_element_factory_make("ffmpegcolorspace", "csp");
sinkpipe = gst_pipeline_new("sinkp");
gst_bin_add_many(GST_BIN(sinkpipe), csp, appsink, NULL);
gst_element_link_filtered(csp, appsink, gst_caps_new_simple("video/x-raw-rgb", NULL));
pad = gst_element_get_static_pad(csp, "sink");
gst_element_add_pad(sinkpipe, gst_ghost_pad_new("ghost", pad));
g_object_unref(pad);

//...

g_object_set(G_OBJECT(playbin), "video-sink", sinkpipe, NULL);

//...

g_signal_connect(appsink, "new-buffer", G_CALLBACK(GetFrame), (gpointer) mark);

//...

//caps_struct can be retrieved via writing data probe
//search for it in streamer manual

cout<<"Getting frame resolution"<<endl;
gst_structure_get_int(caps_struct, "width", &(mark->frame_w));
gst_structure_get_int(caps_struct, "height", &(mark->frame_h));
gst_structure_get_int(caps_struct, "depth", &depth);

mark->GeneratePoints();
frame = cvCreateImage(cvSize(mark->frame_w, mark->frame_h), depth/3, 3);


//callback function

gboolean Core::GetFrame(GstAppSink *fks, gpointer frame)
{

IplImage* frame_temp = frame
IplImage* frame_temp_two = cvCloneImage(frame_temp);

GstBuffer* buf;
buf = gst_app_sink_pull_buffer(fks);
frame_temp_two->imageData = (char*) GST_BUFFER_DATA(buf);
cvConvertImage(frame_temp_two, frame_temp, CV_CVTIMG_SWAP_RB);
ProcessFrame(frame_temp);
gst_buffer_unref(buf);
return true;
}

我希望这会对某人有所帮助.

Answer 2

这是我对Gstreamer 1.4.0和OpenCV 2.4.9的完整源代码解决方案.

它用于gst_parse_launch()解析您将提供的正常命令行gst-launch.Gstreamer管道将帧转换为RGB888格式,然后将它们提供给OpenCV,以便转换尽可能简单.

OpenCV帧处理不是在new_sample()回调中完成的,但它只从gstreamer抓取帧并将其推送到队列,然后将在主线程中使用.这样我们就可以从OpenCV调用例如imshow()来实际渲染图像到屏幕.

通过删除调试打印等〜约150行可以减少到<100行代码.

可能应该在deque读/写周围添加线程同步

#include <gst/gst.h>
#include <gst/app/gstappsink.h>
#include <stdlib.h>

#include "opencv2/opencv.hpp"
using namespace cv;

// TODO: use synchronized deque
std::deque<Mat> frameQueue;

GstFlowReturn
new_preroll(GstAppSink *appsink, gpointer data) {
  g_print ("Got preroll!\n");
  return GST_FLOW_OK;
}

GstFlowReturn
new_sample(GstAppSink *appsink, gpointer data) {
  static int framecount = 0;
  framecount++;

  GstSample *sample = gst_app_sink_pull_sample(appsink);
  GstCaps *caps = gst_sample_get_caps(sample);
  GstBuffer *buffer = gst_sample_get_buffer(sample);
  const GstStructure *info = gst_sample_get_info(sample);

  // ---- Read frame and convert to opencv format ---------------

  GstMapInfo map;
  gst_buffer_map (buffer, &map, GST_MAP_READ);

  // convert gstreamer data to OpenCV Mat, you could actually
  // resolve height / width from caps...
  Mat frame(Size(320, 240), CV_8UC3, (char*)map.data, Mat::AUTO_STEP);
  int frameSize = map.size;

  // TODO: synchronize this....
  frameQueue.push_back(frame);

  gst_buffer_unmap(buffer, &map);

  // ------------------------------------------------------------

  // print dot every 30 frames
  if (framecount%30 == 0) {
    g_print (".");
  }

  // show caps on first frame
  if (framecount == 1) {
    g_print ("%s\n", gst_caps_to_string(caps));
  }

  gst_sample_unref (sample);
  return GST_FLOW_OK;
}

static gboolean
my_bus_callback (GstBus *bus, GstMessage *message, gpointer data) {
  g_print ("Got %s message\n", GST_MESSAGE_TYPE_NAME (message));
  switch (GST_MESSAGE_TYPE (message)) {
    case GST_MESSAGE_ERROR: {
      GError *err;
      gchar *debug;

      gst_message_parse_error (message, &err, &debug);
      g_print ("Error: %s\n", err->message);
      g_error_free (err);
      g_free (debug);    
      break;
    }
    case GST_MESSAGE_EOS:
      /* end-of-stream */
      break;
    default:
      /* unhandled message */
      break;
  }
  /* we want to be notified again the next time there is a message
   * on the bus, so returning TRUE (FALSE means we want to stop watching
   * for messages on the bus and our callback should not be called again)
   */
  return TRUE;
}

int
main (int argc, char *argv[])
{
  GError *error = NULL;

  gst_init (&argc, &argv);

  gchar *descr = g_strdup(
    "videotestsrc  pattern=ball ! "
    "video/x-raw,format=RGB ! "
    "videoconvert ! "
    "appsink name=sink sync=true"
  );
  GstElement *pipeline = gst_parse_launch (descr, &error);

  if (error != NULL) {
    g_print ("could not construct pipeline: %s\n", error->message);
    g_error_free (error);
    exit (-1);
  }

  /* get sink */
  GstElement *sink = gst_bin_get_by_name (GST_BIN (pipeline), "sink");

  gst_app_sink_set_emit_signals((GstAppSink*)sink, true);
  gst_app_sink_set_drop((GstAppSink*)sink, true);
  gst_app_sink_set_max_buffers((GstAppSink*)sink, 1);
  GstAppSinkCallbacks callbacks = { NULL, new_preroll, new_sample };
  gst_app_sink_set_callbacks (GST_APP_SINK(sink), &callbacks, NULL, NULL);

  GstBus *bus;
  guint bus_watch_id;
  bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline));
  bus_watch_id = gst_bus_add_watch (bus, my_bus_callback, NULL);
  gst_object_unref (bus);

  gst_element_set_state (GST_ELEMENT (pipeline), GST_STATE_PLAYING);

  namedWindow("edges",1);
  while(1) {
    g_main_iteration(false);

      // TODO: synchronize...
    if (frameQueue.size() > 0) {
      // this lags pretty badly even when grabbing frames from webcam
      Mat frame = frameQueue.front();
      Mat edges;
      cvtColor(frame, edges, CV_RGB2GRAY);
      GaussianBlur(edges, edges, Size(7,7), 1.5, 1.5);
      Canny(edges, edges, 0, 30, 3);
      imshow("edges", edges);
      cv::waitKey(30);
      frameQueue.clear();
    }
  }  

  gst_element_set_state (GST_ELEMENT (pipeline), GST_STATE_NULL);
  gst_object_unref (GST_OBJECT (pipeline));

  return 0;  
}

要在OSX/Linux上编译make make这样的文件应该:

GST_FLAGS=$(shell pkg-config --cflags --libs gstreamer-gl-1.0 gstreamer-tag-1.0 gstreamer-net-1.0 gstreamer-sdp-1.0 \
  gstreamer-1.0 gstreamer-allocators-1.0 gstreamer-insertbin-1.0 gstreamer-plugins-base-1.0 \
  gstreamer-codecparsers-1.0 gstreamer-base-1.0 gstreamer-app-1.0 gstreamer-check-1.0 \
  gstreamer-controller-1.0 gstreamer-video-1.0 gstreamer-fft-1.0 gstreamer-mpegts-1.0 \
  gstreamer-pbutils-1.0 gstreamer-rtp-1.0 gstreamer-rtsp-1.0 \
  gstreamer-riff-1.0 gstreamer-audio-1.0 gstreamer-plugins-bad-1.0 opencv)

OPENCV_FLAGS=$(shell pkg-config --cflags --libs opencv)

all: gst_opencv

gst_opencv: gst_opencv
    g++ $(GST_FLAGS) $(OPENCV_FLAGS) gst_opencv -o gst_opencv

clean:
    rm -f gst_opencv