GStreamer 时间戳 (PTS) 对于捕获的帧不是单调递增的

Question

我编写了一些代码来使用 GStreamer 1.0 (PyGST) 从网络摄像头捕获帧。对我来说，知道确切的捕获时间很重要。为此，我设置了 v4l2src 属性 do-timestamp，并使用 appsink 将缓冲区 PTS 写入文本文件。

然而，时间戳并不是单调递增的。例如，帧16的时间戳是0.88199秒，帧17的时间戳是0.77462秒，即比前一帧早0.10737秒。（我有一张显示问题的图，但缺乏发布它所需的声誉。）

捕获的 GstBuffer 的 PTS 并不总是单调递增，这是否正确？如果这不是正常行为，有人知道我搞砸了什么吗？

我使用 Logitech c920 网络摄像头。这些帧在相机上采用 h.264 编码。代码大致如下：

import gi
gi.require_version('Gst', '1.0')
from gi.repository import GObject, Gst, Gtk
GObject.threads_init()
Gst.init(None)

class Webcam:
def __init__(self, video_dev='/dev/video0', fps=30):

    ts_log_fname = 'webcam_timestamps.log'
    vid_fname = 'webcam.mkv'
    self.ts_log = open(ts_log_fname, 'w')
    self.ts_log.write('video filename: %s\n '
                      '\nframe_number, cam_running_ts\n' % vid_fname)

    self.n_frames = 0

    # Create GStreamer pipline
    self.pipeline = Gst.Pipeline()

    # Create bus to get events from GStreamer pipeline
    self.bus = self.pipeline.get_bus()
    self.bus.add_signal_watch()
    self.bus.connect('message::error', self.on_error)
    self.bus.enable_sync_message_emission()
    self.bus.connect('sync-message::element', self.on_sync_message)

    ###########################
    # Callable function
    ###########################
    def on_new_sample(appsink):
        """
        Function called from the pipeline by appsink.
        Writes the timestampes of frame capture to a log file.
        """
        # Get the buffer
        smp = appsink.emit('pull-sample')
        buf = smp.get_buffer()
        # Nanoseconds to seconds
        timestamp = np.float64(1e-9) * buf.pts
        self.n_frames += 1
        self.ts_log.write('%d,%0.9f\n' % (self.n_frames, timestamp))
        return False

    ###########################
    # Create GStreamer elements
    ###########################
    # Video source:
    self.v4l2src0 = Gst.ElementFactory.make('v4l2src', None)
    self.v4l2src0.set_property('device', video_dev)
    self.v4l2src0.set_property('do-timestamp', 'true')
    # Video source filters:
    vid0caps = Gst.Caps.from_string('video/x-h264,width=%d,height=%d,'
                                    'framerate=%d/1' % (1280, 720, fps))
    self.vid0filter = Gst.ElementFactory.make('capsfilter', None)
    self.vid0filter.set_property('caps', vid0caps)
    # Parse video:
    self.vid0parse = Gst.ElementFactory.make('h264parse', None)
    # Split:
    self.tee0 = Gst.ElementFactory.make('tee', None)
    self.tee0.set_property('name', 't0')
    ####
    # Display branch
    ####
    # Decode
    self.vid0decode = Gst.ElementFactory.make('avdec_h264', None)
    # Scale to display size:
    self.disp0scale = Gst.ElementFactory.make('videoscale', None)
    # Display filter caps:
    disp0caps = Gst.Caps.from_string('video/x-raw,width=%d,height=%d' %
                                     (800, 600))
    # Sinks:
    self.disp0sink = Gst.ElementFactory.make('autovideosink', None)
    self.disp0sink.set_property('filter-caps', disp0caps)
    ####
    # File branch
    ####
    self.mux = Gst.ElementFactory.make('matroskamux', None)
    self.file0sink = Gst.ElementFactory.make('filesink', None)
    self.file0sink.set_property('location', vid_fname)
    self.file0sink.set_property('sync', False)
    ####
    # Timestamp branch
    ####
    # Create appsink
    self.ts0sink = Gst.ElementFactory.make('appsink', None)
    # Setting properties of appsink
    ts0caps = vid0caps  # use same caps as for camera
    self.ts0sink.set_property('caps', ts0caps)
    self.ts0sink.set_property("max-buffers", 20)  # Limit memory usage
    # Tell sink to emit signals
    self.ts0sink.set_property('emit-signals', True)
    self.ts0sink.set_property('sync', False)  # No sync
    # Connect appsink to my function (writing timestamps)
    self.ts0sink.connect('new-sample', on_new_sample)

    self.queue0 = Gst.ElementFactory.make('queue', None)
    self.queue1 = Gst.ElementFactory.make('queue', None)                
    self.disp_queue = Gst.ElementFactory.make('queue', None)
    self.file_queue = Gst.ElementFactory.make('queue', None)
    self.ts_queue = Gst.ElementFactory.make('queue', None)

    # Add elements to the pipeline
    self.pipeline.add(self.v4l2src0)
    self.pipeline.add(self.vid0filter)
    self.pipeline.add(self.vid0parse)
    self.pipeline.add(self.tee0)
    self.pipeline.add(self.vid0decode)
    self.pipeline.add(self.disp0scale)
    self.pipeline.add(self.disp0sink)
    self.pipeline.add(self.mux)
    self.pipeline.add(self.file0sink)
    self.pipeline.add(self.ts0sink)
    self.pipeline.add(self.queue0)
    self.pipeline.add(self.queue1)
    self.pipeline.add(self.disp_queue)
    self.pipeline.add(self.file_queue)
    self.pipeline.add(self.ts_queue)

    ###############
    # Link elements
    ###############
    # video source
    if not self.v4l2src0.link(self.vid0filter):
        print('video source to video filter link failed')          
    if not self.vid0filter.link(self.vid0parse):
        print('video filter to video parse link failed')
    if not self.vid0parse.link(self.tee0):
        print('video parse to tee link failed')    
    # tee
    if not self.tee0.link(self.disp_queue):
        print('tee to display queue link failed')
    if not self.tee0.link(self.file_queue):
        print('tee to file queue link failed')
    if not self.tee0.link(self.ts_queue):
        print('tee to ts queue link failed')
    # video display sink
    if not self.disp_queue.link(self.vid0decode):
        print('dispaly queue to video decode link failed')
    if not self.vid0decode.link(self.disp0scale):
        print('decode to videoscale link failed')
    if not self.disp0scale.link(self.queue0):
        print('disp0scale to queue0 link failed')            
    if not self.queue0.link_filtered(self.disp0sink, disp0caps):
        print('queue0 to display-sink link failed')
    # file sink
    if not self.file_queue.link(self.mux):
        print('file queue to mux link failed')           
    if not self.mux.link(self.queue1):
        print('mux to queue1 link failed')            
    if not self.queue1.link(self.file0sink):
        print('queue1 to file-sink link failed')
    # timestamp sink
    if not self.ts_queue.link(self.ts0sink):
        print('ts queue to ts-sink link failed')

def run(self):
    self.offset_t = datetime.now().timestamp() - self.t_start
    self.pipeline.set_state(Gst.State.PLAYING)

def quit(self):
    self.pipeline.set_state(Gst.State.NULL)
    self.ts_log.close()

def on_sync_message(self, bus, msg):
    if msg.get_structure().get_name() == 'prepare-window-handle':
        msg.src.set_property('force-aspect-ratio', True)

def on_error(self, bus, msg):
    print('on_error():', msg.parse_error())

Answer 1

正如 mpr 所建议的，在流式传输 h.264 时会出现非单调时间戳。它似乎也发生在流的开头。当流式传输原始视频时，问题就消失了。

每 30 帧左右的 300 毫秒跳跃似乎是 Logitech C920 摄像头特有的。当我使用板载笔记本电脑摄像头时，跳跃更小，为 130 毫秒，甚至更少，每 85 帧左右。