MF SinkWriter 写入示例失败

Question

我正在尝试使用 MediaFoundation 将 ID3D11Texture2D 编码为 mp4。下面是我目前的代码。

初始化接收器写入器

private int InitializeSinkWriter(String outputFile, int videoWidth, int videoHeight)
    {
        IMFMediaType mediaTypeIn = null;
        IMFMediaType mediaTypeOut = null;
        IMFAttributes attributes = null;

        int hr = 0;

        if (Succeeded(hr)) hr = (int)MFExtern.MFCreateAttributes(out attributes, 1);
        if (Succeeded(hr)) hr = (int)attributes.SetUINT32(MFAttributesClsid.MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, 1);            
        if (Succeeded(hr)) hr = (int)attributes.SetUINT32(MFAttributesClsid.MF_LOW_LATENCY, 1);

        // Create the sink writer 
        if (Succeeded(hr)) hr = (int)MFExtern.MFCreateSinkWriterFromURL(outputFile, null, attributes, out sinkWriter);

        // Create the output type
        if (Succeeded(hr)) hr = (int)MFExtern.MFCreateMediaType(out mediaTypeOut);
        if (Succeeded(hr)) hr = (int)mediaTypeOut.SetGUID(MFAttributesClsid.MF_MT_MAJOR_TYPE, MFMediaType.Video);
        if (Succeeded(hr)) hr = (int)mediaTypeOut.SetGUID(MFAttributesClsid.MF_TRANSCODE_CONTAINERTYPE, MFTranscodeContainerType.MPEG4);
        if (Succeeded(hr)) hr = (int)mediaTypeOut.SetGUID(MFAttributesClsid.MF_MT_SUBTYPE, MFMediaType.H264);
        if (Succeeded(hr)) hr = (int)mediaTypeOut.SetUINT32(MFAttributesClsid.MF_MT_AVG_BITRATE, videoBitRate);
        if (Succeeded(hr)) hr = (int)mediaTypeOut.SetUINT32(MFAttributesClsid.MF_MT_INTERLACE_MODE, (int)MFVideoInterlaceMode.Progressive);            

        if (Succeeded(hr)) hr = (int)MFExtern.MFSetAttributeSize(mediaTypeOut, MFAttributesClsid.MF_MT_FRAME_SIZE, videoWidth, videoHeight);
        if (Succeeded(hr)) hr = (int)MFExtern.MFSetAttributeRatio(mediaTypeOut, MFAttributesClsid.MF_MT_FRAME_RATE, VIDEO_FPS, 1);
        if (Succeeded(hr)) hr = (int)MFExtern.MFSetAttributeRatio(mediaTypeOut, MFAttributesClsid.MF_MT_PIXEL_ASPECT_RATIO, 1, 1);
        if (Succeeded(hr)) hr = (int)sinkWriter.AddStream(mediaTypeOut, out streamIndex);



        // Create the input type 
        if (Succeeded(hr)) hr = (int)MFExtern.MFCreateMediaType(out mediaTypeIn);
        if (Succeeded(hr)) hr = (int)mediaTypeIn.SetGUID(MFAttributesClsid.MF_MT_MAJOR_TYPE, MFMediaType.Video);
        if (Succeeded(hr)) hr = (int)mediaTypeIn.SetGUID(MFAttributesClsid.MF_MT_SUBTYPE, MFMediaType.ARGB32);
        if (Succeeded(hr)) hr = (int)mediaTypeIn.SetUINT32(MFAttributesClsid.MF_SA_D3D11_AWARE, 1);
        if (Succeeded(hr)) hr = (int)mediaTypeIn.SetUINT32(MFAttributesClsid.MF_MT_INTERLACE_MODE, (int)MFVideoInterlaceMode.Progressive);
        if (Succeeded(hr)) hr = (int)MFExtern.MFSetAttributeSize(mediaTypeIn, MFAttributesClsid.MF_MT_FRAME_SIZE, videoWidth, videoHeight);
        if (Succeeded(hr)) hr = (int)MFExtern.MFSetAttributeRatio(mediaTypeIn, MFAttributesClsid.MF_MT_FRAME_RATE, VIDEO_FPS, 1);
        if (Succeeded(hr)) hr = (int)MFExtern.MFSetAttributeRatio(mediaTypeIn, MFAttributesClsid.MF_MT_PIXEL_ASPECT_RATIO, 1, 1);
        if (Succeeded(hr)) hr = (int)sinkWriter.SetInputMediaType(streamIndex, mediaTypeIn, null);


        // Start accepting data
        if (Succeeded(hr)) hr = (int)sinkWriter.BeginWriting();


        COMBase.SafeRelease(mediaTypeOut);
        COMBase.SafeRelease(mediaTypeIn);

        return hr;
    }

书写框

 int hr = 0;
        IMFSample sample = null;
        IMFMediaBuffer buffer = null;
        IMF2DBuffer p2Dbuffer = null;
        object texNativeObject = Marshal.GetObjectForIUnknown(surface.NativePointer);

        if (Succeeded(hr)) hr = (int)MFExtern.MFCreateDXGISurfaceBuffer(new Guid("6f15aaf2-d208-4e89-9ab4-489535d34f9c"), texNativeObject, 0, false, out p2Dbuffer);

        buffer = MFVideoEncoderST.ReinterpretCast<IMF2DBuffer,IMFMediaBuffer>(p2Dbuffer);
        int length=0;
        if (Succeeded(hr)) hr = (int)p2Dbuffer.GetContiguousLength(out length);
        if (Succeeded(hr)) hr = (int)buffer.SetCurrentLength(length);


        if (Succeeded(hr)) hr = (int)MFExtern.MFCreateVideoSampleFromSurface(null, out sample);

        if (Succeeded(hr)) hr = (int)sample.AddBuffer(buffer);
        if (Succeeded(hr)) hr = (int)sample.SetSampleTime(prevRecordingDuration);
        if (Succeeded(hr)) hr = (int)sample.SetSampleDuration((recordDuration - prevRecordingDuration));

        if (Succeeded(hr)) hr = (int)sinkWriter.WriteSample(streamIndex, sample);


        COMBase.SafeRelease(sample);
        COMBase.SafeRelease(buffer);

使用 MFTRACE 我收到以下错误。

    02:48:04.99463 CMFSinkWriterDetours::WriteSample @024BEA18 Stream Index 0x0, Sample @17CEACE0, Time 571ms, Duration 16ms, Buffers 1, Size 4196352B,2088,2008 02:48:04.99465 CMFSinkWriterDetours::WriteSample @024BEA18 failed hr=0x887A0005 (null)2088,2008 
02:48:05.01090 CMFSinkWriterDetours::WriteSample @024BEA18 Stream Index 0x0, Sample @17CE9FC0, Time 587ms, Duration 17ms, Buffers 1, Size 4196352B,2088,2008 02:48:05.01091 CMFSinkWriterDetours::WriteSample @024BEA18 failed hr=0x887A0005 (null)2088,2008 
02:48:05.02712 CMFSinkWriterDetours::WriteSample @024BEA18 Stream Index 0x0, Sample @17CEACE0, Time 604ms, Duration 16ms, Buffers 1, Size 4196352B,2088,2008 02:48:05.02713 CMFSinkWriterDetours::WriteSample @024BEA18 failed hr=0x887A0005 (null)

谁能告诉我我的代码有什么问题？我只能生成 0 字节的 mp4 文件。

Answer 1

我在这里遇到了一些潜在的问题。Roman提到了两个大的，所以我会详细说明。我还有一些其他的批评/建议给你。

不使用 IMFDXGIDeviceManager

为了在 Media Foundation 中使用硬件加速，您需要创建一个 DirectX 设备管理器对象，IDirect3DDeviceManager9用于 DX9 或在您的情况下IMFDXGIDeviceManager用于 DXGI。我强烈建议阅读该接口的所有 MSDN 文档。这是必要的原因是因为必须在所有正在使用的协作硬件 MF 变换之间共享同一个 DX 设备，因为它们都需要访问设备控制的共享 GPU 内存，并且每个设备都需要在设备工作时对其进行独占控制，所以需要一个锁定系统。设备管理器对象提供该锁定系统，也是将 DX 设备提供给一个或多个转换的标准方式。对于 DXGI，您可以使用MFCreateDXGIDeviceManager.

从那里，您需要创建您的 DX11 设备，并IMFDXGIDeviceManager::ResetDevice使用您的 DX11 设备进行呼叫。然后您需要为 Sink Writer 本身设置设备管理器，这在您上面提供的代码中没有完成。这是这样完成的：

// ... inside your InitializeSinkWriter function that you listed above

// I'm assuming you've already created and set up the DXGI device manager elsewhere
IMFDXGIDeviceManager pDeviceManager;

// Passing 3 as the argument because we're adding 3 attributes immediately, saves re-allocations
if (Succeeded(hr)) hr = (int)MFExtern.MFCreateAttributes(out attributes, 3);
if (Succeeded(hr)) hr = (int)attributes.SetUINT32(MFAttributesClsid.MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, 1);            
if (Succeeded(hr)) hr = (int)attributes.SetUINT32(MFAttributesClsid.MF_LOW_LATENCY, 1);

// Here's the key piece!
if (Succeeded(hr)) hr = (int)attributes.SetUnknown(MFAttributesClsid.MF_SINK_WRITER_D3D_MANAGER, pDeviceManager);

// Create the sink writer 
if (Succeeded(hr)) hr = (int)MFExtern.MFCreateSinkWriterFromURL(outputFile, null, attributes, out sinkWriter);

这实际上将为硬件编码器启用 D3D11 支持，并允许它访问读取Texture2D您传入的内容。值得注意的是，它MF_SINK_WRITER_D3D_MANAGER适用于 DX9 和 DXGI 设备管理器。

编码器缓冲IMFSample同一纹理的多个实例

这也是您问题的潜在原因 - 至少它会导致许多意外行为，即使它不是明显问题的原因。根据 Roman 的评论，许多编码器将缓冲多个帧作为其编码过程的一部分。使用 Sink Writer 时您不会看到这种行为，因为它会为您处理所有细节工作。但是，您要完成的任务（即，将 D3D11 纹理作为输入帧发送）的级别足够低，您开始不得不担心 Sink Writer 使用的编码器 MFT 的内部细节。

大多数视频编码器MFTS将使用一些尺寸的内部缓冲器来存储最后Ñ经由提供的样品IMFTransform::ProcessInput。这具有副作用，即在生成任何输出之前，必须提供多个样本作为输入。视频编码器需要按顺序访问多个样本，因为它们使用后续帧来确定如何对当前帧进行编码。换句话说，如果解码器正在处理第 0 帧，则它可能还需要查看第 1、2 和 3 帧。从技术角度来看，这是因为诸如帧间预测和运动估计之类的事情。一旦编码器处理完最旧的样本，它就会生成一个输出缓冲区（另一个IMFSample对象，但这次在输出端通过IMFTransform::ProcessOutput) 然后丢弃它正在处理的输入样本（通过调用IUnknown::Release），然后请求更多输入，并最终移动到下一帧。您可以在 MSDN 文章Processing Data in the Encoder 中阅读有关此过程的更多信息

正如 Roman 所暗示的那样，这意味着您正在封装一个ID3D11Texture2Dinside an IMFMediaBufferinside an IMFSample，然后将其传递给 Sink Writer。作为编码过程的一部分，该样本很可能被编码器缓冲。随着编码器的工作，其中的内容Texture2D可能会发生变化，这可能会导致各种问题。即使这不会导致程序错误，它肯定会导致非常奇怪的编码视频输出。想象一下，如果编码器试图预测下一帧中一帧的视觉内容如何变化，然后两帧的实际视觉内容从编码器下方更新出来！

之所以会出现这个特定问题，是因为编码器只有一个指向您的IMFSample实例的指针引用，它最终只是指向您的ID3D11Texture2D对象的指针本身，而该对象是一种指向可变图形内存的指针引用。最终，由于程序的其他部分，该图形内存的内容会发生变化，但由于更新的 GPU 纹理始终相同，因此您发送给编码器的每个样本都指向同一个纹理。这意味着每当您通过更改 GPU 内存更新纹理时，所有活动IMFSample对象都将反映这些更改，因为它们都有效地指向相同的 GPU 纹理。

要解决此问题，您需要分配多个ID3D11Texture2D对象，以便IMFSample在将其提供给 Sink Writer 时可以将一个纹理与一个纹理配对。这将通过使每个样本指向唯一的纹理来解决所有样本指向同一个 GPU 纹理的问题。但是，您不一定知道需要创建多少纹理，因此处理此问题的最安全方法是编写自己的纹理分配器。这仍然可以在 C# 中完成，因为它的价值，MediaFoundation.NET 已经定义了您需要使用的接口。

分配器应该维护一个“空闲”SharpDX.Texture2D对象列表——那些当前未被 Sink Writer / 编码器使用的对象。您的程序应该能够从分配器请求新的纹理对象，在这种情况下，它将从空闲列表中返回一个对象，或者创建一个新的纹理来满足请求。

下一个问题是知道IMFSample对象何时被编码器丢弃，以便您可以将附加的纹理添加回空闲列表。碰巧的是，MFCreateVideoSampleFromSurface您当前使用的函数分配了实现IMFTrackedSample接口的样本。您将需要该接口，以便在样本被释放时收到通知，以便您可以回收Texture2D对象。

诀窍是你必须告诉样本你是分配器。首先，您的分配器类需要实现IMFAsyncCallback. 如果您通过在样本上设置分配器类IMFTrackedSample::SetAllocator，IMFAsyncCallback::Invoke则将调用分配器的方法，并IMFAsyncResult在编码器释放样本时将其作为参数传递。下面是分配器类的一般示例。

sealed class TextureAllocator : IMFAsyncCallback, IDisposable
{
    private ConcurrentStack<SharpDX.Direct3D11.Texture2D> m_freeStack;
    private static readonly Guid s_IID_ID3D11Texture2D = new Guid("6f15aaf2-d208-4e89-9ab4-489535d34f9c");

    // If all textures are the exact same size and color format,
    // consider making those parameters private class members and
    // requiring they be specified as arguments to the constructor.
    public TextureAllocator()
    {
        m_freeStack = new ConcurrentStack<SharpDX.Direct3D11.Texture2D>();
    }

    private bool disposedValue = false;
    private void Dispose(bool disposing)
    {
        if(!disposedValue)
        {
            if(disposing)
            {
                // Dispose managed resources here
            }

            if(m_freeStack != null)
            {
                SharpDX.Direct3D11.Texture2D texture;
                while(m_freeStack.TryPop(out texture))
                {
                    texture.Dispose();
                }
                m_freeStack = null;
            }

            disposedValue = true;
        }
    }

    public void Dispose()
    {
        Dispose(true);
        GC.SuppressFinalize(this);
    }

    ~TextureAllocator()
    {
        Dispose(false);
    }

    private SharpDX.Direct3D11.Texture2D InternalAllocateNewTexture()
    {
        // Allocate a new texture with your format, size, etc here.
    }

    public SharpDX.Direct3D11.Texture2D AllocateTexture()
    {
        SharpDX.Direct3D11.Texture2D existingTexture;
        if(m_freeStack.TryPop(out existingTexture))
        {
            return existingTexture;
        }
        else
        {
            return InternalAllocateNewTexture();
        }
    }

    public IMFSample CreateSampleAndAllocateTexture()
    {
        IMFSample pSample;
        IMFTrackedSample pTrackedSample;
        HResult hr;

        // Create the video sample. This function returns an IMFTrackedSample per MSDN
        hr = MFExtern.MFCreateVideoSampleFromSurface(null, out pSample);
        MFError.ThrowExceptionForHR(hr);

        // Query the IMFSample to see if it implements IMFTrackedSample
        pTrackedSample = pSample as IMFTrackedSample;
        if(pTrackedSample == null)
        {
            // Throw an exception if we didn't get an IMFTrackedSample
            // but this shouldn't happen in practice.
            throw new InvalidCastException("MFCreateVideoSampleFromSurface returned a sample that did not implement IMFTrackedSample");
        }

        // Use our own class to allocate a texture
        SharpDX.Direct3D11.Texture2D availableTexture = AllocateTexture();
        // Convert the texture's native ID3D11Texture2D pointer into
        // an IUnknown (represented as as System.Object)
        object texNativeObject = Marshal.GetObjectForIUnknown(availableTexture.NativePointer);

        // Create the media buffer from the texture
        IMFMediaBuffer p2DBuffer;
        hr = MFExtern.MFCreateDXGISurfaceBuffer(s_IID_ID3D11Texture2D, texNativeObject, 0, false, out p2DBuffer);
        // Release the object-as-IUnknown we created above
        COMBase.SafeRelease(texNativeObject);
        // If media buffer creation failed, throw an exception
        MFError.ThrowExceptionForHR(hr);

        // Set the owning instance of this class as the allocator
        // for IMFTrackedSample to notify when the sample is released
        pTrackedSample.SetAllocator(this, null);

        // Attach the created buffer to the sample
        pTrackedSample.AddBuffer(p2DBuffer);

        return pTrackedSample;
    }

    // This is public so any textures you allocate but don't make IMFSamples 
    // out of can be returned to the allocator manually.
    public void ReturnFreeTexture(SharpDX.Direct3D11.Texture2D freeTexture)
    {
        m_freeStack.Push(freeTexture);
    }

    // IMFAsyncCallback.GetParameters
    // This is allowed to return E_NOTIMPL as a way of specifying
    // there are no special parameters.
    public HResult GetParameters(out MFAsync pdwFlags, out MFAsyncCallbackQueue pdwQueue)
    {
        pdwFlags = MFAsync.None;
        pdwQueue = MFAsyncCallbackQueue.Standard;
        return HResult.E_NOTIMPL;
    }

    public HResult Invoke(IMFAsyncResult pResult)
    {
        object pUnkObject;
        IMFSample pSample = null;
        IMFMediaBuffer pBuffer = null;
        IMFDXGIBuffer pDXGIBuffer = null;

        // Get the IUnknown out of the IMFAsyncResult if there is one
        HResult hr = pResult.GetObject(out pUnkObject);
        if(Succeeded(hr))
        {
            pSample = pUnkObject as IMFSample;
        }

        if(pSample != null)
        {
            // Based on your implementation, there should only be one 
            // buffer attached to one sample, so we can always grab the
            // first buffer. You could add some error checking here to make
            // sure the sample has a buffer count that is 1.
            hr = pSample.GetBufferByIndex(0, out pBuffer);
        }

        if(Succeeded(hr))
        {
            // Query the IMFMediaBuffer to see if it implements IMFDXGIBuffer
            pDXGIBuffer = pBuffer as IMFDXGIBuffer;
        }

        if(pDXGIBuffer != null)
        {
           // Got an IMFDXGIBuffer, so we can extract the internal 
           // ID3D11Texture2D and make a new SharpDX.Texture2D wrapper.
           hr = pDXGIBuffer.GetResource(s_IID_ID3D11Texture2D, out pUnkObject);
        }

        if(Succeeded(hr))
        {
           // If we got here, pUnkObject is the native D3D11 Texture2D as
           // a System.Object, but it's unlikely you have an interface 
           // definition for ID3D11Texture2D handy, so we can't just cast
           // the object to the proper interface.

           // Happily, SharpDX supports wrapping System.Object within
           // SharpDX.ComObject which makes things pretty easy.
           SharpDX.ComObject comWrapper = new SharpDX.ComObject(pUnkObject);

           // If this doesn't work, or you're using something like SlimDX
           // which doesn't support object wrapping the same way, the below
           // code is an alternative way.
           /*
           IntPtr pD3DTexture2D = Marshal.GetIUnknownForObject(pUnkObject);
           // Create your wrapper object here, like this for SharpDX
           SharpDX.ComObject comWrapper = new SharpDX.ComObject(pD3DTexture2D);
           // or like this for SlimDX
           SlimDX.Direct3D11.Texture2D.FromPointer(pD3DTexture2D);
           Marshal.Release(pD3DTexture2D);
           */

           // You might need to query comWrapper for a SharpDX.DXGI.Resource
           // first, then query that for the SharpDX.Direct3D11.Texture2D.
           SharpDX.Direct3D11.Texture2D texture = comWrapper.QueryInterface<SharpDX.Direct3D11.Texture2D>();
           if(texture != null)
           {
               // Now you can add "texture" back to the allocator's free list
               ReturnFreeTexture(texture);
           }
        }
    }
}

设置MF_SA_D3D_AWARE在水槽作家输入介质类型

我不认为这会导致HRESULT您遇到的坏事，但无论如何这都不是正确的做法。MF_SA_D3D_AWARE（以及它的 DX11 对应物，MF_SA_D3D11_AWARE）是由IMFTransform对象设置的属性，用于通知您变换支持分别通过 DX9 或 DX11 进行的图形加速。无需在 Sink Writer 的输入媒体类型上设置此项。

不SafeRelease开texNativeObject

我建议调用COMBase.SafeRelease()，texNativeObject否则可能会泄漏内存。那，或者你会不必要地延长那个 COM 对象的生命周期，直到 GC 为你清理引用计数

不必要的铸造

这是上面代码的一部分：

buffer = MFVideoEncoderST.ReinterpretCast<IMF2DBuffer,IMFMediaBuffer>(p2Dbuffer);
int length=0;
if (Succeeded(hr)) hr = (int)p2Dbuffer.GetContiguousLength(out length);
if (Succeeded(hr)) hr = (int)buffer.SetCurrentLength(length);

我不确定您的ReinterpretCast函数在做什么，但是如果您确实需要QueryInterface在 C# 中执行样式转换，则可以使用as运算符或常规转换。

// pMediaBuffer is of type IMFMediaBuffer and has been created elsewhere
IMF2DBuffer p2DBuffer = pMediaBuffer as IMF2DBuffer;
if(p2DBuffer != null)
{
    // pMediaBuffer is an IMFMediaBuffer that also implements IMF2DBuffer
}
else
{
    // pMediaBuffer does not implement IMF2DBuffer
}