问题描述
我正在尝试使用 IMFTransform 将纹理编码为 H264。我可以使用 SinkWriter 将纹理写入和编码到文件中,并且可以播放视频和所有内容,效果很好。但我正在尝试学习如何使用 IMFTransform,以便我可以自己访问编码的 IMFSamples。
不幸的是,我并没有走得太远,因为 ProcessInput 失败了,"The buffer was too small to carry out the requested action."
作为 HRESULT。
我不知道它指的是哪个“缓冲区”,并且对该错误进行搜索绝对没有结果。除了 Processinput()
之外,没有其他调用返回错误的 HRESULT,并且 SinkWriter 工作正常。所以我完全零线索问题是什么。
#include "main.h"
#include "WinDesktopDup.h"
#include <iostream>
#include <wmcodecdsp.h>
WinDesktopDup dup;
void SetupDpiAwareness()
{
if (!SetProcessDpiAwarenessContext(DPI_AWAREnesS_CONTEXT_SYstem_AWARE))
printf("SetProcessDpiAwarenessContext Failed\n");
}
const UINT32 VIDEO_WIDTH = 3840;
const UINT32 VIDEO_HEIGHT = 2160;
const UINT32 VIDEO_FPS = 120;
const UINT64 VIDEO_FRAME_DURATION = 10 * 1000 * 1000 / VIDEO_FPS;
const UINT32 VIDEO_BIT_RATE = 800000;
const GUID VIDEO_ENCODING_FORMAT = MFVideoFormat_H264;
const GUID VIDEO_INPUT_FORMAT = MFVideoFormat_ARGB32;
const UINT32 VIDEO_PELS = VIDEO_WIDTH * VIDEO_HEIGHT;
const UINT32 VIDEO_FRAME_COUNT = 20 * VIDEO_FPS;
template <class T>
void SafeRelease(T** ppT) {
if (*ppT) {
(*ppT)->Release();
*ppT = NULL;
}
}
bool usingEncoder;
IMFMediaType* pMediaTypeOut = NULL;
IMFMediaType* pMediaTypeIn = NULL;
HRESULT SetMediaType()
{
// Set the output media type.
HRESULT hr = MFCreateMediaType(&pMediaTypeOut);
if (!SUCCEEDED(hr)) { printf("MFCreateMediaType Failed\n"); }
hr = pMediaTypeOut->SetGUID(MF_MT_MAJOR_TYPE,MFMediaType_Video);
if (!SUCCEEDED(hr)) { printf("SetGUID Failed\n"); }
hr = pMediaTypeOut->SetGUID(MF_MT_SUBTYPE,VIDEO_ENCODING_FORMAT);
if (!SUCCEEDED(hr)) { printf("SetGUID (2) Failed\n"); }
hr = pMediaTypeOut->SetUINT32(MF_MT_AVG_BITRATE,VIDEO_BIT_RATE);
if (!SUCCEEDED(hr)) { printf("SetUINT32 (3) Failed\n"); }
hr = pMediaTypeOut->SetUINT32(MF_MT_INTERLACE_MODE,MFVideoInterlace_Progressive);
if (!SUCCEEDED(hr)) { printf("SetUINT32 (4) Failed\n"); }
hr = MFSetAttributeSize(pMediaTypeOut,MF_MT_FRAME_SIZE,VIDEO_WIDTH,VIDEO_HEIGHT);
if (!SUCCEEDED(hr)) { printf("MFSetAttributeSize Failed\n"); }
hr = MFSetAttributeratio(pMediaTypeOut,MF_MT_FRAME_RATE,VIDEO_FPS,1);
if (!SUCCEEDED(hr)) { printf("MFSetAttributeratio Failed\n"); }
hr = MFSetAttributeratio(pMediaTypeOut,MF_MT_PIXEL_ASPECT_RATIO,1,1);
if (!SUCCEEDED(hr)) { printf("MFSetAttributeratio (2) Failed\n"); }
// Set the input media type.
hr = MFCreateMediaType(&pMediaTypeIn);
if (!SUCCEEDED(hr)) { printf("MFCreateMediaType Failed\n"); }
hr = pMediaTypeIn->SetGUID(MF_MT_MAJOR_TYPE,MFMediaType_Video);
if (!SUCCEEDED(hr)) { printf("SetGUID (3) Failed\n"); }
hr = pMediaTypeIn->SetGUID(MF_MT_SUBTYPE,VIDEO_INPUT_FORMAT);
if (!SUCCEEDED(hr)) { printf("SetGUID (4) Failed\n"); }
hr = pMediaTypeIn->SetUINT32(MF_MT_INTERLACE_MODE,MFVideoInterlace_Progressive);
if (!SUCCEEDED(hr)) { printf("SetUINT32 (5) Failed\n"); }
hr = MFSetAttributeSize(pMediaTypeIn,VIDEO_HEIGHT);
if (!SUCCEEDED(hr)) { printf("MFSetAttributeSize (2) Failed\n"); }
hr = MFSetAttributeratio(pMediaTypeIn,1);
if (!SUCCEEDED(hr)) { printf("MFSetAttributeratio (3) Failed\n"); }
hr = MFSetAttributeratio(pMediaTypeIn,1);
if (!SUCCEEDED(hr)) { printf("MFSetAttributeratio (4) Failed\n"); }
return hr;
}
HRESULT InitializeSinkWriter(IMFSinkWriter** ppWriter,DWORD* pStreamIndex)
{
IMFdxgiDeviceManager* pDeviceManager = NULL;
UINT resetToken;
IMFAttributes* attributes;
*ppWriter = NULL;
*pStreamIndex = NULL;
IMFSinkWriter* pSinkWriter = NULL;
DWORD streamIndex;
HRESULT hr = MFCreatedxgiDeviceManager(&resetToken,&pDeviceManager);
if (!SUCCEEDED(hr)) { printf("MFCreatedxgiDeviceManager Failed\n"); }
hr = pDeviceManager->ResetDevice(dup.D3DDevice,resetToken);
if (!SUCCEEDED(hr)) { printf("ResetDevice Failed\n"); }
hr = MFCreateAttributes(&attributes,3);
if (!SUCCEEDED(hr)) { printf("MFCreateAttributes Failed\n"); }
hr = attributes->SetUINT32(MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS,1);
if (!SUCCEEDED(hr)) { printf("SetUINT32 Failed\n"); }
hr = attributes->SetUINT32(MF_LOW_LATENCY,1);
if (!SUCCEEDED(hr)) { printf("SetUINT32 (2) Failed\n"); }
hr = attributes->SetUnkNown(MF_SINK_WRITER_D3D_MANAGER,pDeviceManager);
if (!SUCCEEDED(hr)) { printf("SetUnkNown Failed\n"); }
hr = MFCreateSinkWriterFromURL(L"output.mp4",NULL,attributes,&pSinkWriter);
if (!SUCCEEDED(hr)) { printf("MFCreateSinkWriterFromURL Failed\n"); }
hr = pSinkWriter->AddStream(pMediaTypeOut,&streamIndex);
if (!SUCCEEDED(hr)) { printf("AddStream Failed\n"); }
hr = pSinkWriter->SetInputMediaType(streamIndex,pMediaTypeIn,NULL);
if (!SUCCEEDED(hr)) { printf("SetInputMediaType Failed\n"); }
// Tell the sink writer to start accepting data.
hr = pSinkWriter->BeginWriting();
if (!SUCCEEDED(hr)) { printf("BeginWriting Failed\n"); }
// Return the pointer to the caller.
*ppWriter = pSinkWriter;
(*ppWriter)->AddRef();
*pStreamIndex = streamIndex;
SafeRelease(&pSinkWriter);
SafeRelease(&pMediaTypeOut);
SafeRelease(&pMediaTypeIn);
return hr;
}
IUnkNown* _transformUnk;
IMFTransform* pMFTransform;
HRESULT InitializeEncoder(DWORD* pStreamIndex)
{
HRESULT hr = CoCreateInstance(CLSID_CMSH264EncoderMFT,CLSCTX_INPROC_SERVER,IID_IUnkNown,(void**)&_transformUnk);
if (!SUCCEEDED(hr)) { printf("CoCreateInstance Failed\n"); }
hr = _transformUnk->QueryInterface(IID_PPV_ARGS(&pMFTransform));
if (!SUCCEEDED(hr)) { printf("QueryInterface Failed\n"); }
hr = pMFTransform->SetoutputType(0,pMediaTypeOut,0);
if (!SUCCEEDED(hr)) { printf("SetoutputType Failed\n"); }
hr = pMFTransform->SetInputType(0,0);
if (!SUCCEEDED(hr)) { printf("SetInputType Failed\n"); }
DWORD mftStatus = 0;
hr = pMFTransform->GetInputStatus(0,&mftStatus);
if (!SUCCEEDED(hr)) { printf("GetInputStatus Failed\n"); }
if (MFT_INPUT_STATUS_ACCEPT_DATA != mftStatus)
printf("MFT_INPUT_STATUS_ACCEPT_DATA\n");
hr = pMFTransform->ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING,NULL);
if (!SUCCEEDED(hr)) { printf("MFT_MESSAGE_NOTIFY_BEGIN_STREAMING Failed\n"); }
hr = pMFTransform->ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM,NULL);
if (!SUCCEEDED(hr)) { printf("MFT_MESSAGE_NOTIFY_START_OF_STREAM Failed\n"); }
SafeRelease(&pSinkWriter);
SafeRelease(&pMediaTypeOut);
SafeRelease(&pMediaTypeIn);
return hr;
}
ID3D11Texture2D* texture;
HRESULT WriteFrame(IMFSinkWriter* pWriter,DWORD streamIndex,const LONGLONG& rtStart)
{
IMFSample* pSample = NULL;
IMFMediaBuffer* pBuffer = NULL;
HRESULT hr;
hr = MFCreatedxgiSurfaceBuffer(__uuidof(ID3D11Texture2D),texture,false,&pBuffer);
if (!SUCCEEDED(hr)) { printf("MFCreatedxgiSurfaceBuffer Failed\n"); }
DWORD len;
hr = ((IMF2DBuffer*)pBuffer)->GetContiguousLength(&len);
if (!SUCCEEDED(hr)) { printf("GetContiguousLength Failed\n"); }
hr = pBuffer->SetCurrentLength(len);
if (!SUCCEEDED(hr)) { printf("SetCurrentLength Failed\n"); }
// Create a media sample and add the buffer to the sample.
hr = MFCreateSample(&pSample);
if (!SUCCEEDED(hr)) { printf("MFCreateSample Failed\n"); }
hr = pSample->AddBuffer(pBuffer);
if (!SUCCEEDED(hr)) { printf("AddBuffer Failed\n"); }
// Set the time stamp and the duration.
hr = pSample->SetSampleTime(rtStart);
if (!SUCCEEDED(hr)) { printf("SetSampleTime Failed\n"); }
hr = pSample->SetSampleDuration(VIDEO_FRAME_DURATION);
if (!SUCCEEDED(hr)) { printf("SetSampleDuration Failed\n"); }
// Send the sample to the Sink Writer or Encoder.
if (!usingEncoder)
{
hr = pWriter->WriteSample(streamIndex,pSample);
if (!SUCCEEDED(hr)) { printf("WriteSample Failed\n"); }
}
else
{
hr = pMFTransform->ProcessInput(0,pSample,0);
if (!SUCCEEDED(hr)) { printf("ProcessInput Failed\n"); }
}
SafeRelease(&pSample);
SafeRelease(&pBuffer);
return hr;
}
int APIENTRY main(HINSTANCE hInstance,HINSTANCE hPrevInstance,LPSTR lpCmdLine,int nShowCmd)
{
SetupDpiAwareness();
auto err = dup.Initialize();
// Initialize MF
CoInitializeEx(0,COINIT_APARTMENTTHREADED); // Need to call this once when a thread is using COM or it wont work
MFStartup(MF_VERSION); // Need to call this too for Media Foundation related memes
IMFSinkWriter* pSinkWriter = NULL;
DWORD stream = 0;
LONGLONG rtStart = 0;
usingEncoder = true; // True if we want to encode with IMFTransform,false if we want to write with SinkWriter
HRESULT hr = SetMediaType();
if (!SUCCEEDED(hr)) { printf("SetMediaType Failed\n"); }
if (!usingEncoder)
{
hr = InitializeSinkWriter(&pSinkWriter,&stream);
if (!SUCCEEDED(hr)) { printf("InitializeSinkWriter Failed\n"); }
}
else
{
hr = pMediaTypeIn->SetGUID(MF_MT_SUBTYPE,MFVideoFormat_IYUV); // Using MFVideoFormat_ARGB32 causes SetInputType() to fail
hr = InitializeEncoder(&stream);
if (!SUCCEEDED(hr)) { printf("InitializeEncoder Failed\n"); }
}
const int CAPTURE_LENGTH = 10;
int total_frames = VIDEO_FPS * CAPTURE_LENGTH;
for (int i = 0; i < 1; i++)
{
texture = dup.CaptureNext();
if (texture != nullptr)
{
hr = WriteFrame(pSinkWriter,stream,rtStart);
if (!SUCCEEDED(hr))
printf("WriteFrame Failed\n");
rtStart += VIDEO_FRAME_DURATION;
texture->Release();
}
else
{
i--;
}
}
if (Failed(hr))
{
std::cout << "Failure" << std::endl;
}
if (SUCCEEDED(hr)) {
hr = pSinkWriter->Finalize();
}
SafeRelease(&pSinkWriter);
MFShutdown();
CoUninitialize();
}
解决方法
Here’s documentation 用于您在代码中使用的 Microsoft 基于 CPU 的软件 h.264 编码器。
它不支持 MFVideoFormat_ARGB32
输入。它根本不支持任何 RGB 格式。该转换仅支持输入视频的 YUV 格式。
顺便说一句,如果您用硬件编码器替换 MFT,它们很可能会公开与 Microsoft 软件相同的一组功能,我认为它们不支持 RGB。而且,由于所有硬件转换都是异步的,因此您需要稍微不同的工作流程来直接驱动它们。
sink writer 工作正常的原因,它在引擎盖下创建并托管了 2 个 MFT,从 RGB 到 YUV 的格式转换器,另一个是编码器。
您有以下选择。
-
在将帧传递给编码器之前,使用另一个 MFT 将 RGBA 转换为 NV12。
-
使用像素着色器(使用 2 个不同的像素着色器将纹理四边形渲染到 NV12 纹理的 2 个平面)或使用单个计算着色器(为视频的每个 2x2 块分配 1 个线程,编写每个块 6 个字节,4 个到
R8_UNORM
带有亮度的输出纹理,另外 2 个字节到R8G8_UNORM
带有颜色数据的输出纹理)。 -
使用接收器编写器,但使用
MFCreateSinkWriterFromMediaSink
API 而不是MFCreateSinkWriterFromURL
创建它。实现IMFMediaSink
COM 接口,也为它的视频流实现IMFStreamSink
,并且框架将调用IMFStreamSink.ProcessSample
,一旦它们可用,就会在系统内存中为您提供编码视频样本。