Fix FFmpeg hwdevice setup for Vulkan.

Author: Themaister

tests/video_encode_test.cpp

@@ -21,7 +21,7 @@ int main()
 	options.width = 640;
 	options.height = 480;
 	options.frame_timebase = { 1, 60 };
-	options.encoder = "h264_pyro";
+	options.encoder = "hevc_vulkan";
 	options.low_latency = true;
 	options.realtime = true;
 
@@ -33,7 +33,8 @@ int main()
 	ctx.set_system_handles(handles);
 	if (!ctx.init_instance_and_device(nullptr, 0, nullptr, 0,
 	                                  Vulkan::CONTEXT_CREATION_ENABLE_VIDEO_ENCODE_BIT |
-	                                  Vulkan::CONTEXT_CREATION_ENABLE_VIDEO_H264_BIT))
+	                                  Vulkan::CONTEXT_CREATION_ENABLE_VIDEO_H264_BIT |
+	                                  Vulkan::CONTEXT_CREATION_ENABLE_VIDEO_H265_BIT))
 	{
 		return 1;
 	}

video/ffmpeg_decode.cpp

@@ -1056,7 +1056,7 @@ void VideoDecoder::Impl::begin_audio_stream()
 
 bool VideoDecoder::Impl::init_video_decoder_post_device()
 {
-	if (!hw.init_codec_context(video.av_codec, device, video.av_ctx, opts.hwdevice))
+	if (!hw.init_codec_context(video.av_codec, device, video.av_ctx, opts.hwdevice, false))
 		LOGW("Failed to init hardware decode context. Falling back to software.\n");
 
 	if (avcodec_open2(video.av_ctx, video.av_codec, nullptr) < 0)

video/ffmpeg_encode.cpp

@@ -912,7 +912,7 @@ bool VideoEncoder::Impl::init_video_codec_av(const AVCodec *codec)
 {
 	if (avcodec_get_hw_config(codec, 0) != nullptr)
 	{
-		if (!hw.init_codec_context(codec, device, nullptr, nullptr))
+		if (!hw.init_codec_context(codec, device, nullptr, nullptr, true))
 		{
 			LOGW("Failed to init HW encoder context, falling back to software.\n");
 			return false;

video/ffmpeg_hw_device.cpp

@@ -157,18 +157,36 @@ struct FFmpegHWDevice::Impl
 		}
 	}
 
-	bool init_hw_device(const AVCodec *av_codec, const char *type)
+	bool init_hw_device(const AVCodec *av_codec, const char *type, bool encode)
 	{
 #ifdef HAVE_FFMPEG_VULKAN
-		bool use_vulkan = device->get_device_features().supports_video_decode_queue;
-		if (use_vulkan)
+		bool use_vulkan = false;
+
+		if (encode)
 		{
-			if (av_codec->id == AV_CODEC_ID_H264)
-				use_vulkan = device->get_device_features().supports_video_decode_h264;
-			else if (av_codec->id == AV_CODEC_ID_HEVC)
-				use_vulkan = device->get_device_features().supports_video_decode_h265;
-			else
-				use_vulkan = false;
+			use_vulkan = device->get_device_features().supports_video_encode_queue;
+			if (use_vulkan)
+			{
+				if (av_codec->id == AV_CODEC_ID_H264)
+					use_vulkan = device->get_device_features().supports_video_encode_h264;
+				else if (av_codec->id == AV_CODEC_ID_HEVC)
+					use_vulkan = device->get_device_features().supports_video_encode_h265;
+				else
+					use_vulkan = false;
+			}
+		}
+		else
+		{
+			use_vulkan = device->get_device_features().supports_video_decode_queue;
+			if (use_vulkan)
+			{
+				if (av_codec->id == AV_CODEC_ID_H264)
+					use_vulkan = device->get_device_features().supports_video_decode_h264;
+				else if (av_codec->id == AV_CODEC_ID_HEVC)
+					use_vulkan = device->get_device_features().supports_video_decode_h265;
+				else
+					use_vulkan = false;
+			}
 		}
 #endif
 
@@ -263,7 +281,7 @@ struct FFmpegHWDevice::Impl
 	}
 
 	bool init_codec_context(const AVCodec *av_codec, Vulkan::Device *device_,
-	                        AVCodecContext *av_ctx, const char *type)
+	                        AVCodecContext *av_ctx, const char *type, bool encode)
 	{
 		if (device && (device != device_ || av_codec != cached_av_codec))
 		{
@@ -277,7 +295,7 @@ struct FFmpegHWDevice::Impl
 		device = device_;
 		cached_av_codec = av_codec;
 
-		if (!init_hw_device(av_codec, type))
+		if (!init_hw_device(av_codec, type, encode))
 			return false;
 
 		if (av_ctx && (hw_config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) != 0)
@@ -337,11 +355,11 @@ FFmpegHWDevice::~FFmpegHWDevice()
 }
 
 bool FFmpegHWDevice::init_codec_context(const AVCodec *codec, Vulkan::Device *device,
-                                        AVCodecContext *ctx, const char *type)
+                                        AVCodecContext *ctx, const char *type, bool encode)
 {
 	if (!impl)
 		impl.reset(new Impl);
-	return impl->init_codec_context(codec, device, ctx, type);
+	return impl->init_codec_context(codec, device, ctx, type, encode);
 }
 
 bool FFmpegHWDevice::init_frame_context(AVCodecContext *ctx,

video/ffmpeg_hw_device.hpp

@@ -40,7 +40,7 @@ struct FFmpegHWDevice
 	FFmpegHWDevice();
 	~FFmpegHWDevice();
 
-	bool init_codec_context(const AVCodec *codec, Vulkan::Device *device, AVCodecContext *ctx, const char *type);
+	bool init_codec_context(const AVCodec *codec, Vulkan::Device *device, AVCodecContext *ctx, const char *type, bool encode);
 	bool init_frame_context(AVCodecContext *ctx, unsigned width, unsigned height, int sw_pixel_format);
 	int get_hw_device_type() const;
 	int get_pix_fmt() const;