Merge branch 'pipeline-binary'

Author: Themaister

third_party/fossilize

@@ -16,6 +16,7 @@ add_granite_internal_lib(granite-vulkan
         render_pass.cpp render_pass.hpp
         buffer.cpp buffer.hpp
         indirect_layout.cpp indirect_layout.hpp
+        pipeline_cache.cpp pipeline_cache.hpp
         semaphore.cpp semaphore.hpp
         memory_allocator.cpp memory_allocator.hpp
         fence.hpp fence.cpp

third_party/khronos/vulkan-headers

@@ -1142,7 +1142,7 @@ Pipeline CommandBuffer::build_compute_pipeline(Device *device, const DeferredPip
 		info.flags |= VK_PIPELINE_CREATE_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT;
 
 	auto start_ts = Util::get_current_time_nsecs();
-	VkResult vr = table.vkCreateComputePipelines(device->get_device(), compile.cache, 1, &info, nullptr, &compute_pipeline);
+	VkResult vr = device->pipeline_binary_cache.create_pipeline(&info, compile.cache, &compute_pipeline);
 	auto end_ts = Util::get_current_time_nsecs();
 	log_compile_time("compute", compile.hash, end_ts - start_ts, vr, mode);
 
@@ -1542,7 +1542,7 @@ Pipeline CommandBuffer::build_graphics_pipeline(Device *device, const DeferredPi
 		pipe.flags |= VK_PIPELINE_CREATE_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT;
 
 	auto start_ts = Util::get_current_time_nsecs();
-	VkResult res = table.vkCreateGraphicsPipelines(device->get_device(), compile.cache, 1, &pipe, nullptr, &pipeline);
+	VkResult res = device->pipeline_binary_cache.create_pipeline(&pipe, compile.cache, &pipeline);
 	auto end_ts = Util::get_current_time_nsecs();
 	log_compile_time("graphics", compile.hash, end_ts - start_ts, res, mode);
 

third_party/spirv-headers

@@ -1330,10 +1330,10 @@ bool Context::create_device(VkPhysicalDevice gpu_, VkSurfaceKHR surface,
 		}
 	}
 
-	if (has_extension(VK_NV_COMPUTE_SHADER_DERIVATIVES_EXTENSION_NAME))
+	if (has_extension(VK_KHR_COMPUTE_SHADER_DERIVATIVES_EXTENSION_NAME))
 	{
-		enabled_extensions.push_back(VK_NV_COMPUTE_SHADER_DERIVATIVES_EXTENSION_NAME);
-		ADD_CHAIN(ext.compute_shader_derivative_features, COMPUTE_SHADER_DERIVATIVES_FEATURES_NV);
+		enabled_extensions.push_back(VK_KHR_COMPUTE_SHADER_DERIVATIVES_EXTENSION_NAME);
+		ADD_CHAIN(ext.compute_shader_derivative_features, COMPUTE_SHADER_DERIVATIVES_FEATURES_KHR);
 	}
 
 	if (has_extension(VK_KHR_PERFORMANCE_QUERY_EXTENSION_NAME))
@@ -1454,6 +1454,20 @@ bool Context::create_device(VkPhysicalDevice gpu_, VkSurfaceKHR surface,
 		enabled_extensions.push_back(VK_EXT_LOAD_STORE_OP_NONE_EXTENSION_NAME);
 	}
 
+	// Pipeline binaries are currently borked in VVL.
+	if ((flags & CONTEXT_CREATION_ENABLE_PIPELINE_BINARY_BIT) != 0 &&
+	    has_extension(VK_KHR_PIPELINE_BINARY_EXTENSION_NAME))
+	{
+		enabled_extensions.push_back(VK_KHR_PIPELINE_BINARY_EXTENSION_NAME);
+		ADD_CHAIN(ext.pipeline_binary_features, PIPELINE_BINARY_FEATURES_KHR);
+	}
+
+	if (has_extension(VK_KHR_MAINTENANCE_5_EXTENSION_NAME))
+	{
+		enabled_extensions.push_back(VK_KHR_MAINTENANCE_5_EXTENSION_NAME);
+		ADD_CHAIN(ext.maintenance5_features, MAINTENANCE_5_FEATURES_KHR);
+	}
+
 	if ((flags & CONTEXT_CREATION_ENABLE_ADVANCED_WSI_BIT) != 0 && requires_swapchain)
 	{
 		if (has_extension(VK_KHR_PRESENT_ID_EXTENSION_NAME))
@@ -1643,8 +1657,26 @@ bool Context::create_device(VkPhysicalDevice gpu_, VkSurfaceKHR surface,
 	if (has_extension(VK_EXT_MESH_SHADER_EXTENSION_NAME))
 		ADD_CHAIN(ext.mesh_shader_properties, MESH_SHADER_PROPERTIES_EXT);
 
+	// Pipeline binaries are currently borked in VVL.
+	if ((flags & CONTEXT_CREATION_ENABLE_PIPELINE_BINARY_BIT) != 0 &&
+	    has_extension(VK_KHR_PIPELINE_BINARY_EXTENSION_NAME))
+		ADD_CHAIN(ext.pipeline_binary_properties, PIPELINE_BINARY_PROPERTIES_KHR);
+
 	vkGetPhysicalDeviceProperties2(gpu, &props);
 
+	// If a layer or driver doesn't tell us that internal cache is preferred,
+	// go ahead and take full control over the cache.
+	if (!ext.pipeline_binary_properties.pipelineBinaryPrefersInternalCache &&
+	    ext.pipeline_binary_features.pipelineBinaries &&
+	    ext.pipeline_binary_properties.pipelineBinaryInternalCacheControl)
+	{
+		ext.pipeline_binary_internal_cache_control.sType =
+				VK_STRUCTURE_TYPE_DEVICE_PIPELINE_BINARY_INTERNAL_CACHE_CONTROL_KHR;
+		ext.pipeline_binary_internal_cache_control.disableInternalCache = VK_TRUE;
+		ext.pipeline_binary_internal_cache_control.pNext = device_info.pNext;
+		device_info.pNext = &ext.pipeline_binary_internal_cache_control;
+	}
+
 	if (ext.device_api_core_version < VK_API_VERSION_1_2)
 	{
 		ext.driver_id = driver_properties.driverID;
@@ -1775,5 +1807,11 @@ bool Context::descriptor_set_layout_is_supported(const VkDescriptorSetLayoutCrea
 	vkGetDescriptorSetLayoutSupport(device, set_layout, &support);
 	return support.supported == VK_TRUE;
 }
+
+void Context::physical_device_feature_query(VkPhysicalDeviceFeatures2 *pdf2_)
+{
+	if (gpu && vkGetPhysicalDeviceFeatures2)
+		vkGetPhysicalDeviceFeatures2(gpu, pdf2_);
+}
 #endif
 }

third_party/spirv-tools

@@ -84,11 +84,16 @@ struct DeviceFeatures
 	VkPhysicalDeviceVulkan13Properties vk13_props = {};
 
 	// KHR
+	VkPhysicalDeviceComputeShaderDerivativesFeaturesKHR compute_shader_derivative_features = {};
 	VkPhysicalDevicePerformanceQueryFeaturesKHR performance_query_features = {};
 	VkPhysicalDevicePresentIdFeaturesKHR present_id_features = {};
 	VkPhysicalDevicePresentWaitFeaturesKHR present_wait_features = {};
 	VkPhysicalDeviceFragmentShaderBarycentricFeaturesKHR barycentric_features = {};
 	VkPhysicalDeviceVideoMaintenance1FeaturesKHR video_maintenance1_features = {};
+	VkPhysicalDevicePipelineBinaryFeaturesKHR pipeline_binary_features = {};
+	VkPhysicalDevicePipelineBinaryPropertiesKHR pipeline_binary_properties = {};
+	VkDevicePipelineBinaryInternalCacheControlKHR pipeline_binary_internal_cache_control = {};
+	VkPhysicalDeviceMaintenance5FeaturesKHR maintenance5_features = {};
 
 	// EXT
 	VkPhysicalDeviceExternalMemoryHostPropertiesEXT host_memory_properties = {};
@@ -105,7 +110,6 @@ struct DeviceFeatures
 	VkPhysicalDeviceImageCompressionControlSwapchainFeaturesEXT image_compression_control_swapchain_features = {};
 
 	// Vendor
-	VkPhysicalDeviceComputeShaderDerivativesFeaturesNV compute_shader_derivative_features = {};
 	VkPhysicalDeviceDeviceGeneratedCommandsFeaturesNV device_generated_commands_features = {};
 	VkPhysicalDeviceDeviceGeneratedCommandsComputeFeaturesNV device_generated_commands_compute_features = {};
 	VkPhysicalDeviceDeviceGeneratedCommandsPropertiesNV device_generated_commands_properties = {};
@@ -148,7 +152,8 @@ enum ContextCreationFlagBits
 	CONTEXT_CREATION_ENABLE_VIDEO_DECODE_BIT = 1 << 1,
 	CONTEXT_CREATION_ENABLE_VIDEO_ENCODE_BIT = 1 << 2,
 	CONTEXT_CREATION_ENABLE_VIDEO_H264_BIT = 1 << 3,
-	CONTEXT_CREATION_ENABLE_VIDEO_H265_BIT = 1 << 4
+	CONTEXT_CREATION_ENABLE_VIDEO_H265_BIT = 1 << 4,
+	CONTEXT_CREATION_ENABLE_PIPELINE_BINARY_BIT = 1 << 5
 };
 using ContextCreationFlags = uint32_t;
 
@@ -389,6 +394,7 @@ class Context
 	Fossilize::FeatureFilter feature_filter;
 	bool format_is_supported(VkFormat format, VkFormatFeatureFlags features) override;
 	bool descriptor_set_layout_is_supported(const VkDescriptorSetLayoutCreateInfo *set_layout) override;
+	void physical_device_feature_query(VkPhysicalDeviceFeatures2 *pdf2) override;
 #endif
 
 	bool init_profile();

third_party/volk

@@ -80,6 +80,7 @@ static const QueueIndices queue_flush_order[] = {
 Device::Device()
     : framebuffer_allocator(this)
     , transient_allocator(this)
+	, pipeline_binary_cache(this)
 #ifdef GRANITE_VULKAN_SYSTEM_HANDLES
 	, shader_manager(this)
 	, resource_manager(this)
@@ -710,8 +711,11 @@ void Device::bake_program(Program &program, const ImmutableSamplerBank *sampler_
 	program.set_pipeline_layout(request_pipeline_layout(layout, &ext_immutable_samplers));
 }
 
-bool Device::init_pipeline_cache(const uint8_t *data, size_t size)
+bool Device::init_pipeline_cache(const uint8_t *data, size_t size, bool persistent_mapping)
 {
+	if (ext.pipeline_binary_features.pipelineBinaries)
+		return pipeline_binary_cache.init_from_payload(data, size, persistent_mapping);
+
 	static const auto uuid_size = sizeof(gpu_props.pipelineCacheUUID);
 	static const auto hash_size = sizeof(Util::Hash);
 
@@ -746,10 +750,10 @@ bool Device::init_pipeline_cache(const uint8_t *data, size_t size)
 		}
 	}
 
-	if (pipeline_cache != VK_NULL_HANDLE)
-		table->vkDestroyPipelineCache(device, pipeline_cache, nullptr);
-	pipeline_cache = VK_NULL_HANDLE;
-	return table->vkCreatePipelineCache(device, &info, nullptr, &pipeline_cache) == VK_SUCCESS;
+	if (legacy_pipeline_cache != VK_NULL_HANDLE)
+		table->vkDestroyPipelineCache(device, legacy_pipeline_cache, nullptr);
+	legacy_pipeline_cache = VK_NULL_HANDLE;
+	return table->vkCreatePipelineCache(device, &info, nullptr, &legacy_pipeline_cache) == VK_SUCCESS;
 }
 
 void Device::init_pipeline_cache()
@@ -760,10 +764,16 @@ void Device::init_pipeline_cache()
 	auto file = system_handles.filesystem->open_readonly_mapping("cache://pipeline_cache.bin");
 	if (file)
 	{
+		if (ext.pipeline_binary_features.pipelineBinaries)
+			persistent_pipeline_cache = file;
+
 		auto size = file->get_size();
 		auto *mapped = file->data<uint8_t>();
-		if (mapped && !init_pipeline_cache(mapped, size))
+		if (mapped && !init_pipeline_cache(mapped, size, bool(persistent_pipeline_cache)))
+		{
 			LOGE("Failed to initialize pipeline cache.\n");
+			persistent_pipeline_cache.reset();
+		}
 	}
 	else if (!init_pipeline_cache(nullptr, 0))
 		LOGE("Failed to initialize pipeline cache.\n");
@@ -772,13 +782,13 @@ void Device::init_pipeline_cache()
 
 size_t Device::get_pipeline_cache_size()
 {
-	if (pipeline_cache == VK_NULL_HANDLE)
-		return 0;
+	if (legacy_pipeline_cache == VK_NULL_HANDLE)
+		return pipeline_binary_cache.get_serialized_size();
 
 	static const auto uuid_size = sizeof(gpu_props.pipelineCacheUUID);
 	static const auto hash_size = sizeof(Util::Hash);
 	size_t size = 0;
-	if (table->vkGetPipelineCacheData(device, pipeline_cache, &size, nullptr) != VK_SUCCESS)
+	if (table->vkGetPipelineCacheData(device, legacy_pipeline_cache, &size, nullptr) != VK_SUCCESS)
 	{
 		LOGE("Failed to get pipeline cache data.\n");
 		return 0;
@@ -789,8 +799,8 @@ size_t Device::get_pipeline_cache_size()
 
 bool Device::get_pipeline_cache_data(uint8_t *data, size_t size)
 {
-	if (pipeline_cache == VK_NULL_HANDLE)
-		return false;
+	if (legacy_pipeline_cache == VK_NULL_HANDLE)
+		return pipeline_binary_cache.serialize(data, size);
 
 	static const auto uuid_size = sizeof(gpu_props.pipelineCacheUUID);
 	static const auto hash_size = sizeof(Util::Hash);
@@ -803,7 +813,7 @@ bool Device::get_pipeline_cache_data(uint8_t *data, size_t size)
 	memcpy(data, gpu_props.pipelineCacheUUID, uuid_size);
 	data = hash_data + hash_size;
 
-	if (table->vkGetPipelineCacheData(device, pipeline_cache, &size, data) != VK_SUCCESS)
+	if (table->vkGetPipelineCacheData(device, legacy_pipeline_cache, &size, data) != VK_SUCCESS)
 	{
 		LOGE("Failed to get pipeline cache data.\n");
 		return false;
@@ -823,6 +833,14 @@ void Device::flush_pipeline_cache()
 	if (!system_handles.filesystem)
 		return;
 
+	if (ext.pipeline_binary_features.pipelineBinaries &&
+	    !pipeline_binary_cache.has_new_binary_entries() &&
+	    persistent_pipeline_cache)
+	{
+		LOGI("No new pipelines have been observed, skipping serialize.\n");
+		return;
+	}
+
 	size_t size = get_pipeline_cache_size();
 	if (!size)
 	{
@@ -844,6 +862,8 @@ void Device::flush_pipeline_cache()
 		LOGE("Failed to get pipeline cache data.\n");
 		return;
 	}
+
+	persistent_pipeline_cache.reset();
 #endif
 }
 
@@ -1910,7 +1930,7 @@ CommandBufferHandle Device::request_command_buffer_nolock(unsigned thread_index,
 	info.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
 	table->vkBeginCommandBuffer(cmd, &info);
 	add_frame_counter_nolock();
-	CommandBufferHandle handle(handle_pool.command_buffers.allocate(this, cmd, pipeline_cache, type));
+	CommandBufferHandle handle(handle_pool.command_buffers.allocate(this, cmd, legacy_pipeline_cache, type));
 	handle->set_thread_index(thread_index);
 
 	if (profiled)
@@ -1960,7 +1980,7 @@ CommandBufferHandle Device::request_secondary_command_buffer_for_thread(unsigned
 
 	table->vkBeginCommandBuffer(cmd, &info);
 	add_frame_counter_nolock();
-	CommandBufferHandle handle(handle_pool.command_buffers.allocate(this, cmd, pipeline_cache, type));
+	CommandBufferHandle handle(handle_pool.command_buffers.allocate(this, cmd, legacy_pipeline_cache, type));
 	handle->set_thread_index(thread_index);
 	handle->set_is_secondary();
 	return handle;
@@ -2018,12 +2038,6 @@ Device::~Device()
 
 	managers.timestamps.log_simple();
 
-	if (pipeline_cache != VK_NULL_HANDLE)
-	{
-		flush_pipeline_cache();
-		table->vkDestroyPipelineCache(device, pipeline_cache, nullptr);
-	}
-
 #ifdef GRANITE_VULKAN_SYSTEM_HANDLES
 	flush_shader_manager_cache();
 #endif
@@ -2032,6 +2046,10 @@ Device::~Device()
 	flush_pipeline_state();
 #endif
 
+	if (legacy_pipeline_cache != VK_NULL_HANDLE || ext.pipeline_binary_features.pipelineBinaries)
+		flush_pipeline_cache();
+	table->vkDestroyPipelineCache(device, legacy_pipeline_cache, nullptr);
+
 	framebuffer_allocator.clear();
 	transient_allocator.clear();
 

vulkan/CMakeLists.txt

@@ -39,6 +39,7 @@
 #include "query_pool.hpp"
 #include "buffer_pool.hpp"
 #include "indirect_layout.hpp"
+#include "pipeline_cache.hpp"
 #include <memory>
 #include <vector>
 #include <functional>
@@ -241,7 +242,9 @@ class Device
 
 	size_t get_pipeline_cache_size();
 	bool get_pipeline_cache_data(uint8_t *data, size_t size);
-	bool init_pipeline_cache(const uint8_t *data, size_t size);
+	// If persistent_mapping is true, the data pointer lifetime is live as long as the device is.
+	// Useful for read-only file mmap.
+	bool init_pipeline_cache(const uint8_t *data, size_t size, bool persistent_mapping = false);
 
 	// Frame-pushing interface.
 	void next_frame_context();
@@ -439,6 +442,7 @@ class Device
 	// in query_initialization_progress().
 	ShaderManager &get_shader_manager();
 	ResourceManager &get_resource_manager();
+	Granite::FileMappingHandle persistent_pipeline_cache;
 #endif
 
 	// Useful for loading screens or otherwise figuring out
@@ -737,7 +741,8 @@ class Device
 
 	FramebufferAllocator framebuffer_allocator;
 	TransientAttachmentAllocator transient_allocator;
-	VkPipelineCache pipeline_cache = VK_NULL_HANDLE;
+	VkPipelineCache legacy_pipeline_cache = VK_NULL_HANDLE;
+	PipelineCache pipeline_binary_cache;
 
 	void init_pipeline_cache();
 	void flush_pipeline_cache();

vulkan/command_buffer.cpp

@@ -315,7 +315,7 @@ bool Device::fossilize_replay_graphics_pipeline(Fossilize::Hash hash, VkGraphics
 	}
 
 	VkPipeline pipeline = VK_NULL_HANDLE;
-	VkResult res = table->vkCreateGraphicsPipelines(device, pipeline_cache, 1, &info, nullptr, &pipeline);
+	VkResult res = pipeline_binary_cache.create_pipeline(&info, legacy_pipeline_cache, &pipeline);
 	if (res != VK_SUCCESS)
 	{
 		LOGE("Failed to create graphics pipeline!\n");
@@ -362,7 +362,7 @@ bool Device::fossilize_replay_compute_pipeline(Fossilize::Hash hash, VkComputePi
 	LOGI("Replaying compute pipeline.\n");
 #endif
 	VkPipeline pipeline = VK_NULL_HANDLE;
-	VkResult res = table->vkCreateComputePipelines(device, pipeline_cache, 1, &info, nullptr, &pipeline);
+	VkResult res = pipeline_binary_cache.create_pipeline(&info, legacy_pipeline_cache, &pipeline);
 	if (res != VK_SUCCESS)
 	{
 		LOGE("Failed to create compute pipeline!\n");