diff --git a/src/gpu/vulkan/SDL_gpu_vulkan.c b/src/gpu/vulkan/SDL_gpu_vulkan.c index e196954758..cbb7fc72bd 100644 --- a/src/gpu/vulkan/SDL_gpu_vulkan.c +++ b/src/gpu/vulkan/SDL_gpu_vulkan.c @@ -538,6 +538,7 @@ struct VulkanMemoryAllocation VkDeviceSize usedSpace; Uint8 *mapPointer; SDL_Mutex *memoryLock; + SDL_AtomicInt referenceCount; // Used to avoid defrag races }; typedef struct VulkanMemoryAllocator @@ -1080,10 +1081,18 @@ typedef struct VulkanCommandBuffer Sint32 usedBufferCount; Sint32 usedBufferCapacity; + VulkanBuffer **buffersUsedInPendingTransfers; + Sint32 buffersUsedInPendingTransfersCount; + Sint32 buffersUsedInPendingTransfersCapacity; + VulkanTexture **usedTextures; Sint32 usedTextureCount; Sint32 usedTextureCapacity; + VulkanTexture **texturesUsedInPendingTransfers; + Sint32 texturesUsedInPendingTransfersCount; + Sint32 texturesUsedInPendingTransfersCapacity; + VulkanSampler **usedSamplers; Sint32 usedSamplerCount; Sint32 usedSamplerCapacity; @@ -1253,6 +1262,10 @@ struct VulkanRenderer SDL_Mutex *descriptorSetLayoutFetchLock; SDL_Mutex *windowLock; + // We don't want transfer commands to block each other, + // but we want all transfers to block during defrag. + SDL_RWLock *defragLock; + Uint8 defragInProgress; VulkanMemoryAllocation **allocationsToDefrag; @@ -1884,6 +1897,8 @@ static Uint8 VULKAN_INTERNAL_AllocateMemory( allocation->freeRegionCount = 0; allocation->freeRegionCapacity = 1; + SDL_SetAtomicInt(&allocation->referenceCount, 0); + allocation->allocator = allocator; result = renderer->vkAllocateMemory( @@ -2421,7 +2436,7 @@ static Uint8 VULKAN_INTERNAL_BindMemoryForBuffer( // Resource tracking -#define TRACK_RESOURCE(resource, type, array, count, capacity) \ +#define TRACK_RESOURCE(resource, type, array, count, capacity, refcountvar) \ for (Sint32 i = commandBuffer->count - 1; i >= 0; i -= 1) { \ if (commandBuffer->array[i] == resource) { \ return; \ @@ -2436,7 +2451,8 @@ static Uint8 VULKAN_INTERNAL_BindMemoryForBuffer( } \ commandBuffer->array[commandBuffer->count] = resource; \ commandBuffer->count += 1; \ - SDL_AtomicIncRef(&resource->referenceCount) + SDL_AtomicIncRef(&refcountvar) + static void VULKAN_INTERNAL_TrackBuffer( VulkanCommandBuffer *commandBuffer, @@ -2447,7 +2463,23 @@ static void VULKAN_INTERNAL_TrackBuffer( VulkanBuffer *, usedBuffers, usedBufferCount, - usedBufferCapacity); + usedBufferCapacity, + buffer->referenceCount); +} + +// Use this function when a GPU buffer is part of a transfer operation. +// Note that this isn't for transfer buffers, those don't need to refcount their allocations. +static void VULKAN_INTERNAL_TrackBufferTransfer( + VulkanCommandBuffer *commandBuffer, + VulkanBuffer *buffer) +{ + TRACK_RESOURCE( + buffer, + VulkanBuffer *, + buffersUsedInPendingTransfers, + buffersUsedInPendingTransfersCount, + buffersUsedInPendingTransfersCapacity, + buffer->usedRegion->allocation->referenceCount); } static void VULKAN_INTERNAL_TrackTexture( @@ -2459,7 +2491,22 @@ static void VULKAN_INTERNAL_TrackTexture( VulkanTexture *, usedTextures, usedTextureCount, - usedTextureCapacity); + usedTextureCapacity, + texture->referenceCount); +} + +// Use this when a texture is part of a transfer operation. +static void VULKAN_INTERNAL_TrackTextureTransfer( + VulkanCommandBuffer *commandBuffer, + VulkanTexture *texture) +{ + TRACK_RESOURCE( + texture, + VulkanTexture *, + texturesUsedInPendingTransfers, + texturesUsedInPendingTransfersCount, + texturesUsedInPendingTransfersCapacity, + texture->usedRegion->allocation->referenceCount); } static void VULKAN_INTERNAL_TrackSampler( @@ -2471,7 +2518,8 @@ static void VULKAN_INTERNAL_TrackSampler( VulkanSampler *, usedSamplers, usedSamplerCount, - usedSamplerCapacity); + usedSamplerCapacity, + sampler->referenceCount); } static void VULKAN_INTERNAL_TrackGraphicsPipeline( @@ -2483,7 +2531,8 @@ static void VULKAN_INTERNAL_TrackGraphicsPipeline( VulkanGraphicsPipeline *, usedGraphicsPipelines, usedGraphicsPipelineCount, - usedGraphicsPipelineCapacity); + usedGraphicsPipelineCapacity, + graphicsPipeline->referenceCount); } static void VULKAN_INTERNAL_TrackComputePipeline( @@ -2495,7 +2544,8 @@ static void VULKAN_INTERNAL_TrackComputePipeline( VulkanComputePipeline *, usedComputePipelines, usedComputePipelineCount, - usedComputePipelineCapacity); + usedComputePipelineCapacity, + computePipeline->referenceCount); } static void VULKAN_INTERNAL_TrackFramebuffer( @@ -2507,7 +2557,8 @@ static void VULKAN_INTERNAL_TrackFramebuffer( VulkanFramebuffer *, usedFramebuffers, usedFramebufferCount, - usedFramebufferCapacity); + usedFramebufferCapacity, + framebuffer->referenceCount); } static void VULKAN_INTERNAL_TrackUniformBuffer( @@ -6893,7 +6944,7 @@ static SDL_GPUTexture *VULKAN_CreateTexture( texture); VULKAN_INTERNAL_TrackTexture(barrierCommandBuffer, texture); if (!VULKAN_Submit((SDL_GPUCommandBuffer *)barrierCommandBuffer)) { - VULKAN_ReleaseTexture((SDL_GPURenderer *)renderer, (SDL_GPUTexture *)container); + VULKAN_ReleaseTexture((SDL_GPURenderer *)renderer, (SDL_GPUTexture *)container); return NULL; } } @@ -8791,8 +8842,9 @@ static void VULKAN_UploadToTexture( VulkanTextureSubresource *vulkanTextureSubresource; VkBufferImageCopy imageCopy; - // Note that the transfer buffer does not need a barrier, as it is synced by the client + SDL_LockRWLockForReading(renderer->defragLock); + // Note that the transfer buffer does not need a barrier, as it is synced by the client vulkanTextureSubresource = VULKAN_INTERNAL_PrepareTextureSubresourceForWrite( renderer, vulkanCommandBuffer, @@ -8832,6 +8884,9 @@ static void VULKAN_UploadToTexture( VULKAN_INTERNAL_TrackBuffer(vulkanCommandBuffer, transferBufferContainer->activeBuffer); VULKAN_INTERNAL_TrackTexture(vulkanCommandBuffer, vulkanTextureSubresource->parent); + VULKAN_INTERNAL_TrackTextureTransfer(vulkanCommandBuffer, vulkanTextureSubresource->parent); + + SDL_UnlockRWLock(renderer->defragLock); } static void VULKAN_UploadToBuffer( @@ -8846,8 +8901,9 @@ static void VULKAN_UploadToBuffer( VulkanBufferContainer *bufferContainer = (VulkanBufferContainer *)destination->buffer; VkBufferCopy bufferCopy; - // Note that the transfer buffer does not need a barrier, as it is synced by the client + SDL_LockRWLockForReading(renderer->defragLock); + // Note that the transfer buffer does not need a barrier, as it is synced by the client VulkanBuffer *vulkanBuffer = VULKAN_INTERNAL_PrepareBufferForWrite( renderer, vulkanCommandBuffer, @@ -8855,6 +8911,7 @@ static void VULKAN_UploadToBuffer( cycle, VULKAN_BUFFER_USAGE_MODE_COPY_DESTINATION); + bufferCopy.srcOffset = source->offset; bufferCopy.dstOffset = destination->offset; bufferCopy.size = destination->size; @@ -8874,6 +8931,9 @@ static void VULKAN_UploadToBuffer( VULKAN_INTERNAL_TrackBuffer(vulkanCommandBuffer, transferBufferContainer->activeBuffer); VULKAN_INTERNAL_TrackBuffer(vulkanCommandBuffer, vulkanBuffer); + VULKAN_INTERNAL_TrackBufferTransfer(vulkanCommandBuffer, vulkanBuffer); + + SDL_UnlockRWLock(renderer->defragLock); } // Readback @@ -8889,6 +8949,9 @@ static void VULKAN_DownloadFromTexture( VulkanTextureSubresource *vulkanTextureSubresource; VulkanBufferContainer *transferBufferContainer = (VulkanBufferContainer *)destination->transfer_buffer; VkBufferImageCopy imageCopy; + + SDL_LockRWLockForReading(renderer->defragLock); + vulkanTextureSubresource = VULKAN_INTERNAL_FetchTextureSubresource( textureContainer, source->layer, @@ -8932,6 +8995,9 @@ static void VULKAN_DownloadFromTexture( VULKAN_INTERNAL_TrackBuffer(vulkanCommandBuffer, transferBufferContainer->activeBuffer); VULKAN_INTERNAL_TrackTexture(vulkanCommandBuffer, vulkanTextureSubresource->parent); + VULKAN_INTERNAL_TrackTextureTransfer(vulkanCommandBuffer, vulkanTextureSubresource->parent); + + SDL_UnlockRWLock(renderer->defragLock); } static void VULKAN_DownloadFromBuffer( @@ -8945,8 +9011,9 @@ static void VULKAN_DownloadFromBuffer( VulkanBufferContainer *transferBufferContainer = (VulkanBufferContainer *)destination->transfer_buffer; VkBufferCopy bufferCopy; - // Note that transfer buffer does not need a barrier, as it is synced by the client + SDL_LockRWLockForReading(renderer->defragLock); + // Note that transfer buffer does not need a barrier, as it is synced by the client VULKAN_INTERNAL_BufferTransitionFromDefaultUsage( renderer, vulkanCommandBuffer, @@ -8972,6 +9039,9 @@ static void VULKAN_DownloadFromBuffer( VULKAN_INTERNAL_TrackBuffer(vulkanCommandBuffer, transferBufferContainer->activeBuffer); VULKAN_INTERNAL_TrackBuffer(vulkanCommandBuffer, bufferContainer->activeBuffer); + VULKAN_INTERNAL_TrackBufferTransfer(vulkanCommandBuffer, bufferContainer->activeBuffer); + + SDL_UnlockRWLock(renderer->defragLock); } static void VULKAN_CopyTextureToTexture( @@ -8989,6 +9059,8 @@ static void VULKAN_CopyTextureToTexture( VulkanTextureSubresource *dstSubresource; VkImageCopy imageCopy; + SDL_LockRWLockForReading(renderer->defragLock); + srcSubresource = VULKAN_INTERNAL_FetchTextureSubresource( (VulkanTextureContainer *)source->texture, source->layer, @@ -9050,6 +9122,10 @@ static void VULKAN_CopyTextureToTexture( VULKAN_INTERNAL_TrackTexture(vulkanCommandBuffer, srcSubresource->parent); VULKAN_INTERNAL_TrackTexture(vulkanCommandBuffer, dstSubresource->parent); + VULKAN_INTERNAL_TrackTextureTransfer(vulkanCommandBuffer, srcSubresource->parent); + VULKAN_INTERNAL_TrackTextureTransfer(vulkanCommandBuffer, dstSubresource->parent); + + SDL_UnlockRWLock(renderer->defragLock); } static void VULKAN_CopyBufferToBuffer( @@ -9065,6 +9141,8 @@ static void VULKAN_CopyBufferToBuffer( VulkanBufferContainer *dstContainer = (VulkanBufferContainer *)destination->buffer; VkBufferCopy bufferCopy; + SDL_LockRWLockForReading(renderer->defragLock); + VulkanBuffer *dstBuffer = VULKAN_INTERNAL_PrepareBufferForWrite( renderer, vulkanCommandBuffer, @@ -9103,6 +9181,10 @@ static void VULKAN_CopyBufferToBuffer( VULKAN_INTERNAL_TrackBuffer(vulkanCommandBuffer, srcContainer->activeBuffer); VULKAN_INTERNAL_TrackBuffer(vulkanCommandBuffer, dstBuffer); + VULKAN_INTERNAL_TrackBufferTransfer(vulkanCommandBuffer, srcContainer->activeBuffer); + VULKAN_INTERNAL_TrackBufferTransfer(vulkanCommandBuffer, dstBuffer); + + SDL_UnlockRWLock(renderer->defragLock); } static void VULKAN_GenerateMipmaps( @@ -9116,8 +9198,10 @@ static void VULKAN_GenerateMipmaps( VulkanTextureSubresource *dstTextureSubresource; VkImageBlit blit; + SDL_LockRWLockForReading(renderer->defragLock); + // Blit each slice sequentially. Barriers, barriers everywhere! - for (Uint32 layerOrDepthIndex = 0; layerOrDepthIndex < container->header.info.layer_count_or_depth; layerOrDepthIndex += 1) + for (Uint32 layerOrDepthIndex = 0; layerOrDepthIndex < container->header.info.layer_count_or_depth; layerOrDepthIndex += 1) { for (Uint32 level = 1; level < container->header.info.num_levels; level += 1) { Uint32 layer = container->header.info.type == SDL_GPU_TEXTURETYPE_3D ? 0 : layerOrDepthIndex; Uint32 depth = container->header.info.type == SDL_GPU_TEXTURETYPE_3D ? layerOrDepthIndex : 0; @@ -9196,7 +9280,13 @@ static void VULKAN_GenerateMipmaps( VULKAN_INTERNAL_TrackTexture(vulkanCommandBuffer, srcTextureSubresource->parent); VULKAN_INTERNAL_TrackTexture(vulkanCommandBuffer, dstTextureSubresource->parent); + VULKAN_INTERNAL_TrackTextureTransfer(vulkanCommandBuffer, srcTextureSubresource->parent); + VULKAN_INTERNAL_TrackTextureTransfer(vulkanCommandBuffer, dstTextureSubresource->parent); + } + } + + SDL_UnlockRWLock(renderer->defragLock); } static void VULKAN_EndCopyPass( @@ -9406,11 +9496,21 @@ static bool VULKAN_INTERNAL_AllocateCommandBuffer( commandBuffer->usedBuffers = SDL_malloc( commandBuffer->usedBufferCapacity * sizeof(VulkanBuffer *)); + commandBuffer->buffersUsedInPendingTransfersCapacity = 4; + commandBuffer->buffersUsedInPendingTransfersCount = 0; + commandBuffer->buffersUsedInPendingTransfers = SDL_malloc( + commandBuffer->buffersUsedInPendingTransfersCapacity * sizeof(VulkanBuffer *)); + commandBuffer->usedTextureCapacity = 4; commandBuffer->usedTextureCount = 0; commandBuffer->usedTextures = SDL_malloc( commandBuffer->usedTextureCapacity * sizeof(VulkanTexture *)); + commandBuffer->texturesUsedInPendingTransfersCapacity = 4; + commandBuffer->texturesUsedInPendingTransfersCount = 0; + commandBuffer->texturesUsedInPendingTransfers = SDL_malloc( + commandBuffer->texturesUsedInPendingTransfersCapacity * sizeof(VulkanTexture *)); + commandBuffer->usedSamplerCapacity = 4; commandBuffer->usedSamplerCount = 0; commandBuffer->usedSamplers = SDL_malloc( @@ -10479,11 +10579,21 @@ static void VULKAN_INTERNAL_CleanCommandBuffer( } commandBuffer->usedBufferCount = 0; + for (Sint32 i = 0; i < commandBuffer->buffersUsedInPendingTransfersCount; i += 1) { + (void)SDL_AtomicDecRef(&commandBuffer->usedBuffers[i]->usedRegion->allocation->referenceCount); + } + commandBuffer->buffersUsedInPendingTransfersCount = 0; + for (Sint32 i = 0; i < commandBuffer->usedTextureCount; i += 1) { (void)SDL_AtomicDecRef(&commandBuffer->usedTextures[i]->referenceCount); } commandBuffer->usedTextureCount = 0; + for (Sint32 i = 0; i < commandBuffer->texturesUsedInPendingTransfersCount; i += 1){ + (void)SDL_AtomicDecRef(&commandBuffer->usedTextures[i]->usedRegion->allocation->referenceCount); + } + commandBuffer->texturesUsedInPendingTransfersCount = 0; + for (Sint32 i = 0; i < commandBuffer->usedSamplerCount; i += 1) { (void)SDL_AtomicDecRef(&commandBuffer->usedSamplers[i]->referenceCount); } @@ -10869,8 +10979,30 @@ static bool VULKAN_INTERNAL_DefragmentMemory( commandBuffer->isDefrag = 1; SDL_LockMutex(renderer->allocatorLock); + SDL_LockRWLockForWriting(renderer->defragLock); - VulkanMemoryAllocation *allocation = renderer->allocationsToDefrag[renderer->allocationsToDefragCount - 1]; + // Find an allocation that doesn't have any pending transfer operations + Sint32 indexToDefrag = -1; + for (Sint32 i = renderer->allocationsToDefragCount - 1; i >= 0; i -= 1) { + if (SDL_GetAtomicInt(&renderer->allocationsToDefrag[i]->referenceCount) == 0) { + indexToDefrag = i; + break; + } + } + + if (indexToDefrag == -1) { + // Nothing is available to defrag, but it's not an error + SDL_UnlockRWLock(renderer->defragLock); + SDL_UnlockMutex(renderer->allocatorLock); + return true; + } + + VulkanMemoryAllocation *allocation = renderer->allocationsToDefrag[indexToDefrag]; + + // Plug the hole + if ((Uint32) indexToDefrag != renderer->allocationsToDefragCount - 1) { + renderer->allocationsToDefrag[indexToDefrag] = renderer->allocationsToDefrag[renderer->allocationsToDefragCount - 1]; + } renderer->allocationsToDefragCount -= 1; /* For each used region in the allocation @@ -10890,6 +11022,7 @@ static bool VULKAN_INTERNAL_DefragmentMemory( currentRegion->vulkanBuffer->container != NULL ? currentRegion->vulkanBuffer->container->debugName : NULL); if (newBuffer == NULL) { + SDL_UnlockRWLock(renderer->defragLock); SDL_UnlockMutex(renderer->allocatorLock); SDL_LogError(SDL_LOG_CATEGORY_GPU, "%s", "Failed to allocate defrag buffer!"); return false; @@ -10955,6 +11088,7 @@ static bool VULKAN_INTERNAL_DefragmentMemory( ¤tRegion->vulkanTexture->container->header.info); if (newTexture == NULL) { + SDL_UnlockRWLock(renderer->defragLock); SDL_UnlockMutex(renderer->allocatorLock); SDL_LogError(SDL_LOG_CATEGORY_GPU, "%s", "Failed to allocate defrag buffer!"); return false; @@ -11029,6 +11163,7 @@ static bool VULKAN_INTERNAL_DefragmentMemory( } } + SDL_UnlockRWLock(renderer->defragLock); SDL_UnlockMutex(renderer->allocatorLock); return true;