/* * Copyright © 2016 Red Hat. * Copyright © 2016 Bas Nieuwenhuizen * * based in part on anv driver which is: * Copyright © 2015 Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. */ #include "tu_private.h" #include "fdl/fd6_format_table.h" #include "util/debug.h" #include "util/u_atomic.h" #include "util/format/u_format.h" #include "vk_format.h" #include "vk_util.h" #include "drm-uapi/drm_fourcc.h" #include "tu_cs.h" static uint32_t tu6_plane_count(VkFormat format) { switch (format) { default: return 1; case VK_FORMAT_G8_B8R8_2PLANE_420_UNORM: case VK_FORMAT_D32_SFLOAT_S8_UINT: return 2; case VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM: return 3; } } static VkFormat tu6_plane_format(VkFormat format, uint32_t plane) { switch (format) { case VK_FORMAT_G8_B8R8_2PLANE_420_UNORM: /* note: with UBWC, and Y plane UBWC is different from R8_UNORM */ return plane ? VK_FORMAT_R8G8_UNORM : VK_FORMAT_R8_UNORM; case VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM: return VK_FORMAT_R8_UNORM; case VK_FORMAT_D32_SFLOAT_S8_UINT: return plane ? VK_FORMAT_S8_UINT : VK_FORMAT_D32_SFLOAT; default: return format; } } static uint32_t tu6_plane_index(VkFormat format, VkImageAspectFlags aspect_mask) { switch (aspect_mask) { default: return 0; case VK_IMAGE_ASPECT_PLANE_1_BIT: return 1; case VK_IMAGE_ASPECT_PLANE_2_BIT: return 2; case VK_IMAGE_ASPECT_STENCIL_BIT: return format == VK_FORMAT_D32_SFLOAT_S8_UINT; } } static void compose_swizzle(unsigned char *swiz, const VkComponentMapping *mapping) { unsigned char src_swiz[4] = { swiz[0], swiz[1], swiz[2], swiz[3] }; VkComponentSwizzle vk_swiz[4] = { mapping->r, mapping->g, mapping->b, mapping->a }; for (int i = 0; i < 4; i++) { switch (vk_swiz[i]) { case VK_COMPONENT_SWIZZLE_IDENTITY: swiz[i] = src_swiz[i]; break; case VK_COMPONENT_SWIZZLE_R...VK_COMPONENT_SWIZZLE_A: swiz[i] = src_swiz[vk_swiz[i] - VK_COMPONENT_SWIZZLE_R]; break; case VK_COMPONENT_SWIZZLE_ZERO: swiz[i] = A6XX_TEX_ZERO; break; case VK_COMPONENT_SWIZZLE_ONE: swiz[i] = A6XX_TEX_ONE; break; default: unreachable("unexpected swizzle"); } } } static uint32_t tu6_texswiz(const VkComponentMapping *comps, const struct tu_sampler_ycbcr_conversion *conversion, VkFormat format, VkImageAspectFlagBits aspect_mask, bool has_z24uint_s8uint) { unsigned char swiz[4] = { A6XX_TEX_X, A6XX_TEX_Y, A6XX_TEX_Z, A6XX_TEX_W, }; switch (format) { case VK_FORMAT_G8B8G8R8_422_UNORM: case VK_FORMAT_B8G8R8G8_422_UNORM: case VK_FORMAT_G8_B8R8_2PLANE_420_UNORM: case VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM: swiz[0] = A6XX_TEX_Z; swiz[1] = A6XX_TEX_X; swiz[2] = A6XX_TEX_Y; break; case VK_FORMAT_BC1_RGB_UNORM_BLOCK: case VK_FORMAT_BC1_RGB_SRGB_BLOCK: /* same hardware format is used for BC1_RGB / BC1_RGBA */ swiz[3] = A6XX_TEX_ONE; break; case VK_FORMAT_D24_UNORM_S8_UINT: if (aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT) { if (!has_z24uint_s8uint) { /* using FMT6_8_8_8_8_UINT, so need to pick out the W channel and * swizzle (0,0,1) in the rest (see "Conversion to RGBA"). */ swiz[0] = A6XX_TEX_W; swiz[1] = A6XX_TEX_ZERO; swiz[2] = A6XX_TEX_ZERO; swiz[3] = A6XX_TEX_ONE; } else { /* using FMT6_Z24_UINT_S8_UINT, which is (d, s, 0, 1), so need to * swizzle away the d. */ swiz[0] = A6XX_TEX_Y; swiz[1] = A6XX_TEX_ZERO; } } break; default: break; } compose_swizzle(swiz, comps); if (conversion) compose_swizzle(swiz, &conversion->components); return A6XX_TEX_CONST_0_SWIZ_X(swiz[0]) | A6XX_TEX_CONST_0_SWIZ_Y(swiz[1]) | A6XX_TEX_CONST_0_SWIZ_Z(swiz[2]) | A6XX_TEX_CONST_0_SWIZ_W(swiz[3]); } void tu_cs_image_ref(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer) { tu_cs_emit(cs, iview->PITCH); tu_cs_emit(cs, iview->layer_size >> 6); tu_cs_emit_qw(cs, iview->base_addr + iview->layer_size * layer); } void tu_cs_image_stencil_ref(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer) { tu_cs_emit(cs, iview->stencil_PITCH); tu_cs_emit(cs, iview->stencil_layer_size >> 6); tu_cs_emit_qw(cs, iview->stencil_base_addr + iview->stencil_layer_size * layer); } void tu_cs_image_ref_2d(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer, bool src) { tu_cs_emit_qw(cs, iview->base_addr + iview->layer_size * layer); /* SP_PS_2D_SRC_PITCH has shifted pitch field */ tu_cs_emit(cs, iview->PITCH << (src ? 9 : 0)); } void tu_cs_image_flag_ref(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer) { tu_cs_emit_qw(cs, iview->ubwc_addr + iview->ubwc_layer_size * layer); tu_cs_emit(cs, iview->FLAG_BUFFER_PITCH); } void tu_image_view_init(struct tu_image_view *iview, const VkImageViewCreateInfo *pCreateInfo, bool has_z24uint_s8uint) { TU_FROM_HANDLE(tu_image, image, pCreateInfo->image); const VkImageSubresourceRange *range = &pCreateInfo->subresourceRange; VkFormat format = pCreateInfo->format; VkImageAspectFlagBits aspect_mask = pCreateInfo->subresourceRange.aspectMask; const struct VkSamplerYcbcrConversionInfo *ycbcr_conversion = vk_find_struct_const(pCreateInfo->pNext, SAMPLER_YCBCR_CONVERSION_INFO); const struct tu_sampler_ycbcr_conversion *conversion = ycbcr_conversion ? tu_sampler_ycbcr_conversion_from_handle(ycbcr_conversion->conversion) : NULL; iview->image = image; memset(iview->descriptor, 0, sizeof(iview->descriptor)); struct fdl_layout *layout = &image->layout[tu6_plane_index(image->vk_format, aspect_mask)]; uint32_t width = u_minify(layout->width0, range->baseMipLevel); uint32_t height = u_minify(layout->height0, range->baseMipLevel); uint32_t storage_depth = tu_get_layerCount(image, range); if (pCreateInfo->viewType == VK_IMAGE_VIEW_TYPE_3D) { storage_depth = u_minify(image->layout[0].depth0, range->baseMipLevel); } uint32_t depth = storage_depth; if (pCreateInfo->viewType == VK_IMAGE_VIEW_TYPE_CUBE || pCreateInfo->viewType == VK_IMAGE_VIEW_TYPE_CUBE_ARRAY) { /* Cubes are treated as 2D arrays for storage images, so only divide the * depth by 6 for the texture descriptor. */ depth /= 6; } uint64_t base_addr = image->bo->iova + image->bo_offset + fdl_surface_offset(layout, range->baseMipLevel, range->baseArrayLayer); uint64_t ubwc_addr = image->bo->iova + image->bo_offset + fdl_ubwc_offset(layout, range->baseMipLevel, range->baseArrayLayer); uint32_t pitch = fdl_pitch(layout, range->baseMipLevel); uint32_t ubwc_pitch = fdl_ubwc_pitch(layout, range->baseMipLevel); uint32_t layer_size = fdl_layer_stride(layout, range->baseMipLevel); if (aspect_mask != VK_IMAGE_ASPECT_COLOR_BIT) format = tu6_plane_format(format, tu6_plane_index(format, aspect_mask)); struct tu_native_format fmt = tu6_format_texture(format, layout->tile_mode); /* note: freedreno layout assumes no TILE_ALL bit for non-UBWC color formats * this means smaller mipmap levels have a linear tile mode. * Depth/stencil formats have non-linear tile mode. */ fmt.tile_mode = fdl_tile_mode(layout, range->baseMipLevel); bool ubwc_enabled = fdl_ubwc_enabled(layout, range->baseMipLevel); bool is_d24s8 = (format == VK_FORMAT_D24_UNORM_S8_UINT || format == VK_FORMAT_X8_D24_UNORM_PACK32); if (is_d24s8 && ubwc_enabled) fmt.fmt = FMT6_Z24_UNORM_S8_UINT_AS_R8G8B8A8; unsigned fmt_tex = fmt.fmt; if (is_d24s8) { if (aspect_mask & VK_IMAGE_ASPECT_DEPTH_BIT) fmt_tex = FMT6_Z24_UNORM_S8_UINT; if (aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT) fmt_tex = has_z24uint_s8uint ? FMT6_Z24_UINT_S8_UINT : FMT6_8_8_8_8_UINT; /* TODO: also use this format with storage descriptor ? */ } iview->descriptor[0] = A6XX_TEX_CONST_0_TILE_MODE(fmt.tile_mode) | COND(vk_format_is_srgb(format), A6XX_TEX_CONST_0_SRGB) | A6XX_TEX_CONST_0_FMT(fmt_tex) | A6XX_TEX_CONST_0_SAMPLES(tu_msaa_samples(layout->nr_samples)) | A6XX_TEX_CONST_0_SWAP(fmt.swap) | tu6_texswiz(&pCreateInfo->components, conversion, format, aspect_mask, has_z24uint_s8uint) | A6XX_TEX_CONST_0_MIPLVLS(tu_get_levelCount(image, range) - 1); iview->descriptor[1] = A6XX_TEX_CONST_1_WIDTH(width) | A6XX_TEX_CONST_1_HEIGHT(height); iview->descriptor[2] = A6XX_TEX_CONST_2_PITCHALIGN(layout->pitchalign - 6) | A6XX_TEX_CONST_2_PITCH(pitch) | A6XX_TEX_CONST_2_TYPE(tu6_tex_type(pCreateInfo->viewType, false)); iview->descriptor[3] = A6XX_TEX_CONST_3_ARRAY_PITCH(layer_size); iview->descriptor[4] = base_addr; iview->descriptor[5] = (base_addr >> 32) | A6XX_TEX_CONST_5_DEPTH(depth); if (layout->tile_all) iview->descriptor[3] |= A6XX_TEX_CONST_3_TILE_ALL; if (format == VK_FORMAT_G8_B8R8_2PLANE_420_UNORM || format == VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM) { /* chroma offset re-uses MIPLVLS bits */ assert(tu_get_levelCount(image, range) == 1); if (conversion) { if (conversion->chroma_offsets[0] == VK_CHROMA_LOCATION_MIDPOINT) iview->descriptor[0] |= A6XX_TEX_CONST_0_CHROMA_MIDPOINT_X; if (conversion->chroma_offsets[1] == VK_CHROMA_LOCATION_MIDPOINT) iview->descriptor[0] |= A6XX_TEX_CONST_0_CHROMA_MIDPOINT_Y; } uint64_t base_addr[3]; iview->descriptor[3] |= A6XX_TEX_CONST_3_TILE_ALL; if (ubwc_enabled) { iview->descriptor[3] |= A6XX_TEX_CONST_3_FLAG; /* no separate ubwc base, image must have the expected layout */ for (uint32_t i = 0; i < 3; i++) { base_addr[i] = image->bo->iova + image->bo_offset + fdl_ubwc_offset(&image->layout[i], range->baseMipLevel, range->baseArrayLayer); } } else { for (uint32_t i = 0; i < 3; i++) { base_addr[i] = image->bo->iova + image->bo_offset + fdl_surface_offset(&image->layout[i], range->baseMipLevel, range->baseArrayLayer); } } iview->descriptor[4] = base_addr[0]; iview->descriptor[5] |= base_addr[0] >> 32; iview->descriptor[6] = A6XX_TEX_CONST_6_PLANE_PITCH(fdl_pitch(&image->layout[1], range->baseMipLevel)); iview->descriptor[7] = base_addr[1]; iview->descriptor[8] = base_addr[1] >> 32; iview->descriptor[9] = base_addr[2]; iview->descriptor[10] = base_addr[2] >> 32; assert(pCreateInfo->viewType != VK_IMAGE_VIEW_TYPE_3D); return; } if (ubwc_enabled) { uint32_t block_width, block_height; fdl6_get_ubwc_blockwidth(layout, &block_width, &block_height); iview->descriptor[3] |= A6XX_TEX_CONST_3_FLAG; iview->descriptor[7] = ubwc_addr; iview->descriptor[8] = ubwc_addr >> 32; iview->descriptor[9] |= A6XX_TEX_CONST_9_FLAG_BUFFER_ARRAY_PITCH(layout->ubwc_layer_size >> 2); iview->descriptor[10] |= A6XX_TEX_CONST_10_FLAG_BUFFER_PITCH(ubwc_pitch) | A6XX_TEX_CONST_10_FLAG_BUFFER_LOGW(util_logbase2_ceil(DIV_ROUND_UP(width, block_width))) | A6XX_TEX_CONST_10_FLAG_BUFFER_LOGH(util_logbase2_ceil(DIV_ROUND_UP(height, block_height))); } if (pCreateInfo->viewType == VK_IMAGE_VIEW_TYPE_3D) { iview->descriptor[3] |= A6XX_TEX_CONST_3_MIN_LAYERSZ(layout->slices[image->level_count - 1].size0); } iview->SP_PS_2D_SRC_INFO = A6XX_SP_PS_2D_SRC_INFO( .color_format = fmt.fmt, .tile_mode = fmt.tile_mode, .color_swap = fmt.swap, .flags = ubwc_enabled, .srgb = vk_format_is_srgb(format), .samples = tu_msaa_samples(layout->nr_samples), .samples_average = layout->nr_samples > 1 && !vk_format_is_int(format) && !vk_format_is_depth_or_stencil(format), .unk20 = 1, .unk22 = 1).value; iview->SP_PS_2D_SRC_SIZE = A6XX_SP_PS_2D_SRC_SIZE(.width = width, .height = height).value; /* note: these have same encoding for MRT and 2D (except 2D PITCH src) */ iview->PITCH = A6XX_RB_DEPTH_BUFFER_PITCH(pitch).value; iview->FLAG_BUFFER_PITCH = A6XX_RB_DEPTH_FLAG_BUFFER_PITCH( .pitch = ubwc_pitch, .array_pitch = layout->ubwc_layer_size >> 2).value; iview->base_addr = base_addr; iview->ubwc_addr = ubwc_addr; iview->layer_size = layer_size; iview->ubwc_layer_size = layout->ubwc_layer_size; /* Don't set fields that are only used for attachments/blit dest if COLOR * is unsupported. */ if (!tu6_format_color_supported(format)) return; struct tu_native_format cfmt = tu6_format_color(format, layout->tile_mode); cfmt.tile_mode = fmt.tile_mode; if (is_d24s8 && ubwc_enabled) cfmt.fmt = FMT6_Z24_UNORM_S8_UINT_AS_R8G8B8A8; memset(iview->storage_descriptor, 0, sizeof(iview->storage_descriptor)); iview->storage_descriptor[0] = A6XX_IBO_0_FMT(fmt.fmt) | A6XX_IBO_0_TILE_MODE(fmt.tile_mode); iview->storage_descriptor[1] = A6XX_IBO_1_WIDTH(width) | A6XX_IBO_1_HEIGHT(height); iview->storage_descriptor[2] = A6XX_IBO_2_PITCH(pitch) | A6XX_IBO_2_TYPE(tu6_tex_type(pCreateInfo->viewType, true)); iview->storage_descriptor[3] = A6XX_IBO_3_ARRAY_PITCH(layer_size); iview->storage_descriptor[4] = base_addr; iview->storage_descriptor[5] = (base_addr >> 32) | A6XX_IBO_5_DEPTH(storage_depth); if (ubwc_enabled) { iview->storage_descriptor[3] |= A6XX_IBO_3_FLAG | A6XX_IBO_3_UNK27; iview->storage_descriptor[7] |= ubwc_addr; iview->storage_descriptor[8] |= ubwc_addr >> 32; iview->storage_descriptor[9] = A6XX_IBO_9_FLAG_BUFFER_ARRAY_PITCH(layout->ubwc_layer_size >> 2); iview->storage_descriptor[10] = A6XX_IBO_10_FLAG_BUFFER_PITCH(ubwc_pitch); } iview->extent.width = width; iview->extent.height = height; iview->need_y2_align = (fmt.tile_mode == TILE6_LINEAR && range->baseMipLevel != image->level_count - 1); iview->ubwc_enabled = ubwc_enabled; iview->RB_MRT_BUF_INFO = A6XX_RB_MRT_BUF_INFO(0, .color_tile_mode = cfmt.tile_mode, .color_format = cfmt.fmt, .color_swap = cfmt.swap).value; iview->SP_FS_MRT_REG = A6XX_SP_FS_MRT_REG(0, .color_format = cfmt.fmt, .color_sint = vk_format_is_sint(format), .color_uint = vk_format_is_uint(format)).value; iview->RB_2D_DST_INFO = A6XX_RB_2D_DST_INFO( .color_format = cfmt.fmt, .tile_mode = cfmt.tile_mode, .color_swap = cfmt.swap, .flags = ubwc_enabled, .srgb = vk_format_is_srgb(format)).value; iview->RB_BLIT_DST_INFO = A6XX_RB_BLIT_DST_INFO( .tile_mode = cfmt.tile_mode, .samples = tu_msaa_samples(layout->nr_samples), .color_format = cfmt.fmt, .color_swap = cfmt.swap, .flags = ubwc_enabled).value; if (image->vk_format == VK_FORMAT_D32_SFLOAT_S8_UINT) { layout = &image->layout[1]; iview->stencil_base_addr = image->bo->iova + image->bo_offset + fdl_surface_offset(layout, range->baseMipLevel, range->baseArrayLayer); iview->stencil_layer_size = fdl_layer_stride(layout, range->baseMipLevel); iview->stencil_PITCH = A6XX_RB_STENCIL_BUFFER_PITCH(fdl_pitch(layout, range->baseMipLevel)).value; } } bool ubwc_possible(VkFormat format, VkImageType type, VkImageUsageFlags usage, VkImageUsageFlags stencil_usage, const struct fd_dev_info *info, VkSampleCountFlagBits samples) { /* no UBWC with compressed formats, E5B9G9R9, S8_UINT * (S8_UINT because separate stencil doesn't have UBWC-enable bit) */ if (vk_format_is_compressed(format) || format == VK_FORMAT_E5B9G9R9_UFLOAT_PACK32 || format == VK_FORMAT_S8_UINT) return false; if (!info->a6xx.has_8bpp_ubwc && (format == VK_FORMAT_R8_UNORM || format == VK_FORMAT_R8_SNORM || format == VK_FORMAT_R8_UINT || format == VK_FORMAT_R8_SINT || format == VK_FORMAT_R8_SRGB)) return false; if (type == VK_IMAGE_TYPE_3D) { tu_finishme("UBWC with 3D textures"); return false; } /* Disable UBWC for storage images. * * The closed GL driver skips UBWC for storage images (and additionally * uses linear for writeonly images). We seem to have image tiling working * in freedreno in general, so turnip matches that. freedreno also enables * UBWC on images, but it's not really tested due to the lack of * UBWC-enabled mipmaps in freedreno currently. Just match the closed GL * behavior of no UBWC. */ if ((usage | stencil_usage) & VK_IMAGE_USAGE_STORAGE_BIT) return false; /* Disable UBWC for D24S8 on A630 in some cases * * VK_IMAGE_ASPECT_STENCIL_BIT image view requires to be able to sample * from the stencil component as UINT, however no format allows this * on a630 (the special FMT6_Z24_UINT_S8_UINT format is missing) * * It must be sampled as FMT6_8_8_8_8_UINT, which is not UBWC-compatible * * Additionally, the special AS_R8G8B8A8 format is broken without UBWC, * so we have to fallback to 8_8_8_8_UNORM when UBWC is disabled */ if (!info->a6xx.has_z24uint_s8uint && format == VK_FORMAT_D24_UNORM_S8_UINT && (stencil_usage & (VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT))) return false; if (!info->a6xx.has_z24uint_s8uint && samples > VK_SAMPLE_COUNT_1_BIT) return false; return true; } VKAPI_ATTR VkResult VKAPI_CALL tu_CreateImage(VkDevice _device, const VkImageCreateInfo *pCreateInfo, const VkAllocationCallbacks *alloc, VkImage *pImage) { TU_FROM_HANDLE(tu_device, device, _device); uint64_t modifier = DRM_FORMAT_MOD_INVALID; const VkSubresourceLayout *plane_layouts = NULL; struct tu_image *image; if (pCreateInfo->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT) { const VkImageDrmFormatModifierListCreateInfoEXT *mod_info = vk_find_struct_const(pCreateInfo->pNext, IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INFO_EXT); const VkImageDrmFormatModifierExplicitCreateInfoEXT *drm_explicit_info = vk_find_struct_const(pCreateInfo->pNext, IMAGE_DRM_FORMAT_MODIFIER_EXPLICIT_CREATE_INFO_EXT); assert(mod_info || drm_explicit_info); if (mod_info) { modifier = DRM_FORMAT_MOD_LINEAR; for (unsigned i = 0; i < mod_info->drmFormatModifierCount; i++) { if (mod_info->pDrmFormatModifiers[i] == DRM_FORMAT_MOD_QCOM_COMPRESSED) modifier = DRM_FORMAT_MOD_QCOM_COMPRESSED; } } else { modifier = drm_explicit_info->drmFormatModifier; assert(modifier == DRM_FORMAT_MOD_LINEAR || modifier == DRM_FORMAT_MOD_QCOM_COMPRESSED); plane_layouts = drm_explicit_info->pPlaneLayouts; } } else { const struct wsi_image_create_info *wsi_info = vk_find_struct_const(pCreateInfo->pNext, WSI_IMAGE_CREATE_INFO_MESA); if (wsi_info && wsi_info->scanout) modifier = DRM_FORMAT_MOD_LINEAR; } #ifdef ANDROID const VkNativeBufferANDROID *gralloc_info = vk_find_struct_const(pCreateInfo->pNext, NATIVE_BUFFER_ANDROID); int dma_buf; if (gralloc_info) { VkResult result = tu_gralloc_info(device, gralloc_info, &dma_buf, &modifier); if (result != VK_SUCCESS) return result; } #endif image = vk_object_zalloc(&device->vk, alloc, sizeof(*image), VK_OBJECT_TYPE_IMAGE); if (!image) return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); const VkExternalMemoryImageCreateInfo *external_info = vk_find_struct_const(pCreateInfo->pNext, EXTERNAL_MEMORY_IMAGE_CREATE_INFO); image->shareable = external_info != NULL; image->vk_format = pCreateInfo->format; image->level_count = pCreateInfo->mipLevels; image->layer_count = pCreateInfo->arrayLayers; enum a6xx_tile_mode tile_mode = TILE6_3; bool ubwc_enabled = !(device->physical_device->instance->debug_flags & TU_DEBUG_NOUBWC); /* use linear tiling if requested */ if (pCreateInfo->tiling == VK_IMAGE_TILING_LINEAR || modifier == DRM_FORMAT_MOD_LINEAR) { tile_mode = TILE6_LINEAR; ubwc_enabled = false; } /* Mutable images can be reinterpreted as any other compatible format. * This is a problem with UBWC (compression for different formats is different), * but also tiling ("swap" affects how tiled formats are stored in memory) * Depth and stencil formats cannot be reintepreted as another format, and * cannot be linear with sysmem rendering, so don't fall back for those. * * TODO: * - if the fmt_list contains only formats which are swapped, but compatible * with each other (B8G8R8A8_UNORM and B8G8R8A8_UINT for example), then * tiling is still possible * - figure out which UBWC compressions are compatible to keep it enabled */ if ((pCreateInfo->flags & VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT) && !vk_format_is_depth_or_stencil(image->vk_format)) { const VkImageFormatListCreateInfo *fmt_list = vk_find_struct_const(pCreateInfo->pNext, IMAGE_FORMAT_LIST_CREATE_INFO); bool may_be_swapped = true; if (fmt_list) { may_be_swapped = false; for (uint32_t i = 0; i < fmt_list->viewFormatCount; i++) { if (tu6_format_texture(fmt_list->pViewFormats[i], TILE6_LINEAR).swap) { may_be_swapped = true; break; } } } if (may_be_swapped) tile_mode = TILE6_LINEAR; ubwc_enabled = false; } const VkImageStencilUsageCreateInfo *stencil_usage_info = vk_find_struct_const(pCreateInfo->pNext, IMAGE_STENCIL_USAGE_CREATE_INFO); if (!ubwc_possible(image->vk_format, pCreateInfo->imageType, pCreateInfo->usage, stencil_usage_info ? stencil_usage_info->stencilUsage : pCreateInfo->usage, device->physical_device->info, pCreateInfo->samples)) ubwc_enabled = false; /* expect UBWC enabled if we asked for it */ assert(modifier != DRM_FORMAT_MOD_QCOM_COMPRESSED || ubwc_enabled); for (uint32_t i = 0; i < tu6_plane_count(image->vk_format); i++) { struct fdl_layout *layout = &image->layout[i]; VkFormat format = tu6_plane_format(image->vk_format, i); uint32_t width0 = pCreateInfo->extent.width; uint32_t height0 = pCreateInfo->extent.height; if (i > 0) { switch (image->vk_format) { case VK_FORMAT_G8_B8R8_2PLANE_420_UNORM: case VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM: /* half width/height on chroma planes */ width0 = (width0 + 1) >> 1; height0 = (height0 + 1) >> 1; break; case VK_FORMAT_D32_SFLOAT_S8_UINT: /* no UBWC for separate stencil */ ubwc_enabled = false; break; default: break; } } struct fdl_explicit_layout plane_layout; if (plane_layouts) { /* only expect simple 2D images for now */ if (pCreateInfo->mipLevels != 1 || pCreateInfo->arrayLayers != 1 || pCreateInfo->extent.depth != 1) goto invalid_layout; plane_layout.offset = plane_layouts[i].offset; plane_layout.pitch = plane_layouts[i].rowPitch; /* note: use plane_layouts[0].arrayPitch to support array formats */ } layout->tile_mode = tile_mode; layout->ubwc = ubwc_enabled; if (!fdl6_layout(layout, vk_format_to_pipe_format(format), pCreateInfo->samples, width0, height0, pCreateInfo->extent.depth, pCreateInfo->mipLevels, pCreateInfo->arrayLayers, pCreateInfo->imageType == VK_IMAGE_TYPE_3D, plane_layouts ? &plane_layout : NULL)) { assert(plane_layouts); /* can only fail with explicit layout */ goto invalid_layout; } /* fdl6_layout can't take explicit offset without explicit pitch * add offset manually for extra layouts for planes */ if (!plane_layouts && i > 0) { uint32_t offset = ALIGN_POT(image->total_size, 4096); for (int i = 0; i < pCreateInfo->mipLevels; i++) { layout->slices[i].offset += offset; layout->ubwc_slices[i].offset += offset; } layout->size += offset; } image->total_size = MAX2(image->total_size, layout->size); } const struct util_format_description *desc = util_format_description(image->layout[0].format); if (util_format_has_depth(desc) && !(device->instance->debug_flags & TU_DEBUG_NOLRZ)) { /* Depth plane is the first one */ struct fdl_layout *layout = &image->layout[0]; unsigned width = layout->width0; unsigned height = layout->height0; /* LRZ buffer is super-sampled */ switch (layout->nr_samples) { case 4: width *= 2; FALLTHROUGH; case 2: height *= 2; break; default: break; } unsigned lrz_pitch = align(DIV_ROUND_UP(width, 8), 32); unsigned lrz_height = align(DIV_ROUND_UP(height, 8), 16); image->lrz_height = lrz_height; image->lrz_pitch = lrz_pitch; image->lrz_offset = image->total_size; unsigned lrz_size = lrz_pitch * lrz_height * 2; image->total_size += lrz_size; } *pImage = tu_image_to_handle(image); #ifdef ANDROID if (gralloc_info) return tu_import_memory_from_gralloc_handle(_device, dma_buf, alloc, *pImage); #endif return VK_SUCCESS; invalid_layout: vk_object_free(&device->vk, alloc, image); return vk_error(device, VK_ERROR_INVALID_DRM_FORMAT_MODIFIER_PLANE_LAYOUT_EXT); } VKAPI_ATTR void VKAPI_CALL tu_DestroyImage(VkDevice _device, VkImage _image, const VkAllocationCallbacks *pAllocator) { TU_FROM_HANDLE(tu_device, device, _device); TU_FROM_HANDLE(tu_image, image, _image); if (!image) return; #ifdef ANDROID if (image->owned_memory != VK_NULL_HANDLE) tu_FreeMemory(_device, image->owned_memory, pAllocator); #endif vk_object_free(&device->vk, pAllocator, image); } VKAPI_ATTR void VKAPI_CALL tu_GetImageSubresourceLayout(VkDevice _device, VkImage _image, const VkImageSubresource *pSubresource, VkSubresourceLayout *pLayout) { TU_FROM_HANDLE(tu_image, image, _image); struct fdl_layout *layout = &image->layout[tu6_plane_index(image->vk_format, pSubresource->aspectMask)]; const struct fdl_slice *slice = layout->slices + pSubresource->mipLevel; pLayout->offset = fdl_surface_offset(layout, pSubresource->mipLevel, pSubresource->arrayLayer); pLayout->rowPitch = fdl_pitch(layout, pSubresource->mipLevel); pLayout->arrayPitch = fdl_layer_stride(layout, pSubresource->mipLevel); pLayout->depthPitch = slice->size0; pLayout->size = pLayout->depthPitch * layout->depth0; if (fdl_ubwc_enabled(layout, pSubresource->mipLevel)) { /* UBWC starts at offset 0 */ pLayout->offset = 0; /* UBWC scanout won't match what the kernel wants if we have levels/layers */ assert(image->level_count == 1 && image->layer_count == 1); } } VKAPI_ATTR VkResult VKAPI_CALL tu_GetImageDrmFormatModifierPropertiesEXT( VkDevice device, VkImage _image, VkImageDrmFormatModifierPropertiesEXT* pProperties) { TU_FROM_HANDLE(tu_image, image, _image); /* TODO invent a modifier for tiled but not UBWC buffers */ if (!image->layout[0].tile_mode) pProperties->drmFormatModifier = DRM_FORMAT_MOD_LINEAR; else if (image->layout[0].ubwc_layer_size) pProperties->drmFormatModifier = DRM_FORMAT_MOD_QCOM_COMPRESSED; else pProperties->drmFormatModifier = DRM_FORMAT_MOD_INVALID; return VK_SUCCESS; } VKAPI_ATTR VkResult VKAPI_CALL tu_CreateImageView(VkDevice _device, const VkImageViewCreateInfo *pCreateInfo, const VkAllocationCallbacks *pAllocator, VkImageView *pView) { TU_FROM_HANDLE(tu_device, device, _device); struct tu_image_view *view; view = vk_object_alloc(&device->vk, pAllocator, sizeof(*view), VK_OBJECT_TYPE_IMAGE_VIEW); if (view == NULL) return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); tu_image_view_init(view, pCreateInfo, device->physical_device->info->a6xx.has_z24uint_s8uint); *pView = tu_image_view_to_handle(view); return VK_SUCCESS; } VKAPI_ATTR void VKAPI_CALL tu_DestroyImageView(VkDevice _device, VkImageView _iview, const VkAllocationCallbacks *pAllocator) { TU_FROM_HANDLE(tu_device, device, _device); TU_FROM_HANDLE(tu_image_view, iview, _iview); if (!iview) return; vk_object_free(&device->vk, pAllocator, iview); } void tu_buffer_view_init(struct tu_buffer_view *view, struct tu_device *device, const VkBufferViewCreateInfo *pCreateInfo) { TU_FROM_HANDLE(tu_buffer, buffer, pCreateInfo->buffer); view->buffer = buffer; enum VkFormat vfmt = pCreateInfo->format; enum pipe_format pfmt = vk_format_to_pipe_format(vfmt); const struct tu_native_format fmt = tu6_format_texture(vfmt, TILE6_LINEAR); uint32_t range; if (pCreateInfo->range == VK_WHOLE_SIZE) range = buffer->size - pCreateInfo->offset; else range = pCreateInfo->range; uint32_t elements = range / util_format_get_blocksize(pfmt); static const VkComponentMapping components = { .r = VK_COMPONENT_SWIZZLE_R, .g = VK_COMPONENT_SWIZZLE_G, .b = VK_COMPONENT_SWIZZLE_B, .a = VK_COMPONENT_SWIZZLE_A, }; uint64_t iova = tu_buffer_iova(buffer) + pCreateInfo->offset; memset(&view->descriptor, 0, sizeof(view->descriptor)); view->descriptor[0] = A6XX_TEX_CONST_0_TILE_MODE(TILE6_LINEAR) | A6XX_TEX_CONST_0_SWAP(fmt.swap) | A6XX_TEX_CONST_0_FMT(fmt.fmt) | A6XX_TEX_CONST_0_MIPLVLS(0) | tu6_texswiz(&components, NULL, vfmt, VK_IMAGE_ASPECT_COLOR_BIT, false); COND(vk_format_is_srgb(vfmt), A6XX_TEX_CONST_0_SRGB); view->descriptor[1] = A6XX_TEX_CONST_1_WIDTH(elements & MASK(15)) | A6XX_TEX_CONST_1_HEIGHT(elements >> 15); view->descriptor[2] = A6XX_TEX_CONST_2_UNK4 | A6XX_TEX_CONST_2_UNK31; view->descriptor[4] = iova; view->descriptor[5] = iova >> 32; } VKAPI_ATTR VkResult VKAPI_CALL tu_CreateBufferView(VkDevice _device, const VkBufferViewCreateInfo *pCreateInfo, const VkAllocationCallbacks *pAllocator, VkBufferView *pView) { TU_FROM_HANDLE(tu_device, device, _device); struct tu_buffer_view *view; view = vk_object_alloc(&device->vk, pAllocator, sizeof(*view), VK_OBJECT_TYPE_BUFFER_VIEW); if (!view) return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); tu_buffer_view_init(view, device, pCreateInfo); *pView = tu_buffer_view_to_handle(view); return VK_SUCCESS; } VKAPI_ATTR void VKAPI_CALL tu_DestroyBufferView(VkDevice _device, VkBufferView bufferView, const VkAllocationCallbacks *pAllocator) { TU_FROM_HANDLE(tu_device, device, _device); TU_FROM_HANDLE(tu_buffer_view, view, bufferView); if (!view) return; vk_object_free(&device->vk, pAllocator, view); }