Merge ktxtools into main (KhronosGroup#714)

PR to merge the contents of the ktxtools branch into main.
This PR should be functionally identical to the ktxtools branch. 

Briefs:

    Rebase ktxtools onto main
    Update the CTS golden files to pickup the changes in ASTC and ZSTD versions
    Add CTS testing to MinGW and Windows CI builds
    Pickup changes in image.hpp from main to ktxtools's image.hpp
    Fix any remaining warnings

cmake/docs.cmake

@@ -97,6 +97,7 @@ function( CreateDocLibKTX )
         lib/astc_encode.cpp
         lib/basis_encode.cpp
         lib/basis_transcode.cpp
+        lib/miniz_wrapper.cpp
         lib/strings.c
         lib/glloader.c
         lib/hashlist.c
@@ -120,7 +121,7 @@ function( CreateDocKTXTools )
     set( DOXYGEN_SHOW_FILES NO )
     set( DOXYGEN_FILE_PATTERNS *.cpp )
     set( DOXYGEN_RECURSIVE YES )
-    set( DOXYGEN_EXAMPLE_PATH utils )
+    set( DOXYGEN_EXAMPLE_PATH utils tools )
     set( DOXYGEN_HTML_OUTPUT ktxtools )
     set( DOXYGEN_MAN_EXTENSION .1 )
     set( DOXYGEN_GENERATE_TAGFILE ${docdest}/ktxtools.tag )
@@ -132,6 +133,14 @@ function( CreateDocKTXTools )
         tools/ktx2ktx2/ktx2ktx2.cpp
         tools/ktxsc/ktxsc.cpp
         tools/toktx/toktx.cc
+        tools/ktx/ktx_main.cpp
+        tools/ktx/command_create.cpp
+        tools/ktx/command_encode.cpp
+        tools/ktx/command_extract.cpp
+        tools/ktx/command_help.cpp
+        tools/ktx/command_info.cpp
+        tools/ktx/command_transcode.cpp
+        tools/ktx/command_validate.cpp
     )
     add_docs_cmake(ktxtools.doc)
 endfunction()
@@ -175,12 +184,11 @@ install(
     COMPONENT tools
 )
 
-install( DIRECTORY
-    ${docdest}/man/man1
+install(
+    DIRECTORY ${docdest}/man/man1
     ## Omit those, since at the moment they contain a lot of undesired files generated by Doxygen
     # ${docdest}/man/man3
     # ${docdest}/man/ktx/man3
-DESTINATION
-    "${CMAKE_INSTALL_MANDIR}"
-COMPONENT tools
+    DESTINATION "${CMAKE_INSTALL_MANDIR}"
+    COMPONENT tools
 )

include/ktx.h

@@ -184,7 +184,9 @@ typedef enum ktx_error_code_e {
     KTX_UNSUPPORTED_TEXTURE_TYPE, /*!< The KTX file specifies an unsupported texture type. */
     KTX_UNSUPPORTED_FEATURE,  /*!< Feature not included in in-use library or not yet implemented. */
     KTX_LIBRARY_NOT_LINKED,  /*!< Library dependency (OpenGL or Vulkan) not linked into application. */
-    KTX_ERROR_MAX_ENUM = KTX_LIBRARY_NOT_LINKED /*!< For safety checks. */
+    KTX_DECOMPRESS_LENGTH_ERROR, /*!< Decompressed byte count does not match expected byte size */
+    KTX_DECOMPRESS_CHECKSUM_ERROR, /*!< Checksum mismatch when decompressing */
+    KTX_ERROR_MAX_ENUM = KTX_DECOMPRESS_CHECKSUM_ERROR /*!< For safety checks. */
 } ktx_error_code_e;
 /**
  * @deprecated
@@ -672,8 +674,9 @@ typedef enum ktxSupercmpScheme {
     KTX_SS_NONE = 0,            /*!< No supercompression. */
     KTX_SS_BASIS_LZ = 1,        /*!< Basis LZ supercompression. */
     KTX_SS_ZSTD = 2,            /*!< ZStd supercompression. */
+    KTX_SS_ZLIB = 3,            /*!< ZLIB supercompression. */
     KTX_SS_BEGIN_RANGE = KTX_SS_NONE,
-    KTX_SS_END_RANGE = KTX_SS_ZSTD,
+    KTX_SS_END_RANGE = KTX_SS_ZLIB,
     KTX_SS_BEGIN_VENDOR_RANGE = 0x10000,
     KTX_SS_END_VENDOR_RANGE = 0x1ffff,
     KTX_SS_BEGIN_RESERVED = 0x20000,
@@ -766,9 +769,12 @@ enum ktxTextureCreateFlagBits {
     KTX_TEXTURE_CREATE_RAW_KVDATA_BIT = 0x02,
                                    /*!< Load the raw key-value data instead of
                                         creating a @c ktxHashList from it. */
-    KTX_TEXTURE_CREATE_SKIP_KVDATA_BIT = 0x04
+    KTX_TEXTURE_CREATE_SKIP_KVDATA_BIT = 0x04,
                                    /*!< Skip any key-value data. This overrides
                                         the RAW_KVDATA_BIT. */
+    KTX_TEXTURE_CREATE_CHECK_GLTF_BASISU_BIT = 0x08
+                                   /*!< Load texture compatible with the rules
+                                        of KHR_texture_basisu glTF extension */
 };
 /**
  * @memberof ktxTexture
@@ -1053,6 +1059,9 @@ ktxTexture2_CompressBasis(ktxTexture2* This, ktx_uint32_t quality);
 KTX_API KTX_error_code KTX_APIENTRY
 ktxTexture2_DeflateZstd(ktxTexture2* This, ktx_uint32_t level);
 
+KTX_API KTX_error_code KTX_APIENTRY
+ktxTexture2_DeflateZLIB(ktxTexture2* This, ktx_uint32_t level);
+
 KTX_API void KTX_APIENTRY
 ktxTexture2_GetComponentInfo(ktxTexture2* This, ktx_uint32_t* numComponents,
                              ktx_uint32_t* componentByteLength);
@@ -1682,6 +1691,19 @@ KTX_API KTX_error_code KTX_APIENTRY ktxPrintInfoForStdioStream(FILE* stdioStream
 KTX_API KTX_error_code KTX_APIENTRY ktxPrintInfoForNamedFile(const char* const filename);
 KTX_API KTX_error_code KTX_APIENTRY ktxPrintInfoForMemory(const ktx_uint8_t* bytes, ktx_size_t size);
 
+/*===========================================================*
+ * Utilities for printing info about a KTX2 file.            *
+ *===========================================================*/
+
+KTX_API KTX_error_code KTX_APIENTRY ktxPrintKTX2InfoTextForMemory(const ktx_uint8_t* bytes, ktx_size_t size);
+KTX_API KTX_error_code KTX_APIENTRY ktxPrintKTX2InfoTextForNamedFile(const char* const filename);
+KTX_API KTX_error_code KTX_APIENTRY ktxPrintKTX2InfoTextForStdioStream(FILE* stdioStream);
+KTX_API KTX_error_code KTX_APIENTRY ktxPrintKTX2InfoTextForStream(ktxStream* stream);
+KTX_API KTX_error_code KTX_APIENTRY ktxPrintKTX2InfoJSONForMemory(const ktx_uint8_t* bytes, ktx_size_t size, ktx_uint32_t base_indent, ktx_uint32_t indent_width, bool minified);
+KTX_API KTX_error_code KTX_APIENTRY ktxPrintKTX2InfoJSONForNamedFile(const char* const filename, ktx_uint32_t base_indent, ktx_uint32_t indent_width, bool minified);
+KTX_API KTX_error_code KTX_APIENTRY ktxPrintKTX2InfoJSONForStdioStream(FILE* stdioStream, ktx_uint32_t base_indent, ktx_uint32_t indent_width, bool minified);
+KTX_API KTX_error_code KTX_APIENTRY ktxPrintKTX2InfoJSONForStream(ktxStream* stream, ktx_uint32_t base_indent, ktx_uint32_t indent_width, bool minified);
+
 #ifdef __cplusplus
 }
 #endif

lib/basis_sgd.h

@@ -28,7 +28,7 @@ extern "C" {
 // This must be the same value as cSliceDescFlagsFrameIsIFrame so we can just
 // invert the bit when passing back & forth. As FrameIsIFrame is within
 // a C namespace it can't easily be accessed from a c header.
-enum bu_image_flags__bits_e { eBUImageIsPframe = 0x02 };
+enum bu_image_flags__bits_e { ETC1S_P_FRAME = 0x02 };
 
 typedef uint32_t buFlags;
 

lib/basis_transcode.cpp

@@ -372,6 +372,12 @@ ktxTexture2_transcodeUastc(ktxTexture2* This,
         This->dataSize = prototype->dataSize;
         prototype->pData = 0;
         prototype->dataSize = 0;
+        // Free SGD data
+        This->_private->_sgdByteLength = 0;
+        if (This->_private->_supercompressionGlobalData) {
+            free(This->_private->_supercompressionGlobalData);
+            This->_private->_supercompressionGlobalData = NULL;
+        }
     }
     ktxTexture2_Destroy(prototype);
     return result;

lib/checkheader.c

@@ -27,6 +27,10 @@
 
 #include "ktx.h"
 #include "ktxint.h"
+#include "vkformat_enum.h"
+
+bool isProhibitedFormat(VkFormat format);
+bool isValidFormat(VkFormat format);
 
 /**
  * @internal
@@ -112,7 +116,7 @@ KTX_error_code  ktxCheckHeader1_(KTX_header* pHeader,
         if (pHeader->numberOfArrayElements > 0)
         {
             /* No 3D array textures yet. */
-            return KTX_UNSUPPORTED_TEXTURE_TYPE;
+            return KTX_UNSUPPORTED_FEATURE;
         }
         pSuppInfo->textureDimension = 3;
     }
@@ -192,6 +196,20 @@ KTX_error_code ktxCheckHeader2_(KTX_header2* pHeader,
         return KTX_UNKNOWN_FILE_FORMAT;
     }
 
+    /* Check format */
+    if (isProhibitedFormat(pHeader->vkFormat))
+    {
+        return KTX_FILE_DATA_ERROR;
+    }
+    if (!isValidFormat(pHeader->vkFormat))
+    {
+        return KTX_UNSUPPORTED_FEATURE;
+    }
+    if (pHeader->supercompressionScheme == KTX_SS_BASIS_LZ && pHeader->vkFormat != VK_FORMAT_UNDEFINED)
+    {
+        return KTX_FILE_DATA_ERROR;
+    }
+
     /* Check texture dimensions. KTX files can store 8 types of textures:
        1D, 2D, 3D, cube, and array variants of these. There is currently
        no extension for 3D array textures in any 3D API. */
@@ -208,7 +226,7 @@ KTX_error_code ktxCheckHeader2_(KTX_header2* pHeader,
         if (pHeader->layerCount > 0)
         {
             /* No 3D array textures yet. */
-            return KTX_UNSUPPORTED_TEXTURE_TYPE;
+            return KTX_UNSUPPORTED_FEATURE;
         }
         pSuppInfo->textureDimension = 3;
     }
@@ -228,6 +246,16 @@ KTX_error_code ktxCheckHeader2_(KTX_header2* pHeader,
             /* cube map needs 2D faces */
             return KTX_FILE_DATA_ERROR;
         }
+        if (pHeader->pixelDepth != 0)
+        {
+            /* cube map cannot have depth */
+            return KTX_FILE_DATA_ERROR;
+        }
+        if (pHeader->pixelWidth != pHeader->pixelHeight)
+        {
+            /* cube map needs square faces */
+            return KTX_FILE_DATA_ERROR;
+        }
     }
     else if (pHeader->faceCount != 1)
     {
@@ -246,6 +274,18 @@ KTX_error_code ktxCheckHeader2_(KTX_header2* pHeader,
         pSuppInfo->generateMipmaps = 0;
     }
 
+    // Check supercompression
+    switch (pHeader->supercompressionScheme) {
+      case KTX_SS_NONE:
+      case KTX_SS_BASIS_LZ:
+      case KTX_SS_ZSTD:
+      case KTX_SS_ZLIB:
+        break;
+      default:
+        // Unsupported supercompression
+        return KTX_UNSUPPORTED_FEATURE;
+    }
+
     // This test works for arrays too because height or depth will be 0.
     max_dim = MAX(MAX(pHeader->pixelWidth, pHeader->pixelHeight), pHeader->pixelDepth);
     if (max_dim < ((ktx_uint32_t)1 << (pHeader->levelCount - 1)))

lib/dfdutils/colourspaces.c

@@ -36,7 +36,7 @@ sPrimaryMapping primaryMap[] = {
  * @param[in] latitude tolerance to use while matching. A suitable value might be 0.002
  *                 but it depends on the application.
  */
-khr_df_primaries_e findMapping(Primaries *p, float latitude) {
+khr_df_primaries_e findMapping(const Primaries *p, float latitude) {
     unsigned int i;
     for (i = 0; i < sizeof(primaryMap)/sizeof(sPrimaryMapping); ++i) {
         if (primaryMap[i].primaries.Rx - p->Rx <= latitude && p->Rx - primaryMap[i].primaries.Rx <= latitude &&
@@ -49,3 +49,23 @@ khr_df_primaries_e findMapping(Primaries *p, float latitude) {
     /* No match */
     return KHR_DF_PRIMARIES_UNSPECIFIED;
 }
+
+/**
+ * @brief Get the primaries corresponding to a KDFS primaries enum.
+ *
+ * @param[in]   primaries   the enum identifying the KDFS primaries.
+ * @param[out]  p           pointer to a Primaries struct that will
+ *                          be filled with the primary values.
+ */
+bool getPrimaries(khr_df_primaries_e primaries, Primaries *p) {
+    unsigned int i;
+    for (i = 0; i < sizeof(primaryMap)/sizeof(sPrimaryMapping); ++i) {
+        if (primaryMap[i].dfPrimaryEnum == primaries) {
+            *p = primaryMap[i].primaries;
+            return true;
+        }
+    }
+
+    /* No match */
+    return false;
+}

lib/dfdutils/createdfdtest.c

@@ -37,6 +37,6 @@ int main()
     int channels[] = {0,1,2};
     uint32_t *DFD = createDFDPacked(1, 3, bits, channels, s_UNORM);
 #endif
-    printDFD(DFD);
+    printDFD(DFD, *DFD);
     return 0;
 }

lib/dfdutils/dfd.h

@@ -19,6 +19,7 @@
 #define _DFD_H_
 
 #include <KHR/khr_df.h>
+#include <stdbool.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -126,8 +127,47 @@ enum InterpretDFDResult interpretDFD(const uint32_t *DFD,
                                      InterpretedDFDChannel *A,
                                      uint32_t *wordBytes);
 
+/* Returns the string representation.
+ * If there is no direct match or the value is invalid returns NULL */
+const char* dfdToStringVendorID(khr_df_vendorid_e value);
+
+/* Returns the string representation.
+ * If there is no direct match or the value is invalid returns NULL */
+const char* dfdToStringDescriptorType(khr_df_khr_descriptortype_e value);
+
+/* Returns the string representation.
+ * If there is no direct match or the value is invalid returns NULL */
+const char* dfdToStringVersionNumber(khr_df_versionnumber_e value);
+
+/* Returns the string representation of a bit in a khr_df_flags_e.
+ * If there is no direct match or the value is invalid returns NULL */
+const char* dfdToStringFlagsBit(uint32_t bit_index, bool bit_value);
+
+/* Returns the string representation.
+ * If there is no direct match or the value is invalid returns NULL */
+const char* dfdToStringTransferFunction(khr_df_transfer_e value);
+
+/* Returns the string representation.
+ * If there is no direct match or the value is invalid returns NULL */
+const char* dfdToStringColorPrimaries(khr_df_primaries_e value);
+
+/* Returns the string representation.
+ * If there is no direct match or the value is invalid returns NULL */
+const char* dfdToStringColorModel(khr_df_model_e value);
+
+/* Returns the string representation of a bit in a khr_df_sample_datatype_qualifiers_e.
+ * If there is no direct match or the value is invalid returns NULL */
+const char* dfdToStringSampleDatatypeQualifiersBit(uint32_t bit_index, bool bit_value);
+
+/* Returns the string representation.
+ * If there is no direct match or the value is invalid returns NULL */
+const char* dfdToStringChannelId(khr_df_model_e model, khr_df_model_channels_e value);
+
 /* Print a human-readable interpretation of a data format descriptor. */
-void printDFD(uint32_t *DFD);
+void printDFD(uint32_t *DFD, uint32_t dataSize);
+
+/* Print a JSON interpretation of a data format descriptor. */
+void printDFDJSON(uint32_t *DFD, uint32_t dataSize, uint32_t base_indent, uint32_t indent_width, bool minified);
 
 /* Get the number of components & component size from a DFD for an
  * unpacked format.
@@ -161,7 +201,8 @@ typedef struct _Primaries {
     float Wy; /*!< White y. */
 } Primaries;
 
-khr_df_primaries_e findMapping(Primaries *p, float latitude);
+khr_df_primaries_e findMapping(const Primaries *p, float latitude);
+bool getPrimaries(khr_df_primaries_e primaries, Primaries *p);
 
 #ifdef __cplusplus
 }

lib/dfdutils/interpretdfd.c

@@ -25,8 +25,8 @@
               described as 32-bit words in native endianness.
               Note that this is the whole descriptor, not just
               the basic descriptor block.
- * @param R Information about the decoded red channel, if any.
- * @param G Information about the decoded green channel, if any.
+ * @param R Information about the decoded red channel or the depth channel, if any.
+ * @param G Information about the decoded green channel or the stencil channel, if any.
  * @param B Information about the decoded blue channel, if any.
  * @param A Information about the decoded alpha channel, if any.
  * @param wordBytes Byte size of the channels (unpacked) or total size (packed).
@@ -99,8 +99,8 @@ enum InterpretDFDResult interpretDFD(const uint32_t *DFD,
             if (KHR_DFDSVAL(BDFDB, sampleCounter, QUALIFIERS)
                  & KHR_DF_SAMPLE_DATATYPE_FLOAT) {
                 result |= i_FLOAT_FORMAT_BIT;
-                determinedFloatness = 1;
             }
+            determinedFloatness = 1;
         } else {
             /* Check whether we disagree with our predetermined floatness. */
             /* Note that this could justifiably happen with (say) D24S8. */
@@ -115,8 +115,8 @@ enum InterpretDFDResult interpretDFD(const uint32_t *DFD,
             if (KHR_DFDSVAL(BDFDB, sampleCounter, QUALIFIERS)
                  & KHR_DF_SAMPLE_DATATYPE_SIGNED) {
                 result |= i_SIGNED_FORMAT_BIT;
-                determinedSignedness = 1;
             }
+            determinedSignedness = 1;
         } else {
             /* Check whether we disagree with our predetermined signedness. */
             if (KHR_DFDSVAL(BDFDB, sampleCounter, QUALIFIERS)
@@ -207,6 +207,12 @@ enum InterpretDFDResult interpretDFD(const uint32_t *DFD,
             case KHR_DF_CHANNEL_RGBSDA_BLUE:
                 sampleChannelPtr = B;
                 break;
+            case KHR_DF_CHANNEL_RGBSDA_DEPTH:
+                sampleChannelPtr = R;
+                break;
+            case KHR_DF_CHANNEL_RGBSDA_STENCIL:
+                sampleChannelPtr = G;
+                break;
             case KHR_DF_CHANNEL_RGBSDA_ALPHA:
                 sampleChannelPtr = A;
                 break;
@@ -286,6 +292,12 @@ enum InterpretDFDResult interpretDFD(const uint32_t *DFD,
             case KHR_DF_CHANNEL_RGBSDA_BLUE:
                 sampleChannelPtr = B;
                 break;
+            case KHR_DF_CHANNEL_RGBSDA_DEPTH:
+                sampleChannelPtr = R;
+                break;
+            case KHR_DF_CHANNEL_RGBSDA_STENCIL:
+                sampleChannelPtr = G;
+                break;
             case KHR_DF_CHANNEL_RGBSDA_ALPHA:
                 sampleChannelPtr = A;
                 break;

lib/dfdutils/interpretdfdtest.c

@@ -228,7 +228,7 @@ int main()
     uint32_t *d = createDFDUnpacked(0, 3, 1, 0, s_UNORM);
 #endif
 
-    printDFD(d);
+    printDFD(d, *d);
     t = interpretDFD(d, &R, &G, &B, &A, &wordSize);
 #endif