Utf-8/unicode support in legacy tools and lib. (#800)

Remove all generic text macros.

Fixes #764.

The *NamedFile* functions now accept full utf8 filenames on any platform. Windows applications must use WideCharToMultiByte to convert unicode filenames before calling these functions.

CMakeLists.txt

@@ -411,7 +411,7 @@ if(WIN32)
     # generators (like Visual Studio) to take the exact path and not
     # change it.
     set(CMAKE_RUNTIME_OUTPUT_DIRECTORY $<1:${KTX_BUILD_DIR}/$<CONFIG>>)
-elseif(APPLE)
+elseif(APPLE OR LINUX)
     if(NOT IOS)
         # Set a common RUNTIME_OUTPUT_DIR and LIBRARY_OUTPUT_DIR for all targets,
         # so that INSTALL RPATH is functional in build directory as well.

lib/info.c

@@ -1251,11 +1251,7 @@ ktxPrintKTX2InfoJSONForNamedFile(const char* const filename, ktx_uint32_t base_i
 {
     FILE* file = NULL;
 
-#ifdef _WIN32
-    fopen_s(&file, filename, "rb");
-#else
-    file = fopen(filename, "rb");
-#endif
+    file = ktxFOpenUTF8(filename, "rb");
 
     if (!file)
         return KTX_FILE_OPEN_FAILED;

lib/ktxint.h

@@ -293,6 +293,52 @@ KTX_error_code ktxUncompressZLIBInt(unsigned char* pDest,
 
 KTX_error_code printKTX2Info2(ktxStream* src, KTX_header2* header);
 
+/*
+ * fopen a file identified by a UTF-8 path.
+ */
+#if defined(_WIN32)
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <assert.h>
+#include <windows.h>
+#include <shellapi.h>
+#include <stdlib.h>
+
+// For Windows, we convert the UTF-8 path and mode to UTF-16 path and use _wfopen
+// which correctly handles unicode characters.
+static inline FILE* ktxFOpenUTF8(char const* path, char const* mode) {
+    int wpLen = MultiByteToWideChar(CP_UTF8, 0, path, -1, NULL, 0);
+    int wmLen = MultiByteToWideChar(CP_UTF8, 0, mode, -1, NULL, 0);
+    FILE* fp = NULL;
+    if (wpLen > 0 && wmLen > 0)
+    {
+        wchar_t* wpath = (wchar_t*)malloc(wpLen * sizeof(wchar_t));
+        wchar_t* wmode = (wchar_t*)malloc(wmLen * sizeof(wchar_t));
+        MultiByteToWideChar(CP_UTF8, 0, path, -1, wpath, wpLen);
+        MultiByteToWideChar(CP_UTF8, 0, mode, -1, wmode, wmLen);
+        // Returned errmo_t value is also set in the global errno.
+        // Apps use that for error detail as libktx only returns
+        // KTX_FILE_OPEN_FAILED.
+        (void)_wfopen_s(&fp, wpath, wmode);
+        free(wpath);
+        free(wmode);
+        return fp;
+    } else {
+        assert(KTX_FALSE && "ktxFOpenUTF8 called with zero length path or mode.");
+        return NULL;
+    }
+}
+#else
+// For other platforms there is no need for any conversion, they support UTF-8 natively
+static inline FILE* ktxFOpenUTF8(char const* path, char const* mode) {
+    return fopen(path, mode);
+}
+#endif
+
 #ifdef __cplusplus
 }
 #endif

lib/texture.c

@@ -408,7 +408,7 @@ ktxTexture_CreateFromNamedFile(const char* const filename,
     if (filename == NULL || newTex == NULL)
         return KTX_INVALID_VALUE;
 
-    file = fopen(filename, "rb");
+    file = ktxFOpenUTF8(filename, "rb");
     if (!file)
        return KTX_FILE_OPEN_FAILED;
 

lib/texture1.c

@@ -458,6 +458,9 @@ ktxTexture1_constructFromStdioStream(ktxTexture1* This, FILE* stdioStream,
  * @memberof ktxTexture1 @private
  * @brief Construct a ktxTexture1 from a named KTX file.
  *
+ * The file name must be encoded in utf-8. On Windows convert unicode names
+ * to utf-8 with @c WideCharToMultiByte(CP_UTF8, ...) before calling.
+ *
  * See ktxTextureInt_constructFromStream for details.
  *
  * @param[in] This pointer to a ktxTextureInt-sized block of memory to
@@ -484,7 +487,7 @@ ktxTexture1_constructFromNamedFile(ktxTexture1* This,
     if (This == NULL || filename == NULL)
         return KTX_INVALID_VALUE;
 
-    file = fopen(filename, "rb");
+    file = ktxFOpenUTF8(filename, "rb");
     if (!file)
        return KTX_FILE_OPEN_FAILED;
 
@@ -681,6 +684,9 @@ ktxTexture1_CreateFromStdioStream(FILE* stdioStream,
  * The address of a newly created texture reflecting the contents of the
  * file is written to the location pointed at by @p newTex.
  *
+ * The file name must be encoded in utf-8. On Windows convert unicode names
+ * to utf-8 with @c WideCharToMultiByte(CP_UTF8, ...) before calling.
+ *
  * The create flag KTX_TEXTURE_CREATE_LOAD_IMAGE_DATA_BIT should not be set,
  * if the ktxTexture1 is ultimately to be uploaded to OpenGL or Vulkan. This
  * will minimize memory usage by allowing, for example, loading the images

lib/texture2.c

@@ -1101,6 +1101,9 @@ ktxTexture2_constructFromStdioStream(ktxTexture2* This, FILE* stdioStream,
  * @~English
  * @brief Construct a ktxTexture from a named KTX file.
  *
+ * The file name must be encoded in utf-8. On Windows convert unicode names
+ * to utf-8 with @c WideCharToMultiByte(CP_UTF8, ...) before calling.
+ *
  * See ktxTextureInt_constructFromStream for details.
  *
  * @param[in] This pointer to a ktxTextureInt-sized block of memory to
@@ -1127,7 +1130,7 @@ ktxTexture2_constructFromNamedFile(ktxTexture2* This,
     if (This == NULL || filename == NULL)
         return KTX_INVALID_VALUE;
 
-    file = fopen(filename, "rb");
+    file = ktxFOpenUTF8(filename, "rb");
     if (!file)
        return KTX_FILE_OPEN_FAILED;
 
@@ -1378,6 +1381,9 @@ ktxTexture2_CreateFromStdioStream(FILE* stdioStream,
  * The address of a newly created ktxTexture2 reflecting the contents of the
  * file is written to the location pointed at by @p newTex.
  *
+ * The file name must be encoded in utf-8. On Windows convert unicode names
+ * to utf-8 with @c WideCharToMultiByte(CP_UTF8, ...) before calling.
+ *
  * The create flag KTX_TEXTURE_CREATE_LOAD_IMAGE_DATA_BIT should not be set,
  * if the ktxTexture is ultimately to be uploaded to OpenGL or Vulkan. This
  * will minimize memory usage by allowing, for example, loading the images

lib/writer1.c

@@ -397,6 +397,9 @@ ktxTexture1_WriteToStdioStream(ktxTexture1* This, FILE* dstsstr)
  * @~English
  * @brief Write a ktxTexture object to a named file in KTX format.
  *
+ * The file name must be encoded in utf-8. On Windows convert unicode names
+ * to utf-8 with @c WideCharToMultiByte(CP_UTF8, ...) before calling.
+ *
  * @param[in] This      pointer to the target ktxTexture object.
  * @param[in] dstname   destination file name.
  *
@@ -422,7 +425,7 @@ ktxTexture1_WriteToNamedFile(ktxTexture1* This, const char* const dstname)
     if (!This)
         return KTX_INVALID_VALUE;
 
-    dst = fopen(dstname, "wb");
+    dst = ktxFOpenUTF8(dstname, "wb");
     if (dst) {
         result = ktxTexture1_WriteToStdioStream(This, dst);
         fclose(dst);
@@ -842,6 +845,9 @@ ktxTexture1_WriteKTX2ToStdioStream(ktxTexture1* This, FILE* dstsstr)
  * @~English
  * @brief Write a ktxTexture object to a named file in KTX2 format.
  *
+ * The file name must be encoded in utf-8. On Windows convert unicode names
+ * to utf-8 with @c WideCharToMultiByte(CP_UTF8, ...) before calling.
+ *
  * Callers are strongly urged to include a KTXwriter item in the texture's metadata.
  * It can be added by code, similar to the following, prior to calling this
  * function.
@@ -878,7 +884,7 @@ ktxTexture1_WriteKTX2ToNamedFile(ktxTexture1* This, const char* const dstname)
     if (!This)
         return KTX_INVALID_VALUE;
 
-    dst = fopen(dstname, "wb");
+    dst = ktxFOpenUTF8(dstname, "wb");
     if (dst) {
         result = ktxTexture1_WriteKTX2ToStdioStream(This, dst);
         fclose(dst);

lib/writer2.c

@@ -632,6 +632,9 @@ ktxTexture2_WriteToStdioStream(ktxTexture2* This, FILE* dstsstr)
  * @~English
  * @brief Write a ktxTexture object to a named file in KTX format.
  *
+ * The file name must be encoded in utf-8. On Windows convert unicode names
+ * to utf-8 with @c WideCharToMultiByte(CP_UTF8, ...) before calling.
+ *
  * Callers are strongly urged to include a KTXwriter item in the texture's metadata.
  * It can be added by code, similar to the following, prior to calling this
  * function.
@@ -668,7 +671,7 @@ ktxTexture2_WriteToNamedFile(ktxTexture2* This, const char* const dstname)
     if (!This)
         return KTX_INVALID_VALUE;
 
-    dst = fopen(dstname, "wb");
+    dst = ktxFOpenUTF8(dstname, "wb");
     if (dst) {
         result = ktxTexture2_WriteToStdioStream(This, dst);
         fclose(dst);

tests/ktx2ktx2-tests.cmake

@@ -106,3 +106,15 @@ cnvrtcmpktx( 2d-bc2 pattern_02_bc2.ktx2 pattern_02_bc2.ktx "-f" )
 cnvrtcmpktx( 2d-array-astc texturearray_astc_8x8_unorm.ktx2 texturearray_astc_8x8_unorm.ktx "-f" )
 
 cnvrtcmpktx_implied_out( 2d-bc2 pattern_02_bc2 "-f" )
+
+cnvrtcmpktx( unicode-file-hu hűtő.ktx2 hűtő.ktx "-f" )
+cnvrtcmpktx( unicode-file-jp テクスチャ.ktx2 テクスチャ.ktx "-f" )
+cnvrtcmpktx( unicode-file-ar نَسِيج.ktx2 نَسِيج.ktx "-f" )
+cnvrtcmpktx( unicode-file-zh 质地.ktx2 质地.ktx "-f" )
+cnvrtcmpktx( unicode-file-ko 조직.ktx2 조직.ktx "-f" )
+
+cnvrtcmpktx_implied_out( unicode-file-hu hűtő "-f" )
+cnvrtcmpktx_implied_out( unicode-file-jp テクスチャ "-f" )
+cnvrtcmpktx_implied_out( unicode-file-ar نَسِيج "-f" )
+cnvrtcmpktx_implied_out( unicode-file-zh 质地 "-f" )
+cnvrtcmpktx_implied_out( unicode-file-ko 조직 "-f" )

tests/ktxsc-tests.cmake

@@ -112,3 +112,13 @@ endfunction()
 sccmpktx( zcmp-cubemap skybox_zstd.ktx2 skybox.ktx2 "--zcmp 5" )
 sccmpktxinplacecurdir( zcmp-cubemap skybox_zstd.ktx2 skybox.ktx2 "--zcmp 5" )
 sccmpktxinplacediffdir( zcmp_cubemap skybox_zstd.ktx2 skybox.ktx2 "--zcmp 5" )
+sccmpktxinplacecurdir( unicode-file-hu hűtő_zstd.ktx2 hűtő.ktx2 "--zcmp 5" )
+sccmpktxinplacecurdir( unicode-file-jp テクスチャ_zstd.ktx2 テクスチャ.ktx2 "--zcmp 5" )
+sccmpktxinplacecurdir( unicode-file-ar نَسِيج_zstd.ktx2 نَسِيج.ktx2 "--zcmp 5" )
+sccmpktxinplacecurdir( unicode-file-zh 质地_zstd.ktx2 质地.ktx2 "--zcmp 5" )
+sccmpktxinplacecurdir( unicode-file-ko 조직_zstd.ktx2 조직.ktx2 "--zcmp 5" )
+sccmpktxinplacediffdir( unicode-file-hu hűtő_zstd.ktx2 hűtő.ktx2 "--zcmp 5" )
+sccmpktxinplacediffdir( unicode-file-jp テクスチャ_zstd.ktx2 テクスチャ.ktx2 "--zcmp 5" )
+sccmpktxinplacediffdir( unicode-file-ar نَسِيج_zstd.ktx2 نَسِيج.ktx2 "--zcmp 5" )
+sccmpktxinplacediffdir( unicode-file-zh 质地_zstd.ktx2 质地.ktx2 "--zcmp 5" )
+sccmpktxinplacediffdir( unicode-file-ko 조직_zstd.ktx2 조직.ktx2 "--zcmp 5" )

tests/loadtests/appfwSDL/main.cpp

@@ -20,6 +20,7 @@
 #include <vector>
 #include <stdio.h>
 #include "AppBaseSDL.h"
+#include "platform_utils.h"
 #if defined(EMSCRIPTEN)
 #include <emscripten.h>
 #endif
@@ -55,6 +56,7 @@ main(int argc, char* argv[])
   }
   atexit(SDL_Quit);
 
+  InitUTF8CLI(argc, argv);
   AppBaseSDL::Args args(argv, argv+argc);
 
   if (!theApp->initialize(args))

tests/loadtests/common/TranscodeTargetStrToFmt.cpp

@@ -11,7 +11,7 @@
 #include "TranscodeTargetStrToFmt.h"
 
 ktx_transcode_fmt_e
-TranscodeTargetStrToFmt(_tstring format)
+TranscodeTargetStrToFmt(std::string format)
 {
     if (!format.compare("ETC1_RGB"))
         return KTX_TTF_ETC1_RGB;

tests/loadtests/common/TranscodeTargetStrToFmt.h

@@ -11,14 +11,6 @@
 
 #include <string>
 
-#if !defined(_tstring)
-  #if defined(_UNICODE)
-    #define _tstring std::wstring
-  #else
-    #define _tstring std::string
-  #endif
-#endif
-
-ktx_transcode_fmt_e TranscodeTargetStrToFmt(_tstring format);
+ktx_transcode_fmt_e TranscodeTargetStrToFmt(std::string format);
 
 #endif /* _TRANSCODE_TARGET_STR_TO_FMT_ */

tests/loadtests/common/ltexceptions.h

@@ -15,6 +15,7 @@
  */
 
 #include <stdexcept>
+#include <string>
 
 #define OUT_OF_HOST_MEMORY -1
 #define OUT_OF_DEVICE_MEMORY -2
@@ -27,7 +28,7 @@ class bad_vulkan_alloc : public std::bad_alloc {
         if (which == FRAGMENTED_POOL) {
             message << "Pool fragmented when allocating for " << _message << ".";
         } else {
-            std::string memtype;
+          std::string memtype;
             switch (which) {
               case OUT_OF_HOST_MEMORY: memtype = "host"; break;
               case OUT_OF_DEVICE_MEMORY: memtype = "device"; break;

tests/loadtests/glloadtests/shader-based/EncodeTexture.h

@@ -50,11 +50,11 @@ class EncodeTexture : public GL3LoadTestSample {
     bool bInitialized;
     ktx_transcode_fmt_e transcodeTarget;
     enum encode_fmt_e { EF_ASTC = 1, EF_ETC1S = 2, EF_UASTC = 3 };
-    friend ostream& operator<<(ostream& os, encode_fmt_e format);
+    friend std::ostream& operator<<(std::ostream& os, encode_fmt_e format);
     encode_fmt_e encodeTarget;
 };
 
-inline ostream& operator<<(ostream& os, EncodeTexture::encode_fmt_e format)
+inline std::ostream& operator<<(std::ostream& os, EncodeTexture::encode_fmt_e format)
 {
     switch (format) {
       case EncodeTexture::EF_ASTC:

tests/loadtests/glloadtests/shader-based/InstancedSampleBase.cpp

@@ -16,7 +16,6 @@
  * @author Mark Callow, www.edgewise-consulting.com.
  */
 
-#include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <assert.h>
@@ -34,6 +33,8 @@
 
 #define member_size(type, member) sizeof(((type *)0)->member)
 
+using namespace std;
+
 const GLchar* InstancedSampleBase::pszInstancingFsDeclarations =
     "precision mediump float;\n"
 
@@ -95,7 +96,7 @@ const GLchar* InstancedSampleBase::pszInstancingVsDeclarations =
  */
 InstancedSampleBase::InstancedSampleBase(uint32_t width, uint32_t height,
                            const char* const szArgs,
-                           const std::string sBasePath)
+                           const string sBasePath)
         : GL3LoadTestSample(width, height, szArgs, sBasePath),
           texUnit(GL_TEXTURE0), uniformBufferBindId(0),
           bInitialized(false)
@@ -113,7 +114,7 @@ InstancedSampleBase::InstancedSampleBase(uint32_t width, uint32_t height,
     KTX_error_code ktxresult;
     ktxTexture* kTexture;
     GLenum glerror;
-    std::string ktxfilepath = externalFile ? ktxfilename
+    string ktxfilepath = externalFile ? ktxfilename
                                            : getAssetPath() + ktxfilename;
     ktxresult =
            ktxTexture_CreateFromNamedFile(ktxfilepath.c_str(),
@@ -208,7 +209,7 @@ InstancedSampleBase::run(uint32_t /*msTicks*/)
 //===================================================================
 
 void
-InstancedSampleBase::processArgs(std::string sArgs)
+InstancedSampleBase::processArgs(string sArgs)
 {
     // Options descriptor
     struct argparser::option longopts[] = {
@@ -436,7 +437,7 @@ InstancedSampleBase::prepareProgram(ShaderSource& fs, ShaderSource& vs)
         ssDefine << "#define INSTANCE_COUNT " << instanceCount << "U" << endl;
         // str().c_str() doesn't work because str goes outof scope immediately.
         // Hence this 2 step process.
-        std::string sDefine = ssDefine.str();
+        string sDefine = ssDefine.str();
         vs.insert(vs.begin(), sDefine.c_str());
         makeShader(GL_VERTEX_SHADER, vs, &gnInstancingVs);
         makeShader(GL_FRAGMENT_SHADER, fs, &gnInstancingFs);

tests/loadtests/glloadtests/utils/GLMeshLoader.hpp

@@ -13,7 +13,6 @@
 #include <map>
 #include <stdexcept>
 #include <vector>
-using namespace std;
 
 #ifdef _WIN32
 #include <windows.h>
@@ -211,7 +210,7 @@ class GLMeshLoader {
             std::stringstream message;
 
             message << "  Import via ASSIMP from\"" << filename << "\" failed. "
-                    << importer.GetErrorString() << endl;
+                    << importer.GetErrorString() << std::endl;
             throw std::runtime_error(message.str());
         }
 
@@ -316,7 +315,7 @@ class GLMeshLoader {
     }
 
     void CreateBuffers(glMeshLoader::MeshBuffer& meshBuffer,
-                       vector<glMeshLoader::VertexLayout> layout,
+                       std::vector<glMeshLoader::VertexLayout> layout,
                        float scale)
     {
         std::vector<float> vertexBuffer;