FEAT: loading .txt files with unicode encodings

Text files (.txt) are now loaded just as a string and not as a Rebol code. Unicode decoding is handled correctly if specified in BOM. Files (.txt) are always saved as UTF-8.

fixes: Oldes/Rebol-issues#2424
related to: Oldes/Rebol-issues#1937

Author: Oldes

src/core/b-init.c

@@ -666,10 +666,12 @@ extern const REBYTE Str_Banner[];
 	}
 
 	if (codi->action == CODI_DECODE) {
-		return CODI_TEXT;
+		codi->other = (void*)Decode_UTF_String(codi->data, codi->len, -1, TRUE);
+		return CODI_STRING;
 	}
 
 	if (codi->action == CODI_ENCODE) {
+		//O: This does not happen as in n-system.c only image is allowed to be encoded!
 		return CODI_BINARY;
 	}
 

src/core/n-system.c

@@ -476,6 +476,9 @@ char *evoke_help = "Evoke values:\n"
 	case CODI_BLOCK:
 		Set_Block(D_RET, codi.other);
 		break;
+	case CODI_STRING:
+		Set_String(D_RET, codi.other);
+		break;
 
 	default:
 		Trap0(RE_BAD_MEDIA); // need better!!!

src/include/reb-codec.h

@@ -75,6 +75,7 @@ enum {
 	CODI_IMAGE,
 	CODI_SOUND,
 	CODI_BLOCK,
+	CODI_STRING,			// result is in codi->other as a series (no need to copy).
 };
 
 // Codec commands:

src/tests/units/codecs-test.r3

@@ -73,6 +73,48 @@ Rebol [
 		--assert error? try [save %temp.jpg #{00}]
 ===end-group===
 
+
+===start-group=== "TEXT codec"
+;- loading *.txt file does also conversion CRLF to LF
+;- it is actually same like using read/string
+;@@ https://github.com/Oldes/Rebol-issues/issues/2424
+--test-- "load UCS16-LE txt"
+	--assert all [
+		string? try [str: load %units/files/issue-2186-UTF16-LE.txt]
+		11709824 = checksum str
+	]
+--test-- "load UCS16-BE txt"
+	--assert all [
+		string? try [str: load %units/files/issue-2186-UTF16-BE.txt]
+		11709824 = checksum str
+	]
+--test-- "load UCS32-LE txt"
+	--assert all [
+		string? try [str: load %units/files/issue-2186-UTF32-LE.txt]
+		11709824 = checksum str
+	]
+--test-- "load UCS32-BE txt"
+	--assert all [
+		string? try [str: load %units/files/issue-2186-UTF32-BE.txt]
+		11709824 = checksum str
+	]
+--test-- "load/save issue! as .txt"
+	;@@ https://github.com/Oldes/Rebol-issues/issues/1937
+	;- notice that result is a string! and not an issue!
+	--assert "#00000002" = load save %tmp.txt #00000002
+	--assert  #00000002  = load save %tmp.reb #00000002
+--test-- "load/save block! as .txt"
+	--assert {1 "aha"} = load save %tmp.txt [1 "aha"]
+	--assert [1 "aha"] = load save %tmp.reb [1 "aha"]
+--test-- "load/save binary! as .txt"
+	--assert    "12" = load save %tmp.txt #{3132}
+	--assert #{3132} = load save %tmp.reb #{3132}
+
+	delete %tmp.txt
+	delete %tmp.reb
+
+===end-group===
+
 if find codecs 'wav [
 	codecs/wav/verbose: 3
 	===start-group=== "WAV codec"