FEAT: exported new library functions for better UTF8 text conversions from Rebol extension code

New functions:

RL_Get_UTF8_String
RL_Encode_UTF8
RL_Encode_UTF8_String
RL_Decode_UTF_String

Author: Oldes

src/core/a-lib.c

@@ -612,6 +612,35 @@ RL_API int RL_Get_String(REBSER *series, u32 index, void **str)
 	return len;
 }
 
+RL_API int RL_Get_UTF8_String(REBSER *series, u32 index, void **str)
+/*
+**	Obtain a pointer into an UTF8 encoded string.
+**
+**	Returns:
+**		The length of string is bytes.
+**  Arguments:
+**		series - string series pointer
+**		index - index from beginning (zero-based)
+**		str   - pointer to first character
+**	Notes:
+**		Strings are allowed to move in memory. Therefore, you will want
+**		to make a copy of the string if needed.
+*/
+{
+	int len = (index >= series->tail) ? 0 : series->tail - index;
+
+	if (BYTE_SIZE(series)) {
+		*str = BIN_SKIP(series, index);
+	}
+	else {
+		REBSER *ser = Encode_UTF8_String((void*)UNI_SKIP(series, index), len, TRUE, 0);
+		*str = BIN_HEAD(ser);
+		len = BIN_LEN(ser);
+	}
+
+	return len;
+}
+
 RL_API u32 RL_Map_Word(REBYTE *string)
 /*
 **	Given a word as a string, return its global word identifier.
@@ -917,6 +946,54 @@ RL_API int RL_Callback(RXICBI *cbi)
 	return RL_Event(&evt);	// (returns 0 if queue is full, ignored)
 }
 
+RL_API REBCNT RL_Encode_UTF8(REBYTE *dst, REBINT max, void *src, REBCNT *len, REBFLG uni, REBFLG opts)
+/*
+**	Encode the unicode into UTF8 byte string.
+**		
+**	Returns:
+**		Number of source chars used.
+**		Updates len for dst bytes used.
+**		Does not add a terminator.
+**	Arguments:
+**		max  - The maximum size of the result (UTF8).
+**		uni  - Source string can be byte or unichar sized (uni = TRUE);
+**		opts - Convert LF/CRLF
+*/
+{
+	return Encode_UTF8(dst, max, src, len, uni, opts);
+}
+
+RL_API REBSER* RL_Encode_UTF8_String(void *src, REBCNT len, REBFLG uni, REBFLG opts)
+/*
+**	Encode the unicode into UTF8 byte string.
+**
+**	Returns:
+**		Rebol series value with an UTF8 encoded data.
+**	Arguments:
+**		src  - series as a REBYTE or REBUNI.
+**		len  - number of source bytes to convert.
+**		uni  - Source string can be byte or unichar sized (uni = TRUE);
+**		opts - Convert LF/CRLF
+*/
+{
+	return Encode_UTF8_String(src, len, uni, opts);
+}
+
+RL_API REBSER* RL_Decode_UTF_String(REBYTE *src, REBCNT len, REBINT utf)
+/*
+**	Decode the UTF8 encoded data into Rebol series.
+**
+**	Returns:
+**		Rebol series with char size 1 or 2
+**	Arguments:
+**		src  - UTF8 encoded data
+**		len  - number of source bytes to convert.
+**		utf  - is 0, 8, +/-16, +/-32.
+*/
+{
+	return Decode_UTF_String(src, len, utf);
+}
+
 
 
 #include "reb-lib-lib.h"

src/core/s-unicode.c

@@ -1136,14 +1136,40 @@ ConversionResult ConvertUTF8toUTF32 (
 **		Result can be a shared buffer!
 **
 ***********************************************************************/
+{
+	REBSER * ser;
+	if (VAL_BYTE_SIZE(arg)) {
+		ser = Encode_UTF8_String(VAL_BIN_DATA(arg), len, FALSE, opts);
+	} else {
+		ser = Encode_UTF8_String(VAL_UNI_DATA(arg), len, TRUE, opts);
+	}
+	return ser;
+}
+
+/***********************************************************************
+**
+*/	REBSER *Encode_UTF8_String(void *src, REBCNT len, REBFLG uni, REBFLG opts)
+/*
+**		Do all the details to encode a string as UTF8.
+**		No_copy means do not make a copy.
+**		Result can be a shared buffer!
+**
+***********************************************************************/
 {
 	REBSER *ser = BUF_FORM; // a shared buffer
 	REBCNT size;
 	REBYTE *cp;
 	REBFLG ccr = GET_FLAG(opts, ENC_OPT_CRLF);
 
-	if (VAL_BYTE_SIZE(arg)) {
-		REBYTE *bp = VAL_BIN_DATA(arg);
+	if (uni) {
+		REBUNI *up = (REBUNI*)src;
+
+		size = Length_As_UTF8(up, len, TRUE, (REBOOL)ccr);
+		cp = Reset_Buffer(ser, size + (GET_FLAG(opts, ENC_OPT_BOM) ? 3 : 0));
+		Encode_UTF8(Reset_Buffer(ser, size), size, up, &len, TRUE, ccr);
+	}
+	else {
+		REBYTE *bp = (REBYTE*)src;
 
 		if (Is_Not_ASCII(bp, len)) {
 			size = Length_As_UTF8((REBUNI*)bp, len, FALSE, (REBOOL)ccr);
@@ -1152,13 +1178,6 @@ ConversionResult ConvertUTF8toUTF32 (
 		}
 		else if (GET_FLAG(opts, ENC_OPT_NO_COPY)) return 0;
 		else return Copy_Bytes(bp, len);
-
-	} else {
-		REBUNI *up = VAL_UNI_DATA(arg);
-
-		size = Length_As_UTF8(up, len, TRUE, (REBOOL)ccr);
-		cp = Reset_Buffer(ser, size + (GET_FLAG(opts, ENC_OPT_BOM) ? 3 : 0));
-		Encode_UTF8(Reset_Buffer(ser, size), size, up, &len, TRUE, ccr);
 	}
 
 	SERIES_TAIL(ser) = len;
@@ -1168,6 +1187,17 @@ ConversionResult ConvertUTF8toUTF32 (
 }
 
 #ifdef unused
+/***********************************************************************
+**
+*/	REBSER *Decode_UTF8_String(REBYTE *src, REBCNT len)
+/*
+**		Decode an UTF8 encoded source into series.
+**
+***********************************************************************/
+{
+	return Decode_UTF_String(src, len, 8);
+}
+
 /***********************************************************************
 **
 */	REBSER *Encode_String(void *str, REBCNT len, REBCNT opts)