FEAT: added ENHEX native (string conversion with chars converted to URL-ENCODED format when needed)

related to: metaeducation/rebol-issues#1986

Author: Oldes

src/boot/natives.r

@@ -524,6 +524,11 @@ dehex: native [
 	value [any-string! binary!] {The string to dehex}
 ]
 
+enhex: native [
+	{Converts string into URL-style hex encodeding (%xx) when needed.}
+	value [any-string! binary!] {The string to encode}
+]
+
 get: native [
 	{Gets the value of a word or path, or values of an object.}
 	word {Word, path, object to get}
@@ -940,7 +945,7 @@ do-codec: native [
 	{Evaluate a CODEC function to encode or decode media types.}
 	handle [handle!] "Internal link to codec"
 	action [word!] "Decode, encode, identify"
-	data [binary! image!]
+	data [binary! image! string!]
 ]
 
 set-scheme: native [

src/core/n-strings.c

@@ -28,6 +28,7 @@
 ***********************************************************************/
 
 #include "sys-core.h"
+#include "sys-scan.h"
 #include "sys-deci-funcs.h"
 
 REBCNT z_adler32_z(REBCNT adler, REBYTE *buf, REBCNT len);
@@ -599,6 +600,135 @@ static struct digest {
 }
 
 
+/***********************************************************************
+**
+*/	REBNATIVE(enhex)
+/*
+**		Works for any string.
+**		If source is unicode (wide) string, result is ASCII.
+**
+***********************************************************************/
+{
+	REBVAL *arg = D_ARG(1);
+	REBSER *ser;
+	REBINT lex;
+	REBCNT n;
+	REBYTE encoded[4];
+	REBCNT encoded_size;
+
+	// using FORM buffer for intermediate conversion;
+	// counting with the worst scenario, where each single codepoint
+	// might need 4 bytes of UTF-8 data (%XX%XX%XX%XX)
+	REBYTE *dp = Reset_Buffer(BUF_FORM, 12 * VAL_LEN(arg));
+
+	if (VAL_BYTE_SIZE(arg)) {
+		// byte size is 1, so the series should not contain chars with value over 0x80
+		// see: https://github.com/Oldes/Rebol3/commit/aa939ba41fd71efb9f0a97f85993c95129c7e515
+
+		REBYTE *bp = VAL_BIN_DATA(arg);
+		REBYTE *ep = VAL_BIN_TAIL(arg);		
+
+		while (bp < ep) {
+			REBYTE c = bp[0];
+			bp++;
+
+			switch (GET_LEX_CLASS(c)) {
+			case LEX_CLASS_WORD:
+				if (   (c >= 'a' && c <= 'z')
+					|| (c >= 'A' && c <= 'Z')
+					||  c == '?' || c == '!' || c == '&'
+					||  c == '*' || c == '=' || c == '~'
+				) break; // no conversion
+				goto byte_needs_encoding;
+			case LEX_CLASS_NUMBER:
+				break; // no conversion
+			case LEX_CLASS_SPECIAL:
+				lex = GET_LEX_VALUE(c);
+				if (   lex == LEX_SPECIAL_PERCENT
+					|| lex == LEX_SPECIAL_BACKSLASH
+					|| lex == LEX_SPECIAL_LESSER
+					|| lex == LEX_SPECIAL_GREATER
+				) goto byte_needs_encoding;
+				break; // no conversion
+			case LEX_CLASS_DELIMIT:
+				lex = GET_LEX_VALUE(c);
+				if (    lex <= LEX_DELIMIT_RETURN
+					|| (lex >= LEX_DELIMIT_LEFT_BRACKET && lex <= LEX_DELIMIT_RIGHT_BRACE)
+					||  lex == LEX_DELIMIT_QUOTE
+				) goto byte_needs_encoding;
+				break; // no conversion
+			}
+			// leaving char as is
+			*dp++ = c;
+			continue;
+
+		byte_needs_encoding:
+			*dp++ = '%';
+			*dp++ = Hex_Digits[(c & 0xf0) >> 4];
+            *dp++ = Hex_Digits[ c & 0xf];
+		}
+	}
+	else { // UNICODE variant
+		REBUNI *up = VAL_UNI_DATA(arg);
+		REBUNI *ep = (REBUNI*)VAL_UNI_TAIL(arg);
+		
+		while (up < ep) {
+			REBUNI c = up[0];
+			up++;
+
+			if (c > 0x80) {// all non-ASCII characters *must* be percent encoded
+				encoded_size = Encode_UTF8_Char(encoded, c);
+				goto char_needs_encoding;
+			} else {
+				encoded[0] = cast(REBYTE, c);
+				encoded_size = 1;
+				switch (GET_LEX_CLASS(c)) {
+				case LEX_CLASS_WORD:
+					if (   (c >= 'a' && c <= 'z')
+						|| (c >= 'A' && c <= 'Z')
+						||  c == '?' || c == '!' || c == '&'
+						||  c == '*' || c == '=' || c == '~'
+					) break; // no conversion
+					goto char_needs_encoding;
+				case LEX_CLASS_NUMBER:
+					break; // no conversion
+				case LEX_CLASS_SPECIAL:
+					lex = GET_LEX_VALUE(c);
+					if (   lex == LEX_SPECIAL_PERCENT
+						|| lex == LEX_SPECIAL_BACKSLASH
+						|| lex == LEX_SPECIAL_LESSER
+						|| lex == LEX_SPECIAL_GREATER
+					) goto char_needs_encoding;
+					break; // no conversion
+				case LEX_CLASS_DELIMIT:
+					lex = GET_LEX_VALUE(c);
+					if (    lex <= LEX_DELIMIT_RETURN
+						|| (lex >= LEX_DELIMIT_LEFT_BRACKET && lex <= LEX_DELIMIT_RIGHT_BRACE)
+						||  lex == LEX_DELIMIT_QUOTE
+					) goto char_needs_encoding;
+					break; // no conversion
+				}
+			}
+			// leaving char as is
+			*dp++ = (REBYTE)c;
+			continue;
+
+		char_needs_encoding:
+			for (n = 0; n < encoded_size; ++n) {
+				*dp++ = '%';
+				*dp++ = Hex_Digits[(encoded[n] & 0xf0) >> 4];
+				*dp++ = Hex_Digits[ encoded[n] & 0xf];
+			}
+		}
+	}
+	*dp = 0;
+	ser = Copy_String(BUF_FORM, 0, dp - BIN_HEAD(BUF_FORM));
+	Set_Series(VAL_TYPE(arg), D_RET, ser);
+
+	return R_RET;
+}
+
+
 /***********************************************************************
 **
 */  REBNATIVE(deline)

src/tests/units/series-test.r3

@@ -245,12 +245,24 @@ Rebol [
 ===end-group===
 
 
-===start-group=== "DEHEX"
+===start-group=== "DEHEX / ENHEX"
 
 --test-- "DEHEX UTF-8 encoded data"
 	;@@ https://github.com/rebol/rebol-issues/issues/1986
 	--assert "řek" = to-string dehex to-binary "%c5%99ek"
 
+	--assert "%3x " = dehex "%3x%20"
+	--assert "++"   = dehex "%2b%2b"
+
+--test-- "ENHEX"
+	--assert "%C2%A3"   = enhex "£"
+	--assert "a%20b%5C" = enhex "a b\"
+	--assert "%C5%A1ik" = enhex "šik"
+	--assert "%22%25-.%3C%3E%5C%1F%60%7B%7C%7D~" = enhex {"%-.<>\^_`{|}~}
+	; --assert %%C5%A1ik  = enhex %šik ;<-- this does not work yet!
+	--assert "šik" = to-string dehex enhex to-binary "šik"
+
+
 ===end-group===
 
 ;-- VECTOR related tests moved to %vector-test.r3