File tree 2 files changed +30
-1
lines changed
2 files changed +30
-1
lines changed Original file line number Diff line number Diff line change @@ -113,7 +113,17 @@ load-header: function/with [
113
113
; Commented assert statements are for documentation and testing.
114
114
;
115
115
case /all [
116
- binary? source [tmp: assert-utf8 source]
116
+ binary? source [
117
+ parse source [
118
+ ; utf-16 & utf-32
119
+ #{ 0000FEFF } tmp: (tmp: iconv/to tmp 'utf-32BE 'utf8)
120
+ | #{ FFFE0000 } tmp: (tmp: iconv/to tmp 'utf-32LE 'utf8)
121
+ | #{ FEFF } tmp: (tmp: iconv/to tmp 'utf-16BE 'utf8)
122
+ | #{ FFFE } tmp: (tmp: iconv/to tmp 'utf-16LE 'utf8)
123
+ | ; utf-8 (skip the BOM if found)
124
+ opt [#{ EFBBBF } source: ] (tmp: assert-utf8 source)
125
+ ]
126
+ ]
117
127
string? source [tmp: to binary! source]
118
128
not data: script? tmp [ ; no script header found
119
129
return either required ['no-header] [reduce [none tmp tail tmp]]
Original file line number Diff line number Diff line change @@ -786,6 +786,25 @@ Rebol [
786
786
--assert 5367801 = checksum deline str
787
787
--assert 5367801 = checksum read /string %units/files/quickbrown.bin ;converts CRLF to LF
788
788
789
+ --test-- "LOAD Unicode encoded text with BOM"
790
+ --assert "Writer" = form load #{ FEFF005700720069007400650072 } ;UTF-16BE
791
+ --assert "Writer" = form load #{ FFFE570072006900740065007200 } ;UTF-16LE
792
+ --assert "ěšč" = form load #{ 0000feff0000011b000001610000010d } ;UTF-32BE
793
+ --assert "ěšč" = form load #{ fffe00001b010000610100000d010000 } ;UTF-32LE
794
+ --assert "esc" = form load #{ 0000feff000000650000007300000063 } ;UTF-32BE
795
+ --assert "esc" = form load #{ fffe0000650000007300000063000000 } ;UTF-32LE
796
+ --assert [a b] = load #{ 0000feff000000610000002000000062 }
797
+ --assert [a b] = load #{ fffe0000610000002000000062000000 }
798
+ ;@@ https://github.com/Oldes/Rebol-issues/issues/2280
799
+ --assert "äöü" = form load #{ EFBBBFC3A4C3B6C3BC } ;UTF-8
800
+ --assert "aou" = form load #{ EFBBBF616F75 } ;UTF-8
801
+ --assert "1 2" = load #{ EFBBBF2231203222 }
802
+ --assert "1 2" = load #{ 2231203222 }
803
+ ;@@ https://github.com/Oldes/Rebol-issues/issues/474
804
+ write %temp #{ EFBBBF612062 }
805
+ --assert [a b] = load %temp
806
+ delete %temp
807
+
789
808
===end-group===
790
809
791
810
===start-group=== "ICONV"
You can’t perform that action at this time.
0 commit comments