diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 3fcd9bd..1dc7266 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -15,34 +15,25 @@ env: jobs: test_cover: name: Coverage - runs-on: ubuntu-18.04 + runs-on: ubuntu-22.04 env: CGO_ENABLED: 0 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v5 with: go-version: 1.21 - - name: Restore Go modules from cache - uses: actions/cache@v2 - with: - path: /home/runner/go/pkg/mod - key: deps-${{ hashFiles('go.sum') }} - - - name: Update Go modules - run: go mod download -json - - name: Write coverage profile run: go test -v ./... -coverprofile=./coverage.txt -covermode=atomic -coverpkg=./pkg...,./cli/... - name: Upload coverage results to Codecov - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@v2 with: fail_ci_if_error: false path_to_write_report: ./coverage.txt @@ -50,29 +41,20 @@ jobs: tests: name: Go - runs-on: ubuntu-18.04 + runs-on: ubuntu-22.04 strategy: matrix: go_versions: [ '1.19', '1.20' ] fail-fast: false steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v5 with: go-version: '${{ matrix.go_versions }}' - - name: Restore Go modules from cache - uses: actions/cache@v2 - with: - path: /home/runner/go/pkg/mod - key: deps-${{ hashFiles('go.sum') }} - - - name: Update Go modules - run: go mod download -json - - name: Run tests - run: go test -v -race ./... \ No newline at end of file + run: go test -v -race ./... diff --git a/README.md b/README.md index ddf2c3d..77b2460 100644 --- a/README.md +++ b/README.md @@ -1,35 +1,18 @@ -nspcc-dev fork of go-ordered-json -=============== +.NET-compatible JSON library +============================ -There are some legacy/stupid applications[1] that you need to interoperate with, -and they for whatever reason require that the JSON you're using is ordered in -a particular way (contrary to the JSON specifications). +It's made for 100% compatibility with the JSON variation used by +[Neo blockchain](https://github.com/neo-project/). There are three problems +there: + * it's ordered (that's why it's a fork of [go-ordered-json](https://github.com/virtuald/go-ordered-json)) + * it has different conventions regarding control and "special" symbols + * it has different conventions wrt incorrect UTF-8 -Unfortunately, the golang authors are not willing to support such a broken use -case, so on [their advice](https://groups.google.com/forum/#!topic/golang-dev/zBQwhm3VfvU) -this is a fork of the golang encoding/json package, with the ordered JSON -support originating with a patch from -[Peter Waldschmidt](https://go-review.googlesource.com/c/7930/). +The primary user of this library is [NeoGo](https://github.com/nspcc-dev/neo-go/), +it has to be 100% compatible with C# implementation to correctly process +transactions, that's why we're maintaining this library and solving any +inconsistencies with .NET libraries if found. **If you can, you should avoid using this package**. However, if you can't avoid it, then you are welcome to. Provided under the MIT license, just like golang. - -Known broken applications -------------------------- - -* [1][Windows Communication Foundation Json __type ordering](https://docs.microsoft.com/en-us/dotnet/framework/wcf/feature-details/stand-alone-json-serialization#type-hint-position-in-json-objects) -* [2][NEO node](https://github.com/neo-project/neo/tree/master/src/neo/IO/Json) - -Fork motivations -------------------------- - -NEO project has its own implementation of JSON serializer which is more a JSON -dialect rather than standard-compatible implementation. Until JSON serialisation -format affects contract states we need to be byte-to-byte compatible with the -reference JSON serializer. This fork contains the following compatibility quirks: - -* JSON serializer is ordered (see https://github.com/nspcc-dev/neo-go/pull/2026) (implemented in the original [virtuald/go-ordered-json](https://github.com/virtuald/go-ordered-json) repository) -* JSON serializer escapes non-ascii characters while marshalling (see https://github.com/nspcc-dev/neo-go/pull/2174) - -More compatibility quirks may be added in the future. \ No newline at end of file diff --git a/decode.go b/decode.go index 6dd5657..dddbd40 100644 --- a/decode.go +++ b/decode.go @@ -85,14 +85,13 @@ import ( // // The JSON null value unmarshals into an interface, map, pointer, or slice // by setting that Go value to nil. Because null is often used in JSON to mean -// ``not present,'' unmarshaling a JSON null into any other Go type has no effect +// “not present,” unmarshaling a JSON null into any other Go type has no effect // on the value and produces no error. // // When unmarshaling quoted strings, invalid UTF-8 or // invalid UTF-16 surrogate pairs are not treated as an error. // Instead, they are replaced by the Unicode replacement // character U+FFFD. -// func Unmarshal(data []byte, v interface{}) error { // Check for well-formedness. // Avoids filling out half a data structure @@ -274,8 +273,8 @@ type decodeState struct { Struct string Field string } - savedError error - useNumber bool + savedError error + useNumber bool useOrderedObject bool } diff --git a/decode_test.go b/decode_test.go index c380fc0..1e2430b 100644 --- a/decode_test.go +++ b/decode_test.go @@ -372,12 +372,12 @@ func (b *intWithPtrMarshalText) UnmarshalText(data []byte) error { } type unmarshalTest struct { - in string - ptr interface{} - out interface{} - err error - useNumber bool - golden bool + in string + ptr interface{} + out interface{} + err error + useNumber bool + golden bool useOrderedObject bool } @@ -858,12 +858,12 @@ func TestMarshal(t *testing.T) { var badUTF8 = []struct { in, out string }{ - {"hello\xffworld", `"hello\ufffdworld"`}, + {"hello\xffworld", `"hello\u00FFworld"`}, {"", `""`}, - {"\xff", `"\ufffd"`}, - {"\xff\xff", `"\ufffd\ufffd"`}, - {"a\xffb", `"a\ufffdb"`}, - {"\xe6\x97\xa5\xe6\x9c\xac\xff\xaa\x9e", `"\u65E5\u672C\ufffd\ufffd\ufffd"`}, + {"\xff", `"\u00FF"`}, + {"\xff\xff", `"\u00FF\u00FF"`}, + {"a\xffb", `"a\u00FFb"`}, + {"\xe6\x97\xa5\xe6\x9c\xac\xff\xaa\x9e", `"\u65E5\u672C\u00FF\u00AA\u009E"`}, } func TestMarshalBadUTF8(t *testing.T) { diff --git a/encode.go b/encode.go index 0427122..2271798 100644 --- a/encode.go +++ b/encode.go @@ -81,31 +81,31 @@ import ( // // Examples of struct field tags and their meanings: // -// // Field appears in JSON as key "myName". -// Field int `json:"myName"` +// // Field appears in JSON as key "myName". +// Field int `json:"myName"` // -// // Field appears in JSON as key "myName" and -// // the field is omitted from the object if its value is empty, -// // as defined above. -// Field int `json:"myName,omitempty"` +// // Field appears in JSON as key "myName" and +// // the field is omitted from the object if its value is empty, +// // as defined above. +// Field int `json:"myName,omitempty"` // -// // Field appears in JSON as key "Field" (the default), but -// // the field is skipped if empty. -// // Note the leading comma. -// Field int `json:",omitempty"` +// // Field appears in JSON as key "Field" (the default), but +// // the field is skipped if empty. +// // Note the leading comma. +// Field int `json:",omitempty"` // -// // Field is ignored by this package. -// Field int `json:"-"` +// // Field is ignored by this package. +// Field int `json:"-"` // -// // Field appears in JSON as key "-". -// Field int `json:"-,"` +// // Field appears in JSON as key "-". +// Field int `json:"-,"` // // The "string" option signals that a field is stored as JSON inside a // JSON-encoded string. It applies only to fields of string, floating point, // integer, or boolean types. This extra level of encoding is sometimes used // when communicating with JavaScript programs: // -// Int64String int64 `json:",string"` +// Int64String int64 `json:",string"` // // The key name will be used if it's a non-empty string consisting of // only Unicode letters, digits, and ASCII punctuation except quotation @@ -161,7 +161,6 @@ import ( // JSON cannot represent cyclic data structures and Marshal does not // handle them. Passing cyclic structures to Marshal will result in // an infinite recursion. -// func Marshal(v interface{}) ([]byte, error) { e := &encodeState{} err := e.marshal(v, encOpts{escapeHTML: true}) @@ -917,9 +916,15 @@ func (e *encodeState) string(s string, escapeHTML bool) int { case '\\': e.WriteByte('\\') e.WriteByte(b) + case 0x08: + e.WriteByte('\\') + e.WriteByte('b') case '\n': e.WriteByte('\\') e.WriteByte('n') + case 0x0c: + e.WriteByte('\\') + e.WriteByte('f') case '\r': e.WriteByte('\\') e.WriteByte('r') @@ -945,7 +950,9 @@ func (e *encodeState) string(s string, escapeHTML bool) int { if start < i { e.WriteString(s[start:i]) } - e.WriteString(`\ufffd`) + e.WriteString(`\u00`) + e.WriteByte(hex[s[i]>>4]) + e.WriteByte(hex[s[i]&0xF]) i += size start = i continue @@ -1004,9 +1011,15 @@ func (e *encodeState) stringBytes(s []byte, escapeHTML bool) int { case '\\': e.WriteByte('\\') e.WriteByte(b) + case 0x08: + e.WriteByte('\\') + e.WriteByte('b') case '\n': e.WriteByte('\\') e.WriteByte('n') + case 0x0c: + e.WriteByte('\\') + e.WriteByte('f') case '\r': e.WriteByte('\\') e.WriteByte('r') @@ -1032,7 +1045,9 @@ func (e *encodeState) stringBytes(s []byte, escapeHTML bool) int { if start < i { e.Write(s[start:i]) } - e.WriteString(`\ufffd`) + e.WriteString(`\u00`) + e.WriteByte(hex[s[i]>>4]) + e.WriteByte(hex[s[i]&0xF]) i += size start = i continue diff --git a/encode_test.go b/encode_test.go index 7faf9b3..d433908 100644 --- a/encode_test.go +++ b/encode_test.go @@ -231,7 +231,6 @@ func (CText) MarshalText() ([]byte, error) { return []byte(`"<&>"`), nil } - func TestMarshaler_NeoGo_PR2174(t *testing.T) { source := "IOU（欠条币）：一种支持负数的NEP-17（非严格意义上的）资产，合约无存储区，账户由区块链浏览器统计" b, err := Marshal(source) @@ -578,40 +577,16 @@ var encodeStringTests = []struct { in string out string }{ - {"\x00", `"\u0000"`}, - {"\x01", `"\u0001"`}, - {"\x02", `"\u0002"`}, - {"\x03", `"\u0003"`}, - {"\x04", `"\u0004"`}, - {"\x05", `"\u0005"`}, - {"\x06", `"\u0006"`}, - {"\x07", `"\u0007"`}, - {"\x08", `"\u0008"`}, - {"\x09", `"\t"`}, - {"\x0a", `"\n"`}, - {"\x0b", `"\u000B"`}, - {"\x0c", `"\u000C"`}, - {"\x0d", `"\r"`}, - {"\x0e", `"\u000E"`}, - {"\x0f", `"\u000F"`}, - {"\x10", `"\u0010"`}, - {"\x11", `"\u0011"`}, - {"\x12", `"\u0012"`}, - {"\x13", `"\u0013"`}, - {"\x14", `"\u0014"`}, - {"\x15", `"\u0015"`}, - {"\x16", `"\u0016"`}, - {"\x17", `"\u0017"`}, - {"\x18", `"\u0018"`}, - {"\x19", `"\u0019"`}, - {"\x1a", `"\u001A"`}, - {"\x1b", `"\u001B"`}, - {"\x1c", `"\u001C"`}, - {"\x1d", `"\u001D"`}, - {"\x1e", `"\u001E"`}, - {"\x1f", `"\u001F"`}, - {"'", `"\u0027"`}, - {"\"", `"\u0022"`}, + {"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", `"\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000B\f\r\u000E\u000F\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017\u0018\u0019\u001A\u001B\u001C\u001D\u001E\u001F"`}, + {"\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", `" !\u0022#$%\u0026\u0027()*\u002B,-./0123456789:;\u003C=\u003E?"`}, + {"\x40\x41\x44\x45\x44\x45\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51\x54\x55\x54\x55\x56\x57\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f", `"@ADEDEFGHIJKLMNOPQTUTUVWXYZ[\\]^_"`}, + {"\x60\x61\x66\x67\x64\x65\x66\x67\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f\x70\x71\x76\x77\x74\x75\x76\x77\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f", `"\u0060afgdefghijklmnopqvwtuvwxyz{|}~\u007F"`}, + {"\x80\x81\x88\x89\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x98\x99\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f", `"\u0080\u0081\u0088\u0089\u0084\u0085\u0086\u0087\u0088\u0089\u008A\u008B\u008C\u008D\u008E\u008F\u0090\u0091\u0098\u0099\u0094\u0095\u0096\u0097\u0098\u0099\u009A\u009B\u009C\u009D\u009E\u009F"`}, + {"\xa0\xa1\xaa\xab\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xba\xbb\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf", `"\u00A0\u00A1\u00AA\u00AB\u00A4\u00A5\u00A6\u00A7\u00A8\u00A9\u00AA\u00AB\u00AC\u00AD\u00AE\u00AF\u00B0\u00B1\u00BA\u00BB\u00B4\u00B5\u00B6\u00B7\u00B8\u00B9\u00BA\u00BB\u00BC\u00BD\u00BE\u00BF"`}, + {"\xc0\xc1\xcc\xcd\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xdc\xdd\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf", `"\u00C0\u00C1\u00CC\u00CD\u00C4\u00C5\u00C6\u00C7\u00C8\u00C9\u00CA\u00CB\u00CC\u00CD\u00CE\u00CF\u00D0\u00D1\u00DC\u00DD\u00D4\u00D5\u00D6\u00D7\u00D8\u00D9\u00DA\u00DB\u00DC\u00DD\u00DE\u00DF"`}, + {"\xe0\xe1\xee\xef\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xfe\xff\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff", `"\u00E0\u00E1\u00EE\u00EF\u00E4\u00E5\u00E6\u00E7\u00E8\u00E9\u00EA\u00EB\u00EC\u00ED\u00EE\u00EF\u00F0\u00F1\u00FE\u00FF\u00F4\u00F5\u00F6\u00F7\u00F8\u00F9\u00FA\u00FB\u00FC\u00FD\u00FE\u00FF"`}, + {"\xff\xd0", `"\u00FF\u00D0"`}, + {"测试", `"\u6D4B\u8BD5"`}, } func TestEncodeString(t *testing.T) { diff --git a/example_test.go b/example_test.go index 611fe3c..1baf3d8 100644 --- a/example_test.go +++ b/example_test.go @@ -6,8 +6,8 @@ package json_test import ( "bytes" - "github.com/virtuald/go-ordered-json" "fmt" + "github.com/virtuald/go-ordered-json" "io" "log" "os" @@ -312,4 +312,4 @@ func ExampleOrderedObject() { // name=André-Marie Ampère born=1777 died=1836 // Encoded: // {"name":"Hans Christian Ørsted","born":1777,"died":1851,"nationality":"Danish"} -} \ No newline at end of file +} diff --git a/fold.go b/fold.go index 9e17012..ab249b2 100644 --- a/fold.go +++ b/fold.go @@ -24,8 +24,9 @@ const ( // 4) simpleLetterEqualFold, no specials, no non-letters. // // The letters S and K are special because they map to 3 runes, not just 2: -// * S maps to s and to U+017F 'ſ' Latin small letter long s -// * k maps to K and to U+212A 'K' Kelvin sign +// - S maps to s and to U+017F 'ſ' Latin small letter long s +// - k maps to K and to U+212A 'K' Kelvin sign +// // See https://play.golang.org/p/tTxjOc0OGo // // The returned function is specialized for matching against s and diff --git a/stream.go b/stream.go index 744dcd5..041f911 100644 --- a/stream.go +++ b/stream.go @@ -282,41 +282,41 @@ type Member struct { // OrderedObject is used to enable decoding of arbitrary JSON objects while preserving // the order of the keys. Unmarshal and Decoder.Decode are supported. // -// var o OrderedObject -// Unmarshal(json, &o) // decode JSON object, while preserving key order +// var o OrderedObject +// Unmarshal(json, &o) // decode JSON object, while preserving key order // -// var oa []OrderedObject -// Unmarshal(json, &oa) // decode an array of JSON objects, while preserving key order +// var oa []OrderedObject +// Unmarshal(json, &oa) // decode an array of JSON objects, while preserving key order // -// var v interface{} -// d := new Decoder(json) -// d.UseOrderedObject() // decode all JSON objects as OrderedObject rather than map[string]interface{} -// d.Decode(&v) +// var v interface{} +// d := new Decoder(json) +// d.UseOrderedObject() // decode all JSON objects as OrderedObject rather than map[string]interface{} +// d.Decode(&v) // -// type A struct { -// B bool -// Inner OrderedObject -// I int -// } -// var a A -// Unmarshal(&a) // decode A as a JSON object with Inner as a nested object, preserving key order +// type A struct { +// B bool +// Inner OrderedObject +// I int +// } +// var a A +// Unmarshal(&a) // decode A as a JSON object with Inner as a nested object, preserving key order // // OrderedObject can also be used to encode a JSON object in // a specified order. Marshal and Encoder.Encode are supported. // -// var o OrderedObject -// Marshal(o) // encode JSON object, each with keys in OrderedObject order +// var o OrderedObject +// Marshal(o) // encode JSON object, each with keys in OrderedObject order // -// var oa []OrderedObject -// Marshal(oa) // encode an array of JSON objects, with keys in OrderedObject order +// var oa []OrderedObject +// Marshal(oa) // encode an array of JSON objects, with keys in OrderedObject order // -// type A struct { -// B bool -// Inner OrderedObject -// I int -// } -// var a A = createA() -// Marshal(a) // encode A as a JSON object with Inner as a nested object +// type A struct { +// B bool +// Inner OrderedObject +// I int +// } +// var a A = createA() +// Marshal(a) // encode A as a JSON object with Inner as a nested object type OrderedObject []Member // A Token holds a value of one of these types: @@ -327,7 +327,6 @@ type OrderedObject []Member // Number, for JSON numbers // string, for JSON string literals // nil, for JSON null -// type Token interface{} const ( diff --git a/stream_test.go b/stream_test.go index f3b5327..657448b 100644 --- a/stream_test.go +++ b/stream_test.go @@ -100,8 +100,8 @@ func TestEncoderSetEscapeHTML(t *testing.T) { want string }{ {"c", c, `"\u003C\u0026\u003E"`, `"<&>"`}, - {"ct", ct, `"\u0022\u003C\u0026\u003E\u0022"`, `"\u0022<&>\u0022"`}, - {`"<&>"`, "<&>", `"\u003C\u0026\u003E"`, `"<&>"`}, + {"ct", ct, `"\u0022\u003C\u0026\u003E\u0022"`, `"\u0022<\u0026>\u0022"`}, + {`"<&>"`, "<&>", `"\u003C\u0026\u003E"`, `"<\u0026>"`}, } { var buf bytes.Buffer enc := NewEncoder(&buf) diff --git a/tables.go b/tables.go index fc14704..4ac200d 100644 --- a/tables.go +++ b/tables.go @@ -19,12 +19,12 @@ var safeSet = [utf8.RuneSelf]bool{ '#': true, '$': true, '%': true, - '&': true, + '&': false, '\'': false, '(': true, ')': true, '*': true, - '+': true, + '+': false, ',': true, '-': true, '.': true, @@ -77,7 +77,7 @@ var safeSet = [utf8.RuneSelf]bool{ ']': true, '^': true, '_': true, - '`': true, + '`': false, 'a': true, 'b': true, 'c': true, @@ -108,7 +108,7 @@ var safeSet = [utf8.RuneSelf]bool{ '|': true, '}': true, '~': true, - '\u007f': true, + '\u007f': false, } // htmlSafeSet holds the value true if the ASCII character with the given @@ -130,7 +130,7 @@ var htmlSafeSet = [utf8.RuneSelf]bool{ '(': true, ')': true, '*': true, - '+': true, + '+': false, ',': true, '-': true, '.': true, @@ -183,7 +183,7 @@ var htmlSafeSet = [utf8.RuneSelf]bool{ ']': true, '^': true, '_': true, - '`': true, + '`': false, 'a': true, 'b': true, 'c': true, @@ -214,5 +214,5 @@ var htmlSafeSet = [utf8.RuneSelf]bool{ '|': true, '}': true, '~': true, - '\u007f': true, + '\u007f': false, }