From 0c8e4d90ca8ba84a45b805507b03e34f7425ee71 Mon Sep 17 00:00:00 2001 From: Andrew Sisley Date: Wed, 21 Aug 2024 12:35:13 -0400 Subject: [PATCH 1/2] Add ns precision support to time values Was previously only second precision, which is not particularly useful for many applications (such as createdAt timestamps used to allow multiple similar documents to be created within the same second). --- client/document.go | 19 +++- client/value.go | 3 +- internal/db/merge_test.go | 3 +- tests/integration/index/create_unique_test.go | 2 +- .../create/field_kinds/date_time_test.go | 104 ++++++++++++++++++ .../update/field_kinds/date_time_test.go | 4 +- .../query/one_to_many/with_cid_doc_id_test.go | 2 +- .../simple/with_group_average_filter_test.go | 10 +- .../query/simple/with_group_test.go | 4 +- tests/integration/results.go | 2 +- 10 files changed, 134 insertions(+), 19 deletions(-) create mode 100644 tests/integration/mutation/create/field_kinds/date_time_test.go diff --git a/client/document.go b/client/document.go index 05cefd02e4..82f0a8bb36 100644 --- a/client/document.go +++ b/client/document.go @@ -27,6 +27,21 @@ import ( ccid "github.com/sourcenetwork/defradb/internal/core/cid" ) +// CborEncodingOptions returns the set of cbor encoding options to be used whenever +// encoding defra documents. +// +// It is the canonical encoding options that ensure consistent serialization of +// indeterministic datastructures, like Go Maps, plus nano-second precision for +// time values (not canon). +func CborEncodingOptions() cbor.EncOptions { + // Important: CanonicalEncOptions ensures consistent serialization of + // indeterministic datastructures, like Go Maps + + opts := cbor.CanonicalEncOptions() + opts.Time = cbor.TimeRFC3339Nano + return opts +} + // This is the main implementation starting point for accessing the internal Document API // which provides API access to the various operations available for Documents, i.e. CRUD. // @@ -659,9 +674,7 @@ func (doc *Document) Bytes() ([]byte, error) { return nil, err } - // Important: CanonicalEncOptions ensures consistent serialization of - // indeterministic datastructures, like Go Maps - em, err := cbor.CanonicalEncOptions().EncMode() + em, err := CborEncodingOptions().EncMode() if err != nil { return nil, err } diff --git a/client/value.go b/client/value.go index bc84205cd9..23fb329132 100644 --- a/client/value.go +++ b/client/value.go @@ -11,7 +11,6 @@ package client import ( - "github.com/fxamacker/cbor/v2" "github.com/sourcenetwork/immutable" ) @@ -60,7 +59,7 @@ func (val *FieldValue) SetType(t CType) { } func (val FieldValue) Bytes() ([]byte, error) { - em, err := cbor.EncOptions{Time: cbor.TimeRFC3339}.EncMode() + em, err := CborEncodingOptions().EncMode() if err != nil { return nil, err } diff --git a/internal/db/merge_test.go b/internal/db/merge_test.go index a78fd59983..55cc172634 100644 --- a/internal/db/merge_test.go +++ b/internal/db/merge_test.go @@ -15,7 +15,6 @@ import ( "testing" "time" - "github.com/fxamacker/cbor/v2" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/linking" cidlink "github.com/ipld/go-ipld-prime/linking/cid" @@ -286,7 +285,7 @@ func (d *dagBuilder) generateCompositeUpdate(lsys *linking.LinkSystem, fields ma } func encodeValue(val any) []byte { - em, err := cbor.EncOptions{Time: cbor.TimeRFC3339}.EncMode() + em, err := client.CborEncodingOptions().EncMode() if err != nil { // safe to panic here as this is a test panic(err) diff --git a/tests/integration/index/create_unique_test.go b/tests/integration/index/create_unique_test.go index 932beb5e40..9d7ffb8471 100644 --- a/tests/integration/index/create_unique_test.go +++ b/tests/integration/index/create_unique_test.go @@ -338,7 +338,7 @@ func TestUniqueQueryWithIndex_UponAddingDocWithSameDateTime_Error(t *testing.T) "birthday": "2000-07-23T03:00:00-00:00" }`, ExpectedError: db.NewErrCanNotIndexNonUniqueFields( - "bae-2000529a-8b27-539b-91e9-c35f431fb78e", + "bae-7e20b26e-5d93-572a-9724-d8f862efbe63", errors.NewKV("birthday", testUtils.MustParseTime("2000-07-23T03:00:00-00:00")), ).Error(), }, diff --git a/tests/integration/mutation/create/field_kinds/date_time_test.go b/tests/integration/mutation/create/field_kinds/date_time_test.go new file mode 100644 index 0000000000..e051aee90f --- /dev/null +++ b/tests/integration/mutation/create/field_kinds/date_time_test.go @@ -0,0 +1,104 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package field_kinds + +import ( + "testing" + "time" + + testUtils "github.com/sourcenetwork/defradb/tests/integration" +) + +func TestMutationCreateFieldKinds_WithDateTime(t *testing.T) { + test := testUtils.TestCase{ + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type User { + time: DateTime + } + `, + }, + testUtils.CreateDoc{ + DocMap: map[string]any{ + "time": "2017-07-23T03:46:56.000Z", + }, + }, + testUtils.Request{ + Request: `query { + User { + time + } + }`, + Results: map[string]any{ + "User": []map[string]any{ + { + "time": time.Date(2017, time.July, 23, 3, 46, 56, 0, time.UTC), + }, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestMutationCreateFieldKinds_WithDateTimesNanoSecondsAppart(t *testing.T) { + test := testUtils.TestCase{ + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type User { + time: DateTime + } + `, + }, + testUtils.CreateDoc{ + DocMap: map[string]any{ + "time": "2017-07-23T03:46:56.000Z", + }, + }, + testUtils.CreateDoc{ + DocMap: map[string]any{ + "time": "2017-07-23T03:46:56.000000001Z", + }, + }, + testUtils.CreateDoc{ + DocMap: map[string]any{ + "time": "2017-07-23T03:46:56.000000002Z", + }, + }, + testUtils.Request{ + Request: `query { + User { + time + } + }`, + Results: map[string]any{ + "User": []map[string]any{ + { + "time": time.Date(2017, time.July, 23, 3, 46, 56, 1, time.UTC), + }, + { + "time": time.Date(2017, time.July, 23, 3, 46, 56, 0, time.UTC), + }, + { + "time": time.Date(2017, time.July, 23, 3, 46, 56, 2, time.UTC), + }, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} diff --git a/tests/integration/mutation/update/field_kinds/date_time_test.go b/tests/integration/mutation/update/field_kinds/date_time_test.go index 1eb8a412bc..b91de56d99 100644 --- a/tests/integration/mutation/update/field_kinds/date_time_test.go +++ b/tests/integration/mutation/update/field_kinds/date_time_test.go @@ -95,11 +95,11 @@ func TestMutationUpdate_WithDateTimeField_MultipleDocs(t *testing.T) { Results: map[string]any{ "update_Users": []map[string]any{ { - "name": "Fred", + "name": "John", "created_at": testUtils.MustParseTime("2031-07-23T03:23:23Z"), }, { - "name": "John", + "name": "Fred", "created_at": testUtils.MustParseTime("2031-07-23T03:23:23Z"), }, }, diff --git a/tests/integration/query/one_to_many/with_cid_doc_id_test.go b/tests/integration/query/one_to_many/with_cid_doc_id_test.go index 91f2d5782b..becc516dbc 100644 --- a/tests/integration/query/one_to_many/with_cid_doc_id_test.go +++ b/tests/integration/query/one_to_many/with_cid_doc_id_test.go @@ -331,7 +331,7 @@ func TestQueryOneToManyWithParentUpdateAndLastCidAndDocID(t *testing.T) { testUtils.Request{ Request: `query { Book ( - cid: "bafyreigyxgn2tss7objjzen5s77w6hijpe6wmmz4z3ercpxdcrq7uwnhl4", + cid: "bafyreihylh2iftquu5vukm2myjrfbkjnpr5vonlp5s5oo22bfrhddkju6e", docID: "bae-5366ba09-54e8-5381-8169-a770aa9282ae" ) { name diff --git a/tests/integration/query/simple/with_group_average_filter_test.go b/tests/integration/query/simple/with_group_average_filter_test.go index d5c145c15f..9fd5c918d9 100644 --- a/tests/integration/query/simple/with_group_average_filter_test.go +++ b/tests/integration/query/simple/with_group_average_filter_test.go @@ -289,6 +289,11 @@ func TestQuerySimpleWithGroupByStringWithRenderedGroupWithFilterAndChildAverageW }`, Results: map[string]any{ "Users": []map[string]any{ + { + "Name": "Alice", + "_avg": float64(0), + "_group": []map[string]any{}, + }, { "Name": "John", "_avg": float64(34), @@ -298,11 +303,6 @@ func TestQuerySimpleWithGroupByStringWithRenderedGroupWithFilterAndChildAverageW }, }, }, - { - "Name": "Alice", - "_avg": float64(0), - "_group": []map[string]any{}, - }, }, }, }, diff --git a/tests/integration/query/simple/with_group_test.go b/tests/integration/query/simple/with_group_test.go index ddb141afab..d12ca3035e 100644 --- a/tests/integration/query/simple/with_group_test.go +++ b/tests/integration/query/simple/with_group_test.go @@ -155,10 +155,10 @@ func TestQuerySimpleWithGroupByDateTime(t *testing.T) { "CreatedAt": testUtils.MustParseTime("2011-07-23T03:46:56-05:00"), }, { - "CreatedAt": testUtils.MustParseTime("2013-07-23T03:46:56-05:00"), + "CreatedAt": testUtils.MustParseTime("2012-07-23T03:46:56-05:00"), }, { - "CreatedAt": testUtils.MustParseTime("2012-07-23T03:46:56-05:00"), + "CreatedAt": testUtils.MustParseTime("2013-07-23T03:46:56-05:00"), }, }, }, diff --git a/tests/integration/results.go b/tests/integration/results.go index 755608394d..e246aa5aa0 100644 --- a/tests/integration/results.go +++ b/tests/integration/results.go @@ -147,7 +147,7 @@ func areResultsEqual(expected any, actual any) bool { case []immutable.Option[string]: return areResultArraysEqual(expectedVal, actual) case time.Time: - return areResultsEqual(expectedVal.Format(time.RFC3339), actual) + return areResultsEqual(expectedVal.Format(time.RFC3339Nano), actual) default: return assert.ObjectsAreEqualValues(expected, actual) } From 836a61b526a97d6b1b2efdaf3b25c290bf637e7a Mon Sep 17 00:00:00 2001 From: Andrew Sisley Date: Wed, 21 Aug 2024 14:20:29 -0400 Subject: [PATCH 2/2] Document breaking change --- docs/data_format_changes/i2927-time-ns-precision.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 docs/data_format_changes/i2927-time-ns-precision.md diff --git a/docs/data_format_changes/i2927-time-ns-precision.md b/docs/data_format_changes/i2927-time-ns-precision.md new file mode 100644 index 0000000000..85b9e856eb --- /dev/null +++ b/docs/data_format_changes/i2927-time-ns-precision.md @@ -0,0 +1,3 @@ +# Add ns precision support to time values + +Adds nanosecond precision to DateTime values. As a result the serialization format of DateTime values has changed.