Skip to content

Commit ca5e771

Browse files
hannahhowardhannahhoward
committed
Data Transfer V2 Integration
Voucher refactor integration (#707) * refactor(storagemarket): update storagemarket * refactor(retrievalmarket): use new voucher system refactor for predictability and correctness using new voucher system * chore(deps): update go-data-transfer v2 * style(lint): prep for pr * docs(retrievalmarket): add comments * chore(deps): update go-statemachine * style(imports): fix imports chore(retrievalmarket): remove old types (#712) feat(ipld): bindnode support for all voucher types (#713) * feat(ipld): new data-transfer ipld vouchers + bindnode * feat(ipld): simplify ipldutils API * feat(ipld): use new bindnode registry in go-ipld-prime Ref: ipld/go-ipld-prime#437 feat(deps): update data transfer 61f0756c feat(deps): update data transfer and other deps update to master data transfer with libp2p v0.22.0 plus associated other deps Update retrievalmarket/types.go Update storagemarket/types.go chore(deps): update to rc candidate chore(dtutils): update for new data transfer configuration interface
1 parent a96e387 commit ca5e771

File tree

81 files changed

+2637
-5964
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

81 files changed

+2637
-5964
lines changed

‎.circleci/config.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ orbs:
66
executors:
77
golang:
88
docker:
9-
- image: cimg/go:1.17.9
9+
- image: cimg/go:1.18.7-node
1010
resource_class: large
1111

1212
commands:

‎CONTRIBUTING.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ import (
8181
ipld "github.com/ipfs/go-ipld-format"
8282
"github.com/stretchr/testify/assert"
8383

84-
datatransfer "github.com/filecoin-project/go-data-transfer"
84+
datatransfer "github.com/filecoin-project/go-data-transfer/v2"
8585

8686
"github.com/filecoin-project/go-fil-markets/filestore/file"
8787
)

‎discovery/impl/local_test.go

-60
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,17 @@
11
package discoveryimpl_test
22

33
import (
4-
"bytes"
54
"context"
65
"testing"
76
"time"
87

98
"github.com/ipfs/go-datastore"
10-
dshelp "github.com/ipfs/go-ipfs-ds-help"
119
"github.com/libp2p/go-libp2p/core/peer"
1210
"github.com/stretchr/testify/assert"
1311
"github.com/stretchr/testify/require"
1412

15-
"github.com/filecoin-project/go-address"
16-
1713
discoveryimpl "github.com/filecoin-project/go-fil-markets/discovery/impl"
18-
"github.com/filecoin-project/go-fil-markets/discovery/migrations"
1914
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
20-
retrievalmigrations "github.com/filecoin-project/go-fil-markets/retrievalmarket/migrations"
2115
"github.com/filecoin-project/go-fil-markets/shared_testutil"
2216
)
2317

@@ -70,57 +64,3 @@ func TestLocal_AddPeer(t *testing.T) {
7064
})
7165
}
7266
}
73-
74-
func TestLocalMigrations(t *testing.T) {
75-
ctx := context.Background()
76-
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
77-
defer cancel()
78-
ds := datastore.NewMapDatastore()
79-
80-
peers := shared_testutil.GeneratePeers(4)
81-
pieceCIDs := shared_testutil.GenerateCids(4)
82-
payloadCids := shared_testutil.GenerateCids(2)
83-
for i, c := range payloadCids {
84-
rps := migrations.RetrievalPeers0{
85-
Peers: []retrievalmigrations.RetrievalPeer0{
86-
{
87-
Address: address.TestAddress,
88-
ID: peers[i*2],
89-
PieceCID: &pieceCIDs[i*2],
90-
},
91-
{
92-
Address: address.TestAddress2,
93-
ID: peers[i*2+1],
94-
PieceCID: &pieceCIDs[i*2+1],
95-
},
96-
},
97-
}
98-
buf := new(bytes.Buffer)
99-
err := rps.MarshalCBOR(buf)
100-
require.NoError(t, err)
101-
err = ds.Put(ctx, dshelp.MultihashToDsKey(c.Hash()), buf.Bytes())
102-
require.NoError(t, err)
103-
}
104-
105-
l, err := discoveryimpl.NewLocal(ds)
106-
require.NoError(t, err)
107-
shared_testutil.StartAndWaitForReady(ctx, t, l)
108-
109-
for i, c := range payloadCids {
110-
expectedPeers := []retrievalmarket.RetrievalPeer{
111-
{
112-
Address: address.TestAddress,
113-
ID: peers[i*2],
114-
PieceCID: &pieceCIDs[i*2],
115-
},
116-
{
117-
Address: address.TestAddress2,
118-
ID: peers[i*2+1],
119-
PieceCID: &pieceCIDs[i*2+1],
120-
},
121-
}
122-
peers, err := l.GetPeers(c)
123-
require.NoError(t, err)
124-
require.Equal(t, expectedPeers, peers)
125-
}
126-
}

‎discovery/migrations/migrations.go

+1-30
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,8 @@
11
package migrations
22

33
import (
4-
versioning "github.com/filecoin-project/go-ds-versioning/pkg"
54
"github.com/filecoin-project/go-ds-versioning/pkg/versioned"
6-
7-
"github.com/filecoin-project/go-fil-markets/discovery"
8-
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
9-
"github.com/filecoin-project/go-fil-markets/retrievalmarket/migrations"
105
)
116

12-
//go:generate cbor-gen-for RetrievalPeers0
13-
14-
// RetrievalPeers0 is version 0 of RetrievalPeers
15-
type RetrievalPeers0 struct {
16-
Peers []migrations.RetrievalPeer0
17-
}
18-
19-
// MigrateRetrievalPeers0To1 migrates a tuple encoded list of retrieval peers to a map encoded list
20-
func MigrateRetrievalPeers0To1(oldRps *RetrievalPeers0) (*discovery.RetrievalPeers, error) {
21-
peers := make([]retrievalmarket.RetrievalPeer, 0, len(oldRps.Peers))
22-
for _, oldRp := range oldRps.Peers {
23-
peers = append(peers, retrievalmarket.RetrievalPeer{
24-
Address: oldRp.Address,
25-
ID: oldRp.ID,
26-
PieceCID: oldRp.PieceCID,
27-
})
28-
}
29-
return &discovery.RetrievalPeers{
30-
Peers: peers,
31-
}, nil
32-
}
33-
347
// RetrievalPeersMigrations are migrations for the store local discovery list of peers we can retrieve from
35-
var RetrievalPeersMigrations = versioned.BuilderList{
36-
versioned.NewVersionedBuilder(MigrateRetrievalPeers0To1, versioning.VersionKey("1")),
37-
}
8+
var RetrievalPeersMigrations = versioned.BuilderList{}

‎discovery/migrations/migrations_cbor_gen.go

-105
This file was deleted.

‎docs/retrievalprovider.mmd

+9-5
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,18 @@ stateDiagram-v2
1616
state "DealStatusCancelled" as DealStatusCancelled
1717
DealStatusUnsealing : On entry runs UnsealData
1818
DealStatusUnsealed : On entry runs UnpauseDeal
19-
DealStatusFundsNeededUnseal : On entry runs TrackTransfer
19+
DealStatusFundsNeededUnseal : On entry runs UpdateFunding
2020
DealStatusFailing : On entry runs CancelDeal
21+
DealStatusFundsNeeded : On entry runs UpdateFunding
22+
DealStatusFundsNeededLastPayment : On entry runs UpdateFunding
2123
DealStatusCompleting : On entry runs CleanupDeal
2224
DealStatusCancelling : On entry runs CancelDeal
2325
[*] --> DealStatusNew
2426
note right of DealStatusNew
2527
The following events are not shown cause they can trigger from any state.
2628
29+
ProviderEventProcessPayment - does not transition state
30+
ProviderEventComplete - transitions state to DealStatusCompleting
2731
ProviderEventDataTransferError - transitions state to DealStatusErrored
2832
ProviderEventMultiStoreError - transitions state to DealStatusErrored
2933
ProviderEventClientCancelled - transitions state to DealStatusCancelling
@@ -35,20 +39,20 @@ stateDiagram-v2
3539
DealStatusUnsealing --> DealStatusUnsealed : ProviderEventUnsealComplete
3640
DealStatusUnsealed --> DealStatusOngoing : ProviderEventBlockSent
3741
DealStatusOngoing --> DealStatusOngoing : ProviderEventBlockSent
38-
DealStatusOngoing --> DealStatusBlocksComplete : ProviderEventBlocksCompleted
3942
DealStatusNew --> DealStatusFundsNeededUnseal : ProviderEventPaymentRequested
4043
DealStatusUnsealed --> DealStatusFundsNeeded : ProviderEventPaymentRequested
4144
DealStatusOngoing --> DealStatusFundsNeeded : ProviderEventPaymentRequested
42-
DealStatusBlocksComplete --> DealStatusFundsNeededLastPayment : ProviderEventPaymentRequested
45+
DealStatusUnsealed --> DealStatusFundsNeededLastPayment : ProviderEventLastPaymentRequested
46+
DealStatusOngoing --> DealStatusFundsNeededLastPayment : ProviderEventLastPaymentRequested
47+
DealStatusFundsNeededUnseal --> DealStatusFailing : ProviderEventSaveVoucherFailed
4348
DealStatusFundsNeeded --> DealStatusFailing : ProviderEventSaveVoucherFailed
4449
DealStatusFundsNeededLastPayment --> DealStatusFailing : ProviderEventSaveVoucherFailed
50+
DealStatusFundsNeededUnseal --> DealStatusFundsNeededUnseal : ProviderEventPartialPaymentReceived
4551
DealStatusFundsNeeded --> DealStatusFundsNeeded : ProviderEventPartialPaymentReceived
4652
DealStatusFundsNeededLastPayment --> DealStatusFundsNeededLastPayment : ProviderEventPartialPaymentReceived
4753
DealStatusFundsNeededUnseal --> DealStatusUnsealing : ProviderEventPaymentReceived
4854
DealStatusFundsNeeded --> DealStatusOngoing : ProviderEventPaymentReceived
4955
DealStatusFundsNeededLastPayment --> DealStatusFinalizing : ProviderEventPaymentReceived
50-
DealStatusBlocksComplete --> DealStatusCompleting : ProviderEventComplete
51-
DealStatusFinalizing --> DealStatusCompleting : ProviderEventComplete
5256
DealStatusCompleting --> DealStatusCompleted : ProviderEventCleanupComplete
5357
DealStatusFailing --> DealStatusErrored : ProviderEventCancelComplete
5458
DealStatusCancelling --> DealStatusCancelled : ProviderEventCancelComplete

‎docs/retrievalprovider.mmd.png

44.4 KB
Loading

‎docs/retrievalprovider.mmd.svg

+3-3
Loading

‎go.mod

+6-7
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,19 @@
11
module github.com/filecoin-project/go-fil-markets
22

3-
go 1.17
3+
go 1.18
44

55
require (
66
github.com/filecoin-project/dagstore v0.5.2
77
github.com/filecoin-project/go-address v0.0.6
88
github.com/filecoin-project/go-cbor-util v0.0.1
99
github.com/filecoin-project/go-commp-utils v0.1.3
10-
github.com/filecoin-project/go-data-transfer v1.15.2
10+
github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc2.0.20230211023954-5f99b068a7f2
1111
github.com/filecoin-project/go-ds-versioning v0.1.2
1212
github.com/filecoin-project/go-fil-commcid v0.1.0
1313
github.com/filecoin-project/go-fil-commp-hashhash v0.1.0
1414
github.com/filecoin-project/go-padreader v0.0.1
1515
github.com/filecoin-project/go-state-types v0.1.12
16-
github.com/filecoin-project/go-statemachine v1.0.2
16+
github.com/filecoin-project/go-statemachine v1.0.3
1717
github.com/filecoin-project/go-statestore v0.2.0
1818
github.com/filecoin-project/specs-actors v0.9.13
1919
github.com/hannahhoward/cbor-gen-for v0.0.0-20200817222906-ea96cece81f1
@@ -25,7 +25,7 @@ require (
2525
github.com/ipfs/go-cidutil v0.1.0
2626
github.com/ipfs/go-datastore v0.6.0
2727
github.com/ipfs/go-filestore v1.2.0
28-
github.com/ipfs/go-graphsync v0.13.2
28+
github.com/ipfs/go-graphsync v0.14.0
2929
github.com/ipfs/go-ipfs-blockstore v1.2.0
3030
github.com/ipfs/go-ipfs-blocksutil v0.0.1
3131
github.com/ipfs/go-ipfs-chunker v0.0.5
@@ -35,7 +35,7 @@ require (
3535
github.com/ipfs/go-ipld-cbor v0.0.6
3636
github.com/ipfs/go-ipld-format v0.4.0
3737
github.com/ipfs/go-log/v2 v2.5.1
38-
github.com/ipfs/go-merkledag v0.6.0
38+
github.com/ipfs/go-merkledag v0.8.1
3939
github.com/ipfs/go-unixfs v0.4.0
4040
github.com/ipld/go-car v0.4.0
4141
github.com/ipld/go-car/v2 v2.5.0
@@ -105,7 +105,6 @@ require (
105105
github.com/libp2p/go-buffer-pool v0.1.0 // indirect
106106
github.com/libp2p/go-cidranger v1.1.0 // indirect
107107
github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect
108-
github.com/libp2p/go-libp2p-core v0.20.1 // indirect
109108
github.com/libp2p/go-msgio v0.2.0 // indirect
110109
github.com/libp2p/go-nat v0.1.0 // indirect
111110
github.com/libp2p/go-netroute v0.2.0 // indirect
@@ -132,9 +131,9 @@ require (
132131
github.com/twmb/murmur3 v1.1.6 // indirect
133132
github.com/urfave/cli/v2 v2.16.3 // indirect
134133
github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
134+
github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542 // indirect
135135
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
136136
go.opentelemetry.io/otel v1.10.0 // indirect
137-
go.opentelemetry.io/otel/sdk v1.10.0 // indirect
138137
go.opentelemetry.io/otel/trace v1.10.0 // indirect
139138
go.uber.org/atomic v1.10.0 // indirect
140139
go.uber.org/multierr v1.8.0 // indirect

0 commit comments

Comments
 (0)