Skip to content

Commit 05d76d3

Browse files
hannahhowardhannahhoward
committed
Data Transfer V2 Integration
Voucher refactor integration (#707) * refactor(storagemarket): update storagemarket * refactor(retrievalmarket): use new voucher system refactor for predictability and correctness using new voucher system * chore(deps): update go-data-transfer v2 * style(lint): prep for pr * docs(retrievalmarket): add comments * chore(deps): update go-statemachine * style(imports): fix imports chore(retrievalmarket): remove old types (#712) feat(ipld): bindnode support for all voucher types (#713) * feat(ipld): new data-transfer ipld vouchers + bindnode * feat(ipld): simplify ipldutils API * feat(ipld): use new bindnode registry in go-ipld-prime Ref: ipld/go-ipld-prime#437 feat(deps): update data transfer 61f0756c feat(deps): update data transfer and other deps update to master data transfer with libp2p v0.22.0 plus associated other deps
1 parent f6bef67 commit 05d76d3

File tree

104 files changed

+2788
-5759
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

104 files changed

+2788
-5759
lines changed

‎.circleci/config.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ orbs:
66
executors:
77
golang:
88
docker:
9-
- image: cimg/go:1.17.9
9+
- image: cimg/go:1.18.7-node
1010
resource_class: large
1111

1212
commands:

‎CONTRIBUTING.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ import (
8181
ipld "github.com/ipfs/go-ipld-format"
8282
"github.com/stretchr/testify/assert"
8383

84-
datatransfer "github.com/filecoin-project/go-data-transfer"
84+
datatransfer "github.com/filecoin-project/go-data-transfer/v2"
8585

8686
"github.com/filecoin-project/go-fil-markets/filestore/file"
8787
)

‎discovery/impl/local_test.go

+1-61
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,17 @@
11
package discoveryimpl_test
22

33
import (
4-
"bytes"
54
"context"
65
"testing"
76
"time"
87

98
"github.com/ipfs/go-datastore"
10-
dshelp "github.com/ipfs/go-ipfs-ds-help"
11-
"github.com/libp2p/go-libp2p-core/peer"
9+
"github.com/libp2p/go-libp2p/core/peer"
1210
"github.com/stretchr/testify/assert"
1311
"github.com/stretchr/testify/require"
1412

15-
"github.com/filecoin-project/go-address"
16-
1713
discoveryimpl "github.com/filecoin-project/go-fil-markets/discovery/impl"
18-
"github.com/filecoin-project/go-fil-markets/discovery/migrations"
1914
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
20-
retrievalmigrations "github.com/filecoin-project/go-fil-markets/retrievalmarket/migrations"
2115
"github.com/filecoin-project/go-fil-markets/shared_testutil"
2216
)
2317

@@ -70,57 +64,3 @@ func TestLocal_AddPeer(t *testing.T) {
7064
})
7165
}
7266
}
73-
74-
func TestLocalMigrations(t *testing.T) {
75-
ctx := context.Background()
76-
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
77-
defer cancel()
78-
ds := datastore.NewMapDatastore()
79-
80-
peers := shared_testutil.GeneratePeers(4)
81-
pieceCIDs := shared_testutil.GenerateCids(4)
82-
payloadCids := shared_testutil.GenerateCids(2)
83-
for i, c := range payloadCids {
84-
rps := migrations.RetrievalPeers0{
85-
Peers: []retrievalmigrations.RetrievalPeer0{
86-
{
87-
Address: address.TestAddress,
88-
ID: peers[i*2],
89-
PieceCID: &pieceCIDs[i*2],
90-
},
91-
{
92-
Address: address.TestAddress2,
93-
ID: peers[i*2+1],
94-
PieceCID: &pieceCIDs[i*2+1],
95-
},
96-
},
97-
}
98-
buf := new(bytes.Buffer)
99-
err := rps.MarshalCBOR(buf)
100-
require.NoError(t, err)
101-
err = ds.Put(ctx, dshelp.MultihashToDsKey(c.Hash()), buf.Bytes())
102-
require.NoError(t, err)
103-
}
104-
105-
l, err := discoveryimpl.NewLocal(ds)
106-
require.NoError(t, err)
107-
shared_testutil.StartAndWaitForReady(ctx, t, l)
108-
109-
for i, c := range payloadCids {
110-
expectedPeers := []retrievalmarket.RetrievalPeer{
111-
{
112-
Address: address.TestAddress,
113-
ID: peers[i*2],
114-
PieceCID: &pieceCIDs[i*2],
115-
},
116-
{
117-
Address: address.TestAddress2,
118-
ID: peers[i*2+1],
119-
PieceCID: &pieceCIDs[i*2+1],
120-
},
121-
}
122-
peers, err := l.GetPeers(c)
123-
require.NoError(t, err)
124-
require.Equal(t, expectedPeers, peers)
125-
}
126-
}

‎discovery/migrations/migrations.go

+1-30
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,8 @@
11
package migrations
22

33
import (
4-
versioning "github.com/filecoin-project/go-ds-versioning/pkg"
54
"github.com/filecoin-project/go-ds-versioning/pkg/versioned"
6-
7-
"github.com/filecoin-project/go-fil-markets/discovery"
8-
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
9-
"github.com/filecoin-project/go-fil-markets/retrievalmarket/migrations"
105
)
116

12-
//go:generate cbor-gen-for RetrievalPeers0
13-
14-
// RetrievalPeers0 is version 0 of RetrievalPeers
15-
type RetrievalPeers0 struct {
16-
Peers []migrations.RetrievalPeer0
17-
}
18-
19-
// MigrateRetrievalPeers0To1 migrates a tuple encoded list of retrieval peers to a map encoded list
20-
func MigrateRetrievalPeers0To1(oldRps *RetrievalPeers0) (*discovery.RetrievalPeers, error) {
21-
peers := make([]retrievalmarket.RetrievalPeer, 0, len(oldRps.Peers))
22-
for _, oldRp := range oldRps.Peers {
23-
peers = append(peers, retrievalmarket.RetrievalPeer{
24-
Address: oldRp.Address,
25-
ID: oldRp.ID,
26-
PieceCID: oldRp.PieceCID,
27-
})
28-
}
29-
return &discovery.RetrievalPeers{
30-
Peers: peers,
31-
}, nil
32-
}
33-
347
// RetrievalPeersMigrations are migrations for the store local discovery list of peers we can retrieve from
35-
var RetrievalPeersMigrations = versioned.BuilderList{
36-
versioned.NewVersionedBuilder(MigrateRetrievalPeers0To1, versioning.VersionKey("1")),
37-
}
8+
var RetrievalPeersMigrations = versioned.BuilderList{}

‎discovery/migrations/migrations_cbor_gen.go

-105
This file was deleted.

‎docs/retrievalprovider.mmd

+9-5
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,18 @@ stateDiagram-v2
1616
state "DealStatusCancelled" as DealStatusCancelled
1717
DealStatusUnsealing : On entry runs UnsealData
1818
DealStatusUnsealed : On entry runs UnpauseDeal
19-
DealStatusFundsNeededUnseal : On entry runs TrackTransfer
19+
DealStatusFundsNeededUnseal : On entry runs UpdateFunding
2020
DealStatusFailing : On entry runs CancelDeal
21+
DealStatusFundsNeeded : On entry runs UpdateFunding
22+
DealStatusFundsNeededLastPayment : On entry runs UpdateFunding
2123
DealStatusCompleting : On entry runs CleanupDeal
2224
DealStatusCancelling : On entry runs CancelDeal
2325
[*] --> DealStatusNew
2426
note right of DealStatusNew
2527
The following events are not shown cause they can trigger from any state.
2628
29+
ProviderEventProcessPayment - does not transition state
30+
ProviderEventComplete - transitions state to DealStatusCompleting
2731
ProviderEventDataTransferError - transitions state to DealStatusErrored
2832
ProviderEventMultiStoreError - transitions state to DealStatusErrored
2933
ProviderEventClientCancelled - transitions state to DealStatusCancelling
@@ -35,20 +39,20 @@ stateDiagram-v2
3539
DealStatusUnsealing --> DealStatusUnsealed : ProviderEventUnsealComplete
3640
DealStatusUnsealed --> DealStatusOngoing : ProviderEventBlockSent
3741
DealStatusOngoing --> DealStatusOngoing : ProviderEventBlockSent
38-
DealStatusOngoing --> DealStatusBlocksComplete : ProviderEventBlocksCompleted
3942
DealStatusNew --> DealStatusFundsNeededUnseal : ProviderEventPaymentRequested
4043
DealStatusUnsealed --> DealStatusFundsNeeded : ProviderEventPaymentRequested
4144
DealStatusOngoing --> DealStatusFundsNeeded : ProviderEventPaymentRequested
42-
DealStatusBlocksComplete --> DealStatusFundsNeededLastPayment : ProviderEventPaymentRequested
45+
DealStatusUnsealed --> DealStatusFundsNeededLastPayment : ProviderEventLastPaymentRequested
46+
DealStatusOngoing --> DealStatusFundsNeededLastPayment : ProviderEventLastPaymentRequested
47+
DealStatusFundsNeededUnseal --> DealStatusFailing : ProviderEventSaveVoucherFailed
4348
DealStatusFundsNeeded --> DealStatusFailing : ProviderEventSaveVoucherFailed
4449
DealStatusFundsNeededLastPayment --> DealStatusFailing : ProviderEventSaveVoucherFailed
50+
DealStatusFundsNeededUnseal --> DealStatusFundsNeededUnseal : ProviderEventPartialPaymentReceived
4551
DealStatusFundsNeeded --> DealStatusFundsNeeded : ProviderEventPartialPaymentReceived
4652
DealStatusFundsNeededLastPayment --> DealStatusFundsNeededLastPayment : ProviderEventPartialPaymentReceived
4753
DealStatusFundsNeededUnseal --> DealStatusUnsealing : ProviderEventPaymentReceived
4854
DealStatusFundsNeeded --> DealStatusOngoing : ProviderEventPaymentReceived
4955
DealStatusFundsNeededLastPayment --> DealStatusFinalizing : ProviderEventPaymentReceived
50-
DealStatusBlocksComplete --> DealStatusCompleting : ProviderEventComplete
51-
DealStatusFinalizing --> DealStatusCompleting : ProviderEventComplete
5256
DealStatusCompleting --> DealStatusCompleted : ProviderEventCleanupComplete
5357
DealStatusFailing --> DealStatusErrored : ProviderEventCancelComplete
5458
DealStatusCancelling --> DealStatusCancelled : ProviderEventCancelComplete

‎docs/retrievalprovider.mmd.png

44.4 KB
Loading

‎docs/retrievalprovider.mmd.svg

+3-3
Loading

‎extern/filecoin-ffi

Submodule filecoin-ffi updated 72 files

0 commit comments

Comments
 (0)