clean up car handling, IPFS integration, import manager, retrieve RPC.

api/api_full.go

@@ -12,13 +12,14 @@ import (
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-bitfield"
 	datatransfer "github.com/filecoin-project/go-data-transfer"
-	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
-	"github.com/filecoin-project/go-fil-markets/storagemarket"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/go-state-types/crypto"
 	"github.com/filecoin-project/go-state-types/dline"
 
+	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
+	"github.com/filecoin-project/go-fil-markets/storagemarket"
+
 	apitypes "github.com/filecoin-project/lotus/api/types"
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/market"
@@ -730,10 +731,21 @@ type Import struct {
 	Key imports.ID
 	Err string
 
-	Root          *cid.Cid
-	Source        string
-	FilePath      string
-	CARv2FilePath string
+	Root *cid.Cid
+
+	// Source is the provenance of the import, e.g. "import", "unknown", else.
+	// Currently useless but may be used in the future.
+	Source string
+
+	// FilePath is the path of the original file. It is important that the file
+	// is retained at this path, because it will be referenced during
+	// the transfer (when we do the UnixFS chunking, we don't duplicate the
+	// leaves, but rather point to chunks of the original data through
+	// positional references).
+	FilePath string
+
+	// CARPath is the path of the CAR file containing the DAG for this import.
+	CARPath string
 }
 
 type DealInfo struct {
@@ -916,7 +928,7 @@ type RetrievalOrder struct {
 	Piece *cid.Cid
 	Size  uint64
 
-	LocalCARV2FilePath string // if specified, get data from a local CARv2 file.
+	FromLocalCAR string // if specified, get data from a local CARv2 file.
 	// TODO: support offset
 	Total                   types.BigInt
 	UnsealPrice             types.BigInt

cli/client.go

@@ -24,19 +24,21 @@ import (
 	"github.com/docker/go-units"
 	"github.com/fatih/color"
 	datatransfer "github.com/filecoin-project/go-data-transfer"
-	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-cidutil/cidenc"
 	"github.com/libp2p/go-libp2p-core/peer"
 	"github.com/multiformats/go-multibase"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/xerrors"
 
+	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
+
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-fil-markets/storagemarket"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
 
+	"github.com/filecoin-project/go-fil-markets/storagemarket"
+
 	"github.com/filecoin-project/lotus/api"
 	lapi "github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/api/v0api"
@@ -1104,8 +1106,8 @@ var clientRetrieveCmd = &cli.Command{
 			for _, i := range imports {
 				if i.Root != nil && i.Root.Equals(file) {
 					order = &lapi.RetrievalOrder{
-						Root:               file,
-						LocalCARV2FilePath: i.CARv2FilePath,
+						Root:         file,
+						FromLocalCAR: i.CARPath,
 
 						Total:       big.Zero(),
 						UnsealPrice: big.Zero(),

documentation/en/api-v0-methods.md

@@ -1467,7 +1467,7 @@ Inputs:
     },
     "Piece": null,
     "Size": 42,
-    "LocalCARV2FilePath": "string value",
+    "FromLocalCAR": "string value",
     "Total": "0",
     "UnsealPrice": "0",
     "PaymentInterval": 42,
@@ -1521,7 +1521,7 @@ Inputs:
     },
     "Piece": null,
     "Size": 42,
-    "LocalCARV2FilePath": "string value",
+    "FromLocalCAR": "string value",
     "Total": "0",
     "UnsealPrice": "0",
     "PaymentInterval": 42,

documentation/en/api-v1-unstable-methods.md

@@ -1497,7 +1497,7 @@ Inputs:
     },
     "Piece": null,
     "Size": 42,
-    "LocalCARV2FilePath": "string value",
+    "FromLocalCAR": "string value",
     "Total": "0",
     "UnsealPrice": "0",
     "PaymentInterval": 42,
@@ -1551,7 +1551,7 @@ Inputs:
     },
     "Piece": null,
     "Size": 42,
-    "LocalCARV2FilePath": "string value",
+    "FromLocalCAR": "string value",
     "Total": "0",
     "UnsealPrice": "0",
     "PaymentInterval": 42,

go.mod

@@ -36,7 +36,7 @@ require (
 	github.com/filecoin-project/go-data-transfer v1.7.3
 	github.com/filecoin-project/go-fil-commcid v0.1.0
 	github.com/filecoin-project/go-fil-commp-hashhash v0.1.0
-	github.com/filecoin-project/go-fil-markets v1.6.3-0.20210811182620-1237843e237c
+	github.com/filecoin-project/go-fil-markets v1.6.3-0.20210816133929-00251d2c79b8
 	github.com/filecoin-project/go-jsonrpc v0.1.4-0.20210217175800-45ea43ac2bec
 	github.com/filecoin-project/go-multistore v0.0.3
 	github.com/filecoin-project/go-padreader v0.0.0-20210723183308-812a16dc01b1
@@ -99,7 +99,7 @@ require (
 	github.com/ipfs/go-unixfs v0.2.6
 	github.com/ipfs/interface-go-ipfs-core v0.2.3
 	github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d
-	github.com/ipld/go-car/v2 v2.0.2
+	github.com/ipld/go-car/v2 v2.0.3-0.20210811121346-c514a30114d7
 	github.com/ipld/go-ipld-prime v0.5.1-0.20201021195245-109253e8a018
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/lib/pq v1.7.0
@@ -129,6 +129,7 @@ require (
 	github.com/multiformats/go-multiaddr-dns v0.3.1
 	github.com/multiformats/go-multibase v0.0.3
 	github.com/multiformats/go-multihash v0.0.15
+	github.com/multiformats/go-varint v0.0.6
 	github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a

go.sum

@@ -289,8 +289,8 @@ github.com/filecoin-project/go-fil-commcid v0.1.0/go.mod h1:Eaox7Hvus1JgPrL5+M3+
 github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 h1:imrrpZWEHRnNqqv0tN7LXep5bFEVOVmQWHJvl2mgsGo=
 github.com/filecoin-project/go-fil-commp-hashhash v0.1.0/go.mod h1:73S8WSEWh9vr0fDJVnKADhfIv/d6dCbAGaAGWbdJEI8=
 github.com/filecoin-project/go-fil-markets v1.0.5-0.20201113164554-c5eba40d5335/go.mod h1:AJySOJC00JRWEZzRG2KsfUnqEf5ITXxeX09BE9N4f9c=
-github.com/filecoin-project/go-fil-markets v1.6.3-0.20210811182620-1237843e237c h1:rHmpzPGgYbr9efLWkRHahkPyjrVCMesVGB8G8vxFmEg=
-github.com/filecoin-project/go-fil-markets v1.6.3-0.20210811182620-1237843e237c/go.mod h1:D5xHWxyuU0EK8wcK4qStO5rjmpH206eb4OdrkWmTdaY=
+github.com/filecoin-project/go-fil-markets v1.6.3-0.20210816133929-00251d2c79b8 h1:p8LqLGy361wm1L5JFWBSijpQ/OG33NUsQvNkSp5gMW0=
+github.com/filecoin-project/go-fil-markets v1.6.3-0.20210816133929-00251d2c79b8/go.mod h1:D5xHWxyuU0EK8wcK4qStO5rjmpH206eb4OdrkWmTdaY=
 github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM=
 github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24=
 github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxlWi3xVvbQP0IT38fvM=
@@ -760,8 +760,9 @@ github.com/ipld/go-car v0.1.1-0.20200923150018-8cdef32e2da4/go.mod h1:xrMEcuSq+D
 github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d h1:iphSzTuPqyDgH7WUVZsdqUnQNzYgIblsVr1zhVNA33U=
 github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d/go.mod h1:2Gys8L8MJ6zkh1gktTSXreY63t4UbyvNp5JaudTyxHQ=
 github.com/ipld/go-car/v2 v2.0.0-beta1.0.20210721090610-5a9d1b217d25/go.mod h1:I2ACeeg6XNBe5pdh5TaR7Ambhfa7If9KXxmXgZsYENU=
-github.com/ipld/go-car/v2 v2.0.2 h1:R1oIAPwrGp26mEFzcGf5bfTZAAHDOkaVnZTEVebaWX4=
 github.com/ipld/go-car/v2 v2.0.2/go.mod h1:I2ACeeg6XNBe5pdh5TaR7Ambhfa7If9KXxmXgZsYENU=
+github.com/ipld/go-car/v2 v2.0.3-0.20210811121346-c514a30114d7 h1:6Z0beJSZNsRY+7udoqUl4gQ/tqtrPuRvDySrlsvbqZA=
+github.com/ipld/go-car/v2 v2.0.3-0.20210811121346-c514a30114d7/go.mod h1:I2ACeeg6XNBe5pdh5TaR7Ambhfa7If9KXxmXgZsYENU=
 github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785/go.mod h1:bDDSvVz7vaK12FNvMeRYnpRFkSUPNQOiCYQezMD/P3w=
 github.com/ipld/go-ipld-prime v0.0.2-0.20200428162820-8b59dc292b8e/go.mod h1:uVIwe/u0H4VdKv3kaN1ck7uCb6yD9cFLS9/ELyXbsw8=
 github.com/ipld/go-ipld-prime v0.5.1-0.20200828233916-988837377a7f/go.mod h1:0xEgdD6MKbZ1vF0GC+YcR/C4SQCAlRuOjIJ2i0HxqzM=

markets/retrievaladapter/client_blockstore.go

@@ -0,0 +1,83 @@
+package retrievaladapter
+
+import (
+	"fmt"
+	"path/filepath"
+	"sync"
+
+	"github.com/ipfs/go-cid"
+	bstore "github.com/ipfs/go-ipfs-blockstore"
+	"github.com/ipld/go-car/v2/blockstore"
+
+	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
+)
+
+// ProxyBlockstoreAccessor is an accessor that returns a fixed blockstore.
+// To be used in combination with IPFS integration.
+type ProxyBlockstoreAccessor struct {
+	Blockstore bstore.Blockstore
+}
+
+var _ retrievalmarket.BlockstoreAccessor = (*ProxyBlockstoreAccessor)(nil)
+
+func NewFixedBlockstoreAccessor(bs bstore.Blockstore) retrievalmarket.BlockstoreAccessor {
+	return &ProxyBlockstoreAccessor{Blockstore: bs}
+}
+
+func (p *ProxyBlockstoreAccessor) Get(_ retrievalmarket.DealID, _ retrievalmarket.PayloadCID) (bstore.Blockstore, error) {
+	return p.Blockstore, nil
+}
+
+func (p *ProxyBlockstoreAccessor) Done(_ retrievalmarket.DealID) error {
+	return nil
+}
+
+type CARBlockstoreAccessor struct {
+	rootdir string
+	lk      sync.Mutex
+	open    map[retrievalmarket.DealID]*blockstore.ReadWrite
+}
+
+var _ retrievalmarket.BlockstoreAccessor = (*CARBlockstoreAccessor)(nil)
+
+func NewCARBlockstoreAccessor(rootdir string) *CARBlockstoreAccessor {
+	return &CARBlockstoreAccessor{
+		rootdir: rootdir,
+		open:    make(map[retrievalmarket.DealID]*blockstore.ReadWrite),
+	}
+}
+
+func (c *CARBlockstoreAccessor) Get(id retrievalmarket.DealID, payloadCid retrievalmarket.PayloadCID) (bstore.Blockstore, error) {
+	c.lk.Lock()
+	defer c.lk.Unlock()
+
+	bs, ok := c.open[id]
+	if ok {
+		return bs, nil
+	}
+
+	path := c.PathFor(id)
+	bs, err := blockstore.OpenReadWrite(path, []cid.Cid{payloadCid}, blockstore.UseWholeCIDs(true))
+	if err != nil {
+		return nil, err
+	}
+	c.open[id] = bs
+	return bs, nil
+}
+
+func (c *CARBlockstoreAccessor) Done(id retrievalmarket.DealID) error {
+	c.lk.Lock()
+	defer c.lk.Unlock()
+
+	bs, ok := c.open[id]
+	if !ok {
+		return nil
+	}
+
+	delete(c.open, id)
+	return bs.Finalize()
+}
+
+func (c *CARBlockstoreAccessor) PathFor(id retrievalmarket.DealID) string {
+	return filepath.Join(c.rootdir, fmt.Sprintf("%d.car", id))
+}

markets/storageadapter/client_blockstore.go

@@ -0,0 +1,101 @@
+package storageadapter
+
+import (
+	"sync"
+
+	"github.com/ipfs/go-cid"
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-fil-markets/storagemarket"
+	"github.com/filecoin-project/go-fil-markets/stores"
+	"github.com/filecoin-project/lotus/node/repo/imports"
+)
+
+// ProxyBlockstoreAccessor is an accessor that returns a fixed blockstore.
+// To be used in combination with IPFS integration.
+type ProxyBlockstoreAccessor struct {
+	Blockstore blockstore.Blockstore
+}
+
+var _ storagemarket.BlockstoreAccessor = (*ProxyBlockstoreAccessor)(nil)
+
+func NewFixedBlockstoreAccessor(bs blockstore.Blockstore) storagemarket.BlockstoreAccessor {
+	return &ProxyBlockstoreAccessor{Blockstore: bs}
+}
+
+func (p *ProxyBlockstoreAccessor) Get(cid storagemarket.PayloadCID) (blockstore.Blockstore, error) {
+	return p.Blockstore, nil
+}
+
+func (p *ProxyBlockstoreAccessor) Done(cid storagemarket.PayloadCID) error {
+	return nil
+}
+
+// ImportsBlockstoreAccessor is a blockstore accessor backed by the
+// imports.Manager.
+type ImportsBlockstoreAccessor struct {
+	m    *imports.Manager
+	lk   sync.Mutex
+	open map[cid.Cid]struct {
+		st   stores.ClosableBlockstore
+		refs int
+	}
+}
+
+var _ storagemarket.BlockstoreAccessor = (*ImportsBlockstoreAccessor)(nil)
+
+func NewImportsBlockstoreAccessor(importmgr *imports.Manager) *ImportsBlockstoreAccessor {
+	return &ImportsBlockstoreAccessor{
+		m: importmgr,
+		open: make(map[cid.Cid]struct {
+			st   stores.ClosableBlockstore
+			refs int
+		}),
+	}
+}
+
+func (s *ImportsBlockstoreAccessor) Get(payloadCID storagemarket.PayloadCID) (blockstore.Blockstore, error) {
+	s.lk.Lock()
+	defer s.lk.Unlock()
+
+	e, ok := s.open[payloadCID]
+	if ok {
+		e.refs++
+		return e.st, nil
+	}
+
+	path, err := s.m.CARPathFor(payloadCID)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to get client blockstore for root %s: %w", payloadCID, err)
+	}
+	if path == "" {
+		return nil, xerrors.Errorf("no client blockstore for root %s", payloadCID)
+	}
+	ret, err := stores.ReadOnlyFilestore(path)
+	if err != nil {
+		return nil, err
+	}
+	e.st = ret
+	s.open[payloadCID] = e
+	return ret, nil
+}
+
+func (s *ImportsBlockstoreAccessor) Done(payloadCID storagemarket.PayloadCID) error {
+	s.lk.Lock()
+	defer s.lk.Unlock()
+
+	e, ok := s.open[payloadCID]
+	if !ok {
+		return nil
+	}
+
+	e.refs--
+	if e.refs == 0 {
+		if err := e.st.Close(); err != nil {
+			log.Warnf("failed to close blockstore: %s", err)
+		}
+		delete(s.open, payloadCID)
+	}
+	return nil
+}

node/builder.go

@@ -6,9 +6,10 @@ import (
 	"os"
 	"time"
 
-	"github.com/filecoin-project/lotus/node/impl/net"
 	metricsi "github.com/ipfs/go-metrics-interface"
 
+	"github.com/filecoin-project/lotus/node/impl/net"
+
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/system"
 
@@ -335,7 +336,6 @@ func Repo(r repo.Repo) Option {
 			Override(new(dtypes.ClientImportMgr), modules.ClientImportMgr),
 
 			Override(new(dtypes.ClientBlockstore), modules.ClientBlockstore),
-			Override(new(dtypes.ClientRetrievalStoreManager), modules.ClientBlockstoreRetrievalStoreManager(false)),
 
 			Override(new(ci.PrivKey), lp2p.PrivKey),
 			Override(new(ci.PubKey), ci.PrivKey.GetPublic),