From b383629e640dc3bb83d80205a36bdfe54caaed54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 16 Aug 2021 00:38:38 +0100 Subject: [PATCH 01/12] clean up car handling, IPFS integration, import manager, retrieve RPC. The goal of this commit was to move the CAR handling logic from go-fil-markets to Lotus via the BlockstoreAccessor abstractions introduced recently. However, as I began to do that, I realised that the ImportMgr component had ended up quite messy. So I cleaned it up. I also noticed that we were doing **a lot of** CAR wrangling and copying. Related code was MVP quality and not production-grade. CARs were being placed in temporary locations, and copied several times, imported into blockstores, etc. All of that resulting in overhead that was not acceptable in master. So I cleaned that up by introducing Blockstore-level abstractions. The system now detects when it's running with IPFS integration, and uses the old strategy of retrieving directly into the IPFS blockstore, instead of using a CARv2 as a buffer. I also ended up refactoring the `lotus client retrieve` operation. It was quite messy and had a bunch of pathways, which are easier to follow. --- api/api_full.go | 26 +- build/openrpc/full.json.gz | Bin 25242 -> 25239 bytes cli/client.go | 10 +- documentation/en/api-v0-methods.md | 4 +- documentation/en/api-v1-unstable-methods.md | 4 +- go.mod | 5 +- go.sum | 7 +- markets/retrievaladapter/client_blockstore.go | 83 ++++ markets/storageadapter/client_blockstore.go | 101 +++++ node/builder.go | 4 +- node/builder_chain.go | 5 +- node/impl/client/car_helpers.go | 91 +++++ node/impl/client/client.go | 358 +++++++++++------- node/impl/client/client_test.go | 66 ++++ node/impl/client/import.go | 108 +++--- node/impl/client/import_test.go | 27 +- .../impl/client/testdata/duplicate_blocks.txt | 1 + node/impl/client/testdata/payload.txt | 49 +++ node/impl/client/testdata/payload2.txt | 49 +++ node/modules/client.go | 62 +-- node/modules/dtypes/storage.go | 5 +- node/modules/graphsync.go | 7 +- node/modules/{ipfsclient.go => ipfs.go} | 13 + node/repo/imports/manager.go | 119 ++++-- .../retrievalstoremgr/retrievalstoremgr.go | 60 --- 25 files changed, 927 insertions(+), 337 deletions(-) create mode 100644 markets/retrievaladapter/client_blockstore.go create mode 100644 markets/storageadapter/client_blockstore.go create mode 100644 node/impl/client/car_helpers.go create mode 100644 node/impl/client/testdata/duplicate_blocks.txt create mode 100644 node/impl/client/testdata/payload.txt create mode 100644 node/impl/client/testdata/payload2.txt rename node/modules/{ipfsclient.go => ipfs.go} (69%) delete mode 100644 node/repo/retrievalstoremgr/retrievalstoremgr.go diff --git a/api/api_full.go b/api/api_full.go index 49208f2ff52..46d1813b72d 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -12,13 +12,14 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-bitfield" datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-fil-markets/retrievalmarket" - "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/go-state-types/dline" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + "github.com/filecoin-project/go-fil-markets/storagemarket" + apitypes "github.com/filecoin-project/lotus/api/types" "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/builtin/market" @@ -730,10 +731,21 @@ type Import struct { Key imports.ID Err string - Root *cid.Cid - Source string - FilePath string - CARv2FilePath string + Root *cid.Cid + + // Source is the provenance of the import, e.g. "import", "unknown", else. + // Currently useless but may be used in the future. + Source string + + // FilePath is the path of the original file. It is important that the file + // is retained at this path, because it will be referenced during + // the transfer (when we do the UnixFS chunking, we don't duplicate the + // leaves, but rather point to chunks of the original data through + // positional references). + FilePath string + + // CARPath is the path of the CAR file containing the DAG for this import. + CARPath string } type DealInfo struct { @@ -916,7 +928,7 @@ type RetrievalOrder struct { Piece *cid.Cid Size uint64 - LocalCARV2FilePath string // if specified, get data from a local CARv2 file. + FromLocalCAR string // if specified, get data from a local CARv2 file. // TODO: support offset Total types.BigInt UnsealPrice types.BigInt diff --git a/build/openrpc/full.json.gz b/build/openrpc/full.json.gz index ff642de6d73c427db605686b02841ec9d0b9f4e8..aa88a71e3ad44fd0c01698fed3172d26848d13d5 100644 GIT binary patch delta 1663 zcmV-_27vjR#Q~Sa0k9<(e~L4Qe&rozv7#y~{e~9nR5dPX7&x^`#^c@CyS4XO#R86_ z8m_XplSZnxcU;2U6_6kXM3kRT~kWe4ei6=cR-WjSJ);d~2$HSS!E-)`qO2^E9mhrexG=XDV)2fBOiAD`|FuQ?*j; zV%@B4bDMup%AqG%YjYYNrb1@Jl+?kxSGA!CWc;3*s77YEF4szxGnsaJ+|t2+(s>4D zPJ>k3wkA>Aa=q6ez?DL`6$fdW@gPrN1O;I#K3hH-iUVe%medQo4U3Zd3T7_b`bPSC z(h1pYdK>DUTbr%He@2(BgrN21q^8$P9TAQDr!J8M^PJ>uOl>V8sxF(2)21RNtCUN^>3KFc! z@Pr^2B;x?%fX;IFDkNy_4y!(hL{>%6x^WP@^rlAD`Ya;7nv!Pmwi59lot~zj3F(+m zneh?c6`$pSz!jb$#^KBtC3d!Vqz?ra)aaj==pBI{M0Bq6FbSItS?pCw#}8c!rpmXo455s@T7oukhry^rF+TTTGDS+%t~QbDeK5jT+qH>Yf3;Ib+ni%j^(vMZ`;2@JTokC> zJ8{H-r5sMR-4$DlDK$}Vay!1l zoO=?`nT*gPV+!Og%JQhWLrbsvI0aMh!?68E#5dKpvQu1cJhmoaYXUAhc{Kk-Kf`id zo1y9{e}_tWa#&@jHi}Usldpn>Rx?fs3&&&Z$V_-4aof5b8?QPD~1ak#t z0hAsrGCi_1uE4I3-tv=WD57PG*9VZUnGly=7>%{P?8h&?15#XO zC~G|2Y`Bu4w%jyYge)RG_h2dr4VdQIkj;ECQHLlXe>_ zFJ6#0tPi1c6Z*}8N~&@dLRp?aJk`j`mdh+bHNYd?sUv?dK&XqrNIjz^A>E}Tlg}F# z60E>O23&|{l!)aUgoKEt9yzI#_Zu;PRlcu7xj=qRL~kTTH|mpiK!NkjT40~5%TryL*w zk(0G->dYKVd8bUQ-8?G#xo64N0jV zv;}ll>#qgkW`>%{ delta 1685 zcmV;G25R}2#Q~bd0k9<(f7^jrA{8eO{mPrnwUvHDi*+g-moyBVT2dKI2cvT_ab*!DSi_JVO5zzNa`ST5DC19r0h+TSLqiTH?kzP$nv#48%_>ay|)6ax- z%qPzH2=9u|@<8AU&k*Bq=8FpZ_!E@{L)QS$uNCiZp$FwTDrMz0?OtH zTz17TLqH&Z=!<@86rG8kWK!0bFm2mrOj}{^ysgdNdLejHz&l9aTGGp)Mx$#2cwsk=^XxY$x#Ten#bk!~R zsUOpF>&kL+%XB@)q30nt?Z!+=oSMT|3>P@crRsM{&7jcPmf4PR9gJ&;dU$kZCb8+1c59rbqFWag~ z|GvBwD7{%^!enV^fn6WNups`zfZCPGglSLaW0w_Ziq$O_1p|Bv zxel_xLk3)kW|WBK8-#?2r5-t~lk^)ge`mh0gS$X}O+;@p_IyI+e2NHgpy!DmGvmZe zQlL+pN06zyodG198~kVN?7fPsxbGo*$2gurj!qCdg3Y&gp@m8AeTF?6-h(-Cj%46lcXFV z0hE))93UL&g0+UU-}^W5w*PZ^ICaQ>P0@e<>tDTouQZL50Ua*^H$Cgn!#Ee4c41;?QI8+phb_$9BVdWD;-@PlaL)k38)7f3R2!w+^Cb=9Uv6H z(=tl3KqO>JZx)DuriLxmhd6MP4;~jSp+YQ{u~^1pnN7wr78jV~f(Z9L3^cf*@ZfF5 zrPZu_ix|pk0717>pJ=))`jb;0WE>g$VFiAuRVp`?^(8bH*|imTRuR4zNw>FMamAC< z9(hHOD<5Te!~@qogdP;1ys1u*Z=gvW{W1$!#u^2l8QeC=Q&3~-iemLQ(F95cxLgS+ z&;(tVsVl;OQ*~S>ivZ|w+Fb;xlaU`40;vd-s2?vI=YlgD5)SA%V*XReWgrPB3<5;B zw*Wy@Daxjk@*l4P1VodgAc+w8Yj@BWoe@kX=k${6eq`{IJRu+foReK4N&@prlc^yo f0iTo8A!q@#lRP4H0kN~OBHlj;LY%he_3H!x*DW0r diff --git a/cli/client.go b/cli/client.go index 1829c588ce0..549589d64b1 100644 --- a/cli/client.go +++ b/cli/client.go @@ -24,7 +24,6 @@ import ( "github.com/docker/go-units" "github.com/fatih/color" datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/ipfs/go-cid" "github.com/ipfs/go-cidutil/cidenc" "github.com/libp2p/go-libp2p-core/peer" @@ -32,11 +31,14 @@ import ( "github.com/urfave/cli/v2" "golang.org/x/xerrors" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/go-fil-markets/storagemarket" + "github.com/filecoin-project/lotus/api" lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/v0api" @@ -1104,8 +1106,8 @@ var clientRetrieveCmd = &cli.Command{ for _, i := range imports { if i.Root != nil && i.Root.Equals(file) { order = &lapi.RetrievalOrder{ - Root: file, - LocalCARV2FilePath: i.CARv2FilePath, + Root: file, + FromLocalCAR: i.CARPath, Total: big.Zero(), UnsealPrice: big.Zero(), diff --git a/documentation/en/api-v0-methods.md b/documentation/en/api-v0-methods.md index 9790a28df2d..345037be475 100644 --- a/documentation/en/api-v0-methods.md +++ b/documentation/en/api-v0-methods.md @@ -1467,7 +1467,7 @@ Inputs: }, "Piece": null, "Size": 42, - "LocalCARV2FilePath": "string value", + "FromLocalCAR": "string value", "Total": "0", "UnsealPrice": "0", "PaymentInterval": 42, @@ -1521,7 +1521,7 @@ Inputs: }, "Piece": null, "Size": 42, - "LocalCARV2FilePath": "string value", + "FromLocalCAR": "string value", "Total": "0", "UnsealPrice": "0", "PaymentInterval": 42, diff --git a/documentation/en/api-v1-unstable-methods.md b/documentation/en/api-v1-unstable-methods.md index 02dfe23f442..0b420645d88 100644 --- a/documentation/en/api-v1-unstable-methods.md +++ b/documentation/en/api-v1-unstable-methods.md @@ -1497,7 +1497,7 @@ Inputs: }, "Piece": null, "Size": 42, - "LocalCARV2FilePath": "string value", + "FromLocalCAR": "string value", "Total": "0", "UnsealPrice": "0", "PaymentInterval": 42, @@ -1551,7 +1551,7 @@ Inputs: }, "Piece": null, "Size": 42, - "LocalCARV2FilePath": "string value", + "FromLocalCAR": "string value", "Total": "0", "UnsealPrice": "0", "PaymentInterval": 42, diff --git a/go.mod b/go.mod index cca6a57d34b..dc0ffd9ef07 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/filecoin-project/go-data-transfer v1.7.3 github.com/filecoin-project/go-fil-commcid v0.1.0 github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 - github.com/filecoin-project/go-fil-markets v1.6.3-0.20210811182620-1237843e237c + github.com/filecoin-project/go-fil-markets v1.6.3-0.20210815233218-8025f5c1a128 github.com/filecoin-project/go-jsonrpc v0.1.4-0.20210217175800-45ea43ac2bec github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20210723183308-812a16dc01b1 @@ -99,7 +99,7 @@ require ( github.com/ipfs/go-unixfs v0.2.6 github.com/ipfs/interface-go-ipfs-core v0.2.3 github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d - github.com/ipld/go-car/v2 v2.0.2 + github.com/ipld/go-car/v2 v2.0.3-0.20210811121346-c514a30114d7 github.com/ipld/go-ipld-prime v0.5.1-0.20201021195245-109253e8a018 github.com/kelseyhightower/envconfig v1.4.0 github.com/lib/pq v1.7.0 @@ -129,6 +129,7 @@ require ( github.com/multiformats/go-multiaddr-dns v0.3.1 github.com/multiformats/go-multibase v0.0.3 github.com/multiformats/go-multihash v0.0.15 + github.com/multiformats/go-varint v0.0.6 github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 github.com/opentracing/opentracing-go v1.2.0 github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a diff --git a/go.sum b/go.sum index 74e3cc5c163..b5ec62d7157 100644 --- a/go.sum +++ b/go.sum @@ -289,8 +289,8 @@ github.com/filecoin-project/go-fil-commcid v0.1.0/go.mod h1:Eaox7Hvus1JgPrL5+M3+ github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 h1:imrrpZWEHRnNqqv0tN7LXep5bFEVOVmQWHJvl2mgsGo= github.com/filecoin-project/go-fil-commp-hashhash v0.1.0/go.mod h1:73S8WSEWh9vr0fDJVnKADhfIv/d6dCbAGaAGWbdJEI8= github.com/filecoin-project/go-fil-markets v1.0.5-0.20201113164554-c5eba40d5335/go.mod h1:AJySOJC00JRWEZzRG2KsfUnqEf5ITXxeX09BE9N4f9c= -github.com/filecoin-project/go-fil-markets v1.6.3-0.20210811182620-1237843e237c h1:rHmpzPGgYbr9efLWkRHahkPyjrVCMesVGB8G8vxFmEg= -github.com/filecoin-project/go-fil-markets v1.6.3-0.20210811182620-1237843e237c/go.mod h1:D5xHWxyuU0EK8wcK4qStO5rjmpH206eb4OdrkWmTdaY= +github.com/filecoin-project/go-fil-markets v1.6.3-0.20210815233218-8025f5c1a128 h1:O8ms+mO9ZJGKvRxa0SI3oV7G7TQT6OMhZjO+oFuUvCA= +github.com/filecoin-project/go-fil-markets v1.6.3-0.20210815233218-8025f5c1a128/go.mod h1:D5xHWxyuU0EK8wcK4qStO5rjmpH206eb4OdrkWmTdaY= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxlWi3xVvbQP0IT38fvM= @@ -760,8 +760,9 @@ github.com/ipld/go-car v0.1.1-0.20200923150018-8cdef32e2da4/go.mod h1:xrMEcuSq+D github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d h1:iphSzTuPqyDgH7WUVZsdqUnQNzYgIblsVr1zhVNA33U= github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d/go.mod h1:2Gys8L8MJ6zkh1gktTSXreY63t4UbyvNp5JaudTyxHQ= github.com/ipld/go-car/v2 v2.0.0-beta1.0.20210721090610-5a9d1b217d25/go.mod h1:I2ACeeg6XNBe5pdh5TaR7Ambhfa7If9KXxmXgZsYENU= -github.com/ipld/go-car/v2 v2.0.2 h1:R1oIAPwrGp26mEFzcGf5bfTZAAHDOkaVnZTEVebaWX4= github.com/ipld/go-car/v2 v2.0.2/go.mod h1:I2ACeeg6XNBe5pdh5TaR7Ambhfa7If9KXxmXgZsYENU= +github.com/ipld/go-car/v2 v2.0.3-0.20210811121346-c514a30114d7 h1:6Z0beJSZNsRY+7udoqUl4gQ/tqtrPuRvDySrlsvbqZA= +github.com/ipld/go-car/v2 v2.0.3-0.20210811121346-c514a30114d7/go.mod h1:I2ACeeg6XNBe5pdh5TaR7Ambhfa7If9KXxmXgZsYENU= github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785/go.mod h1:bDDSvVz7vaK12FNvMeRYnpRFkSUPNQOiCYQezMD/P3w= github.com/ipld/go-ipld-prime v0.0.2-0.20200428162820-8b59dc292b8e/go.mod h1:uVIwe/u0H4VdKv3kaN1ck7uCb6yD9cFLS9/ELyXbsw8= github.com/ipld/go-ipld-prime v0.5.1-0.20200828233916-988837377a7f/go.mod h1:0xEgdD6MKbZ1vF0GC+YcR/C4SQCAlRuOjIJ2i0HxqzM= diff --git a/markets/retrievaladapter/client_blockstore.go b/markets/retrievaladapter/client_blockstore.go new file mode 100644 index 00000000000..84c75fdbde7 --- /dev/null +++ b/markets/retrievaladapter/client_blockstore.go @@ -0,0 +1,83 @@ +package retrievaladapter + +import ( + "fmt" + "path/filepath" + "sync" + + "github.com/ipfs/go-cid" + bstore "github.com/ipfs/go-ipfs-blockstore" + "github.com/ipld/go-car/v2/blockstore" + + "github.com/filecoin-project/go-fil-markets/retrievalmarket" +) + +// ProxyBlockstoreAccessor is an accessor that returns a fixed blockstore. +// To be used in combination with IPFS integration. +type ProxyBlockstoreAccessor struct { + Blockstore bstore.Blockstore +} + +var _ retrievalmarket.BlockstoreAccessor = (*ProxyBlockstoreAccessor)(nil) + +func NewFixedBlockstoreAccessor(bs bstore.Blockstore) retrievalmarket.BlockstoreAccessor { + return &ProxyBlockstoreAccessor{Blockstore: bs} +} + +func (p *ProxyBlockstoreAccessor) Get(_ retrievalmarket.DealID, _ retrievalmarket.PayloadCID) (bstore.Blockstore, error) { + return p.Blockstore, nil +} + +func (p *ProxyBlockstoreAccessor) Done(_ retrievalmarket.DealID) error { + return nil +} + +type CARBlockstoreAccessor struct { + rootdir string + lk sync.Mutex + open map[retrievalmarket.DealID]*blockstore.ReadWrite +} + +var _ retrievalmarket.BlockstoreAccessor = (*CARBlockstoreAccessor)(nil) + +func NewCARBlockstoreAccessor(rootdir string) *CARBlockstoreAccessor { + return &CARBlockstoreAccessor{ + rootdir: rootdir, + open: make(map[retrievalmarket.DealID]*blockstore.ReadWrite), + } +} + +func (c *CARBlockstoreAccessor) Get(id retrievalmarket.DealID, payloadCid retrievalmarket.PayloadCID) (bstore.Blockstore, error) { + c.lk.Lock() + defer c.lk.Unlock() + + bs, ok := c.open[id] + if ok { + return bs, nil + } + + path := c.PathFor(id) + bs, err := blockstore.OpenReadWrite(path, []cid.Cid{payloadCid}, blockstore.UseWholeCIDs(true)) + if err != nil { + return nil, err + } + c.open[id] = bs + return bs, nil +} + +func (c *CARBlockstoreAccessor) Done(id retrievalmarket.DealID) error { + c.lk.Lock() + defer c.lk.Unlock() + + bs, ok := c.open[id] + if !ok { + return nil + } + + delete(c.open, id) + return bs.Finalize() +} + +func (c *CARBlockstoreAccessor) PathFor(id retrievalmarket.DealID) string { + return filepath.Join(c.rootdir, fmt.Sprintf("%d.car", id)) +} diff --git a/markets/storageadapter/client_blockstore.go b/markets/storageadapter/client_blockstore.go new file mode 100644 index 00000000000..5dc0a6aceb4 --- /dev/null +++ b/markets/storageadapter/client_blockstore.go @@ -0,0 +1,101 @@ +package storageadapter + +import ( + "sync" + + "github.com/ipfs/go-cid" + "github.com/ipfs/go-ipfs-blockstore" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-fil-markets/storagemarket" + "github.com/filecoin-project/go-fil-markets/stores" + "github.com/filecoin-project/lotus/node/repo/imports" +) + +// ProxyBlockstoreAccessor is an accessor that returns a fixed blockstore. +// To be used in combination with IPFS integration. +type ProxyBlockstoreAccessor struct { + Blockstore blockstore.Blockstore +} + +var _ storagemarket.BlockstoreAccessor = (*ProxyBlockstoreAccessor)(nil) + +func NewFixedBlockstoreAccessor(bs blockstore.Blockstore) storagemarket.BlockstoreAccessor { + return &ProxyBlockstoreAccessor{Blockstore: bs} +} + +func (p *ProxyBlockstoreAccessor) Get(cid storagemarket.PayloadCID) (blockstore.Blockstore, error) { + return p.Blockstore, nil +} + +func (p *ProxyBlockstoreAccessor) Done(cid storagemarket.PayloadCID) error { + return nil +} + +// ImportsBlockstoreAccessor is a blockstore accessor backed by the +// imports.Manager. +type ImportsBlockstoreAccessor struct { + m *imports.Manager + lk sync.Mutex + open map[cid.Cid]struct { + st stores.ClosableBlockstore + refs int + } +} + +var _ storagemarket.BlockstoreAccessor = (*ImportsBlockstoreAccessor)(nil) + +func NewImportsBlockstoreAccessor(importmgr *imports.Manager) *ImportsBlockstoreAccessor { + return &ImportsBlockstoreAccessor{ + m: importmgr, + open: make(map[cid.Cid]struct { + st stores.ClosableBlockstore + refs int + }), + } +} + +func (s *ImportsBlockstoreAccessor) Get(payloadCID storagemarket.PayloadCID) (blockstore.Blockstore, error) { + s.lk.Lock() + defer s.lk.Unlock() + + e, ok := s.open[payloadCID] + if ok { + e.refs++ + return e.st, nil + } + + path, err := s.m.CARPathFor(payloadCID) + if err != nil { + return nil, xerrors.Errorf("failed to get client blockstore for root %s: %w", payloadCID, err) + } + if path == "" { + return nil, xerrors.Errorf("no client blockstore for root %s", payloadCID) + } + ret, err := stores.ReadOnlyFilestore(path) + if err != nil { + return nil, err + } + e.st = ret + s.open[payloadCID] = e + return ret, nil +} + +func (s *ImportsBlockstoreAccessor) Done(payloadCID storagemarket.PayloadCID) error { + s.lk.Lock() + defer s.lk.Unlock() + + e, ok := s.open[payloadCID] + if !ok { + return nil + } + + e.refs-- + if e.refs == 0 { + if err := e.st.Close(); err != nil { + log.Warnf("failed to close blockstore: %s", err) + } + delete(s.open, payloadCID) + } + return nil +} diff --git a/node/builder.go b/node/builder.go index 397df7d4902..f04678bc8c0 100644 --- a/node/builder.go +++ b/node/builder.go @@ -6,9 +6,10 @@ import ( "os" "time" - "github.com/filecoin-project/lotus/node/impl/net" metricsi "github.com/ipfs/go-metrics-interface" + "github.com/filecoin-project/lotus/node/impl/net" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/system" @@ -335,7 +336,6 @@ func Repo(r repo.Repo) Option { Override(new(dtypes.ClientImportMgr), modules.ClientImportMgr), Override(new(dtypes.ClientBlockstore), modules.ClientBlockstore), - Override(new(dtypes.ClientRetrievalStoreManager), modules.ClientBlockstoreRetrievalStoreManager(false)), Override(new(ci.PrivKey), lp2p.PrivKey), Override(new(ci.PubKey), ci.PrivKey.GetPublic), diff --git a/node/builder_chain.go b/node/builder_chain.go index 8e52eb5cc80..88f1e60c527 100644 --- a/node/builder_chain.go +++ b/node/builder_chain.go @@ -113,12 +113,14 @@ var ChainNode = Options( // Markets (retrieval) Override(new(discovery.PeerResolver), modules.RetrievalResolver), + Override(new(retrievalmarket.BlockstoreAccessor), modules.RetrievalBlockstoreAccessor), Override(new(retrievalmarket.RetrievalClient), modules.RetrievalClient), Override(new(dtypes.ClientDataTransfer), modules.NewClientGraphsyncDataTransfer), // Markets (storage) Override(new(*market.FundManager), market.NewFundManager), Override(new(dtypes.ClientDatastore), modules.NewClientDatastore), + Override(new(storagemarket.BlockstoreAccessor), modules.StorageBlockstoreAccessor), Override(new(storagemarket.StorageClient), modules.StorageClient), Override(new(storagemarket.StorageClientNode), storageadapter.NewClientNodeAdapter), Override(HandleMigrateClientFundsKey, modules.HandleMigrateClientFunds), @@ -168,8 +170,9 @@ func ConfigFullNode(c interface{}) Option { If(cfg.Client.UseIpfs, Override(new(dtypes.ClientBlockstore), modules.IpfsClientBlockstore(ipfsMaddr, cfg.Client.IpfsOnlineMode)), + Override(new(storagemarket.BlockstoreAccessor), modules.IpfsStorageBlockstoreAccessor), If(cfg.Client.IpfsUseForRetrieval, - Override(new(dtypes.ClientRetrievalStoreManager), modules.ClientBlockstoreRetrievalStoreManager(true)), + Override(new(retrievalmarket.BlockstoreAccessor), modules.IpfsRetrievalBlockstoreAccessor), ), ), Override(new(dtypes.Graphsync), modules.Graphsync(cfg.Client.SimultaneousTransfers)), diff --git a/node/impl/client/car_helpers.go b/node/impl/client/car_helpers.go new file mode 100644 index 00000000000..c638b4bef81 --- /dev/null +++ b/node/impl/client/car_helpers.go @@ -0,0 +1,91 @@ +package client + +import ( + "fmt" + "io" + + "github.com/ipfs/go-cid" + cbor "github.com/ipfs/go-ipld-cbor" + "github.com/ipld/go-car/util" + "github.com/multiformats/go-varint" +) + +// ————————————————————————————————————————————————————————— +// +// This code is temporary, and should be deleted when +// https://github.com/ipld/go-car/issues/196 is resolved. +// +// ————————————————————————————————————————————————————————— + +func init() { + cbor.RegisterCborType(CarHeader{}) +} + +type CarHeader struct { + Roots []cid.Cid + Version uint64 +} + +func readHeader(r io.Reader) (*CarHeader, error) { + hb, err := ldRead(r, false) + if err != nil { + return nil, err + } + + var ch CarHeader + if err := cbor.DecodeInto(hb, &ch); err != nil { + return nil, fmt.Errorf("invalid header: %v", err) + } + + return &ch, nil +} + +func writeHeader(h *CarHeader, w io.Writer) error { + hb, err := cbor.DumpObject(h) + if err != nil { + return err + } + + return util.LdWrite(w, hb) +} + +func ldRead(r io.Reader, zeroLenAsEOF bool) ([]byte, error) { + l, err := varint.ReadUvarint(toByteReader(r)) + if err != nil { + // If the length of bytes read is non-zero when the error is EOF then signal an unclean EOF. + if l > 0 && err == io.EOF { + return nil, io.ErrUnexpectedEOF + } + return nil, err + } else if l == 0 && zeroLenAsEOF { + return nil, io.EOF + } + + buf := make([]byte, l) + if _, err := io.ReadFull(r, buf); err != nil { + return nil, err + } + + return buf, nil +} + +type readerPlusByte struct { + io.Reader +} + +func (rb readerPlusByte) ReadByte() (byte, error) { + return readByte(rb) +} + +func readByte(r io.Reader) (byte, error) { + var p [1]byte + _, err := io.ReadFull(r, p[:]) + return p[0], err +} + +func toByteReader(r io.Reader) io.ByteReader { + if br, ok := r.(io.ByteReader); ok { + return br + } + return &readerPlusByte{r} +} diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 6c9909ab3d4..1fbe9966883 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -2,19 +2,19 @@ package client import ( "bufio" + "bytes" "context" "fmt" "io" - "math/rand" "os" "path/filepath" "sort" "time" + bstore "github.com/ipfs/go-ipfs-blockstore" "github.com/ipld/go-car" carv2 "github.com/ipld/go-car/v2" "github.com/ipld/go-car/v2/blockstore" - "golang.org/x/xerrors" "github.com/filecoin-project/go-padreader" @@ -41,6 +41,7 @@ import ( "github.com/filecoin-project/go-commp-utils/ffiwrapper" "github.com/filecoin-project/go-commp-utils/writer" datatransfer "github.com/filecoin-project/go-data-transfer" + "github.com/filecoin-project/go-fil-markets/discovery" "github.com/filecoin-project/go-fil-markets/retrievalmarket" rm "github.com/filecoin-project/go-fil-markets/retrievalmarket" @@ -48,6 +49,8 @@ import ( "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-fil-markets/storagemarket/network" "github.com/filecoin-project/go-fil-markets/stores" + "github.com/filecoin-project/lotus/markets/retrievaladapter" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/specs-actors/v3/actors/builtin/market" @@ -85,13 +88,14 @@ type API struct { Retrieval rm.RetrievalClient Chain *store.ChainStore - Imports dtypes.ClientImportMgr + // accessors for imports and retrievals. + Imports dtypes.ClientImportMgr + RtvlBlockstoreAccessor retrievalmarket.BlockstoreAccessor DataTransfer dtypes.ClientDataTransfer Host host.Host - RetrievalStoreMgr dtypes.ClientRetrievalStoreManager - Repo repo.LockedRepo + Repo repo.LockedRepo } func calcDealExpiration(minDuration uint64, md *dline.Info, startEpoch abi.ChainEpoch) abi.ChainEpoch { @@ -108,7 +112,8 @@ func calcDealExpiration(minDuration uint64, md *dline.Info, startEpoch abi.Chain return exp } -func (a *API) imgr() *imports.Manager { +// importManager converts the injected type to the required type. +func (a *API) importManager() *imports.Manager { return a.Imports } @@ -121,7 +126,6 @@ func (a *API) ClientStatelessDeal(ctx context.Context, params *api.StartDealPara } func (a *API) dealStarter(ctx context.Context, params *api.StartDealParams, isStateless bool) (*cid.Cid, error) { - var CARV2FilePath string if isStateless { if params.Data.TransferType != storagemarket.TTManual { return nil, xerrors.Errorf("invalid transfer type %s for stateless storage deal", params.Data.TransferType) @@ -130,14 +134,13 @@ func (a *API) dealStarter(ctx context.Context, params *api.StartDealParams, isSt return nil, xerrors.New("stateless storage deals can only be initiated with storage price of 0") } } else if params.Data.TransferType == storagemarket.TTGraphsync { - fc, err := a.imgr().FilestoreCARV2FilePathFor(params.Data.Root) + fc, err := a.importManager().CARPathFor(params.Data.Root) if err != nil { return nil, xerrors.Errorf("failed to find CARv2 file path: %w", err) } if fc == "" { return nil, xerrors.New("no CARv2 file path for deal") } - CARV2FilePath = fc } walletKey, err := a.StateAccountKey(ctx, params.Wallet, types.EmptyTSK) @@ -203,7 +206,6 @@ func (a *API) dealStarter(ctx context.Context, params *api.StartDealParams, isSt Rt: st, FastRetrieval: params.FastRetrieval, VerifiedDeal: params.VerifiedDeal, - IndexedCAR: CARV2FilePath, }) if err != nil { @@ -404,7 +406,7 @@ func (a *API) newDealInfoWithTransfer(transferCh *api.DataTransferChannel, v sto func (a *API) ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) { // TODO: check if we have the ENTIRE dag - fc, err := a.imgr().FilestoreCARV2FilePathFor(root) + fc, err := a.importManager().CARPathFor(root) if err != nil { return false, err } @@ -484,22 +486,29 @@ func (a *API) makeRetrievalQuery(ctx context.Context, rp rm.RetrievalPeer, paylo } } -func (a *API) ClientImport(ctx context.Context, ref api.FileRef) (res *api.ImportRes, finalErr error) { - id, err := a.imgr().CreateImport() +// TODO check +func (a *API) ClientImport(ctx context.Context, ref api.FileRef) (res *api.ImportRes, err error) { + var ( + imgr = a.importManager() + id imports.ID + root cid.Cid + carPath string + ) + + id, err = imgr.CreateImport() if err != nil { - return nil, err + return nil, xerrors.Errorf("failed to create import: %w", err) } - var root cid.Cid - var carFile string if ref.IsCAR { - // if user has given us a CAR file -> just ensure it's either a v1 or a - // v2, has one root and save it as it is as markets can do deal making for both. + // user gave us a CAR fil, use it as-is + // validate that it's either a carv1 or carv2, and has one root. f, err := os.Open(ref.Path) if err != nil { return nil, xerrors.Errorf("failed to open CAR file: %w", err) } defer f.Close() //nolint:errcheck + hd, _, err := car.ReadHeader(bufio.NewReader(f)) if err != nil { return nil, xerrors.Errorf("failed to read CAR header: %w", err) @@ -511,97 +520,186 @@ func (a *API) ClientImport(ctx context.Context, ref api.FileRef) (res *api.Impor return nil, xerrors.Errorf("car version must be 1 or 2, is %d", hd.Version) } - carFile = ref.Path + carPath = ref.Path root = hd.Roots[0] } else { - carFile, err = a.imgr().NewTempFile(id) + carPath, err = imgr.AllocateCAR(id) if err != nil { - return nil, xerrors.Errorf("failed to create temp CARv2 file: %w", err) + return nil, xerrors.Errorf("failed to create car path for import: %w", err) } - // make sure to remove the CARv2 file if anything goes wrong from here on. + + // remove the import if something went wrong. defer func() { - if finalErr != nil { - _ = os.Remove(carFile) + if err != nil { + _ = os.Remove(carPath) + _ = imgr.Remove(id) } }() - root, err = a.doImport(ctx, ref.Path, carFile) + // perform the unixfs chunking. + root, err = a.createUnixFSFilestore(ctx, ref.Path, carPath) if err != nil { - return nil, xerrors.Errorf("failed to import normal file to CARv2: %w", err) + return nil, xerrors.Errorf("failed to import file using unixfs: %w", err) } } - if err := a.imgr().AddLabel(id, imports.LSource, "import"); err != nil { + if err = imgr.AddLabel(id, imports.LSource, "import"); err != nil { return nil, err } - if err := a.imgr().AddLabel(id, imports.LFileName, ref.Path); err != nil { + if err = imgr.AddLabel(id, imports.LFileName, ref.Path); err != nil { return nil, err } - if err := a.imgr().AddLabel(id, imports.LCARPath, carFile); err != nil { + if err = imgr.AddLabel(id, imports.LCARPath, carPath); err != nil { return nil, err } - if err := a.imgr().AddLabel(id, imports.LRootCid, root.String()); err != nil { + if err = imgr.AddLabel(id, imports.LRootCid, root.String()); err != nil { return nil, err } - return &api.ImportRes{ Root: root, ImportID: id, }, nil } -func (a *API) ClientRemoveImport(ctx context.Context, importID imports.ID) error { - info, err := a.imgr().Info(importID) +func (a *API) ClientRemoveImport(ctx context.Context, id imports.ID) error { + info, err := a.importManager().Info(id) if err != nil { - return xerrors.Errorf("failed to fetch import info: %w", err) + return xerrors.Errorf("failed to get import metadata: %w", err) } - // remove the CARv2 file if we've created one. - if path := info.Labels[imports.LCARPath]; path != "" { + owner := info.Labels[imports.LCAROwner] + path := info.Labels[imports.LCARPath] + + // CARv2 file was not provided by the user, delete it. + if path != "" && owner == imports.CAROwnerImportMgr { _ = os.Remove(path) } - return a.imgr().Remove(importID) + return a.importManager().Remove(id) } +// ClientImportLocal imports a standard file into this node as a UnixFS payload, +// storing it in a CARv2 file. Note that this method is NOT integrated with the +// IPFS blockstore. That is, if client-side IPFS integration is enabled, this +// method won't import the file into that func (a *API) ClientImportLocal(ctx context.Context, r io.Reader) (cid.Cid, error) { + file := files.NewReaderFile(r) + // write payload to temp file - tmpPath, err := a.imgr().NewTempFile(imports.ID(rand.Uint64())) + id, err := a.importManager().CreateImport() if err != nil { return cid.Undef, err } - defer os.Remove(tmpPath) //nolint:errcheck - tmpF, err := os.Open(tmpPath) - if err != nil { + if err := a.importManager().AddLabel(id, imports.LSource, "import-local"); err != nil { return cid.Undef, err } - defer tmpF.Close() //nolint:errcheck - if _, err := io.Copy(tmpF, r); err != nil { + + path, err := a.importManager().AllocateCAR(id) + if err != nil { return cid.Undef, err } - if err := tmpF.Close(); err != nil { + + // writing a carv2 requires knowing the root ahead of time, which makes + // streaming cases impossible. + // https://github.com/ipld/go-car/issues/196 + // we work around this limitation by informing a placeholder root CID of the + // same length as our unixfs chunking strategy will generate. + // once the DAG is formed and the root is calculated, we overwrite the + // inner carv1 header with the final root. + + b, err := unixFSCidBuilder() + if err != nil { return cid.Undef, err } - res, err := a.ClientImport(ctx, api.FileRef{ - Path: tmpPath, - IsCAR: false, - }) + // placeholder payload needs to be larger than inline CID threshold; 256 + // bytes is a safe value. + placeholderRoot, err := b.Sum(make([]byte, 256)) if err != nil { - return cid.Undef, err + return cid.Undef, xerrors.Errorf("failed to calculate placeholder root: %w", err) + } + + bs, err := blockstore.OpenReadWrite(path, []cid.Cid{placeholderRoot}, blockstore.UseWholeCIDs(true)) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to create carv2 read/write blockstore: %w", err) + } + + root, err := buildUnixFS(ctx, file, bs, false) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to build unixfs dag: %w", err) } - return res.Root, nil + + err = bs.Finalize() + if err != nil { + return cid.Undef, xerrors.Errorf("failed to finalize carv2 read/write blockstore: %w", err) + } + + // record the root in the import manager. + if err := a.importManager().AddLabel(id, imports.LRootCid, root.String()); err != nil { + return cid.Undef, xerrors.Errorf("failed to record root CID in import manager: %w", err) + } + + // now go ahead and overwrite the root in the carv1 header. + reader, err := carv2.OpenReader(path) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to create car reader: %w", err) + } + + // save the header offset. + headerOff := reader.Header.DataOffset + + // read the old header. + dr := reader.DataReader() + header, err := readHeader(dr) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to read car reader: %w", err) + } + _ = reader.Close() // close the CAR reader. + + // write the old header into a buffer. + var oldBuf bytes.Buffer + if err = writeHeader(header, &oldBuf); err != nil { + return cid.Undef, xerrors.Errorf("failed to write header into buffer: %w", err) + } + + // replace the root. + header.Roots = []cid.Cid{root} + + // write the new header into a buffer. + var newBuf bytes.Buffer + err = writeHeader(header, &newBuf) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to write header into buffer: %w", err) + } + + // verify the length matches. + if newBuf.Len() != oldBuf.Len() { + return cid.Undef, xerrors.Errorf("failed to replace carv1 header; length mismatch (old: %d, new: %d)", oldBuf.Len(), newBuf.Len()) + } + + // open the file again, seek to the header position, and write. + f, err := os.OpenFile(path, os.O_WRONLY, 0755) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to open car: %w", err) + } + defer f.Close() + + n, err := f.WriteAt(newBuf.Bytes(), int64(headerOff)) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to write new header to car (bytes written: %d): %w", n, err) + } + return root, nil } -func (a *API) ClientListImports(ctx context.Context) ([]api.Import, error) { - importIDs, err := a.imgr().List() +func (a *API) ClientListImports(_ context.Context) ([]api.Import, error) { + ids, err := a.importManager().List() if err != nil { return nil, xerrors.Errorf("failed to fetch imports: %w", err) } - out := make([]api.Import, len(importIDs)) - for i, id := range importIDs { - info, err := a.imgr().Info(id) + out := make([]api.Import, len(ids)) + for i, id := range ids { + info, err := a.importManager().Info(id) if err != nil { out[i] = api.Import{ Key: id, @@ -611,10 +709,10 @@ func (a *API) ClientListImports(ctx context.Context) ([]api.Import, error) { } ai := api.Import{ - Key: id, - Source: info.Labels[imports.LSource], - FilePath: info.Labels[imports.LFileName], - CARv2FilePath: info.Labels[imports.LCARPath], + Key: id, + Source: info.Labels[imports.LSource], + FilePath: info.Labels[imports.LFileName], + CARPath: info.Labels[imports.LCARPath], } if info.Labels[imports.LRootCid] != "" { @@ -737,8 +835,26 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref } } - var carV2FilePath string - if order.LocalCARV2FilePath == "" { + // summary: + // 1. if we're retrieving from an import, FromLocalCAR will be informed. + // Open as a Filestore and populate the target CAR or UnixFS export from it. + // (cannot use ExtractV1File because user wants a dense CAR, not a ref CAR/filestore) + // 2. if we're using an IPFS blockstore for retrieval, retrieve into it, + // then extract the CAR or UnixFS export from it. + // 3. if we have to retrieve, perform a CARv2 retrieval, then extract + // the CARv1 (with ExtractV1File) or UnixFS export from it. + + // this indicates we're proxying to IPFS. + proxyBss, isIPFS := a.RtvlBlockstoreAccessor.(*retrievaladapter.ProxyBlockstoreAccessor) + carBss, isCAR := a.RtvlBlockstoreAccessor.(*retrievaladapter.CARBlockstoreAccessor) + + if !isIPFS && !isCAR { + finish(xerrors.Errorf("unsupported retrieval blockstore accessor")) + return + } + + carPath := order.FromLocalCAR + if carPath == "" { if order.MinerPeer == nil || order.MinerPeer.ID == "" { mi, err := a.StateMinerInfo(ctx, order.Miner, types.EmptyTSK) if err != nil { @@ -784,22 +900,17 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref } }) - tmpCarv2FilePath, err := a.getTmpCarV2FilePath() - if err != nil { - unsubscribe() - finish(xerrors.Errorf("Retrieve failed: %w", err)) - return - } - - resp, err := a.Retrieval.Retrieve( + id := a.Retrieval.NextID() + id, err = a.Retrieval.Retrieve( ctx, + id, order.Root, params, order.Total, *order.MinerPeer, order.Client, order.Miner, - tmpCarv2FilePath) + ) if err != nil { unsubscribe() @@ -807,7 +918,7 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref return } - err = readSubscribeEvents(ctx, resp.DealID, subscribeEvents, events) + err = readSubscribeEvents(ctx, id, subscribeEvents, events) unsubscribe() if err != nil { @@ -815,74 +926,56 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref return } - carV2FilePath = resp.CarFilePath - // remove the temp CARv2 file when retrieval is complete - defer os.Remove(carV2FilePath) //nolint:errcheck - } else { - carV2FilePath = order.LocalCARV2FilePath - } - - // TODO We only support this currently for the IPFS Retrieval use case - // where users want to write out filecoin retrievals directly to IPFS. - // If users haven' configured the Ipfs retrieval flag, the blockstore we get here will be a "no-op" blockstore. - // write out the CARv2 file to the retrieval block-store (which is really an IPFS node behind the scenes). - rs, err := a.RetrievalStoreMgr.NewStore() - defer a.RetrievalStoreMgr.ReleaseStore(rs) //nolint:errcheck - if err != nil { - finish(xerrors.Errorf("Error setting up new store: %w", err)) - return - } - if rs.IsIPFSRetrieval() { - // write out the CARv1 blocks of the CARv2 file to the IPFS blockstore. - carv2Reader, err := carv2.OpenReader(carV2FilePath) - if err != nil { - finish(err) - return - } - defer carv2Reader.Close() //nolint:errcheck - - if _, err := car.LoadCar(rs.Blockstore(), carv2Reader.DataReader()); err != nil { - finish(err) - return - } + carPath = carBss.PathFor(id) } - // If ref is nil, it only fetches the data into the configured blockstore. - if ref == nil { + switch { + case ref == nil: + // If ref is nil, it only fetches the data into the configured blockstore + // (if fetching from network). finish(nil) return - } - if ref.IsCAR { - // user wants a CAR file, transform the CARv2 to a CARv1 and write it out. + case ref.IsCAR && isIPFS: + // generating a CARv1 from IPFS. f, err := os.OpenFile(ref.Path, os.O_CREATE|os.O_WRONLY, 0644) if err != nil { finish(err) return } - carv2Reader, err := carv2.OpenReader(carV2FilePath) + bs := proxyBss.Blockstore + dags := merkledag.NewDAGService(blockservice.New(bs, offline.Exchange(bs))) + err = car.WriteCar(ctx, dags, []cid.Cid{order.Root}, f) if err != nil { finish(err) return } - defer carv2Reader.Close() //nolint:errcheck - if _, err := io.Copy(f, carv2Reader.DataReader()); err != nil { - finish(err) - return - } - finish(f.Close()) return - } - readOnly, err := blockstore.OpenReadOnly(carV2FilePath, carv2.ZeroLengthSectionAsEOF(true), blockstore.UseWholeCIDs(true)) - if err != nil { + case ref.IsCAR: + // generating a CARv1 from the CARv2 where we stored the retrieval. + err := carv2.ExtractV1File(carPath, ref.Path) finish(err) return } - defer readOnly.Close() //nolint:errcheck - bsvc := blockservice.New(readOnly, offline.Exchange(readOnly)) + + // we are extracting a UnixFS file. + var bs bstore.Blockstore + if isIPFS { + bs = proxyBss.Blockstore + } else { + cbs, err := stores.ReadOnlyFilestore(carPath) + if err != nil { + finish(err) + return + } + defer cbs.Close() //nolint:errcheck + bs = cbs + } + + bsvc := blockservice.New(bs, offline.Exchange(bs)) dag := merkledag.NewDAGService(bsvc) nd, err := dag.Get(ctx, order.Root) @@ -895,9 +988,8 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref finish(xerrors.Errorf("ClientRetrieve: %w", err)) return } - finish(files.WriteTo(file, ref.Path)) - return + finish(files.WriteTo(file, ref.Path)) } // TODO: Come up with a better mechanism for creating the tmp CARv2 file path @@ -1056,7 +1148,7 @@ func (w *lenWriter) Write(p []byte) (n int, err error) { } func (a *API) ClientDealSize(ctx context.Context, root cid.Cid) (api.DataSize, error) { - path, err := a.imgr().FilestoreCARV2FilePathFor(root) + path, err := a.importManager().CARPathFor(root) if err != nil { return api.DataSize{}, xerrors.Errorf("failed to find CARv2 file for root: %w", err) } @@ -1087,7 +1179,7 @@ func (a *API) ClientDealSize(ctx context.Context, root cid.Cid) (api.DataSize, e } func (a *API) ClientDealPieceCID(ctx context.Context, root cid.Cid) (api.DataCIDSize, error) { - path, err := a.imgr().FilestoreCARV2FilePathFor(root) + path, err := a.importManager().CARPathFor(root) if err != nil { return api.DataCIDSize{}, xerrors.Errorf("failed to find CARv2 file for root: %w", err) } @@ -1119,26 +1211,36 @@ func (a *API) ClientDealPieceCID(ctx context.Context, root cid.Cid) (api.DataCID } func (a *API) ClientGenCar(ctx context.Context, ref api.FileRef, outputPath string) error { - id := imports.ID(rand.Uint64()) - tmp, err := a.imgr().NewTempFile(id) + // create a temporary import to represent this job and obtain a staging CAR. + id, err := a.importManager().CreateImport() + if err != nil { + return xerrors.Errorf("failed to create temporary import: %w", err) + } + defer a.importManager().Remove(id) //nolint:errcheck + + tmp, err := a.importManager().AllocateCAR(id) if err != nil { - return xerrors.Errorf("failed to create temp file: %w", err) + return xerrors.Errorf("failed to allocate temporary CAR: %w", err) } defer os.Remove(tmp) //nolint:errcheck - root, err := a.doImport(ctx, ref.Path, tmp) + // geneate and import the UnixFS DAG into a filestore (positional reference) CAR. + root, err := a.createUnixFSFilestore(ctx, ref.Path, tmp) if err != nil { - return xerrors.Errorf("failed to import normal file to CARv2") + return xerrors.Errorf("failed to import file using unixfs: %w", err) } + // open the positional reference CAR as a filestore. fs, err := stores.ReadOnlyFilestore(tmp) if err != nil { return xerrors.Errorf("failed to open filestore from carv2 in path %s: %w", tmp, err) } defer fs.Close() //nolint:errcheck + // build a dense deterministic CAR (dense = containing filled leaves) ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) - allSelector := ssb.ExploreRecursive(selector.RecursionLimitNone(), + allSelector := ssb.ExploreRecursive( + selector.RecursionLimitNone(), ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node() sc := car.NewSelectiveCar(ctx, fs, []car.Dag{{Root: root, Selector: allSelector}}) f, err := os.Create(outputPath) diff --git a/node/impl/client/client_test.go b/node/impl/client/client_test.go index da13c8ef3ca..19b4219c383 100644 --- a/node/impl/client/client_test.go +++ b/node/impl/client/client_test.go @@ -1 +1,67 @@ package client + +import ( + "bytes" + "context" + "embed" + "io/ioutil" + "path/filepath" + "strings" + "testing" + + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + dssync "github.com/ipfs/go-datastore/sync" + "github.com/stretchr/testify/require" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/node/repo/imports" +) + +//go:embed testdata/* +var testdata embed.FS + +func TestImportLocal(t *testing.T) { + ds := dssync.MutexWrap(datastore.NewMapDatastore()) + dir := t.TempDir() + im := imports.NewManager(ds, dir) + ctx := context.Background() + + a := &API{Imports: im} + + b, err := testdata.ReadFile("testdata/payload.txt") + require.NoError(t, err) + + root, err := a.ClientImportLocal(ctx, bytes.NewReader(b)) + require.NoError(t, err) + require.NotEqual(t, cid.Undef, root) + + list, err := a.ClientListImports(ctx) + require.NoError(t, err) + require.Len(t, list, 1) + + it := list[0] + require.Equal(t, root, *it.Root) + require.True(t, strings.HasPrefix(it.CARPath, dir)) + + local, err := a.ClientHasLocal(ctx, root) + require.NoError(t, err) + require.True(t, local) + + order := api.RetrievalOrder{ + Root: root, + FromLocalCAR: it.CARPath, + } + + // retrieve as UnixFS. + out1 := filepath.Join(dir, "retrieval1.data") + // out2 := filepath.Join(dir, "retrieval2.data") + err = a.ClientRetrieve(ctx, order, &api.FileRef{ + Path: out1, + }) + require.NoError(t, err) + + outBytes, err := ioutil.ReadFile(out1) + require.NoError(t, err) + require.Equal(t, b, outBytes) +} diff --git a/node/impl/client/import.go b/node/impl/client/import.go index 8f37442d32f..0296900db91 100644 --- a/node/impl/client/import.go +++ b/node/impl/client/import.go @@ -2,6 +2,8 @@ package client import ( "context" + "fmt" + "io" "io/ioutil" "os" @@ -9,9 +11,10 @@ import ( "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" "github.com/ipfs/go-cidutil" + bstore "github.com/ipfs/go-ipfs-blockstore" chunker "github.com/ipfs/go-ipfs-chunker" offline "github.com/ipfs/go-ipfs-exchange-offline" - files2 "github.com/ipfs/go-ipfs-files" + "github.com/ipfs/go-ipfs-files" ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" "github.com/ipfs/go-unixfs/importer/balanced" @@ -21,9 +24,22 @@ import ( "github.com/filecoin-project/lotus/build" ) -// doImport takes a standard file (src), forms a UnixFS DAG, and writes a -// CARv2 file with positional mapping (backed by the go-filestore library). -func (a *API) doImport(ctx context.Context, src string, dst string) (cid.Cid, error) { +func unixFSCidBuilder() (cid.Builder, error) { + prefix, err := merkledag.PrefixForCidVersion(1) + if err != nil { + return nil, fmt.Errorf("failed to initialize UnixFS CID Builder: %w", err) + } + prefix.MhType = DefaultHashFunction + b := cidutil.InlineBuilder{ + Builder: prefix, + Limit: 126, + } + return b, nil +} + +// createUnixFSFilestore takes a standard file whose path is src, forms a UnixFS DAG, and +// writes a CARv2 file with positional mapping (backed by the go-filestore library). +func (a *API) createUnixFSFilestore(ctx context.Context, srcPath string, dstPath string) (cid.Cid, error) { // This method uses a two-phase approach with a staging CAR blockstore and // a final CAR blockstore. // @@ -31,11 +47,29 @@ func (a *API) doImport(ctx context.Context, src string, dst string) (cid.Cid, er // // TODO: do we need to chunk twice? Isn't the first output already in the // right order? Can't we just copy the CAR file and replace the header? + + src, err := os.Open(srcPath) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to open input file: %w", err) + } + defer src.Close() //nolint:errcheck + + stat, err := src.Stat() + if err != nil { + return cid.Undef, xerrors.Errorf("failed to stat file :%w", err) + } + + file, err := files.NewReaderPathFile(srcPath, src, stat) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to create reader path file: %w", err) + } + f, err := ioutil.TempFile("", "") if err != nil { return cid.Undef, xerrors.Errorf("failed to create temp file: %w", err) } _ = f.Close() // close; we only want the path. + tmp := f.Name() defer os.Remove(tmp) //nolint:errcheck @@ -46,10 +80,7 @@ func (a *API) doImport(ctx context.Context, src string, dst string) (cid.Cid, er return cid.Undef, xerrors.Errorf("failed to create temporary filestore: %w", err) } - bsvc := blockservice.New(fstore, offline.Exchange(fstore)) - dags := merkledag.NewDAGService(bsvc) - - root, err := buildUnixFS(ctx, src, dags) + finalRoot1, err := buildUnixFS(ctx, file, fstore, true) if err != nil { _ = fstore.Close() return cid.Undef, xerrors.Errorf("failed to import file to store to compute root: %w", err) @@ -61,14 +92,17 @@ func (a *API) doImport(ctx context.Context, src string, dst string) (cid.Cid, er // Step 2. We now have the root of the UnixFS DAG, and we can write the // final CAR for real under `dst`. - bs, err := stores.ReadWriteFilestore(dst, root) + bs, err := stores.ReadWriteFilestore(dstPath, finalRoot1) if err != nil { return cid.Undef, xerrors.Errorf("failed to create a carv2 read/write filestore: %w", err) } - bsvc = blockservice.New(bs, offline.Exchange(bs)) - dags = merkledag.NewDAGService(bsvc) - finalRoot, err := buildUnixFS(ctx, src, dags) + // rewind file to the beginning. + if _, err := src.Seek(0, 0); err != nil { + return cid.Undef, xerrors.Errorf("failed to rewind file: %w", err) + } + + finalRoot2, err := buildUnixFS(ctx, file, bs, true) if err != nil { _ = bs.Close() return cid.Undef, xerrors.Errorf("failed to create UnixFS DAG with carv2 blockstore: %w", err) @@ -78,52 +112,34 @@ func (a *API) doImport(ctx context.Context, src string, dst string) (cid.Cid, er return cid.Undef, xerrors.Errorf("failed to finalize car blockstore: %w", err) } - if root != finalRoot { + if finalRoot1 != finalRoot2 { return cid.Undef, xerrors.New("roots do not match") } - return root, nil + return finalRoot1, nil } -// buildUnixFS builds a UnixFS DAG out of the supplied ordinary file, +// buildUnixFS builds a UnixFS DAG out of the supplied reader, // and imports the DAG into the supplied service. -func buildUnixFS(ctx context.Context, src string, dag ipld.DAGService) (cid.Cid, error) { - f, err := os.Open(src) - if err != nil { - return cid.Undef, xerrors.Errorf("failed to open input file: %w", err) - } - defer f.Close() //nolint:errcheck - - stat, err := f.Stat() - if err != nil { - return cid.Undef, xerrors.Errorf("failed to stat file :%w", err) - } - - file, err := files2.NewReaderPathFile(src, f, stat) - if err != nil { - return cid.Undef, xerrors.Errorf("failed to create reader path file: %w", err) - } - - bufDs := ipld.NewBufferedDAG(ctx, dag) - - prefix, err := merkledag.PrefixForCidVersion(1) +func buildUnixFS(ctx context.Context, reader io.Reader, into bstore.Blockstore, filestore bool) (cid.Cid, error) { + b, err := unixFSCidBuilder() if err != nil { return cid.Undef, err } - prefix.MhType = DefaultHashFunction + + bsvc := blockservice.New(into, offline.Exchange(into)) + dags := merkledag.NewDAGService(bsvc) + bufdag := ipld.NewBufferedDAG(ctx, dags) params := ihelper.DagBuilderParams{ - Maxlinks: build.UnixfsLinksPerLevel, - RawLeaves: true, - CidBuilder: cidutil.InlineBuilder{ - Builder: prefix, - Limit: 126, - }, - Dagserv: bufDs, - NoCopy: true, + Maxlinks: build.UnixfsLinksPerLevel, + RawLeaves: true, + CidBuilder: b, + Dagserv: bufdag, + NoCopy: filestore, } - db, err := params.New(chunker.NewSizeSplitter(file, int64(build.UnixfsChunkSize))) + db, err := params.New(chunker.NewSizeSplitter(reader, int64(build.UnixfsChunkSize))) if err != nil { return cid.Undef, err } @@ -132,7 +148,7 @@ func buildUnixFS(ctx context.Context, src string, dag ipld.DAGService) (cid.Cid, return cid.Undef, err } - if err := bufDs.Commit(); err != nil { + if err := bufdag.Commit(); err != nil { return cid.Undef, err } diff --git a/node/impl/client/import_test.go b/node/impl/client/import_test.go index 6057a24ff50..adf6531d088 100644 --- a/node/impl/client/import_test.go +++ b/node/impl/client/import_test.go @@ -1,6 +1,7 @@ package client import ( + "bytes" "context" "io" "io/ioutil" @@ -8,7 +9,6 @@ import ( "strings" "testing" - "github.com/filecoin-project/go-fil-markets/stores" "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" offline "github.com/ipfs/go-ipfs-exchange-offline" @@ -19,6 +19,8 @@ import ( "github.com/ipld/go-car/v2/blockstore" "github.com/stretchr/testify/require" + "github.com/filecoin-project/go-fil-markets/stores" + "github.com/filecoin-project/lotus/node/repo/imports" ) @@ -33,18 +35,15 @@ func TestRoundtripUnixFS_Dense(t *testing.T) { defer os.Remove(carv2File) //nolint:errcheck // import a file to a Unixfs DAG using a CARv2 read/write blockstore. - path, err := blockstore.OpenReadWrite(carv2File, nil, + bs, err := blockstore.OpenReadWrite(carv2File, nil, carv2.ZeroLengthSectionAsEOF(true), blockstore.UseWholeCIDs(true)) require.NoError(t, err) - bsvc := blockservice.New(path, offline.Exchange(path)) - dags := merkledag.NewDAGService(bsvc) - - root, err := buildUnixFS(ctx, inputPath, dags) + root, err := buildUnixFS(ctx, bytes.NewBuffer(inputContents), bs, false) require.NoError(t, err) require.NotEqual(t, cid.Undef, root) - require.NoError(t, path.Finalize()) + require.NoError(t, bs.Finalize()) // reconstruct the file. readOnly, err := blockstore.OpenReadOnly(carv2File, @@ -53,7 +52,7 @@ func TestRoundtripUnixFS_Dense(t *testing.T) { require.NoError(t, err) defer readOnly.Close() //nolint:errcheck - dags = merkledag.NewDAGService(blockservice.New(readOnly, offline.Exchange(readOnly))) + dags := merkledag.NewDAGService(blockservice.New(readOnly, offline.Exchange(readOnly))) nd, err := dags.Get(ctx, root) require.NoError(t, err) @@ -80,18 +79,18 @@ func TestRoundtripUnixFS_Filestore(t *testing.T) { Imports: &imports.Manager{}, } - inputFilePath, inputContents := genInputFile(t) - defer os.Remove(inputFilePath) //nolint:errcheck + inputPath, inputContents := genInputFile(t) + defer os.Remove(inputPath) //nolint:errcheck - path := newTmpFile(t) - defer os.Remove(path) //nolint:errcheck + dst := newTmpFile(t) + defer os.Remove(dst) //nolint:errcheck - root, err := a.doImport(ctx, inputFilePath, path) + root, err := a.createUnixFSFilestore(ctx, inputPath, dst) require.NoError(t, err) require.NotEqual(t, cid.Undef, root) // convert the CARv2 to a normal file again and ensure the contents match - fs, err := stores.ReadOnlyFilestore(path) + fs, err := stores.ReadOnlyFilestore(dst) require.NoError(t, err) defer fs.Close() //nolint:errcheck diff --git a/node/impl/client/testdata/duplicate_blocks.txt b/node/impl/client/testdata/duplicate_blocks.txt new file mode 100644 index 00000000000..53695d7b95f --- /dev/null +++ b/node/impl/client/testdata/duplicate_blocks.txt @@ -0,0 +1 @@ +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccdddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd \ No newline at end of file diff --git a/node/impl/client/testdata/payload.txt b/node/impl/client/testdata/payload.txt new file mode 100644 index 00000000000..fd4a2f3c1ff --- /dev/null +++ b/node/impl/client/testdata/payload.txt @@ -0,0 +1,49 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Vitae semper quis lectus nulla at volutpat diam ut venenatis. Ac tortor dignissim convallis aenean et tortor at. Faucibus ornare suspendisse sed nisi lacus sed. Commodo ullamcorper a lacus vestibulum sed arcu non. Est pellentesque elit ullamcorper dignissim. Quam quisque id diam vel quam. Pretium aenean pharetra magna ac. In nulla posuere sollicitudin aliquam ultrices. Sed arcu non odio euismod lacinia at. Suspendisse ultrices gravida dictum fusce ut placerat orci nulla pellentesque. Feugiat vivamus at augue eget arcu. + +Pellentesque nec nam aliquam sem et tortor. Vitae tortor condimentum lacinia quis vel. Cras pulvinar mattis nunc sed. In massa tempor nec feugiat. Ornare arcu odio ut sem nulla. Diam maecenas sed enim ut sem. Pretium vulputate sapien nec sagittis. Bibendum arcu vitae elementum curabitur vitae nunc sed velit dignissim. Duis ut diam quam nulla porttitor massa. Viverra mauris in aliquam sem fringilla ut morbi. Ullamcorper eget nulla facilisi etiam dignissim. Vulputate mi sit amet mauris commodo quis imperdiet massa tincidunt. Nunc consequat interdum varius sit. Nunc mi ipsum faucibus vitae aliquet nec ullamcorper. Nunc sed augue lacus viverra. Lobortis scelerisque fermentum dui faucibus in ornare quam. Urna neque viverra justo nec ultrices. Varius vel pharetra vel turpis nunc eget lorem dolor sed. + +Feugiat nisl pretium fusce id velit ut tortor pretium. Lorem dolor sed viverra ipsum nunc aliquet bibendum. Ultrices vitae auctor eu augue ut lectus. Pharetra massa massa ultricies mi quis. Nibh cras pulvinar mattis nunc sed blandit libero. Ac felis donec et odio pellentesque diam volutpat. Lectus proin nibh nisl condimentum id venenatis. Quis vel eros donec ac odio. Commodo sed egestas egestas fringilla phasellus faucibus scelerisque eleifend donec. Adipiscing diam donec adipiscing tristique. + +Tempus imperdiet nulla malesuada pellentesque elit eget gravida cum sociis. Libero nunc consequat interdum varius sit. Et pharetra pharetra massa massa. Feugiat pretium nibh ipsum consequat. Amet commodo nulla facilisi nullam vehicula. Ornare arcu dui vivamus arcu felis bibendum ut tristique. At erat pellentesque adipiscing commodo elit at imperdiet dui. Auctor neque vitae tempus quam pellentesque nec nam aliquam sem. Eget velit aliquet sagittis id consectetur. Enim diam vulputate ut pharetra sit amet aliquam id diam. Eget velit aliquet sagittis id consectetur purus ut faucibus pulvinar. Amet porttitor eget dolor morbi. Felis eget velit aliquet sagittis id. Facilisis magna etiam tempor orci eu. Lacus suspendisse faucibus interdum posuere lorem. Pharetra et ultrices neque ornare aenean euismod. Platea dictumst quisque sagittis purus. + +Quis varius quam quisque id diam vel quam elementum. Augue mauris augue neque gravida in fermentum et sollicitudin. Sapien nec sagittis aliquam malesuada bibendum arcu. Urna duis convallis convallis tellus id interdum velit. Tellus in hac habitasse platea dictumst vestibulum. Fames ac turpis egestas maecenas pharetra convallis. Diam volutpat commodo sed egestas egestas fringilla phasellus faucibus. Placerat orci nulla pellentesque dignissim enim sit amet venenatis. Sed adipiscing diam donec adipiscing. Praesent elementum facilisis leo vel fringilla est. Sed enim ut sem viverra aliquet eget sit amet tellus. Proin sagittis nisl rhoncus mattis rhoncus urna neque viverra. Turpis egestas pretium aenean pharetra magna ac placerat vestibulum. Massa id neque aliquam vestibulum morbi blandit cursus risus. Vitae congue eu consequat ac. Egestas erat imperdiet sed euismod nisi porta lorem mollis aliquam. Dolor purus non enim praesent elementum facilisis. Ultrices mi tempus imperdiet nulla malesuada pellentesque elit. In est ante in nibh. + +Facilisis gravida neque convallis a. Urna nunc id cursus metus aliquam eleifend mi. Lacus luctus accumsan tortor posuere ac. Molestie nunc non blandit massa. Iaculis urna id volutpat lacus laoreet non. Cursus vitae congue mauris rhoncus aenean. Nunc vel risus commodo viverra maecenas. A pellentesque sit amet porttitor eget dolor morbi. Leo vel orci porta non pulvinar neque laoreet suspendisse. Sit amet facilisis magna etiam tempor. Consectetur a erat nam at lectus urna duis convallis convallis. Vestibulum morbi blandit cursus risus at ultrices. Dolor purus non enim praesent elementum. Adipiscing elit pellentesque habitant morbi tristique senectus et netus et. Et odio pellentesque diam volutpat commodo sed egestas egestas fringilla. Leo vel fringilla est ullamcorper eget nulla. Dui ut ornare lectus sit amet. Erat pellentesque adipiscing commodo elit at imperdiet dui accumsan sit. + +Tristique senectus et netus et. Pellentesque diam volutpat commodo sed egestas egestas fringilla. Mauris pharetra et ultrices neque ornare aenean. Amet tellus cras adipiscing enim. Convallis aenean et tortor at risus viverra adipiscing at. Proin sagittis nisl rhoncus mattis rhoncus urna neque viverra justo. Dictumst vestibulum rhoncus est pellentesque elit. Fringilla ut morbi tincidunt augue interdum velit euismod in pellentesque. Dictum at tempor commodo ullamcorper a lacus vestibulum. Sed viverra tellus in hac habitasse platea. Sed id semper risus in hendrerit. In hendrerit gravida rutrum quisque non tellus orci ac. Sit amet risus nullam eget. Sit amet est placerat in egestas erat imperdiet sed. In nisl nisi scelerisque eu ultrices. Sit amet mattis vulputate enim nulla aliquet. + +Dignissim suspendisse in est ante in nibh mauris cursus. Vitae proin sagittis nisl rhoncus. Id leo in vitae turpis massa sed elementum. Lobortis elementum nibh tellus molestie nunc non blandit massa enim. Arcu dictum varius duis at consectetur. Suspendisse faucibus interdum posuere lorem ipsum dolor sit amet consectetur. Imperdiet nulla malesuada pellentesque elit eget gravida cum sociis. Sed adipiscing diam donec adipiscing. Purus sit amet volutpat consequat mauris nunc congue nisi vitae. Elementum nisi quis eleifend quam adipiscing vitae proin sagittis nisl. Mattis ullamcorper velit sed ullamcorper morbi tincidunt ornare massa. Sit amet nisl purus in mollis nunc sed. Turpis tincidunt id aliquet risus feugiat in ante. Id diam maecenas ultricies mi eget mauris pharetra et ultrices. + +Aliquam purus sit amet luctus venenatis lectus magna fringilla urna. Id diam vel quam elementum pulvinar. Elementum sagittis vitae et leo duis. Viverra aliquet eget sit amet tellus cras adipiscing enim eu. Et tortor at risus viverra adipiscing at in tellus integer. Purus in massa tempor nec feugiat. Augue neque gravida in fermentum et sollicitudin ac orci. Sodales ut eu sem integer vitae justo eget magna fermentum. Netus et malesuada fames ac. Augue interdum velit euismod in. Sed elementum tempus egestas sed sed risus pretium. Mattis vulputate enim nulla aliquet porttitor lacus luctus. Dui vivamus arcu felis bibendum ut tristique et egestas quis. + +Viverra justo nec ultrices dui sapien. Quisque egestas diam in arcu cursus euismod quis viverra nibh. Nam libero justo laoreet sit amet cursus sit amet. Lacus sed viverra tellus in hac habitasse. Blandit aliquam etiam erat velit scelerisque in. Ut sem nulla pharetra diam sit amet nisl suscipit adipiscing. Diam sollicitudin tempor id eu nisl nunc. Eget duis at tellus at urna condimentum mattis. Urna porttitor rhoncus dolor purus non enim praesent elementum facilisis. Sed turpis tincidunt id aliquet risus feugiat. Est velit egestas dui id ornare arcu odio ut sem. Nibh sit amet commodo nulla facilisi nullam vehicula. Sit amet consectetur adipiscing elit duis tristique sollicitudin. Eu facilisis sed odio morbi. Massa id neque aliquam vestibulum morbi. In eu mi bibendum neque egestas congue quisque egestas. Massa sed elementum tempus egestas sed sed risus. Quam elementum pulvinar etiam non. At augue eget arcu dictum varius duis at consectetur lorem. + +Penatibus et magnis dis parturient montes nascetur ridiculus. Dictumst quisque sagittis purus sit amet volutpat consequat. Bibendum at varius vel pharetra. Sed adipiscing diam donec adipiscing tristique risus nec feugiat in. Phasellus faucibus scelerisque eleifend donec pretium. Vitae tortor condimentum lacinia quis vel eros. Ac tincidunt vitae semper quis lectus nulla at volutpat diam. Eget sit amet tellus cras adipiscing. Morbi tristique senectus et netus. Nullam vehicula ipsum a arcu cursus vitae congue mauris rhoncus. Auctor urna nunc id cursus metus aliquam eleifend. Ultrices vitae auctor eu augue. Eu non diam phasellus vestibulum lorem sed risus ultricies. Fames ac turpis egestas sed tempus. Volutpat blandit aliquam etiam erat. Dictum varius duis at consectetur lorem. Sit amet volutpat consequat mauris nunc congue. Volutpat sed cras ornare arcu dui vivamus arcu felis. + +Scelerisque fermentum dui faucibus in ornare quam viverra. Interdum velit laoreet id donec ultrices tincidunt arcu. Netus et malesuada fames ac. Netus et malesuada fames ac turpis. Suscipit tellus mauris a diam maecenas sed enim ut sem. Id velit ut tortor pretium. Neque aliquam vestibulum morbi blandit cursus risus at. Cum sociis natoque penatibus et magnis dis parturient. Lobortis elementum nibh tellus molestie nunc non blandit. Ipsum dolor sit amet consectetur adipiscing elit duis tristique. Amet nisl purus in mollis. Amet massa vitae tortor condimentum lacinia quis vel eros donec. Proin sagittis nisl rhoncus mattis rhoncus urna neque viverra justo. + +Nullam ac tortor vitae purus faucibus. Dis parturient montes nascetur ridiculus mus mauris. Molestie at elementum eu facilisis sed odio morbi. Scelerisque felis imperdiet proin fermentum leo vel orci porta. Lectus proin nibh nisl condimentum id venenatis a. Eget nullam non nisi est sit amet facilisis. Hendrerit gravida rutrum quisque non tellus orci ac auctor. Ut faucibus pulvinar elementum integer enim. Rhoncus dolor purus non enim praesent elementum facilisis. Enim sed faucibus turpis in eu mi bibendum. Faucibus nisl tincidunt eget nullam. + +Cursus risus at ultrices mi tempus imperdiet nulla malesuada pellentesque. Pretium nibh ipsum consequat nisl vel pretium lectus quam. Semper viverra nam libero justo laoreet sit amet cursus sit. Augue eget arcu dictum varius duis at consectetur lorem donec. Et malesuada fames ac turpis. Erat nam at lectus urna duis convallis convallis. Dictum sit amet justo donec enim. Urna condimentum mattis pellentesque id nibh tortor id. Morbi tempus iaculis urna id. Lectus proin nibh nisl condimentum id venenatis a condimentum. Nibh sit amet commodo nulla facilisi nullam vehicula. Dui faucibus in ornare quam. Gravida arcu ac tortor dignissim convallis aenean. Consectetur adipiscing elit pellentesque habitant morbi tristique. Pulvinar elementum integer enim neque volutpat ac tincidunt vitae. Pharetra pharetra massa massa ultricies mi quis hendrerit. Dictum at tempor commodo ullamcorper a lacus vestibulum sed. Mattis pellentesque id nibh tortor id. Ultricies integer quis auctor elit sed vulputate. Pretium vulputate sapien nec sagittis aliquam malesuada. + +Auctor augue mauris augue neque gravida. Porttitor lacus luctus accumsan tortor posuere ac ut. Urna neque viverra justo nec ultrices dui. Sit amet est placerat in egestas. Urna nec tincidunt praesent semper feugiat nibh sed pulvinar. Tincidunt eget nullam non nisi est sit amet facilisis magna. Elementum tempus egestas sed sed risus pretium quam vulputate dignissim. Fermentum posuere urna nec tincidunt praesent semper feugiat nibh sed. Porttitor eget dolor morbi non arcu risus quis. Non quam lacus suspendisse faucibus interdum. Venenatis cras sed felis eget velit aliquet sagittis id. Arcu ac tortor dignissim convallis aenean et. Morbi tincidunt ornare massa eget egestas purus. Ac feugiat sed lectus vestibulum mattis ullamcorper velit sed ullamcorper. Vestibulum morbi blandit cursus risus at ultrices. Volutpat blandit aliquam etiam erat velit scelerisque. + +Et egestas quis ipsum suspendisse. Amet consectetur adipiscing elit duis. Purus ut faucibus pulvinar elementum integer enim neque. Cursus vitae congue mauris rhoncus aenean vel elit scelerisque mauris. Tincidunt eget nullam non nisi est. Aliquam purus sit amet luctus. Dui ut ornare lectus sit amet est placerat in. Fringilla ut morbi tincidunt augue interdum velit euismod in. Felis eget nunc lobortis mattis aliquam faucibus purus in. Suspendisse interdum consectetur libero id faucibus nisl. + +Scelerisque fermentum dui faucibus in ornare quam. Lectus proin nibh nisl condimentum id venenatis a condimentum vitae. Fames ac turpis egestas integer eget aliquet nibh praesent tristique. Arcu non sodales neque sodales ut etiam sit. Pharetra convallis posuere morbi leo urna. Nec dui nunc mattis enim ut tellus. Nunc sed augue lacus viverra vitae. Consequat id porta nibh venenatis cras sed felis. Dolor sit amet consectetur adipiscing. Tellus rutrum tellus pellentesque eu tincidunt tortor aliquam nulla. + +Metus aliquam eleifend mi in nulla posuere. Blandit massa enim nec dui nunc mattis enim. Aliquet nibh praesent tristique magna. In aliquam sem fringilla ut. Magna fermentum iaculis eu non. Eget aliquet nibh praesent tristique magna sit amet purus. Ultrices gravida dictum fusce ut placerat orci. Fermentum posuere urna nec tincidunt praesent. Enim tortor at auctor urna nunc. Ridiculus mus mauris vitae ultricies leo integer malesuada nunc vel. Sed id semper risus in hendrerit gravida rutrum. Vestibulum lectus mauris ultrices eros in cursus turpis. Et sollicitudin ac orci phasellus egestas tellus rutrum. Pellentesque elit ullamcorper dignissim cras tincidunt lobortis feugiat vivamus at. Metus vulputate eu scelerisque felis imperdiet proin fermentum leo. Porta non pulvinar neque laoreet suspendisse. Suscipit adipiscing bibendum est ultricies integer quis auctor elit sed. Euismod in pellentesque massa placerat duis ultricies lacus sed. Pellentesque adipiscing commodo elit at imperdiet dui accumsan sit amet. + +Pellentesque eu tincidunt tortor aliquam nulla facilisi. Commodo nulla facilisi nullam vehicula ipsum a arcu. Commodo quis imperdiet massa tincidunt nunc pulvinar sapien et. Faucibus purus in massa tempor. Purus semper eget duis at tellus at urna condimentum. Vivamus at augue eget arcu dictum. Lacus vel facilisis volutpat est velit egestas dui id. Malesuada fames ac turpis egestas maecenas pharetra. Nunc faucibus a pellentesque sit amet porttitor eget dolor. Ultricies tristique nulla aliquet enim. Vel risus commodo viverra maecenas accumsan lacus vel facilisis volutpat. Dignissim diam quis enim lobortis scelerisque. Donec ultrices tincidunt arcu non sodales neque sodales ut etiam. + +Vitae proin sagittis nisl rhoncus mattis rhoncus urna neque. Fermentum leo vel orci porta non. At elementum eu facilisis sed. Quis enim lobortis scelerisque fermentum. Fermentum odio eu feugiat pretium nibh ipsum consequat. Habitant morbi tristique senectus et netus et. Enim praesent elementum facilisis leo vel fringilla est ullamcorper. Egestas quis ipsum suspendisse ultrices gravida dictum. Nam libero justo laoreet sit amet cursus sit amet. Viverra tellus in hac habitasse platea dictumst vestibulum. Varius vel pharetra vel turpis nunc eget. Nullam non nisi est sit amet facilisis magna. Ullamcorper eget nulla facilisi etiam dignissim diam. Ante metus dictum at tempor commodo ullamcorper a lacus. + +Etiam non quam lacus suspendisse. Ut venenatis tellus in metus vulputate eu scelerisque felis. Pulvinar sapien et ligula ullamcorper malesuada proin libero. Consequat interdum varius sit amet mattis. Nunc eget lorem dolor sed viverra ipsum nunc aliquet. Potenti nullam ac tortor vitae purus faucibus ornare. Urna et pharetra pharetra massa massa ultricies mi quis hendrerit. Purus in mollis nunc sed id. Pharetra vel turpis nunc eget lorem dolor sed viverra. Et netus et malesuada fames ac turpis. Libero id faucibus nisl tincidunt eget nullam non nisi. Cursus sit amet dictum sit amet. Porttitor lacus luctus accumsan tortor. + +Volutpat diam ut venenatis tellus in metus vulputate eu scelerisque. Sed viverra tellus in hac habitasse. Aliquam sem et tortor consequat id. Pellentesque habitant morbi tristique senectus et netus et. Consectetur purus ut faucibus pulvinar elementum. Aliquam malesuada bibendum arcu vitae elementum curabitur vitae nunc sed. Malesuada bibendum arcu vitae elementum curabitur vitae nunc sed. Sollicitudin tempor id eu nisl nunc mi ipsum. Fringilla phasellus faucibus scelerisque eleifend donec pretium vulputate sapien nec. Quis eleifend quam adipiscing vitae proin sagittis nisl rhoncus. Bibendum neque egestas congue quisque egestas. A iaculis at erat pellentesque adipiscing commodo elit at imperdiet. Pulvinar etiam non quam lacus. Adipiscing commodo elit at imperdiet. Scelerisque eu ultrices vitae auctor. Sed cras ornare arcu dui vivamus arcu felis bibendum ut. Ornare lectus sit amet est. + +Consequat semper viverra nam libero justo laoreet sit. Imperdiet sed euismod nisi porta lorem mollis aliquam ut porttitor. Cras sed felis eget velit aliquet sagittis id consectetur. Dolor morbi non arcu risus quis. Adipiscing tristique risus nec feugiat in fermentum posuere urna. Dolor magna eget est lorem ipsum dolor. Mauris pharetra et ultrices neque ornare aenean euismod. Nulla facilisi etiam dignissim diam quis. Ultrices tincidunt arcu non sodales. Fames ac turpis egestas maecenas pharetra convallis posuere morbi leo. Interdum varius sit amet mattis vulputate. Tincidunt praesent semper feugiat nibh sed pulvinar. Quisque sagittis purus sit amet volutpat. + +Sed vulputate odio ut enim blandit. Vitae auctor eu augue ut lectus arcu bibendum. Consectetur adipiscing elit pellentesque habitant morbi tristique senectus et. Scelerisque eu ultrices vitae auctor eu augue. Etiam dignissim diam quis enim lobortis scelerisque fermentum dui faucibus. Tellus integer feugiat scelerisque varius. Vulputate enim nulla aliquet porttitor lacus luctus accumsan tortor. Amet nisl purus in mollis. Scelerisque viverra mauris in aliquam sem fringilla ut morbi tincidunt. Semper eget duis at tellus at. Erat velit scelerisque in dictum non consectetur a erat nam. Gravida rutrum quisque non tellus orci. Morbi blandit cursus risus at. Mauris sit amet massa vitae. Non odio euismod lacinia at quis risus sed vulputate. Fermentum posuere urna nec tincidunt praesent. Ut eu sem integer vitae justo eget magna fermentum iaculis. Ullamcorper velit sed ullamcorper morbi tincidunt ornare massa. Arcu cursus euismod quis viverra nibh. Arcu dui vivamus arcu felis bibendum. + +Eros in cursus turpis massa tincidunt dui ut. Urna condimentum mattis pellentesque id nibh tortor id aliquet lectus. Nibh venenatis cras sed felis. Ac felis donec et odio pellentesque diam. Ultricies lacus sed turpis tincidunt id aliquet risus. Diam volutpat commodo sed egestas. Dignissim sodales ut eu sem integer vitae. Pellentesque eu tincidunt tortor aliquam nulla facilisi. Et tortor consequat id porta nibh venenatis cras sed. \ No newline at end of file diff --git a/node/impl/client/testdata/payload2.txt b/node/impl/client/testdata/payload2.txt new file mode 100644 index 00000000000..16fb150f5b2 --- /dev/null +++ b/node/impl/client/testdata/payload2.txt @@ -0,0 +1,49 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Vitae semper quis lectus nulla at volutpat diam ut venenatis. Ac tortor dignissim convallis aenean et tortor at. Faucibus ornare suspendisse sed nisi lacus sed. Commodo ullamcorper a lacus vestibulum sed arcu non. Est pellentesque elit ullamcorper dignissim. Quam quisque id diam vel quam. Pretium aenean pharetra magna ac. In nulla posuere sollicitudin aliquam ultrices. Sed arcu non odio euismod lacinia at. Suspendisse ultrices gravida dictum fusce ut placerat orci nulla pellentesque. Feugiat vivamus at augue eget arcu. + +Pellentesque nec nam aliquam sem et tortor. Vitae tortor condimentum lacinia quis vel. Cras pulvinar mattis nunc sed. In massa tempor nec feugiat. Ornare arcu odio ut sem nulla. Diam maecenas sed enim ut sem. Pretium vulputate sapien nec sagittis. Bibendum arcu vitae elementum curabitur vitae nunc sed velit dignissim. Duis ut diam quam nulla porttitor massa. Viverra mauris in aliquam sem fringilla ut morbi. Ullamcorper eget nulla facilisi etiam dignissim. Vulputate mi sit amet mauris commodo quis imperdiet massa tincidunt. Nunc consequat interdum varius sit. Nunc mi ipsum faucibus vitae aliquet nec ullamcorper. Nunc sed augue lacus viverra. Lobortis scelerisque fermentum dui faucibus in ornare quam. Urna neque viverra justo nec ultrices. Varius vel pharetra vel turpis nunc eget lorem dolor sed. + +Feugiat nisl pretium fusce id velit ut tortor pretium. Lorem dolor sed viverra ipsum nunc aliquet bibendum. Ultrices vitae auctor eu augue ut lectus. Pharetra massa massa ultricies mi quis. Nibh cras pulvinar mattis nunc sed blandit libero. Ac felis donec et odio pellentesque diam volutpat. Lectus proin nibh nisl condimentum id venenatis. Quis vel eros donec ac odio. Commodo sed egestas egestas fringilla phasellus faucibus scelerisque eleifend donec. Adipiscing diam donec adipiscing tristique. + +Tempus imperdiet nulla malesuada pellentesque elit eget gravida cum sociis. Libero nunc consequat interdum varius sit. Et pharetra pharetra massa massa. Feugiat pretium nibh ipsum consequat. Amet commodo nulla facilisi nullam vehicula. Ornare arcu dui vivamus arcu felis bibendum ut tristique. At erat pellentesque adipiscing commodo elit at imperdiet dui. Auctor neque vitae tempus quam pellentesque nec nam aliquam sem. Eget velit aliquet sagittis id consectetur. Enim diam vulputate ut pharetra sit amet aliquam id diam. Eget velit aliquet sagittis id consectetur purus ut faucibus pulvinar. Amet porttitor eget dolor morbi. Felis eget velit aliquet sagittis id. Facilisis magna etiam tempor orci eu. Lacus suspendisse faucibus interdum posuere lorem. Pharetra et ultrices neque ornare aenean euismod. Platea dictumst quisque sagittis purus. + +Quis varius quam quisque id diam vel quam elementum. Augue mauris augue neque gravida in fermentum et sollicitudin. Sapien nec sagittis aliquam malesuada bibendum arcu. Urna duis convallis convallis tellus id interdum velit. Tellus in hac habitasse platea dictumst vestibulum. Fames ac turpis egestas maecenas pharetra convallis. Diam volutpat commodo sed egestas egestas fringilla phasellus faucibus. Placerat orci nulla pellentesque dignissim enim sit amet venenatis. Sed adipiscing diam donec adipiscing. Praesent elementum facilisis leo vel fringilla est. Sed enim ut sem viverra aliquet eget sit amet tellus. Proin sagittis nisl rhoncus mattis rhoncus urna neque viverra. Turpis egestas pretium aenean pharetra magna ac placerat vestibulum. Massa id neque aliquam vestibulum morbi blandit cursus risus. Vitae congue eu consequat ac. Egestas erat imperdiet sed euismod nisi porta lorem mollis aliquam. Dolor purus non enim praesent elementum facilisis. Ultrices mi tempus imperdiet nulla malesuada pellentesque elit. In est ante in nibh. + +Facilisis gravida neque convallis a. Urna nunc id cursus metus aliquam eleifend mi. Lacus luctus accumsan tortor posuere ac. Molestie nunc non blandit massa. Iaculis urna id volutpat lacus laoreet non. Cursus vitae congue mauris rhoncus aenean. Nunc vel risus commodo viverra maecenas. A pellentesque sit amet porttitor eget dolor morbi. Leo vel orci porta non pulvinar neque laoreet suspendisse. Sit amet facilisis magna etiam tempor. Consectetur a erat nam at lectus urna duis convallis convallis. Vestibulum morbi blandit cursus risus at ultrices. Dolor purus non enim praesent elementum. Adipiscing elit pellentesque habitant morbi tristique senectus et netus et. Et odio pellentesque diam volutpat commodo sed egestas egestas fringilla. Leo vel fringilla est ullamcorper eget nulla. Dui ut ornare lectus sit amet. Erat pellentesque adipiscing commodo elit at imperdiet dui accumsan sit. + +Tristique senectus et netus et. Pellentesque diam volutpat commodo sed egestas egestas fringilla. Mauris pharetra et ultrices neque ornare aenean. Amet tellus cras adipiscing enim. Convallis aenean et tortor at risus viverra adipiscing at. Proin sagittis nisl rhoncus mattis rhoncus urna neque viverra justo. Dictumst vestibulum rhoncus est pellentesque elit. Fringilla ut morbi tincidunt augue interdum velit euismod in pellentesque. Dictum at tempor commodo ullamcorper a lacus vestibulum. Sed viverra tellus in hac habitasse platea. Sed id semper risus in hendrerit. In hendrerit gravida rutrum quisque non tellus orci ac. Sit amet risus nullam eget. Sit amet est placerat in egestas erat imperdiet sed. In nisl nisi scelerisque eu ultrices. Sit amet mattis vulputate enim nulla aliquet. + +Dignissim suspendisse in est ante in nibh mauris cursus. Vitae proin sagittis nisl rhoncus. Id leo in vitae turpis massa sed elementum. Lobortis elementum nibh tellus molestie nunc non blandit massa enim. Arcu dictum varius duis at consectetur. Suspendisse faucibus interdum posuere lorem ipsum dolor sit amet consectetur. Imperdiet nulla malesuada pellentesque elit eget gravida cum sociis. Sed adipiscing diam donec adipiscing. Purus sit amet volutpat consequat mauris nunc congue nisi vitae. Elementum nisi quis eleifend quam adipiscing vitae proin sagittis nisl. Mattis ullamcorper velit sed ullamcorper morbi tincidunt ornare massa. Sit amet nisl purus in mollis nunc sed. Turpis tincidunt id aliquet risus feugiat in ante. Id diam maecenas ultricies mi eget mauris pharetra et ultrices. + +Aliquam purus sit amet luctus venenatis lectus magna fringilla urna. Id diam vel quam elementum pulvinar. Elementum sagittis vitae et leo duis. Viverra aliquet eget sit amet tellus cras adipiscing enim eu. Et tortor at risus viverra adipiscing at in tellus integer. Purus in massa tempor nec feugiat. Augue neque gravida in fermentum et sollicitudin ac orci. Sodales ut eu sem integer vitae justo eget magna fermentum. Netus et malesuada fames ac. Augue interdum velit euismod in. Sed elementum tempus egestas sed sed risus pretium. Mattis vulputate enim nulla aliquet porttitor lacus luctus. Dui vivamus arcu felis bibendum ut tristique et egestas quis. + +Viverra justo nec ultrices dui sapien. Quisque egestas diam in arcu cursus euismod quis viverra nibh. Nam libero justo laoreet sit amet cursus sit amet. Lacus sed viverra tellus in hac habitasse. Blandit aliquam etiam erat velit scelerisque in. Ut sem nulla pharetra diam sit amet nisl suscipit adipiscing. Diam sollicitudin tempor id eu nisl nunc. Eget duis at tellus at urna condimentum mattis. Urna porttitor rhoncus dolor purus non enim praesent elementum facilisis. Sed turpis tincidunt id aliquet risus feugiat. Est velit egestas dui id ornare arcu odio ut sem. Nibh sit amet commodo nulla facilisi nullam vehicula. Sit amet consectetur adipiscing elit duis tristique sollicitudin. Eu facilisis sed odio morbi. Massa id neque aliquam vestibulum morbi. In eu mi bibendum neque egestas congue quisque egestas. Massa sed elementum tempus egestas sed sed risus. Quam elementum pulvinar etiam non. At augue eget arcu dictum varius duis at consectetur lorem. + +Penatibus et magnis dis parturient montes nascetur ridiculus. Dictumst quisque sagittis purus sit amet volutpat consequat. Bibendum at varius vel pharetra. Sed adipiscing diam donec adipiscing tristique risus nec feugiat in. Phasellus faucibus scelerisque eleifend donec pretium. Vitae tortor condimentum lacinia quis vel eros. Ac tincidunt vitae semper quis lectus nulla at volutpat diam. Eget sit amet tellus cras adipiscing. Morbi tristique senectus et netus. Nullam vehicula ipsum a arcu cursus vitae congue mauris rhoncus. Auctor urna nunc id cursus metus aliquam eleifend. Ultrices vitae auctor eu augue. Eu non diam phasellus vestibulum lorem sed risus ultricies. Fames ac turpis egestas sed tempus. Volutpat blandit aliquam etiam erat. Dictum varius duis at consectetur lorem. Sit amet volutpat consequat mauris nunc congue. Volutpat sed cras ornare arcu dui vivamus arcu felis. + +Scelerisque fermentum dui faucibus in ornare quam viverra. Interdum velit laoreet id donec ultrices tincidunt arcu. Netus et malesuada fames ac. Netus et malesuada fames ac turpis. Suscipit tellus mauris a diam maecenas sed enim ut sem. Id velit ut tortor pretium. Neque aliquam vestibulum morbi blandit cursus risus at. Cum sociis natoque penatibus et magnis dis parturient. Lobortis elementum nibh tellus molestie nunc non blandit. Ipsum dolor sit amet consectetur adipiscing elit duis tristique. Amet nisl purus in mollis. Amet massa vitae tortor condimentum lacinia quis vel eros donec. Proin sagittis nisl rhoncus mattis rhoncus urna neque viverra justo. + +Nullam ac tortor vitae purus faucibus. Dis parturient montes nascetur ridiculus mus mauris. Molestie at elementum eu facilisis sed odio morbi. Scelerisque felis imperdiet proin fermentum leo vel orci porta. Lectus proin nibh nisl condimentum id venenatis a. Eget nullam non nisi est sit amet facilisis. Hendrerit gravida rutrum quisque non tellus orci ac auctor. Ut faucibus pulvinar elementum integer enim. Rhoncus dolor purus non enim praesent elementum facilisis. Enim sed faucibus turpis in eu mi bibendum. Faucibus nisl tincidunt eget nullam. + +Cursus risus at ultrices mi tempus imperdiet nulla malesuada pellentesque. Pretium nibh ipsum consequat nisl vel pretium lectus quam. Semper viverra nam libero justo laoreet sit amet cursus sit. Augue eget arcu dictum varius duis at consectetur lorem donec. Et malesuada fames ac turpis. Erat nam at lectus urna duis convallis convallis. Dictum sit amet justo donec enim. Urna condimentum mattis pellentesque id nibh tortor id. Morbi tempus iaculis urna id. Lectus proin nibh nisl condimentum id venenatis a condimentum. Nibh sit amet commodo nulla facilisi nullam vehicula. Dui faucibus in ornare quam. Gravida arcu ac tortor dignissim convallis aenean. Consectetur adipiscing elit pellentesque habitant morbi tristique. Pulvinar elementum integer enim neque volutpat ac tincidunt vitae. Pharetra pharetra massa massa ultricies mi quis hendrerit. Dictum at tempor commodo ullamcorper a lacus vestibulum sed. Mattis pellentesque id nibh tortor id. Ultricies integer quis auctor elit sed vulputate. Pretium vulputate sapien nec sagittis aliquam malesuada. + +Auctor augue mauris augue neque gravida. Porttitor lacus luctus accumsan tortor posuere ac ut. Urna neque viverra justo nec ultrices dui. Sit amet est placerat in egestas. Urna nec tincidunt praesent semper feugiat nibh sed pulvinar. Tincidunt eget nullam non nisi est sit amet facilisis magna. Elementum tempus egestas sed sed risus pretium quam vulputate dignissim. Fermentum posuere urna nec tincidunt praesent semper feugiat nibh sed. Porttitor eget dolor morbi non arcu risus quis. Non quam lacus suspendisse faucibus interdum. Venenatis cras sed felis eget velit aliquet sagittis id. Arcu ac tortor dignissim convallis aenean et. Morbi tincidunt ornare massa eget egestas purus. Ac feugiat sed lectus vestibulum mattis ullamcorper velit sed ullamcorper. Vestibulum morbi blandit cursus risus at ultrices. Volutpat blandit aliquam etiam erat velit scelerisque. + +Et egestas quis ipsum suspendisse. Amet consectetur adipiscing elit duis. Purus ut faucibus pulvinar elementum integer enim neque. Cursus vitae congue mauris rhoncus aenean vel elit scelerisque mauris. Tincidunt eget nullam non nisi est. Aliquam purus sit amet luctus. Dui ut ornare lectus sit amet est placerat in. Fringilla ut morbi tincidunt augue interdum velit euismod in. Felis eget nunc lobortis mattis aliquam faucibus purus in. Suspendisse interdum consectetur libero id faucibus nisl. + +Scelerisque fermentum dui faucibus in ornare quam. Lectus proin nibh nisl condimentum id venenatis a condimentum vitae. Fames ac turpis egestas integer eget aliquet nibh praesent tristique. Arcu non sodales neque sodales ut etiam sit. Pharetra convallis posuere morbi leo urna. Nec dui nunc mattis enim ut tellus. Nunc sed augue lacus viverra vitae. Consequat id porta nibh venenatis cras sed felis. Dolor sit amet consectetur adipiscing. Tellus rutrum tellus pellentesque eu tincidunt tortor aliquam nulla. + +Metus aliquam eleifend mi in nulla posuere. Blandit massa enim nec dui nunc mattis enim. Aliquet nibh praesent tristique magna. In aliquam sem fringilla ut. Magna fermentum iaculis eu non. Eget aliquet nibh praesent tristique magna sit amet purus. Ultrices gravida dictum fusce ut placerat orci. Fermentum posuere urna nec tincidunt praesent. Enim tortor at auctor urna nunc. Ridiculus mus mauris vitae ultricies leo integer malesuada nunc vel. Sed id semper risus in hendrerit gravida rutrum. Vestibulum lectus mauris ultrices eros in cursus turpis. Et sollicitudin ac orci phasellus egestas tellus rutrum. Pellentesque elit ullamcorper dignissim cras tincidunt lobortis feugiat vivamus at. Metus vulputate eu scelerisque felis imperdiet proin fermentum leo. Porta non pulvinar neque laoreet suspendisse. Suscipit adipiscing bibendum est ultricies integer quis auctor elit sed. Euismod in pellentesque massa placerat duis ultricies lacus sed. Pellentesque adipiscing commodo elit at imperdiet dui accumsan sit amet. + +Pellentesque eu tincidunt tortor aliquam nulla facilisi. Commodo nulla facilisi nullam vehicula ipsum a arcu. Commodo quis imperdiet massa tincidunt nunc pulvinar sapien et. Faucibus purus in massa tempor. Purus semper eget duis at tellus at urna condimentum. Vivamus at augue eget arcu dictum. Lacus vel facilisis volutpat est velit egestas dui id. Malesuada fames ac turpis egestas maecenas pharetra. Nunc faucibus a pellentesque sit amet porttitor eget dolor. Ultricies tristique nulla aliquet enim. Vel risus commodo viverra maecenas accumsan lacus vel facilisis volutpat. Dignissim diam quis enim lobortis scelerisque. Donec ultrices tincidunt arcu non sodales neque sodales ut etiam. + +Vitae proin sagittis nisl rhoncus mattis rhoncus urna neque. Fermentum leo vel orci porta non. At elementum eu facilisis sed. Quis enim lobortis scelerisque fermentum. Fermentum odio eu feugiat pretium nibh ipsum consequat. Habitant morbi tristique senectus et netus et. Enim praesent elementum facilisis leo vel fringilla est ullamcorper. Egestas quis ipsum suspendisse ultrices gravida dictum. Nam libero justo laoreet sit amet cursus sit amet. Viverra tellus in hac habitasse platea dictumst vestibulum. Varius vel pharetra vel turpis nunc eget. Nullam non nisi est sit amet facilisis magna. Ullamcorper eget nulla facilisi etiam dignissim diam. Ante metus dictum at tempor commodo ullamcorper a lacus. + +Etiam non quam lacus suspendisse. Ut venenatis tellus in metus vulputate eu scelerisque felis. Pulvinar sapien et ligula ullamcorper malesuada proin libero. Consequat interdum varius sit amet mattis. Nunc eget lorem dolor sed viverra ipsum nunc aliquet. Potenti nullam ac tortor vitae purus faucibus ornare. Urna et pharetra pharetra massa massa ultricies mi quis hendrerit. Purus in mollis nunc sed id. Pharetra vel turpis nunc eget lorem dolor sed viverra. Et netus et malesuada fames ac turpis. Libero id faucibus nisl tincidunt eget nullam non nisi. Cursus sit amet dictum sit amet. Porttitor lacus luctus accumsan tortor. + +Volutpat diam ut venenatis tellus in metus vulputate eu scelerisque. Sed viverra tellus in hac habitasse. Aliquam sem et tortor consequat id. Pellentesque habitant morbi tristique senectus et netus et. Consectetur purus ut faucibus pulvinar elementum. Aliquam malesuada bibendum arcu vitae elementum curabitur vitae nunc sed. Malesuada bibendum arcu vitae elementum curabitur vitae nunc sed. Sollicitudin tempor id eu nisl nunc mi ipsum. Fringilla phasellus faucibus scelerisque eleifend donec pretium vulputate sapien nec. Quis eleifend quam adipiscing vitae proin sagittis nisl rhoncus. Bibendum neque egestas congue quisque egestas. A iaculis at erat pellentesque adipiscing commodo elit at imperdiet. Pulvinar etiam non quam lacus. Adipiscing commodo elit at imperdiet. Scelerisque eu ultrices vitae auctor. Sed cras ornare arcu dui vivamus arcu felis bibendum ut. Ornare lectus sit amet est. + +Consequat semper viverra nam libero justo laoreet sit. Imperdiet sed euismod nisi porta lorem mollis aliquam ut porttitor. Cras sed felis eget velit aliquet sagittis id consectetur. Dolor morbi non arcu risus quis. Adipiscing tristique risus nec feugiat in fermentum posuere urna. Dolor magna eget est lorem ipsum dolor. Mauris pharetra et ultrices neque ornare aenean euismod. Nulla facilisi etiam dignissim diam quis. Ultrices tincidunt arcu non sodales. Fames ac turpis egestas maecenas pharetra convallis posuere morbi leo. Interdum varius sit amet mattis vulputate. Tincidunt praesent semper feugiat nibh sed pulvinar. Quisque sagittis purus sit amet volutpat. + +Sed vulputate odio ut enim blandit. Vitae auctor eu augue ut lectus arcu bibendum. Consectetur adipiscing elit pellentesque habitant morbi tristique senectus et. Scelerisque eu ultrices vitae auctor eu augue. Etiam dignissim diam quis enim lobortis scelerisque fermentum dui faucibus. Tellus integer feugiat scelerisque varius. Vulputate enim nulla aliquet porttitor lacus luctus accumsan tortor. Amet nisl purus in mollis. Scelerisque viverra mauris in aliquam sem fringilla ut morbi tincidunt. Semper eget duis at tellus at. Erat velit scelerisque in dictum non consectetur a erat nam. Gravida rutrum quisque non tellus orci. Morbi blandit cursus risus at. Mauris sit amet massa vitae. Non odio euismod lacinia at quis risus sed vulputate. Fermentum posuere urna nec tincidunt praesent. Ut eu sem integer vitae justo eget magna fermentum iaculis. Ullamcorper velit sed ullamcorper morbi tincidunt ornare massa. Arcu cursus euismod quis viverra nibh. Arcu dui vivamus arcu felis bibendum. + +Eros in cursus turpis massa tincidunt dui ut. Aarsh shah is simply an amazing person. Urna condimentum mattis pellentesque id nibh tortor id aliquet lectus. Nibh venenatis cras sed felis. Ac felis donec et odio pellentesque diam. Ultricies lacus sed turpis tincidunt id aliquet risus. Diam volutpat commodo sed egestas. Dignissim sodales ut eu sem integer vitae. Pellentesque eu tincidunt tortor aliquam nulla facilisi. Et tortor consequat id porta nibh venenatis cras sed. \ No newline at end of file diff --git a/node/modules/client.go b/node/modules/client.go index c0f8747b3de..7ca97f0e0da 100644 --- a/node/modules/client.go +++ b/node/modules/client.go @@ -14,6 +14,11 @@ import ( dtimpl "github.com/filecoin-project/go-data-transfer/impl" dtnet "github.com/filecoin-project/go-data-transfer/network" dtgstransport "github.com/filecoin-project/go-data-transfer/transport/graphsync" + "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/namespace" + "github.com/libp2p/go-libp2p-core/host" + "github.com/filecoin-project/go-fil-markets/discovery" discoveryimpl "github.com/filecoin-project/go-fil-markets/discovery/impl" "github.com/filecoin-project/go-fil-markets/retrievalmarket" @@ -23,12 +28,9 @@ import ( storageimpl "github.com/filecoin-project/go-fil-markets/storagemarket/impl" "github.com/filecoin-project/go-fil-markets/storagemarket/impl/requestvalidation" smnet "github.com/filecoin-project/go-fil-markets/storagemarket/network" - "github.com/filecoin-project/go-state-types/abi" - "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/namespace" - "github.com/libp2p/go-libp2p-core/host" - "github.com/filecoin-project/lotus/blockstore" + "github.com/filecoin-project/lotus/markets/storageadapter" + "github.com/filecoin-project/lotus/chain/market" "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/markets" @@ -39,7 +41,6 @@ import ( "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/node/repo/imports" - "github.com/filecoin-project/lotus/node/repo/retrievalstoremgr" ) func HandleMigrateClientFunds(lc fx.Lifecycle, ds dtypes.MetadataDS, wallet full.WalletAPI, fundMgr *market.FundManager) { @@ -76,12 +77,19 @@ func HandleMigrateClientFunds(lc fx.Lifecycle, ds dtypes.MetadataDS, wallet full }) } -func ClientImportMgr(ds dtypes.MetadataDS, r repo.LockedRepo) dtypes.ClientImportMgr { - return imports.NewManager(namespace.Wrap(ds, datastore.NewKey("/client")), r.Path()) +func ClientImportMgr(ds dtypes.MetadataDS, r repo.LockedRepo) (dtypes.ClientImportMgr, error) { + // store the imports under the repo's `imports` subdirectory. + dir := filepath.Join(r.Path(), "imports") + if err := os.MkdirAll(dir, 0755); err != nil { + return nil, xerrors.Errorf("failed to create directory %s: %w", dir, err) + } + + ns := namespace.Wrap(ds, datastore.NewKey("/client")) + return imports.NewManager(ns, dir), nil } -// TODO Ge this to work when we work on IPFS integration. What we effectively need here is a cross-shard/cross-CAR files index for the Storage client's imports. -func ClientBlockstore(imgr dtypes.ClientImportMgr) dtypes.ClientBlockstore { +// TODO this should be removed. +func ClientBlockstore() dtypes.ClientBlockstore { // in most cases this is now unused in normal operations -- however, it's important to preserve for the IPFS use case return blockstore.WrapIDStore(blockstore.FromDatastore(datastore.NewMapDatastore())) } @@ -151,13 +159,30 @@ func NewClientDatastore(ds dtypes.MetadataDS) dtypes.ClientDatastore { return namespace.Wrap(ds, datastore.NewKey("/deals/client")) } -func StorageClient(lc fx.Lifecycle, h host.Host, dataTransfer dtypes.ClientDataTransfer, discovery *discoveryimpl.Local, deals dtypes.ClientDatastore, scn storagemarket.StorageClientNode, j journal.Journal) (storagemarket.StorageClient, error) { +// StorageBlockstoreAccessor returns the default storage blockstore accessor +// from the import manager. +func StorageBlockstoreAccessor(importmgr dtypes.ClientImportMgr) storagemarket.BlockstoreAccessor { + return storageadapter.NewImportsBlockstoreAccessor(importmgr) +} + +// RetrievalBlockstoreAccessor returns the default retrieval blockstore accessor +// using the subdirectory `retrievals` under the repo. +func RetrievalBlockstoreAccessor(r repo.LockedRepo) (retrievalmarket.BlockstoreAccessor, error) { + dir := filepath.Join(r.Path(), "retrievals") + if err := os.MkdirAll(dir, 0755); err != nil { + return nil, xerrors.Errorf("failed to create directory %s: %w", dir, err) + } + return retrievaladapter.NewCARBlockstoreAccessor(dir), nil +} + +func StorageClient(lc fx.Lifecycle, h host.Host, dataTransfer dtypes.ClientDataTransfer, discovery *discoveryimpl.Local, + deals dtypes.ClientDatastore, scn storagemarket.StorageClientNode, accessor storagemarket.BlockstoreAccessor, j journal.Journal) (storagemarket.StorageClient, error) { // go-fil-markets protocol retries: // 1s, 5s, 25s, 2m5s, 5m x 11 ~= 1 hour marketsRetryParams := smnet.RetryParameters(time.Second, 5*time.Minute, 15, 5) net := smnet.NewFromLibp2pHost(h, marketsRetryParams) - c, err := storageimpl.NewClient(net, dataTransfer, discovery, deals, scn, storageimpl.DealPollingInterval(time.Second)) + c, err := storageimpl.NewClient(net, dataTransfer, discovery, deals, scn, accessor, storageimpl.DealPollingInterval(time.Second)) if err != nil { return nil, err } @@ -180,12 +205,12 @@ func StorageClient(lc fx.Lifecycle, h host.Host, dataTransfer dtypes.ClientDataT // RetrievalClient creates a new retrieval client attached to the client blockstore func RetrievalClient(lc fx.Lifecycle, h host.Host, r repo.LockedRepo, dt dtypes.ClientDataTransfer, payAPI payapi.PaychAPI, resolver discovery.PeerResolver, - ds dtypes.MetadataDS, chainAPI full.ChainAPI, stateAPI full.StateAPI, j journal.Journal) (retrievalmarket.RetrievalClient, error) { + ds dtypes.MetadataDS, chainAPI full.ChainAPI, stateAPI full.StateAPI, accessor retrievalmarket.BlockstoreAccessor, j journal.Journal) (retrievalmarket.RetrievalClient, error) { adapter := retrievaladapter.NewRetrievalClientNode(payAPI, chainAPI, stateAPI) network := rmnet.NewFromLibp2pHost(h) - client, err := retrievalimpl.NewClient(network, - dt, adapter, resolver, namespace.Wrap(ds, datastore.NewKey("/retrievals/client"))) + ds = namespace.Wrap(ds, datastore.NewKey("/retrievals/client")) + client, err := retrievalimpl.NewClient(network, dt, adapter, resolver, ds, accessor) if err != nil { return nil, err } @@ -202,10 +227,3 @@ func RetrievalClient(lc fx.Lifecycle, h host.Host, r repo.LockedRepo, dt dtypes. }) return client, nil } - -// ClientBlockstoreRetrievalStoreManager is the default version of the RetrievalStoreManager that runs on multistore -func ClientBlockstoreRetrievalStoreManager(isIpfsRetrieval bool) func(bs dtypes.ClientBlockstore) (dtypes.ClientRetrievalStoreManager, error) { - return func(bs dtypes.ClientBlockstore) (dtypes.ClientRetrievalStoreManager, error) { - return retrievalstoremgr.NewBlockstoreRetrievalStoreManager(bs, isIpfsRetrieval), nil - } -} diff --git a/node/modules/dtypes/storage.go b/node/modules/dtypes/storage.go index b7f0a4af8aa..7ccf8946671 100644 --- a/node/modules/dtypes/storage.go +++ b/node/modules/dtypes/storage.go @@ -8,13 +8,13 @@ import ( format "github.com/ipfs/go-ipld-format" datatransfer "github.com/filecoin-project/go-data-transfer" + "github.com/filecoin-project/go-statestore" + "github.com/filecoin-project/go-fil-markets/piecestore" "github.com/filecoin-project/go-fil-markets/storagemarket/impl/requestvalidation" - "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/node/repo/imports" - "github.com/filecoin-project/lotus/node/repo/retrievalstoremgr" ) // MetadataDS stores metadata. By default it's namespaced under /metadata in @@ -74,7 +74,6 @@ type ClientBlockstore blockstore.BasicBlockstore type ClientDealStore *statestore.StateStore type ClientRequestValidator *requestvalidation.UnifiedRequestValidator type ClientDatastore datastore.Batching -type ClientRetrievalStoreManager retrievalstoremgr.RetrievalStoreManager type Graphsync graphsync.GraphExchange diff --git a/node/modules/graphsync.go b/node/modules/graphsync.go index a7f62db76ce..cbd67ad1fa3 100644 --- a/node/modules/graphsync.go +++ b/node/modules/graphsync.go @@ -1,9 +1,6 @@ package modules import ( - "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/filecoin-project/lotus/node/modules/helpers" - "github.com/filecoin-project/lotus/node/repo" "github.com/ipfs/go-graphsync" graphsyncimpl "github.com/ipfs/go-graphsync/impl" gsnet "github.com/ipfs/go-graphsync/network" @@ -11,6 +8,10 @@ import ( "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/peer" "go.uber.org/fx" + + "github.com/filecoin-project/lotus/node/modules/dtypes" + "github.com/filecoin-project/lotus/node/modules/helpers" + "github.com/filecoin-project/lotus/node/repo" ) // Graphsync creates a graphsync instance from the given loader and storer diff --git a/node/modules/ipfsclient.go b/node/modules/ipfs.go similarity index 69% rename from node/modules/ipfsclient.go rename to node/modules/ipfs.go index 24c5c96783e..14960830923 100644 --- a/node/modules/ipfsclient.go +++ b/node/modules/ipfs.go @@ -1,16 +1,29 @@ package modules import ( + bstore "github.com/ipfs/go-ipfs-blockstore" "go.uber.org/fx" "golang.org/x/xerrors" "github.com/multiformats/go-multiaddr" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/lotus/blockstore" + "github.com/filecoin-project/lotus/markets/retrievaladapter" + "github.com/filecoin-project/lotus/markets/storageadapter" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" ) +func IpfsStorageBlockstoreAccessor(ipfsBlockstore dtypes.ClientBlockstore) storagemarket.BlockstoreAccessor { + return storageadapter.NewFixedBlockstoreAccessor(bstore.Blockstore(ipfsBlockstore)) +} + +func IpfsRetrievalBlockstoreAccessor(ipfsBlockstore dtypes.ClientBlockstore) retrievalmarket.BlockstoreAccessor { + return retrievaladapter.NewFixedBlockstoreAccessor(bstore.Blockstore(ipfsBlockstore)) +} + // IpfsClientBlockstore returns a ClientBlockstore implementation backed by an IPFS node. // If ipfsMaddr is empty, a local IPFS node is assumed considering IPFS_PATH configuration. // If ipfsMaddr is not empty, it will connect to the remote IPFS node with the provided multiaddress. diff --git a/node/repo/imports/manager.go b/node/repo/imports/manager.go index 0bf14b015b2..3050f43c6a0 100644 --- a/node/repo/imports/manager.go +++ b/node/repo/imports/manager.go @@ -3,10 +3,10 @@ package imports import ( "encoding/json" "fmt" - "io/ioutil" + "os" + "path/filepath" "strconv" - "github.com/filecoin-project/go-fil-markets/shared" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore/query" logging "github.com/ipfs/go-log/v2" @@ -14,6 +14,8 @@ import ( "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" + + "github.com/filecoin-project/go-fil-markets/shared" ) var log = logging.Logger("importmgr") @@ -34,42 +36,98 @@ type LabelKey = string type LabelValue = string const ( - LSource = "source" // Function which created the import - LRootCid = "root" // Root CID - LFileName = "filename" // Local file path of the source file. - LCARPath = "carpath" // Path of the CARv2 file containing the imported data. + CAROwnerImportMgr = "importmgr" + CAROwnerUser = "user" +) + +const ( + LSource = LabelKey("source") // Function which created the import + LRootCid = LabelKey("root") // Root CID + LFileName = LabelKey("filename") // Local file path of the source file. + LCARPath = LabelKey("car_path") // Path of the CARv2 file containing the imported data. + LCAROwner = LabelKey("car_owner") // Owner of the CAR; "importmgr" is us; "user" or empty is them. ) func NewManager(ds datastore.Batching, rootDir string) *Manager { ds = namespace.Wrap(ds, datastore.NewKey("/stores")) ds = datastore.NewLogDatastore(ds, "storess") - return &Manager{ + m := &Manager{ ds: ds, rootDir: rootDir, counter: shared.NewTimeCounter(), } + + return m } -type StoreMeta struct { +type Meta struct { Labels map[LabelKey]LabelValue } -// CreateImport initializes a new import and returns its ID. -func (m *Manager) CreateImport() (ID, error) { - id := ID(m.counter.Next()) +// CreateImport initializes a new import, returning its ID and optionally a +// CAR path where to place the data, if requested. +func (m *Manager) CreateImport() (id ID, err error) { + id = ID(m.counter.Next()) - meta, err := json.Marshal(&StoreMeta{Labels: map[LabelKey]LabelValue{ + meta := &Meta{Labels: map[LabelKey]LabelValue{ LSource: "unknown", - }}) + }} + + metajson, err := json.Marshal(meta) if err != nil { - return 0, xerrors.Errorf("marshaling empty store metadata: %w", err) + return 0, xerrors.Errorf("marshaling store metadata: %w", err) + } + + err = m.ds.Put(id.dsKey(), metajson) + if err != nil { + return 0, xerrors.Errorf("failed to insert import metadata: %w", err) } - err = m.ds.Put(id.dsKey(), meta) return id, err } +// AllocateCAR creates a new CAR allocated to the supplied import under the +// root directory. +func (m *Manager) AllocateCAR(id ID) (path string, err error) { + meta, err := m.ds.Get(id.dsKey()) + if err != nil { + return "", xerrors.Errorf("getting metadata form datastore: %w", err) + } + + var sm Meta + if err := json.Unmarshal(meta, &sm); err != nil { + return "", xerrors.Errorf("unmarshaling store meta: %w", err) + } + + // refuse if a CAR path already exists. + if curr := sm.Labels[LCARPath]; curr != "" { + return "", xerrors.Errorf("import CAR already exists at %s: %w", curr, err) + } + + path = filepath.Join(m.rootDir, fmt.Sprintf("%d.car", id)) + file, err := os.Create(path) + if err != nil { + return "", xerrors.Errorf("failed to create car file for import: %w", err) + } + + // close the file before returning the path. + if err := file.Close(); err != nil { + return "", xerrors.Errorf("failed to close temp file: %w", err) + } + + // record the path and ownership. + sm.Labels[LCARPath] = path + sm.Labels[LCAROwner] = CAROwnerImportMgr + + if meta, err = json.Marshal(sm); err != nil { + return "", xerrors.Errorf("marshaling store metadata: %w", err) + } + + err = m.ds.Put(id.dsKey(), meta) + return path, err +} + // AddLabel adds a label associated with an import, such as the source, // car path, CID, etc. func (m *Manager) AddLabel(id ID, key LabelKey, value LabelValue) error { @@ -78,7 +136,7 @@ func (m *Manager) AddLabel(id ID, key LabelKey, value LabelValue) error { return xerrors.Errorf("getting metadata form datastore: %w", err) } - var sm StoreMeta + var sm Meta if err := json.Unmarshal(meta, &sm); err != nil { return xerrors.Errorf("unmarshaling store meta: %w", err) } @@ -120,13 +178,13 @@ func (m *Manager) List() ([]ID, error) { } // Info returns the metadata known to this store for the specified import ID. -func (m *Manager) Info(id ID) (*StoreMeta, error) { +func (m *Manager) Info(id ID) (*Meta, error) { meta, err := m.ds.Get(id.dsKey()) if err != nil { return nil, xerrors.Errorf("getting metadata form datastore: %w", err) } - var sm StoreMeta + var sm Meta if err := json.Unmarshal(meta, &sm); err != nil { return nil, xerrors.Errorf("unmarshaling store meta: %w", err) } @@ -139,20 +197,19 @@ func (m *Manager) Remove(id ID) error { if err := m.ds.Delete(id.dsKey()); err != nil { return xerrors.Errorf("removing import metadata: %w", err) } - return nil } -func (m *Manager) FilestoreCARV2FilePathFor(dagRoot cid.Cid) (string, error) { - importIDs, err := m.List() +func (m *Manager) CARPathFor(dagRoot cid.Cid) (string, error) { + ids, err := m.List() if err != nil { return "", xerrors.Errorf("failed to fetch import IDs: %w", err) } - for _, importID := range importIDs { - info, err := m.Info(importID) + for _, id := range ids { + info, err := m.Info(id) if err != nil { - log.Errorf("failed to fetch info, importID=%d: %s", importID, err) + log.Errorf("failed to fetch info, importID=%d: %s", id, err) continue } if info.Labels[LRootCid] == "" { @@ -170,17 +227,3 @@ func (m *Manager) FilestoreCARV2FilePathFor(dagRoot cid.Cid) (string, error) { return "", nil } - -func (m *Manager) NewTempFile(importID ID) (string, error) { - file, err := ioutil.TempFile(m.rootDir, fmt.Sprintf("%d", importID)) - if err != nil { - return "", xerrors.Errorf("failed to create temp file: %w", err) - } - - // close the file as we need to return the path here. - if err := file.Close(); err != nil { - return "", xerrors.Errorf("failed to close temp file: %w", err) - } - - return file.Name(), nil -} diff --git a/node/repo/retrievalstoremgr/retrievalstoremgr.go b/node/repo/retrievalstoremgr/retrievalstoremgr.go deleted file mode 100644 index 11806225853..00000000000 --- a/node/repo/retrievalstoremgr/retrievalstoremgr.go +++ /dev/null @@ -1,60 +0,0 @@ -package retrievalstoremgr - -import ( - "github.com/filecoin-project/lotus/blockstore" -) - -// RetrievalStore references a store for a retrieval deal. -type RetrievalStore interface { - IsIPFSRetrieval() bool - Blockstore() blockstore.BasicBlockstore -} - -// RetrievalStoreManager manages stores for retrieval deals, abstracting -// the underlying storage mechanism. -type RetrievalStoreManager interface { - NewStore() (RetrievalStore, error) - ReleaseStore(RetrievalStore) error -} - -// BlockstoreRetrievalStoreManager is a blockstore used for retrieval. -type BlockstoreRetrievalStoreManager struct { - bs blockstore.BasicBlockstore - isIpfsRetrieval bool -} - -var _ RetrievalStoreManager = &BlockstoreRetrievalStoreManager{} - -// NewBlockstoreRetrievalStoreManager returns a new blockstore based RetrievalStoreManager -func NewBlockstoreRetrievalStoreManager(bs blockstore.BasicBlockstore, isIpfsRetrieval bool) RetrievalStoreManager { - return &BlockstoreRetrievalStoreManager{ - bs: bs, - isIpfsRetrieval: isIpfsRetrieval, - } -} - -// NewStore creates a new store (just uses underlying blockstore) -func (brsm *BlockstoreRetrievalStoreManager) NewStore() (RetrievalStore, error) { - return &blockstoreRetrievalStore{ - bs: brsm.bs, - isIpfsRetrieval: brsm.isIpfsRetrieval, - }, nil -} - -// ReleaseStore for this implementation does nothing -func (brsm *BlockstoreRetrievalStoreManager) ReleaseStore(RetrievalStore) error { - return nil -} - -type blockstoreRetrievalStore struct { - bs blockstore.BasicBlockstore - isIpfsRetrieval bool -} - -func (brs *blockstoreRetrievalStore) Blockstore() blockstore.BasicBlockstore { - return brs.bs -} - -func (brs *blockstoreRetrievalStore) IsIPFSRetrieval() bool { - return brs.isIpfsRetrieval -} From 7fe6aeb7e4ef5a22dcf1919868a38efa70d096d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 16 Aug 2021 01:09:24 +0100 Subject: [PATCH 02/12] update deps. --- go.mod | 2 +- go.sum | 4 ++-- node/repo/imports/manager.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index dc0ffd9ef07..da8e0d6b22a 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/filecoin-project/go-data-transfer v1.7.3 github.com/filecoin-project/go-fil-commcid v0.1.0 github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 - github.com/filecoin-project/go-fil-markets v1.6.3-0.20210815233218-8025f5c1a128 + github.com/filecoin-project/go-fil-markets v1.6.3-0.20210816000803-594907446f64 github.com/filecoin-project/go-jsonrpc v0.1.4-0.20210217175800-45ea43ac2bec github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20210723183308-812a16dc01b1 diff --git a/go.sum b/go.sum index b5ec62d7157..871021dedc9 100644 --- a/go.sum +++ b/go.sum @@ -289,8 +289,8 @@ github.com/filecoin-project/go-fil-commcid v0.1.0/go.mod h1:Eaox7Hvus1JgPrL5+M3+ github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 h1:imrrpZWEHRnNqqv0tN7LXep5bFEVOVmQWHJvl2mgsGo= github.com/filecoin-project/go-fil-commp-hashhash v0.1.0/go.mod h1:73S8WSEWh9vr0fDJVnKADhfIv/d6dCbAGaAGWbdJEI8= github.com/filecoin-project/go-fil-markets v1.0.5-0.20201113164554-c5eba40d5335/go.mod h1:AJySOJC00JRWEZzRG2KsfUnqEf5ITXxeX09BE9N4f9c= -github.com/filecoin-project/go-fil-markets v1.6.3-0.20210815233218-8025f5c1a128 h1:O8ms+mO9ZJGKvRxa0SI3oV7G7TQT6OMhZjO+oFuUvCA= -github.com/filecoin-project/go-fil-markets v1.6.3-0.20210815233218-8025f5c1a128/go.mod h1:D5xHWxyuU0EK8wcK4qStO5rjmpH206eb4OdrkWmTdaY= +github.com/filecoin-project/go-fil-markets v1.6.3-0.20210816000803-594907446f64 h1:Lpm55IhGvDupCjesT1gnr9OM4cYH3PZF7PpRMbUZW6M= +github.com/filecoin-project/go-fil-markets v1.6.3-0.20210816000803-594907446f64/go.mod h1:D5xHWxyuU0EK8wcK4qStO5rjmpH206eb4OdrkWmTdaY= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxlWi3xVvbQP0IT38fvM= diff --git a/node/repo/imports/manager.go b/node/repo/imports/manager.go index 3050f43c6a0..6620238765c 100644 --- a/node/repo/imports/manager.go +++ b/node/repo/imports/manager.go @@ -217,7 +217,7 @@ func (m *Manager) CARPathFor(dagRoot cid.Cid) (string, error) { } c, err := cid.Parse(info.Labels[LRootCid]) if err != nil { - log.Errorf("failed to parse Root cid %s: %w", info.Labels[LRootCid], err) + log.Errorf("failed to parse root cid %s: %s", info.Labels[LRootCid], err) continue } if c.Equals(dagRoot) { From 52992af96946931f556cb73b7b6e75b0e790091d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 16 Aug 2021 14:39:44 +0100 Subject: [PATCH 03/12] fix test failure. --- node/impl/client/client.go | 68 +++++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 1fbe9966883..853f0c648f8 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -12,6 +12,7 @@ import ( "time" bstore "github.com/ipfs/go-ipfs-blockstore" + unixfile "github.com/ipfs/go-unixfs/file" "github.com/ipld/go-car" carv2 "github.com/ipld/go-car/v2" "github.com/ipld/go-car/v2/blockstore" @@ -25,8 +26,6 @@ import ( offline "github.com/ipfs/go-ipfs-exchange-offline" files "github.com/ipfs/go-ipfs-files" "github.com/ipfs/go-merkledag" - unixfile "github.com/ipfs/go-unixfs/file" - basicnode "github.com/ipld/go-ipld-prime/node/basic" "github.com/ipld/go-ipld-prime/traversal/selector" "github.com/ipld/go-ipld-prime/traversal/selector/builder" @@ -49,6 +48,7 @@ import ( "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-fil-markets/storagemarket/network" "github.com/filecoin-project/go-fil-markets/stores" + "github.com/filecoin-project/lotus/markets/retrievaladapter" "github.com/filecoin-project/go-state-types/abi" @@ -784,7 +784,7 @@ type retrievalSubscribeEvent struct { state rm.ClientDealState } -func readSubscribeEvents(ctx context.Context, dealID retrievalmarket.DealID, subscribeEvents chan retrievalSubscribeEvent, events chan marketevents.RetrievalEvent) error { +func consumeAllEvents(ctx context.Context, dealID retrievalmarket.DealID, subscribeEvents chan retrievalSubscribeEvent, events chan marketevents.RetrievalEvent) error { for { var subscribeEvent retrievalSubscribeEvent select { @@ -845,16 +845,18 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref // the CARv1 (with ExtractV1File) or UnixFS export from it. // this indicates we're proxying to IPFS. - proxyBss, isIPFS := a.RtvlBlockstoreAccessor.(*retrievaladapter.ProxyBlockstoreAccessor) - carBss, isCAR := a.RtvlBlockstoreAccessor.(*retrievaladapter.CARBlockstoreAccessor) - - if !isIPFS && !isCAR { - finish(xerrors.Errorf("unsupported retrieval blockstore accessor")) - return - } + proxyBss, retrieveIntoIPFS := a.RtvlBlockstoreAccessor.(*retrievaladapter.ProxyBlockstoreAccessor) + carBss, retrieveIntoCAR := a.RtvlBlockstoreAccessor.(*retrievaladapter.CARBlockstoreAccessor) carPath := order.FromLocalCAR if carPath == "" { + if !retrieveIntoIPFS && !retrieveIntoCAR { + // we actually need to retrieve from the network, but we don't + // recognize the blockstore accessor. + finish(xerrors.Errorf("unsupported retrieval blockstore accessor")) + return + } + if order.MinerPeer == nil || order.MinerPeer.ID == "" { mi, err := a.StateMinerInfo(ctx, order.Miner, types.EmptyTSK) if err != nil { @@ -891,7 +893,7 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref subscribeCtx, cancel := context.WithCancel(ctx) defer cancel() unsubscribe := a.Retrieval.SubscribeToEvents(func(event rm.ClientEvent, state rm.ClientDealState) { - // We'll check the deal IDs inside readSubscribeEvents. + // We'll check the deal IDs inside consumeAllEvents. if state.PayloadCID.Equals(order.Root) { select { case <-subscribeCtx.Done(): @@ -918,7 +920,7 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref return } - err = readSubscribeEvents(ctx, id, subscribeEvents, events) + err = consumeAllEvents(ctx, id, subscribeEvents, events) unsubscribe() if err != nil { @@ -926,35 +928,39 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref return } - carPath = carBss.PathFor(id) + if retrieveIntoCAR { + carPath = carBss.PathFor(id) + } } - switch { - case ref == nil: + if ref == nil { // If ref is nil, it only fetches the data into the configured blockstore // (if fetching from network). finish(nil) return + } - case ref.IsCAR && isIPFS: - // generating a CARv1 from IPFS. - f, err := os.OpenFile(ref.Path, os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - finish(err) - return - } + // Are we outputting a CAR? + if ref.IsCAR { + if retrieveIntoIPFS { + // generating a CARv1 from IPFS. + f, err := os.OpenFile(ref.Path, os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + finish(err) + return + } - bs := proxyBss.Blockstore - dags := merkledag.NewDAGService(blockservice.New(bs, offline.Exchange(bs))) - err = car.WriteCar(ctx, dags, []cid.Cid{order.Root}, f) - if err != nil { - finish(err) + bs := proxyBss.Blockstore + dags := merkledag.NewDAGService(blockservice.New(bs, offline.Exchange(bs))) + err = car.WriteCar(ctx, dags, []cid.Cid{order.Root}, f) + if err != nil { + finish(err) + return + } + finish(f.Close()) return } - finish(f.Close()) - return - case ref.IsCAR: // generating a CARv1 from the CARv2 where we stored the retrieval. err := carv2.ExtractV1File(carPath, ref.Path) finish(err) @@ -963,7 +969,7 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref // we are extracting a UnixFS file. var bs bstore.Blockstore - if isIPFS { + if retrieveIntoIPFS { bs = proxyBss.Blockstore } else { cbs, err := stores.ReadOnlyFilestore(carPath) From c56db9bfa23d56d9cfb64ebbf1dede816ec6dc5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 16 Aug 2021 14:40:46 +0100 Subject: [PATCH 04/12] upgrade go-fil-markets. --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index da8e0d6b22a..b79542cf5ba 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/filecoin-project/go-data-transfer v1.7.3 github.com/filecoin-project/go-fil-commcid v0.1.0 github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 - github.com/filecoin-project/go-fil-markets v1.6.3-0.20210816000803-594907446f64 + github.com/filecoin-project/go-fil-markets v1.6.3-0.20210816133929-00251d2c79b8 github.com/filecoin-project/go-jsonrpc v0.1.4-0.20210217175800-45ea43ac2bec github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20210723183308-812a16dc01b1 diff --git a/go.sum b/go.sum index 871021dedc9..a65c8bf808f 100644 --- a/go.sum +++ b/go.sum @@ -289,8 +289,8 @@ github.com/filecoin-project/go-fil-commcid v0.1.0/go.mod h1:Eaox7Hvus1JgPrL5+M3+ github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 h1:imrrpZWEHRnNqqv0tN7LXep5bFEVOVmQWHJvl2mgsGo= github.com/filecoin-project/go-fil-commp-hashhash v0.1.0/go.mod h1:73S8WSEWh9vr0fDJVnKADhfIv/d6dCbAGaAGWbdJEI8= github.com/filecoin-project/go-fil-markets v1.0.5-0.20201113164554-c5eba40d5335/go.mod h1:AJySOJC00JRWEZzRG2KsfUnqEf5ITXxeX09BE9N4f9c= -github.com/filecoin-project/go-fil-markets v1.6.3-0.20210816000803-594907446f64 h1:Lpm55IhGvDupCjesT1gnr9OM4cYH3PZF7PpRMbUZW6M= -github.com/filecoin-project/go-fil-markets v1.6.3-0.20210816000803-594907446f64/go.mod h1:D5xHWxyuU0EK8wcK4qStO5rjmpH206eb4OdrkWmTdaY= +github.com/filecoin-project/go-fil-markets v1.6.3-0.20210816133929-00251d2c79b8 h1:p8LqLGy361wm1L5JFWBSijpQ/OG33NUsQvNkSp5gMW0= +github.com/filecoin-project/go-fil-markets v1.6.3-0.20210816133929-00251d2c79b8/go.mod h1:D5xHWxyuU0EK8wcK4qStO5rjmpH206eb4OdrkWmTdaY= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxlWi3xVvbQP0IT38fvM= From 3a6dad0dd9e388a658138e7296cc3a7b8c85a959 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 16 Aug 2021 14:49:58 +0100 Subject: [PATCH 05/12] fix lint errors. --- markets/storageadapter/client_blockstore.go | 2 +- node/impl/client/client.go | 2 +- node/impl/client/import.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/markets/storageadapter/client_blockstore.go b/markets/storageadapter/client_blockstore.go index 5dc0a6aceb4..4239251d34b 100644 --- a/markets/storageadapter/client_blockstore.go +++ b/markets/storageadapter/client_blockstore.go @@ -4,7 +4,7 @@ import ( "sync" "github.com/ipfs/go-cid" - "github.com/ipfs/go-ipfs-blockstore" + blockstore "github.com/ipfs/go-ipfs-blockstore" "golang.org/x/xerrors" "github.com/filecoin-project/go-fil-markets/storagemarket" diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 853f0c648f8..9a268bc7ff3 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -682,7 +682,7 @@ func (a *API) ClientImportLocal(ctx context.Context, r io.Reader) (cid.Cid, erro if err != nil { return cid.Undef, xerrors.Errorf("failed to open car: %w", err) } - defer f.Close() + defer f.Close() //nolint:errcheck n, err := f.WriteAt(newBuf.Bytes(), int64(headerOff)) if err != nil { diff --git a/node/impl/client/import.go b/node/impl/client/import.go index 0296900db91..367cc73c7fb 100644 --- a/node/impl/client/import.go +++ b/node/impl/client/import.go @@ -14,7 +14,7 @@ import ( bstore "github.com/ipfs/go-ipfs-blockstore" chunker "github.com/ipfs/go-ipfs-chunker" offline "github.com/ipfs/go-ipfs-exchange-offline" - "github.com/ipfs/go-ipfs-files" + files "github.com/ipfs/go-ipfs-files" ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" "github.com/ipfs/go-unixfs/importer/balanced" From 33d3dd6f830ce5a13b5d004b72a484421ba8b8d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 16 Aug 2021 15:04:59 +0100 Subject: [PATCH 06/12] TestImportLocal: more comprehensive test. --- node/impl/client/client.go | 2 +- node/impl/client/client_test.go | 63 +++++++++++++++++++++++++++++++-- 2 files changed, 62 insertions(+), 3 deletions(-) diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 9a268bc7ff3..475ed7e507d 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -1230,7 +1230,7 @@ func (a *API) ClientGenCar(ctx context.Context, ref api.FileRef, outputPath stri } defer os.Remove(tmp) //nolint:errcheck - // geneate and import the UnixFS DAG into a filestore (positional reference) CAR. + // generate and import the UnixFS DAG into a filestore (positional reference) CAR. root, err := a.createUnixFSFilestore(ctx, ref.Path, tmp) if err != nil { return xerrors.Errorf("failed to import file using unixfs: %w", err) diff --git a/node/impl/client/client_test.go b/node/impl/client/client_test.go index 19b4219c383..d57604db4c1 100644 --- a/node/impl/client/client_test.go +++ b/node/impl/client/client_test.go @@ -9,9 +9,17 @@ import ( "strings" "testing" + "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" + blockstore "github.com/ipfs/go-ipfs-blockstore" + offline "github.com/ipfs/go-ipfs-exchange-offline" + files "github.com/ipfs/go-ipfs-files" + "github.com/ipfs/go-merkledag" + unixfile "github.com/ipfs/go-unixfs/file" + "github.com/ipld/go-car" + carv2 "github.com/ipld/go-car/v2" "github.com/stretchr/testify/require" "github.com/filecoin-project/lotus/api" @@ -54,8 +62,8 @@ func TestImportLocal(t *testing.T) { } // retrieve as UnixFS. - out1 := filepath.Join(dir, "retrieval1.data") - // out2 := filepath.Join(dir, "retrieval2.data") + out1 := filepath.Join(dir, "retrieval1.data") // as unixfs + out2 := filepath.Join(dir, "retrieval2.data") // as car err = a.ClientRetrieve(ctx, order, &api.FileRef{ Path: out1, }) @@ -64,4 +72,55 @@ func TestImportLocal(t *testing.T) { outBytes, err := ioutil.ReadFile(out1) require.NoError(t, err) require.Equal(t, b, outBytes) + + err = a.ClientRetrieve(ctx, order, &api.FileRef{ + Path: out2, + IsCAR: true, + }) + require.NoError(t, err) + + // open the CARv2 being custodied by the import manager + orig, err := carv2.OpenReader(it.CARPath) + require.NoError(t, err) + + // open the CARv1 we just exported + exported, err := carv2.OpenReader(out2) + require.NoError(t, err) + + require.EqualValues(t, 1, exported.Version) + require.EqualValues(t, 2, orig.Version) + + origRoots, err := orig.Roots() + require.NoError(t, err) + require.Len(t, origRoots, 1) + + exportedRoots, err := exported.Roots() + require.NoError(t, err) + require.Len(t, exportedRoots, 1) + + require.EqualValues(t, origRoots, exportedRoots) + + // recreate the unixfs dag, and see if it matches the original file byte by byte + // import the car into a memory blockstore, then export the unixfs file. + bs := blockstore.NewBlockstore(datastore.NewMapDatastore()) + _, err = car.LoadCar(bs, exported.DataReader()) + require.NoError(t, err) + + dag := merkledag.NewDAGService(blockservice.New(bs, offline.Exchange(bs))) + + nd, err := dag.Get(ctx, exportedRoots[0]) + require.NoError(t, err) + + file, err := unixfile.NewUnixfsFile(ctx, dag, nd) + require.NoError(t, err) + + exportedPath := filepath.Join(dir, "exported.data") + err = files.WriteTo(file, exportedPath) + require.NoError(t, err) + + exportedBytes, err := ioutil.ReadFile(exportedPath) + require.NoError(t, err) + + // compare original file to recreated unixfs file. + require.Equal(t, b, exportedBytes) } From a823d70a0b5c988e179229a0bf800e9e65f07c38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 16 Aug 2021 15:31:02 +0100 Subject: [PATCH 07/12] sanity check imports on start. --- node/repo/imports/manager.go | 39 ++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/node/repo/imports/manager.go b/node/repo/imports/manager.go index 6620238765c..d972ffb7712 100644 --- a/node/repo/imports/manager.go +++ b/node/repo/imports/manager.go @@ -58,6 +58,45 @@ func NewManager(ds datastore.Batching, rootDir string) *Manager { counter: shared.NewTimeCounter(), } + log.Info("sanity checking imports") + + ids, err := m.List() + if err != nil { + log.Warnw("failed to enumerate imports on initialization", "error", err) + return m + } + + var broken int + for _, id := range ids { + log := log.With("id", id) + + info, err := m.Info(id) + if err != nil { + log.Warnw("failed to query metadata for import; skipping", "error", err) + continue + } + + log = log.With("source", info.Labels[LSource], "root", info.Labels[LRootCid], "original", info.Labels[LFileName]) + + path, ok := info.Labels[LCARPath] + if !ok { + broken++ + log.Warnw("import lacks carv2 path; import will not work; please reimport") + continue + } + + stat, err := os.Stat(path) + if err != nil { + broken++ + log.Warnw("import has missing/broken carv2; please reimport", "error", err) + continue + } + + log.Infow("import ok", "size", stat.Size()) + } + + log.Infow("sanity check completed", "broken", broken, "total", len(ids)) + return m } From 6d237677789e9005b57ff9bfd434fed5d914ac68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 16 Aug 2021 16:46:37 +0100 Subject: [PATCH 08/12] fix storage deal-making from ipfs. --- node/impl/client/client.go | 43 ++++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 475ed7e507d..854e840e757 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -50,6 +50,7 @@ import ( "github.com/filecoin-project/go-fil-markets/stores" "github.com/filecoin-project/lotus/markets/retrievaladapter" + "github.com/filecoin-project/lotus/markets/storageadapter" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/specs-actors/v3/actors/builtin/market" @@ -89,8 +90,9 @@ type API struct { Chain *store.ChainStore // accessors for imports and retrievals. - Imports dtypes.ClientImportMgr - RtvlBlockstoreAccessor retrievalmarket.BlockstoreAccessor + Imports dtypes.ClientImportMgr + StorageBlockstoreAccessor storagemarket.BlockstoreAccessor + RtvlBlockstoreAccessor retrievalmarket.BlockstoreAccessor DataTransfer dtypes.ClientDataTransfer Host host.Host @@ -1154,24 +1156,33 @@ func (w *lenWriter) Write(p []byte) (n int, err error) { } func (a *API) ClientDealSize(ctx context.Context, root cid.Cid) (api.DataSize, error) { - path, err := a.importManager().CARPathFor(root) - if err != nil { - return api.DataSize{}, xerrors.Errorf("failed to find CARv2 file for root: %w", err) - } - if len(path) == 0 { - return api.DataSize{}, xerrors.New("no CARv2 file for root") - } + var bs bstore.Blockstore - fs, err := stores.ReadOnlyFilestore(path) - if err != nil { - return api.DataSize{}, xerrors.Errorf("failed to open filestore from carv2 in path %s: %w", path, err) + // pick the source blockstore; either the ipfs blockstore, or an import CARv2 file. + proxyBss, storeFromIPFS := a.StorageBlockstoreAccessor.(*storageadapter.ProxyBlockstoreAccessor) + if !storeFromIPFS { + path, err := a.importManager().CARPathFor(root) + if err != nil { + return api.DataSize{}, xerrors.Errorf("failed to find carv2 import for root: %w", err) + } + if len(path) == 0 { + return api.DataSize{}, xerrors.New("no carv2 import for root") + } + + fs, err := stores.ReadOnlyFilestore(path) + if err != nil { + return api.DataSize{}, xerrors.Errorf("failed to open filestore from carv2 in path %s: %w", path, err) + } + defer fs.Close() //nolint:errcheck + + } else { + bs = proxyBss.Blockstore } - defer fs.Close() //nolint:errcheck - dag := merkledag.NewDAGService(blockservice.New(fs, offline.Exchange(fs))) - w := lenWriter(0) + dag := merkledag.NewDAGService(blockservice.New(bs, offline.Exchange(bs))) - err = car.WriteCar(ctx, dag, []cid.Cid{root}, &w) + w := lenWriter(0) + err := car.WriteCar(ctx, dag, []cid.Cid{root}, &w) if err != nil { return api.DataSize{}, err } From 9d603a9df42fd9d04677f997688babf84eb72f83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 16 Aug 2021 16:53:07 +0100 Subject: [PATCH 09/12] cleaner logic to make deals from ipfs. --- node/impl/client/client.go | 38 +++++++++++--------------------------- 1 file changed, 11 insertions(+), 27 deletions(-) diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 854e840e757..e3c141723cf 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "os" - "path/filepath" "sort" "time" @@ -1000,17 +999,6 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref finish(files.WriteTo(file, ref.Path)) } -// TODO: Come up with a better mechanism for creating the tmp CARv2 file path -func (a *API) getTmpCarV2FilePath() (string, error) { - carsPath := filepath.Join(a.Repo.Path(), DefaultDAGStoreDir, "retrieval-cars") - - if err := os.MkdirAll(carsPath, 0755); err != nil { - return "", xerrors.Errorf("failed to create dir") - } - - return filepath.Join(carsPath, fmt.Sprintf("%d.car", time.Now().UnixNano())), nil -} - func (a *API) ClientListRetrievals(ctx context.Context) ([]api.RetrievalInfo, error) { deals, err := a.Retrieval.ListDeals() if err != nil { @@ -1159,29 +1147,25 @@ func (a *API) ClientDealSize(ctx context.Context, root cid.Cid) (api.DataSize, e var bs bstore.Blockstore // pick the source blockstore; either the ipfs blockstore, or an import CARv2 file. - proxyBss, storeFromIPFS := a.StorageBlockstoreAccessor.(*storageadapter.ProxyBlockstoreAccessor) - if !storeFromIPFS { - path, err := a.importManager().CARPathFor(root) + switch acc := a.StorageBlockstoreAccessor.(type) { + case *storageadapter.ImportsBlockstoreAccessor: + var err error + bs, err = acc.Get(root) if err != nil { - return api.DataSize{}, xerrors.Errorf("failed to find carv2 import for root: %w", err) - } - if len(path) == 0 { - return api.DataSize{}, xerrors.New("no carv2 import for root") + return api.DataSize{}, xerrors.Errorf("no import found for root %s: %w", root, err) } + defer acc.Done(root) //nolint:errcheck - fs, err := stores.ReadOnlyFilestore(path) - if err != nil { - return api.DataSize{}, xerrors.Errorf("failed to open filestore from carv2 in path %s: %w", path, err) - } - defer fs.Close() //nolint:errcheck + case *storageadapter.ProxyBlockstoreAccessor: + bs = acc.Blockstore - } else { - bs = proxyBss.Blockstore + default: + return api.DataSize{}, xerrors.Errorf("unsupported blockstore accessor type: %T", acc) } dag := merkledag.NewDAGService(blockservice.New(bs, offline.Exchange(bs))) - w := lenWriter(0) + var w lenWriter err := car.WriteCar(ctx, dag, []cid.Cid{root}, &w) if err != nil { return api.DataSize{}, err From 8c97e07b0551df7f745f8216fe9549fe642d8fb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 16 Aug 2021 17:36:46 +0100 Subject: [PATCH 10/12] port additional client methods to StorageBlockstoreAccessor. --- node/impl/client/client.go | 73 +++++++++++++++++++------------------- 1 file changed, 36 insertions(+), 37 deletions(-) diff --git a/node/impl/client/client.go b/node/impl/client/client.go index e3c141723cf..68217e4b2ac 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -405,15 +405,12 @@ func (a *API) newDealInfoWithTransfer(transferCh *api.DataTransferChannel, v sto } } -func (a *API) ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) { - // TODO: check if we have the ENTIRE dag - fc, err := a.importManager().CARPathFor(root) +func (a *API) ClientHasLocal(_ context.Context, root cid.Cid) (bool, error) { + _, onDone, err := a.dealBlockstore(root) if err != nil { return false, err } - if len(fc) == 0 { - return false, nil - } + onDone() return true, nil } @@ -487,7 +484,6 @@ func (a *API) makeRetrievalQuery(ctx context.Context, rp rm.RetrievalPeer, paylo } } -// TODO check func (a *API) ClientImport(ctx context.Context, ref api.FileRef) (res *api.ImportRes, err error) { var ( imgr = a.importManager() @@ -1144,29 +1140,16 @@ func (w *lenWriter) Write(p []byte) (n int, err error) { } func (a *API) ClientDealSize(ctx context.Context, root cid.Cid) (api.DataSize, error) { - var bs bstore.Blockstore - - // pick the source blockstore; either the ipfs blockstore, or an import CARv2 file. - switch acc := a.StorageBlockstoreAccessor.(type) { - case *storageadapter.ImportsBlockstoreAccessor: - var err error - bs, err = acc.Get(root) - if err != nil { - return api.DataSize{}, xerrors.Errorf("no import found for root %s: %w", root, err) - } - defer acc.Done(root) //nolint:errcheck - - case *storageadapter.ProxyBlockstoreAccessor: - bs = acc.Blockstore - - default: - return api.DataSize{}, xerrors.Errorf("unsupported blockstore accessor type: %T", acc) + bs, onDone, err := a.dealBlockstore(root) + if err != nil { + return api.DataSize{}, err } + defer onDone() dag := merkledag.NewDAGService(blockservice.New(bs, offline.Exchange(bs))) var w lenWriter - err := car.WriteCar(ctx, dag, []cid.Cid{root}, &w) + err = car.WriteCar(ctx, dag, []cid.Cid{root}, &w) if err != nil { return api.DataSize{}, err } @@ -1180,21 +1163,13 @@ func (a *API) ClientDealSize(ctx context.Context, root cid.Cid) (api.DataSize, e } func (a *API) ClientDealPieceCID(ctx context.Context, root cid.Cid) (api.DataCIDSize, error) { - path, err := a.importManager().CARPathFor(root) - if err != nil { - return api.DataCIDSize{}, xerrors.Errorf("failed to find CARv2 file for root: %w", err) - } - if len(path) == 0 { - return api.DataCIDSize{}, xerrors.New("no CARv2 file for root") - } - - fs, err := stores.ReadOnlyFilestore(path) + bs, onDone, err := a.dealBlockstore(root) if err != nil { - return api.DataCIDSize{}, xerrors.Errorf("failed to open filestore from carv2 in path %s: %w", path, err) + return api.DataCIDSize{}, err } - defer fs.Close() //nolint:errcheck + defer onDone() - dag := merkledag.NewDAGService(blockservice.New(fs, offline.Exchange(fs))) + dag := merkledag.NewDAGService(blockservice.New(bs, offline.Exchange(bs))) w := &writer.Writer{} bw := bufio.NewWriterSize(w, int(writer.CommPBuf)) @@ -1316,3 +1291,27 @@ func (a *API) ClientGetDealStatus(ctx context.Context, statusCode uint64) (strin return ststr, nil } + +// dealBlockstore picks the source blockstore for a storage deal; either the +// IPFS blockstore, or an import CARv2 file. It also returns a function that +// must be called when done. +func (a *API) dealBlockstore(root cid.Cid) (bstore.Blockstore, func(), error) { + switch acc := a.StorageBlockstoreAccessor.(type) { + case *storageadapter.ImportsBlockstoreAccessor: + bs, err := acc.Get(root) + if err != nil { + return nil, nil, xerrors.Errorf("no import found for root %s: %w", root, err) + } + + doneFn := func() { + _ = acc.Done(root) //nolint:errcheck + } + return bs, doneFn, nil + + case *storageadapter.ProxyBlockstoreAccessor: + return acc.Blockstore, func() {}, nil + + default: + return nil, nil, xerrors.Errorf("unsupported blockstore accessor type: %T", acc) + } +} From f0255c53262753369165b084ce47b9ffe0d4aa57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 16 Aug 2021 17:44:03 +0100 Subject: [PATCH 11/12] further fixes. --- node/impl/client/client.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 68217e4b2ac..f06a62f908b 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -135,13 +135,16 @@ func (a *API) dealStarter(ctx context.Context, params *api.StartDealParams, isSt return nil, xerrors.New("stateless storage deals can only be initiated with storage price of 0") } } else if params.Data.TransferType == storagemarket.TTGraphsync { - fc, err := a.importManager().CARPathFor(params.Data.Root) + bs, onDone, err := a.dealBlockstore(params.Data.Root) if err != nil { - return nil, xerrors.Errorf("failed to find CARv2 file path: %w", err) + return nil, xerrors.Errorf("failed to find blockstore for root CID: %w", err) } - if fc == "" { - return nil, xerrors.New("no CARv2 file path for deal") + if has, err := bs.Has(params.Data.Root); err != nil { + return nil, xerrors.Errorf("failed to query blockstore for root CID: %w", err) + } else if !has { + return nil, xerrors.Errorf("failed to find root CID in blockstore: %w", err) } + onDone() } walletKey, err := a.StateAccountKey(ctx, params.Wallet, types.EmptyTSK) From 66b19ea1b72f1e19680a6824999968ecbfc55088 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Mon, 16 Aug 2021 18:18:23 +0100 Subject: [PATCH 12/12] fix test. --- node/impl/client/client_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/node/impl/client/client_test.go b/node/impl/client/client_test.go index d57604db4c1..834c980ab14 100644 --- a/node/impl/client/client_test.go +++ b/node/impl/client/client_test.go @@ -23,6 +23,7 @@ import ( "github.com/stretchr/testify/require" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/markets/storageadapter" "github.com/filecoin-project/lotus/node/repo/imports" ) @@ -35,7 +36,10 @@ func TestImportLocal(t *testing.T) { im := imports.NewManager(ds, dir) ctx := context.Background() - a := &API{Imports: im} + a := &API{ + Imports: im, + StorageBlockstoreAccessor: storageadapter.NewImportsBlockstoreAccessor(im), + } b, err := testdata.ReadFile("testdata/payload.txt") require.NoError(t, err)