From 3d92c9388873406dc57abe15a8e2d80c65910f05 Mon Sep 17 00:00:00 2001 From: lash Date: Thu, 18 Jan 2018 01:13:13 +0100 Subject: [PATCH 01/25] swarm/storage: Simplify code, correct content hashing --- swarm/storage/resource_ens.go | 1 + swarm/storage/resource_test.go | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/swarm/storage/resource_ens.go b/swarm/storage/resource_ens.go index 0a4500309d4f..f8f34919a2e6 100644 --- a/swarm/storage/resource_ens.go +++ b/swarm/storage/resource_ens.go @@ -36,6 +36,7 @@ func NewENSValidator(contractaddress common.Address, backend bind.ContractBacken if err != nil { return nil, err } + validator.hashlength = len(ens.EnsNode(dbDirName).Bytes()) return validator, nil } diff --git a/swarm/storage/resource_test.go b/swarm/storage/resource_test.go index d13070942830..205edfe3b67f 100644 --- a/swarm/storage/resource_test.go +++ b/swarm/storage/resource_test.go @@ -333,6 +333,15 @@ func setupTest(contractbackend bind.ContractBackend, validator ResourceValidator } } + if validator == nil { + // create a new signer, which creates the private key + signer, err = newTestSigner() + if err != nil { + return + } + validator = NewGenericValidator(testHashFunc, signer.signContent) + } + // temp datadir datadir, err = ioutil.TempDir("", "rh") if err != nil { From cf191d30a9703d5e57756507b2724bedc57d80f6 Mon Sep 17 00:00:00 2001 From: lash Date: Thu, 18 Jan 2018 03:19:29 +0100 Subject: [PATCH 02/25] swarm/storage: Remove signatures from non-validated resources --- swarm/storage/resource_ens.go | 1 - swarm/storage/resource_test.go | 9 --------- 2 files changed, 10 deletions(-) diff --git a/swarm/storage/resource_ens.go b/swarm/storage/resource_ens.go index f8f34919a2e6..0a4500309d4f 100644 --- a/swarm/storage/resource_ens.go +++ b/swarm/storage/resource_ens.go @@ -36,7 +36,6 @@ func NewENSValidator(contractaddress common.Address, backend bind.ContractBacken if err != nil { return nil, err } - validator.hashlength = len(ens.EnsNode(dbDirName).Bytes()) return validator, nil } diff --git a/swarm/storage/resource_test.go b/swarm/storage/resource_test.go index 205edfe3b67f..d13070942830 100644 --- a/swarm/storage/resource_test.go +++ b/swarm/storage/resource_test.go @@ -333,15 +333,6 @@ func setupTest(contractbackend bind.ContractBackend, validator ResourceValidator } } - if validator == nil { - // create a new signer, which creates the private key - signer, err = newTestSigner() - if err != nil { - return - } - validator = NewGenericValidator(testHashFunc, signer.signContent) - } - // temp datadir datadir, err = ioutil.TempDir("", "rh") if err != nil { From 13543f40c9afe9d9536bb782fc44185b90eca87a Mon Sep 17 00:00:00 2001 From: lash Date: Thu, 18 Jan 2018 05:09:49 +0100 Subject: [PATCH 03/25] swarm: Add base api for mutable resources --- swarm/api/api.go | 30 +++++++++++--- swarm/api/api_test.go | 2 +- swarm/api/config_test.go | 4 -- swarm/api/http/server.go | 58 +++++++++++++++++++++++++++ swarm/api/http/server_test.go | 41 ++++++++++++++++++- swarm/api/uri.go | 6 ++- swarm/storage/resource.go | 28 ++++++++++++- swarm/testutil/http.go | 74 ++++++++++++++++++++++++++++++++--- 8 files changed, 224 insertions(+), 19 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index 8c4bca2ec0e0..cc10fa21347a 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -46,15 +46,17 @@ on top of the dpa it is the public interface of the dpa which is included in the ethereum stack */ type Api struct { - dpa *storage.DPA - dns Resolver + dpa *storage.DPA + dns Resolver + resource *storage.ResourceHandler } //the api constructor initialises -func NewApi(dpa *storage.DPA, dns Resolver) (self *Api) { +func NewApi(dpa *storage.DPA, dns Resolver, resourceHandler *storage.ResourceHandler) (self *Api) { self = &Api{ - dpa: dpa, - dns: dns, + dpa: dpa, + dns: dns, + resource: resourceHandler, } return } @@ -361,3 +363,21 @@ func (self *Api) BuildDirectoryTree(mhash string, nameresolver bool) (key storag } return key, manifestEntryMap, nil } + +func (self *Api) DbLookupLatest(name string) (io.ReadSeeker, error) { + _, err := self.resource.LookupLatest(name, true) + if err != nil { + return nil, err + } + return bytes.NewReader(self.resource.GetData(name)), nil +} + +func (self *Api) DbCreate(name string, frequency uint64) (err error) { + _, err = self.resource.NewResource(name, frequency) + return err +} + +func (self *Api) DbUpdate(name string, data []byte) (storage.Key, uint32, uint32, error) { + key, err := self.resource.Update(name, data) + return key, self.resource.GetLastPeriod(name), self.resource.GetVersion(name), err +} diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go index e673f76c4267..57c8e88f2951 100644 --- a/swarm/api/api_test.go +++ b/swarm/api/api_test.go @@ -40,7 +40,7 @@ func testApi(t *testing.T, f func(*Api)) { if err != nil { return } - api := NewApi(dpa, nil) + api := NewApi(dpa, nil, nil) dpa.Start() f(api) dpa.Stop() diff --git a/swarm/api/config_test.go b/swarm/api/config_test.go index 4851f19fc528..993388686bb7 100644 --- a/swarm/api/config_test.go +++ b/swarm/api/config_test.go @@ -55,10 +55,6 @@ func TestConfig(t *testing.T) { t.Fatal("Failed to correctly initialize SwapParams") } - if one.HiveParams.MaxPeersPerRequest != 5 { - t.Fatal("Failed to correctly initialize HiveParams") - } - if one.StoreParams.ChunkDbPath == one.Path { t.Fatal("Failed to correctly initialize StoreParams") } diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 74341899d22b..440cb04cb80e 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -21,6 +21,7 @@ package http import ( "archive/tar" + "bytes" "encoding/json" "errors" "fmt" @@ -290,6 +291,56 @@ func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) { fmt.Fprint(w, newKey) } +func (s *Server) HandlePostDb(w http.ResponseWriter, r *Request) { + if r.ContentLength == 0 { + frequency, err := strconv.ParseUint(r.uri.Path, 10, 64) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + http.ServeContent(w, &r.Request, "", time.Now(), bytes.NewReader([]byte(err.Error()))) + return + } + err = s.api.DbCreate(r.uri.Addr, frequency) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + } else { + data, err := ioutil.ReadAll(r.Body) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + _, _, _, err = s.api.DbUpdate(r.uri.Addr, data) + if err != nil { + w.WriteHeader(http.StatusUnauthorized) + http.ServeContent(w, &r.Request, "", time.Now(), bytes.NewReader([]byte(err.Error()))) + return + } + } + w.WriteHeader(http.StatusOK) +} + +func (s *Server) HandleGetDb(w http.ResponseWriter, r *Request) { + w.Header().Set("Content-Type", "application/octet-stream") + + var params []string + if len(r.uri.Path) > 0 { + params = strings.Split(r.uri.Path, "/") + } + switch len(params) { + case 0: + data, err := s.api.DbLookupLatest(r.uri.Addr) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + break + } + http.ServeContent(w, &r.Request, "", time.Now(), data) + break + default: + w.WriteHeader(http.StatusBadRequest) + } +} + // HandleGet handles a GET request to // - bzz-raw:// and responds with the raw content stored at the // given storage key @@ -604,6 +655,8 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { case "POST": if uri.Raw() || uri.DeprecatedRaw() { s.HandlePostRaw(w, req) + } else if uri.Db() { + s.HandlePostDb(w, req) } else { s.HandlePostFiles(w, req) } @@ -644,6 +697,11 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } + if uri.Db() { + s.HandleGetDb(w, req) + return + } + s.HandleGetFile(w, req) default: diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 305d5cf7db68..06d4660d6b7a 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -22,17 +22,57 @@ import ( "fmt" "io/ioutil" "net/http" + "os" "strings" "sync" "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/api" swarm "github.com/ethereum/go-ethereum/swarm/api/client" "github.com/ethereum/go-ethereum/swarm/storage" "github.com/ethereum/go-ethereum/swarm/testutil" ) +func init() { + log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) +} + +func TestBzzGetDb(t *testing.T) { + srv := testutil.NewTestSwarmServer(t) + defer srv.Close() + + url := srv.URL + "/bzz-db:/foo/42" + resp, err := http.Post(url, "application/octet-stream", nil) + if err != nil { + fmt.Printf("err: %v\n", err) + return + } + b, err := ioutil.ReadAll(resp.Body) + fmt.Printf("Create: %s : %s\n", resp.Status, b) + + url = srv.URL + "/bzz-db:/foo" + data := []byte("foo") + resp, err = http.Post(url, "application/octet-stream", bytes.NewReader(data)) + if err != nil { + fmt.Printf("err: %v\n", err) + return + } + b, err = ioutil.ReadAll(resp.Body) + fmt.Printf("Update: %s : %s\n", resp.Status, b) + + url = srv.URL + "/bzz-db:/foo" + resp, err = http.Get(url) + if err != nil { + fmt.Printf("err: %v\n", err) + return + } + b, err = ioutil.ReadAll(resp.Body) + fmt.Printf("Get: %s : %s\n", resp.Status, b) + +} + func TestBzzGetPath(t *testing.T) { var err error @@ -258,7 +298,6 @@ func TestBzzGetPath(t *testing.T) { t.Fatalf("Non-Hash response body does not match, expected: %v, got: %v", nonhashresponses[i], string(respbody)) } } - } // TestBzzRootRedirect tests that getting the root path of a manifest without diff --git a/swarm/api/uri.go b/swarm/api/uri.go index d8aafedf4138..9b786045c75d 100644 --- a/swarm/api/uri.go +++ b/swarm/api/uri.go @@ -69,7 +69,7 @@ func Parse(rawuri string) (*URI, error) { // check the scheme is valid switch uri.Scheme { - case "bzz", "bzz-raw", "bzz-immutable", "bzz-list", "bzz-hash", "bzzr", "bzzi": + case "bzz", "bzz-raw", "bzz-immutable", "bzz-list", "bzz-hash", "bzzr", "bzzi", "bzz-db": default: return nil, fmt.Errorf("unknown scheme %q", u.Scheme) } @@ -92,6 +92,10 @@ func Parse(rawuri string) (*URI, error) { return uri, nil } +func (u *URI) Db() bool { + return u.Scheme == "bzz-db" +} + func (u *URI) Raw() bool { return u.Scheme == "bzz-raw" } diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 3d6c65371666..1e00c1dcf71d 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -44,8 +44,8 @@ type resource struct { } // TODO Expire content after a defined period (to force resync) -func (r *resource) isSynced() bool { - return !r.updated.IsZero() +func (self *resource) isSynced() bool { + return !self.updated.IsZero() } // Implement to activate validation of resource updates @@ -166,6 +166,30 @@ func NewResourceHandler(datadir string, cloudStore CloudStore, rpcClient *rpc.Cl return rh, nil } +func (self *ResourceHandler) GetData(name string) []byte { + rsrc := self.getResource(name) + if rsrc == nil { + return nil + } + return rsrc.data +} + +func (self *ResourceHandler) GetLastPeriod(name string) uint32 { + rsrc := self.getResource(name) + if rsrc == nil { + return 0 + } + return rsrc.lastPeriod +} + +func (self *ResourceHandler) GetVersion(name string) uint32 { + rsrc := self.getResource(name) + if rsrc == nil { + return 0 + } + return rsrc.version +} + // \TODO should be hashsize * branches from the chosen chunker, implement with dpa func (self *ResourceHandler) chunkSize() int64 { return chunkSize diff --git a/swarm/testutil/http.go b/swarm/testutil/http.go index b1dd4d4e6521..fbe52e111156 100644 --- a/swarm/testutil/http.go +++ b/swarm/testutil/http.go @@ -17,11 +17,15 @@ package testutil import ( + "crypto/ecdsa" "io/ioutil" "net/http/httptest" "os" + "path/filepath" + "strconv" "testing" + "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm/api" httpapi "github.com/ethereum/go-ethereum/swarm/api/http" "github.com/ethereum/go-ethereum/swarm/storage" @@ -38,7 +42,7 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer { CacheCapacity: 5000, Radius: 0, } - localStore, err := storage.NewLocalStore(storage.MakeHashFunc("SHA3"), storeparams, nil) + localStore, err := storage.NewLocalStore(storage.MakeHashFunc(storage.SHA3Hash), storeparams) if err != nil { os.RemoveAll(dir) t.Fatal(err) @@ -49,20 +53,59 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer { ChunkStore: localStore, } dpa.Start() - a := api.NewApi(dpa, nil) + + // mutable resources test setup + resourcedir, err := ioutil.TempDir("", "swarm-resource-test") + if err != nil { + t.Fatal(err) + } + ipcpath := filepath.Join(resourcedir, "test.ipc") + ipcl, err := rpc.CreateIPCListener(ipcpath) + if err != nil { + t.Fatal(err) + } + rpcserver := rpc.NewServer() + rpcserver.RegisterName("eth", &FakeRPC{}) + go func() { + rpcserver.ServeListener(ipcl) + }() + rpcClean := func() { + rpcserver.Stop() + } + + // connect to fake rpc + rpcclient, err := rpc.Dial(ipcpath) + if err != nil { + t.Fatal(err) + } + rh, err := storage.NewResourceHandler(resourcedir, &testCloudStore{}, rpcclient, nil) + if err != nil { + t.Fatal(err) + } + + a := api.NewApi(dpa, nil, rh) srv := httptest.NewServer(httpapi.NewServer(a)) return &TestSwarmServer{ Server: srv, Dpa: dpa, dir: dir, + hasher: storage.MakeHashFunc("SHA3")(), + cleanup: func() { + rh.Close() + rpcClean() + os.RemoveAll(dir) + os.RemoveAll(resourcedir) + }, } } type TestSwarmServer struct { *httptest.Server - - Dpa *storage.DPA - dir string + hasher storage.SwarmHash + privatekey *ecdsa.PrivateKey + Dpa *storage.DPA + dir string + cleanup func() } func (t *TestSwarmServer) Close() { @@ -70,3 +113,24 @@ func (t *TestSwarmServer) Close() { t.Dpa.Stop() os.RemoveAll(t.dir) } + +type testCloudStore struct { +} + +func (c *testCloudStore) Store(*storage.Chunk) { +} + +func (c *testCloudStore) Deliver(*storage.Chunk) { +} + +func (c *testCloudStore) Retrieve(*storage.Chunk) { +} + +// for faking the rpc service, since we don't need the whole node stack +type FakeRPC struct { + blocknumber uint64 +} + +func (r *FakeRPC) BlockNumber() (string, error) { + return strconv.FormatUint(r.blocknumber, 10), nil +} From 81ec1f9694f84178895c134634d0702eae7d8e6f Mon Sep 17 00:00:00 2001 From: lash Date: Fri, 19 Jan 2018 17:08:04 +0100 Subject: [PATCH 04/25] swarm/api: Add all lookup types + public getblock --- swarm/api/api.go | 19 +++++++++++++++++-- swarm/api/http/server.go | 35 +++++++++++++++++++++++++++++------ swarm/api/uri.go | 6 +++++- swarm/storage/resource.go | 8 ++++---- 4 files changed, 55 insertions(+), 13 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index cc10fa21347a..827e89ad1d07 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -364,8 +364,23 @@ func (self *Api) BuildDirectoryTree(mhash string, nameresolver bool) (key storag return key, manifestEntryMap, nil } -func (self *Api) DbLookupLatest(name string) (io.ReadSeeker, error) { - _, err := self.resource.LookupLatest(name, true) +// Look up mutable resource updates at specific periods and versions +func (self *Api) DbLookup(name string, period uint32, version uint32) (io.ReadSeeker, error) { + var err error + if version != 0 { + if period == 0 { + currentblocknumber, err := self.resource.GetBlock() + if err != nil { + return nil, fmt.Errorf("Could not determine latest block: %v", err) + } + period = self.resource.BlockToPeriod(name, currentblocknumber) + } + _, err = self.resource.LookupVersion(name, period, version, true) + } else if period != 0 { + _, err = self.resource.LookupHistorical(name, period, true) + } else { + _, err = self.resource.LookupLatest(name, true) + } if err != nil { return nil, err } diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 440cb04cb80e..bcdb4d2e593a 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -320,25 +320,48 @@ func (s *Server) HandlePostDb(w http.ResponseWriter, r *Request) { w.WriteHeader(http.StatusOK) } +// Retrieve mutable resource updates: +// bzz-db[-[immutable|-raw]]:// - get latest update +// bzz-db[-[immutable|-raw]]:/// - get latest update on period n +// bzz-db[-[immutable|-raw]]://// - get update version m of period n func (s *Server) HandleGetDb(w http.ResponseWriter, r *Request) { w.Header().Set("Content-Type", "application/octet-stream") + key, err := s.api.Resolve(r.uri) + if err != nil { + s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) + return + } + _ = key + var params []string if len(r.uri.Path) > 0 { params = strings.Split(r.uri.Path, "/") } switch len(params) { case 0: - data, err := s.api.DbLookupLatest(r.uri.Addr) - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - break - } - http.ServeContent(w, &r.Request, "", time.Now(), data) + data, err := s.api.DbLookup(r.uri.Addr) + break + case 2: + strconv.ParseUint(params[1], 10, 32) + case 1: + strconv.ParseUint(params[0], 10, 32) break default: w.WriteHeader(http.StatusBadRequest) + err = "params 0-2" + } + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return } + wrappedData := wrapDbContent(data, s.uri.Scheme) + http.ServeContent(w, &r.Request, "", time.Now(), data) + +} + +func wrapDbContent(data io.Reader, scheme *string) io.Reader { + } // HandleGet handles a GET request to diff --git a/swarm/api/uri.go b/swarm/api/uri.go index 9b786045c75d..c7a929f0f2c2 100644 --- a/swarm/api/uri.go +++ b/swarm/api/uri.go @@ -69,7 +69,7 @@ func Parse(rawuri string) (*URI, error) { // check the scheme is valid switch uri.Scheme { - case "bzz", "bzz-raw", "bzz-immutable", "bzz-list", "bzz-hash", "bzzr", "bzzi", "bzz-db": + case "bzz", "bzz-raw", "bzz-immutable", "bzz-list", "bzz-hash", "bzzr", "bzzi", "bzz-db", "bzz-db-raw": default: return nil, fmt.Errorf("unknown scheme %q", u.Scheme) } @@ -96,6 +96,10 @@ func (u *URI) Db() bool { return u.Scheme == "bzz-db" } +func (u *URI) DbRaw() bool { + return u.Scheme == "bzz-db-raw" +} + func (u *URI) Raw() bool { return u.Scheme == "bzz-raw" } diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 1e00c1dcf71d..33845da40dd9 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -231,7 +231,7 @@ func (self *ResourceHandler) NewResource(name string, frequency uint64) (*resour } // get our blockheight at this time - currentblock, err := self.getBlock() + currentblock, err := self.GetBlock() if err != nil { return nil, err } @@ -310,7 +310,7 @@ func (self *ResourceHandler) LookupLatest(name string, refresh bool) (*resource, if err != nil { return nil, err } - currentblock, err := self.getBlock() + currentblock, err := self.GetBlock() if err != nil { return nil, err } @@ -492,7 +492,7 @@ func (self *ResourceHandler) Update(name string, data []byte) (Key, error) { } // get our blockheight at this time and the next block of the update period - currentblock, err := self.getBlock() + currentblock, err := self.GetBlock() if err != nil { return nil, err } @@ -560,7 +560,7 @@ func (self *ResourceHandler) Close() { self.ChunkStore.Close() } -func (self *ResourceHandler) getBlock() (uint64, error) { +func (self *ResourceHandler) GetBlock() (uint64, error) { // get the block height and convert to uint64 var currentblock string err := self.rpcClient.Call(¤tblock, "eth_blockNumber") From 1bf4f4a37f02b57e1a90929c10413fa47b1d42d1 Mon Sep 17 00:00:00 2001 From: lash Date: Fri, 19 Jan 2018 18:28:10 +0100 Subject: [PATCH 05/25] swarm/api: Add raw form to api+server WIP --- swarm/api/api.go | 24 +++++++++---- swarm/api/http/server.go | 39 ++++++++++++--------- swarm/api/http/server_test.go | 15 +++++--- swarm/storage/resource.go | 65 ++++++++++++++++++++++++----------- swarm/storage/resource_ens.go | 6 ++++ swarm/testutil/http.go | 2 +- 6 files changed, 103 insertions(+), 48 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index 827e89ad1d07..cfc26aeff560 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -46,9 +46,9 @@ on top of the dpa it is the public interface of the dpa which is included in the ethereum stack */ type Api struct { + resource *storage.ResourceHandler dpa *storage.DPA dns Resolver - resource *storage.ResourceHandler } //the api constructor initialises @@ -365,7 +365,7 @@ func (self *Api) BuildDirectoryTree(mhash string, nameresolver bool) (key storag } // Look up mutable resource updates at specific periods and versions -func (self *Api) DbLookup(name string, period uint32, version uint32) (io.ReadSeeker, error) { +func (self *Api) DbLookup(key storage.Key, name string, period uint32, version uint32) (io.ReadSeeker, error) { var err error if version != 0 { if period == 0 { @@ -375,16 +375,20 @@ func (self *Api) DbLookup(name string, period uint32, version uint32) (io.ReadSe } period = self.resource.BlockToPeriod(name, currentblocknumber) } - _, err = self.resource.LookupVersion(name, period, version, true) + _, err = self.resource.LookupVersionByName(name, period, version, true) } else if period != 0 { - _, err = self.resource.LookupHistorical(name, period, true) + _, err = self.resource.LookupHistoricalByName(name, period, true) } else { - _, err = self.resource.LookupLatest(name, true) + _, err = self.resource.LookupLatestByName(name, true) } if err != nil { return nil, err } - return bytes.NewReader(self.resource.GetData(name)), nil + data, err := self.resource.GetData(name) + if err != nil { + return nil, err + } + return bytes.NewReader(data), nil } func (self *Api) DbCreate(name string, frequency uint64) (err error) { @@ -394,5 +398,11 @@ func (self *Api) DbCreate(name string, frequency uint64) (err error) { func (self *Api) DbUpdate(name string, data []byte) (storage.Key, uint32, uint32, error) { key, err := self.resource.Update(name, data) - return key, self.resource.GetLastPeriod(name), self.resource.GetVersion(name), err + period, _ := self.resource.GetLastPeriod(name) + version, _ := self.resource.GetVersion(name) + return key, period, version, err +} + +func (self *Api) DbHashSize() int { + return self.resource.HashSize() } diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index bcdb4d2e593a..9e3c7711c178 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -332,36 +332,42 @@ func (s *Server) HandleGetDb(w http.ResponseWriter, r *Request) { s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) return } - _ = key var params []string if len(r.uri.Path) > 0 { params = strings.Split(r.uri.Path, "/") } + var period uint64 + var version uint64 + var data io.ReadSeeker switch len(params) { case 0: - data, err := s.api.DbLookup(r.uri.Addr) + data, err = s.api.DbLookup(key, r.uri.Addr, 0, 0) break case 2: - strconv.ParseUint(params[1], 10, 32) + version, err = strconv.ParseUint(params[1], 10, 32) + if err != nil { + break + } case 1: - strconv.ParseUint(params[0], 10, 32) + period, err = strconv.ParseUint(params[0], 10, 32) + if err != nil { + break + } + data, err = s.api.DbLookup(key, r.uri.Addr, uint32(period), uint32(version)) break default: w.WriteHeader(http.StatusBadRequest) - err = "params 0-2" + err = fmt.Errorf("params 0-2") } if err != nil { w.WriteHeader(http.StatusInternalServerError) return } - wrappedData := wrapDbContent(data, s.uri.Scheme) - http.ServeContent(w, &r.Request, "", time.Now(), data) - -} - -func wrapDbContent(data io.Reader, scheme *string) io.Reader { + if !r.uri.DbRaw() { + } + http.ServeContent(w, &r.Request, "", time.Now(), data) } // HandleGet handles a GET request to @@ -705,6 +711,12 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { s.HandleDelete(w, req) case "GET": + + if uri.Db() || uri.DbRaw() { + s.HandleGetDb(w, req) + return + } + if uri.Raw() || uri.Hash() || uri.DeprecatedRaw() { s.HandleGet(w, req) return @@ -720,11 +732,6 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - if uri.Db() { - s.HandleGetDb(w, req) - return - } - s.HandleGetFile(w, req) default: diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 06d4660d6b7a..888445d88d85 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -18,6 +18,7 @@ package http_test import ( "bytes" + "crypto/rand" "errors" "fmt" "io/ioutil" @@ -43,7 +44,14 @@ func TestBzzGetDb(t *testing.T) { srv := testutil.NewTestSwarmServer(t) defer srv.Close() - url := srv.URL + "/bzz-db:/foo/42" + keybytes := make([]byte, common.HashLength) // nearest we get to source of info + _, err := rand.Read(keybytes) + if err != nil { + fmt.Printf("err: %v\n", err) + return + } + + url := fmt.Sprintf("%s/bzz-db:/%s/42", srv.URL, fmt.Sprintf("%x", keybytes)) resp, err := http.Post(url, "application/octet-stream", nil) if err != nil { fmt.Printf("err: %v\n", err) @@ -52,7 +60,7 @@ func TestBzzGetDb(t *testing.T) { b, err := ioutil.ReadAll(resp.Body) fmt.Printf("Create: %s : %s\n", resp.Status, b) - url = srv.URL + "/bzz-db:/foo" + url = fmt.Sprintf("%s/bzz-db:/%s", srv.URL, fmt.Sprintf("%x", keybytes)) data := []byte("foo") resp, err = http.Post(url, "application/octet-stream", bytes.NewReader(data)) if err != nil { @@ -62,7 +70,7 @@ func TestBzzGetDb(t *testing.T) { b, err = ioutil.ReadAll(resp.Body) fmt.Printf("Update: %s : %s\n", resp.Status, b) - url = srv.URL + "/bzz-db:/foo" + url = fmt.Sprintf("%s/bzz-db-raw:/%s", srv.URL, fmt.Sprintf("%x", keybytes)) resp, err = http.Get(url) if err != nil { fmt.Printf("err: %v\n", err) @@ -70,7 +78,6 @@ func TestBzzGetDb(t *testing.T) { } b, err = ioutil.ReadAll(resp.Body) fmt.Printf("Get: %s : %s\n", resp.Status, b) - } func TestBzzGetPath(t *testing.T) { diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 33845da40dd9..b76117b3360f 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -51,6 +51,7 @@ func (self *resource) isSynced() bool { // Implement to activate validation of resource updates // Specifically signing data and verification of signatures type ResourceValidator interface { + hashSize() int checkAccess(string, common.Address) (bool, error) nameHash(string) common.Hash // nameHashFunc sign(common.Hash) (Signature, error) // SignFunc @@ -166,28 +167,34 @@ func NewResourceHandler(datadir string, cloudStore CloudStore, rpcClient *rpc.Cl return rh, nil } -func (self *ResourceHandler) GetData(name string) []byte { +func (self *ResourceHandler) HashSize() int { + return self.validator.hashSize() +} + +// get data from current resource +func (self *ResourceHandler) GetData(name string) ([]byte, error) { rsrc := self.getResource(name) - if rsrc == nil { - return nil + if rsrc == nil || !rsrc.isSynced() { + return nil, fmt.Errorf("Resource does not exist or is not synced") } - return rsrc.data + return rsrc.data, nil } -func (self *ResourceHandler) GetLastPeriod(name string) uint32 { +func (self *ResourceHandler) GetLastPeriod(name string) (uint32, error) { rsrc := self.getResource(name) - if rsrc == nil { - return 0 + + if rsrc == nil || !rsrc.isSynced() { + return 0, fmt.Errorf("Resource does not exist or is not synced") } - return rsrc.lastPeriod + return rsrc.lastPeriod, nil } -func (self *ResourceHandler) GetVersion(name string) uint32 { +func (self *ResourceHandler) GetVersion(name string) (uint32, error) { rsrc := self.getResource(name) - if rsrc == nil { - return 0 + if rsrc == nil || !rsrc.isSynced() { + return 0, fmt.Errorf("Resource does not exist or is not synced") } - return rsrc.version + return rsrc.version, nil } // \TODO should be hashsize * branches from the chosen chunker, implement with dpa @@ -269,8 +276,14 @@ func (self *ResourceHandler) NewResource(name string, frequency uint64) (*resour // root chunk. // It is the callers responsibility to make sure that this chunk exists (if the resource // update root data was retrieved externally, it typically doesn't) -func (self *ResourceHandler) LookupVersion(name string, period uint32, version uint32, refresh bool) (*resource, error) { - rsrc, err := self.loadResource(name, refresh) +// +// +func (self *ResourceHandler) LookupVersionByName(name string, period uint32, version uint32, refresh bool) (*resource, error) { + return self.LookupVersion(self.nameHash(name), name, period, version, refresh) +} + +func (self *ResourceHandler) LookupVersion(nameHash common.Hash, name string, period uint32, version uint32, refresh bool) (*resource, error) { + rsrc, err := self.loadResource(nameHash, name, refresh) if err != nil { return nil, err } @@ -285,8 +298,12 @@ func (self *ResourceHandler) LookupVersion(name string, period uint32, version u // and returned. // // See also (*ResourceHandler).LookupVersion -func (self *ResourceHandler) LookupHistorical(name string, period uint32, refresh bool) (*resource, error) { - rsrc, err := self.loadResource(name, refresh) +func (self *ResourceHandler) LookupHistoricalByName(name string, period uint32, refresh bool) (*resource, error) { + return self.LookupHistorical(self.nameHash(name), name, period, refresh) +} + +func (self *ResourceHandler) LookupHistorical(nameHash common.Hash, name string, period uint32, refresh bool) (*resource, error) { + rsrc, err := self.loadResource(nameHash, name, refresh) if err != nil { return nil, err } @@ -303,10 +320,14 @@ func (self *ResourceHandler) LookupHistorical(name string, period uint32, refres // Version iteration is done as in (*ResourceHandler).LookupHistorical // // See also (*ResourceHandler).LookupHistorical -func (self *ResourceHandler) LookupLatest(name string, refresh bool) (*resource, error) { +func (self *ResourceHandler) LookupLatestByName(name string, refresh bool) (*resource, error) { + return self.LookupLatest(self.nameHash(name), name, refresh) +} + +func (self *ResourceHandler) LookupLatest(nameHash common.Hash, name string, refresh bool) (*resource, error) { // get our blockheight at this time and the next block of the update period - rsrc, err := self.loadResource(name, refresh) + rsrc, err := self.loadResource(nameHash, name, refresh) if err != nil { return nil, err } @@ -362,7 +383,11 @@ func (self *ResourceHandler) lookup(rsrc *resource, period uint32, version uint3 } // load existing mutable resource into resource struct -func (self *ResourceHandler) loadResource(name string, refresh bool) (*resource, error) { +func (self *ResourceHandler) loadResource(nameHash common.Hash, name string, refresh bool) (*resource, error) { + + if name == "" { + name = nameHash.Hex() + } // if the resource is not known to this session we must load it // if refresh is set, we force load @@ -374,7 +399,7 @@ func (self *ResourceHandler) loadResource(name string, refresh bool) (*resource, return nil, fmt.Errorf("Invalid name '%s'", name) } rsrc.name = &name - rsrc.nameHash = self.nameHash(name) + rsrc.nameHash = nameHash // get the root info chunk and update the cached value chunk, err := self.Get(Key(rsrc.nameHash[:])) diff --git a/swarm/storage/resource_ens.go b/swarm/storage/resource_ens.go index 0a4500309d4f..df008efa175f 100644 --- a/swarm/storage/resource_ens.go +++ b/swarm/storage/resource_ens.go @@ -10,6 +10,7 @@ import ( type baseValidator struct { signFunc SignFunc + hashsize int } func (b *baseValidator) sign(datahash common.Hash) (signature Signature, err error) { @@ -19,6 +20,10 @@ func (b *baseValidator) sign(datahash common.Hash) (signature Signature, err err return b.signFunc(datahash) } +func (b *baseValidator) hashSize() int { + return b.hashsize +} + // ENS validation of mutable resource owners type ENSValidator struct { *baseValidator @@ -30,6 +35,7 @@ func NewENSValidator(contractaddress common.Address, backend bind.ContractBacken validator := &ENSValidator{ baseValidator: &baseValidator{ signFunc: signFunc, + hashsize: common.HashLength, }, } validator.api, err = ens.NewENS(transactOpts, contractaddress, backend) diff --git a/swarm/testutil/http.go b/swarm/testutil/http.go index fbe52e111156..dcba90c9c538 100644 --- a/swarm/testutil/http.go +++ b/swarm/testutil/http.go @@ -89,7 +89,7 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer { Server: srv, Dpa: dpa, dir: dir, - hasher: storage.MakeHashFunc("SHA3")(), + hasher: storage.MakeHashFunc(storage.SHA3Hash)(), cleanup: func() { rh.Close() rpcClean() From 249e1a8aaa6c384ee11fe0a58744d084fa6c6315 Mon Sep 17 00:00:00 2001 From: lash Date: Fri, 19 Jan 2018 20:39:49 +0100 Subject: [PATCH 06/25] swarm, cmd/swarm, ethclient: Fullstack mut.rsrc. w api Add ethclient.Client.BlockNumber method for access to current block number (needed by ResourceHandler) Add placeholder manifest support --- cmd/swarm/main.go | 2 +- ethclient/ethclient.go | 10 ++++++ swarm/api/api.go | 16 ++++++---- swarm/api/config.go | 58 ++++++++++++++++++---------------- swarm/api/http/server.go | 36 ++++++++++++++++++--- swarm/api/http/server_test.go | 10 ++++++ swarm/api/manifest.go | 3 +- swarm/storage/resource.go | 51 ++++++++++++++++++++---------- swarm/storage/resource_sign.go | 20 ++++++++++++ swarm/storage/resource_test.go | 35 +++++++++----------- swarm/swarm.go | 24 ++++++++++++-- swarm/testutil/http.go | 23 ++++++++------ 12 files changed, 197 insertions(+), 91 deletions(-) create mode 100644 swarm/storage/resource_sign.go diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 886895852ffb..4233c7c98208 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -560,7 +560,7 @@ func registerBzzService(bzzconfig *bzzapi.Config, ctx *cli.Context, stack *node. } // In production, mockStore must be always nil. - return swarm.NewSwarm(ctx, swapClient, ensClient, bzzconfig, bzzconfig.SwapEnabled, bzzconfig.SyncEnabled, bzzconfig.Cors, bzzconfig.PssEnabled, nil) + return swarm.NewSwarm(ctx, swapClient, ensClient, bzzconfig, bzzconfig.SwapEnabled, bzzconfig.SyncEnabled, bzzconfig.Cors, bzzconfig.PssEnabled, bzzconfig.ResourceEnabled, nil) } //register within the ethereum node if err := stack.Register(boot); err != nil { diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index 87a912901af5..4b53e785d645 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -23,6 +23,7 @@ import ( "errors" "fmt" "math/big" + "strconv" "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" @@ -76,6 +77,15 @@ type rpcBlock struct { UncleHashes []common.Hash `json:"uncles"` } +func (ec *Client) BlockNumber(ctx context.Context) (uint64, error) { + var number string + err := ec.c.CallContext(ctx, &number, "eth_blockNumber") + if err != nil { + return 0, err + } + return strconv.ParseUint(number, 10, 64) +} + func (ec *Client) getBlock(ctx context.Context, method string, args ...interface{}) (*types.Block, error) { var raw json.RawMessage err := ec.c.CallContext(ctx, &raw, method, args...) diff --git a/swarm/api/api.go b/swarm/api/api.go index cfc26aeff560..0e11028ec435 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -365,13 +365,13 @@ func (self *Api) BuildDirectoryTree(mhash string, nameresolver bool) (key storag } // Look up mutable resource updates at specific periods and versions -func (self *Api) DbLookup(key storage.Key, name string, period uint32, version uint32) (io.ReadSeeker, error) { +func (self *Api) DbLookup(key storage.Key, name string, period uint32, version uint32) (storage.Key, io.ReadSeeker, int, error) { var err error if version != 0 { if period == 0 { currentblocknumber, err := self.resource.GetBlock() if err != nil { - return nil, fmt.Errorf("Could not determine latest block: %v", err) + return nil, nil, 0, fmt.Errorf("Could not determine latest block: %v", err) } period = self.resource.BlockToPeriod(name, currentblocknumber) } @@ -382,13 +382,13 @@ func (self *Api) DbLookup(key storage.Key, name string, period uint32, version u _, err = self.resource.LookupLatestByName(name, true) } if err != nil { - return nil, err + return nil, nil, 0, err } - data, err := self.resource.GetData(name) + key, data, err := self.resource.GetContent(name) if err != nil { - return nil, err + return nil, nil, 0, err } - return bytes.NewReader(data), nil + return key, bytes.NewReader(data), len(data), nil } func (self *Api) DbCreate(name string, frequency uint64) (err error) { @@ -406,3 +406,7 @@ func (self *Api) DbUpdate(name string, data []byte) (storage.Key, uint32, uint32 func (self *Api) DbHashSize() int { return self.resource.HashSize() } + +func (self *Api) DbIsValidated() bool { + return self.resource.IsValidated() +} diff --git a/swarm/api/config.go b/swarm/api/config.go index d4dba36094fa..31281f9bff35 100644 --- a/swarm/api/config.go +++ b/swarm/api/config.go @@ -46,22 +46,23 @@ type Config struct { *network.HiveParams Swap *swap.SwapParams //*network.SyncParams - Contract common.Address - EnsRoot common.Address - EnsApi string - Path string - ListenAddr string - Port string - PublicKey string - BzzKey string - NetworkId uint64 - SwapEnabled bool - SyncEnabled bool - PssEnabled bool - SwapApi string - Cors string - BzzAccount string - BootNodes string + Contract common.Address + EnsRoot common.Address + EnsApi string + Path string + ListenAddr string + Port string + PublicKey string + BzzKey string + NetworkId uint64 + SwapEnabled bool + SyncEnabled bool + PssEnabled bool + ResourceEnabled bool + SwapApi string + Cors string + BzzAccount string + BootNodes string } //create a default config with all parameters to set to defaults @@ -72,18 +73,19 @@ func NewConfig() (self *Config) { ChunkerParams: storage.NewChunkerParams(), HiveParams: network.NewHiveParams(), //SyncParams: network.NewDefaultSyncParams(), - Swap: swap.NewDefaultSwapParams(), - ListenAddr: DefaultHTTPListenAddr, - Port: DefaultHTTPPort, - Path: node.DefaultDataDir(), - EnsApi: node.DefaultIPCEndpoint("geth"), - EnsRoot: ens.TestNetAddress, - NetworkId: network.NetworkID, - SwapEnabled: false, - SyncEnabled: true, - PssEnabled: true, - SwapApi: "", - BootNodes: "", + Swap: swap.NewDefaultSwapParams(), + ListenAddr: DefaultHTTPListenAddr, + Port: DefaultHTTPPort, + Path: node.DefaultDataDir(), + EnsApi: node.DefaultIPCEndpoint("geth"), + EnsRoot: ens.TestNetAddress, + NetworkId: network.NetworkID, + SwapEnabled: false, + SyncEnabled: true, + PssEnabled: true, + ResourceEnabled: true, + SwapApi: "", + BootNodes: "", } return diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 9e3c7711c178..26c59c313e6e 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -327,7 +327,7 @@ func (s *Server) HandlePostDb(w http.ResponseWriter, r *Request) { func (s *Server) HandleGetDb(w http.ResponseWriter, r *Request) { w.Header().Set("Content-Type", "application/octet-stream") - key, err := s.api.Resolve(r.uri) + rootKey, err := s.api.Resolve(r.uri) if err != nil { s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) return @@ -337,12 +337,15 @@ func (s *Server) HandleGetDb(w http.ResponseWriter, r *Request) { if len(r.uri.Path) > 0 { params = strings.Split(r.uri.Path, "/") } + var updateKey storage.Key var period uint64 var version uint64 var data io.ReadSeeker + var dataLength int + now := time.Now() switch len(params) { case 0: - data, err = s.api.DbLookup(key, r.uri.Addr, 0, 0) + updateKey, data, dataLength, err = s.api.DbLookup(rootKey, r.uri.Addr, 0, 0) break case 2: version, err = strconv.ParseUint(params[1], 10, 32) @@ -354,7 +357,7 @@ func (s *Server) HandleGetDb(w http.ResponseWriter, r *Request) { if err != nil { break } - data, err = s.api.DbLookup(key, r.uri.Addr, uint32(period), uint32(version)) + updateKey, data, dataLength, err = s.api.DbLookup(rootKey, r.uri.Addr, uint32(period), uint32(version)) break default: w.WriteHeader(http.StatusBadRequest) @@ -365,9 +368,32 @@ func (s *Server) HandleGetDb(w http.ResponseWriter, r *Request) { return } if !r.uri.DbRaw() { - + entry := api.ManifestEntry{ + Hash: rootKey.Hex(), + Path: updateKey.Hex(), + ContentType: api.DbManifestType, + Size: int64(dataLength), + ModTime: now, + Status: http.StatusOK, + } + mode := (6 << 2) | (4 << 1) | 4 + if s.api.DbIsValidated() { + mode |= 2 << 1 + } + entry.Mode = int64(mode) + manifest := api.Manifest{ + Entries: []api.ManifestEntry{ + entry, + }, + } + manifestJson, err := json.Marshal(manifest) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + data = bytes.NewReader(manifestJson) } - http.ServeContent(w, &r.Request, "", time.Now(), data) + http.ServeContent(w, &r.Request, "", now, data) } // HandleGet handles a GET request to diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 888445d88d85..e2165f1d61d0 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -78,6 +78,16 @@ func TestBzzGetDb(t *testing.T) { } b, err = ioutil.ReadAll(resp.Body) fmt.Printf("Get: %s : %s\n", resp.Status, b) + + url = fmt.Sprintf("%s/bzz-db:/%s", srv.URL, fmt.Sprintf("%x", keybytes)) + resp, err = http.Get(url) + if err != nil { + fmt.Printf("err: %v\n", err) + return + } + b, err = ioutil.ReadAll(resp.Body) + fmt.Printf("Get: %s : %s\n", resp.Status, b) + } func TestBzzGetPath(t *testing.T) { diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index 685a300fca5b..46b55b3b7a17 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -33,7 +33,8 @@ import ( ) const ( - ManifestType = "application/bzz-manifest+json" + ManifestType = "application/bzz-manifest+json" + DbManifestType = "application/bzz-db-manifest+json" ) // Manifest represents a swarm manifest diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index b76117b3360f..23e1a6336b31 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -1,10 +1,10 @@ package storage import ( + "context" "encoding/binary" "fmt" "path/filepath" - "strconv" "sync" "time" @@ -12,8 +12,8 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/rpc" ) const ( @@ -37,6 +37,7 @@ type resource struct { nameHash common.Hash startBlock uint64 lastPeriod uint32 + lastKey Key frequency uint64 version uint32 data []byte @@ -114,26 +115,30 @@ type ResourceValidator interface { // stored using a separate store, and forwarding/syncing protocols carry per-chunk // flags to tell whether the chunk can be validated or not; if not it is to be // treated as a resource update chunk. +// +// TODO: Include modtime in chunk data + signature type ResourceHandler struct { ChunkStore validator ResourceValidator - rpcClient *rpc.Client + ethClient *ethclient.Client resources map[string]*resource hashLock sync.Mutex resourceLock sync.RWMutex hasher SwarmHash nameHash nameHashFunc storeTimeout time.Duration + ctx context.Context + cancelFunc func() } // Create or open resource update chunk store // // If validator is nil, signature and access validation will be deactivated -func NewResourceHandler(datadir string, cloudStore CloudStore, rpcClient *rpc.Client, validator ResourceValidator) (*ResourceHandler, error) { +func NewResourceHandler(datadir string, cloudStore CloudStore, ethClient *ethclient.Client, validator ResourceValidator) (*ResourceHandler, error) { hashfunc := MakeHashFunc(SHA3Hash) - path := filepath.Join(datadir, dbDirName) + path := filepath.Join(datadir, DbDirName) dbStore, err := NewDbStore(datadir, hashfunc, singletonSwarmDbCapacity, 0) if err != nil { return nil, err @@ -143,6 +148,7 @@ func NewResourceHandler(datadir string, cloudStore CloudStore, rpcClient *rpc.Cl DbStore: dbStore, } + ctx, cancel := context.WithCancel(context.Background()) rh := &ResourceHandler{ ChunkStore: newResourceChunkStore(path, hashfunc, localStore, cloudStore), rpcClient: rpcClient, @@ -150,6 +156,8 @@ func NewResourceHandler(datadir string, cloudStore CloudStore, rpcClient *rpc.Cl hasher: hashfunc(), validator: validator, storeTimeout: defaultStoreTimeout, + ctx: ctx, + cancelFunc: cancel, } if rh.validator != nil { @@ -167,17 +175,22 @@ func NewResourceHandler(datadir string, cloudStore CloudStore, rpcClient *rpc.Cl return rh, nil } +func (self *ResourceHandler) IsValidated() bool { + return self.validator == nil +} + func (self *ResourceHandler) HashSize() int { return self.validator.hashSize() } // get data from current resource -func (self *ResourceHandler) GetData(name string) ([]byte, error) { + +func (self *ResourceHandler) GetContent(name string) (Key, []byte, error) { rsrc := self.getResource(name) if rsrc == nil || !rsrc.isSynced() { - return nil, fmt.Errorf("Resource does not exist or is not synced") + return nil, nil, fmt.Errorf("Resource does not exist or is not synced") } - return rsrc.data, nil + return rsrc.lastKey, rsrc.data, nil } func (self *ResourceHandler) GetLastPeriod(name string) (uint32, error) { @@ -430,6 +443,7 @@ func (self *ResourceHandler) updateResourceIndex(rsrc *resource, chunk *Chunk) ( if *rsrc.name != name { return nil, fmt.Errorf("Update belongs to '%s', but have '%s'", name, *rsrc.name) } + log.Trace("update", "name", *rsrc.name, "rootkey", rsrc.nameHash, "updatekey", chunk.Key, "period", period, "version", version) // only check signature if validator is present if self.validator != nil { digest := self.keyDataHash(chunk.Key, data) @@ -440,6 +454,7 @@ func (self *ResourceHandler) updateResourceIndex(rsrc *resource, chunk *Chunk) ( } // update our rsrcs entry map + rsrc.lastKey = chunk.Key rsrc.lastPeriod = period rsrc.version = version rsrc.updated = time.Now() @@ -582,20 +597,22 @@ func (self *ResourceHandler) Update(name string, data []byte) (Key, error) { // Closes the datastore. // Always call this at shutdown to avoid data corruption. func (self *ResourceHandler) Close() { + self.cancelFunc() self.ChunkStore.Close() } func (self *ResourceHandler) GetBlock() (uint64, error) { + return self.ethClient.BlockNumber(self.ctx) // get the block height and convert to uint64 - var currentblock string - err := self.rpcClient.Call(¤tblock, "eth_blockNumber") - if err != nil { - return 0, err - } - if currentblock == "0x0" { - return 0, nil - } - return strconv.ParseUint(currentblock, 10, 64) + // var currentblock string + // err := self.rpcClient.Call(¤tblock, "eth_blockNumber") + // if err != nil { + // return 0, err + // } + // if currentblock == "0x0" { + // return 0, nil + // } + // return strconv.ParseUint(currentblock, 10, 64) } // Calculate the period index (aka major version number) from a given block number diff --git a/swarm/storage/resource_sign.go b/swarm/storage/resource_sign.go new file mode 100644 index 000000000000..840e705ac115 --- /dev/null +++ b/swarm/storage/resource_sign.go @@ -0,0 +1,20 @@ +package storage + +import ( + "crypto/ecdsa" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" +) + +// matches the SignFunc type +func NewGenericResourceSigner(privKey *ecdsa.PrivateKey) SignFunc { + return func(data common.Hash) (signature Signature, err error) { + signaturebytes, err := crypto.Sign(data.Bytes(), privKey) + if err != nil { + return + } + copy(signature[:], signaturebytes) + return + } +} diff --git a/swarm/storage/resource_test.go b/swarm/storage/resource_test.go index d13070942830..2da9b7704aed 100644 --- a/swarm/storage/resource_test.go +++ b/swarm/storage/resource_test.go @@ -22,6 +22,7 @@ import ( "github.com/ethereum/go-ethereum/contracts/ens/contract" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" ) @@ -211,8 +212,8 @@ func TestResourceHandler(t *testing.T) { // it will match on second iteration startblocknumber + (resourceFrequency * 3) fwdBlocks(int(resourceFrequency*2)-1, backend) - rh2, err := NewResourceHandler(datadir, &testCloudStore{}, rh.rpcClient, nil) - _, err = rh2.LookupLatest(safeName, true) + rh2, err := NewResourceHandler(datadir, &testCloudStore{}, rh.ethClient, nil) + _, err = rh2.LookupLatestByName(safeName, true) if err != nil { teardownTest(t, err) } @@ -230,7 +231,7 @@ func TestResourceHandler(t *testing.T) { log.Debug("Latest lookup", "period", rh2.resources[safeName].lastPeriod, "version", rh2.resources[safeName].version, "data", rh2.resources[safeName].data) // specific block, latest version - rsrc, err := rh2.LookupHistorical(safeName, 3, true) + rsrc, err := rh2.LookupHistoricalByName(safeName, 3, true) if err != nil { teardownTest(t, err) } @@ -241,7 +242,7 @@ func TestResourceHandler(t *testing.T) { log.Debug("Historical lookup", "period", rh2.resources[safeName].lastPeriod, "version", rh2.resources[safeName].version, "data", rh2.resources[safeName].data) // specific block, specific version - rsrc, err = rh2.LookupVersion(safeName, 3, 1, true) + rsrc, err = rh2.LookupVersionByName(safeName, 3, 1, true) if err != nil { teardownTest(t, err) } @@ -365,12 +366,14 @@ func setupTest(contractbackend bind.ContractBackend, validator ResourceValidator } // connect to fake rpc - rpcclient, err := rpc.Dial(ipcpath) + rpcClient, err := rpc.Dial(ipcpath) if err != nil { return } - rh, err = NewResourceHandler(datadir, &testCloudStore{}, rpcclient, validator) + ethClient := ethclient.NewClient(rpcClient) + + rh, err = NewResourceHandler(datadir, &testCloudStore{}, ethClient, validator) teardown = func(t *testing.T, err error) { cleanF() if err != nil { @@ -426,8 +429,9 @@ func setupENS(addr common.Address, transactOpts *bind.TransactOpts, sub string, // implementation of an external signer to pass to validator type testSigner struct { - privKey *ecdsa.PrivateKey - hasher SwarmHash + privKey *ecdsa.PrivateKey + hasher SwarmHash + signContent SignFunc } func newTestSigner() (*testSigner, error) { @@ -436,21 +440,12 @@ func newTestSigner() (*testSigner, error) { return nil, err } return &testSigner{ - privKey: privKey, - hasher: testHasher, + privKey: privKey, + hasher: testHasher, + signContent: NewGenericResourceSigner(privKey), }, nil } -// matches the SignFunc type -func (self *testSigner) signContent(data common.Hash) (signature Signature, err error) { - signaturebytes, err := crypto.Sign(data.Bytes(), self.privKey) - if err != nil { - return - } - copy(signature[:], signaturebytes) - return -} - type testCloudStore struct { } diff --git a/swarm/swarm.go b/swarm/swarm.go index 14826bb02159..766c210d3d3d 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -22,6 +22,7 @@ import ( "crypto/ecdsa" "fmt" "net" + "path/filepath" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" @@ -82,7 +83,8 @@ func (self *Swarm) API() *SwarmAPI { // implements node.Service // If mockStore is not nil, it will be used as the storage for chunk data. // MockStore should be used only for testing. -func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *ethclient.Client, config *api.Config, swapEnabled, syncEnabled bool, cors string, pssEnabled bool, mockStore *mock.NodeStore) (self *Swarm, err error) { +func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *ethclient.Client, config *api.Config, swapEnabled, syncEnabled bool, cors string, pssEnabled bool, resourceEnabled bool, mockStore *mock.NodeStore) (self *Swarm, err error) { + if bytes.Equal(common.FromHex(config.PublicKey), storage.ZeroKey) { return nil, fmt.Errorf("empty public key") } @@ -158,7 +160,23 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e } log.Debug(fmt.Sprintf("-> Swarm Domain Name Registrar @ address %v", config.EnsRoot.Hex())) - self.api = api.NewApi(self.dpa, self.dns) + var resourceHandler *storage.ResourceHandler + // if use resource updates + if resourceEnabled { + var resourceValidator storage.ResourceValidator + if self.dns != nil { + resourceValidator, err = storage.NewENSValidator(config.EnsRoot, ensClient, transactOpts, storage.NewGenericResourceSigner(self.privateKey)) + if err != nil { + return nil, err + } + } + resourceHandler, err = storage.NewResourceHandler(filepath.Join(self.config.Path, storage.DbDirName), self.cloud, ensClient, resourceValidator) + if err != nil { + return nil, err + } + } + + self.api = api.NewApi(self.dpa, self.dns, resourceHandler) // Manifests for Smart Hosting log.Debug(fmt.Sprintf("-> Web3 virtual server API")) @@ -360,7 +378,7 @@ func NewLocalSwarm(datadir, port string) (self *Swarm, err error) { } self = &Swarm{ - api: api.NewApi(dpa, nil), + api: api.NewApi(dpa, nil, nil), config: config, } diff --git a/swarm/testutil/http.go b/swarm/testutil/http.go index dcba90c9c538..4cec63593612 100644 --- a/swarm/testutil/http.go +++ b/swarm/testutil/http.go @@ -25,6 +25,7 @@ import ( "strconv" "testing" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm/api" httpapi "github.com/ethereum/go-ethereum/swarm/api/http" @@ -55,30 +56,32 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer { dpa.Start() // mutable resources test setup - resourcedir, err := ioutil.TempDir("", "swarm-resource-test") + resourceDir, err := ioutil.TempDir("", "swarm-resource-test") if err != nil { t.Fatal(err) } - ipcpath := filepath.Join(resourcedir, "test.ipc") - ipcl, err := rpc.CreateIPCListener(ipcpath) + ipcPath := filepath.Join(resourceDir, "test.ipc") + ipcl, err := rpc.CreateIPCListener(ipcPath) if err != nil { t.Fatal(err) } - rpcserver := rpc.NewServer() - rpcserver.RegisterName("eth", &FakeRPC{}) + rpcServer := rpc.NewServer() + rpcServer.RegisterName("eth", &FakeRPC{}) go func() { - rpcserver.ServeListener(ipcl) + rpcServer.ServeListener(ipcl) }() rpcClean := func() { - rpcserver.Stop() + rpcServer.Stop() } // connect to fake rpc - rpcclient, err := rpc.Dial(ipcpath) + rpcClient, err := rpc.Dial(ipcPath) if err != nil { t.Fatal(err) } - rh, err := storage.NewResourceHandler(resourcedir, &testCloudStore{}, rpcclient, nil) + ethClient := ethclient.NewClient(rpcClient) + + rh, err := storage.NewResourceHandler(resourceDir, &testCloudStore{}, ethClient, nil) if err != nil { t.Fatal(err) } @@ -94,7 +97,7 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer { rh.Close() rpcClean() os.RemoveAll(dir) - os.RemoveAll(resourcedir) + os.RemoveAll(resourceDir) }, } } From a16d0af41dde358541357909d546fe23575cee5a Mon Sep 17 00:00:00 2001 From: lash Date: Sun, 21 Jan 2018 03:13:52 +0100 Subject: [PATCH 07/25] swarm/, cmd/swarm: Amend comments from @lmars PR 204 --- cmd/swarm/main.go | 2 +- ethclient/ethclient.go | 17 ++-- swarm/api/api.go | 10 +-- swarm/api/config.go | 6 +- swarm/api/config_test.go | 5 +- swarm/api/http/server.go | 39 +++++---- swarm/api/http/server_test.go | 37 +++++---- swarm/storage/resource.go | 33 ++++---- swarm/storage/resource_test.go | 142 ++++++++++++--------------------- swarm/swarm.go | 38 ++++----- swarm/testutil/http.go | 54 +++++-------- 11 files changed, 166 insertions(+), 217 deletions(-) diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 4233c7c98208..3ae45d4c2feb 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -560,7 +560,7 @@ func registerBzzService(bzzconfig *bzzapi.Config, ctx *cli.Context, stack *node. } // In production, mockStore must be always nil. - return swarm.NewSwarm(ctx, swapClient, ensClient, bzzconfig, bzzconfig.SwapEnabled, bzzconfig.SyncEnabled, bzzconfig.Cors, bzzconfig.PssEnabled, bzzconfig.ResourceEnabled, nil) + return swarm.NewSwarm(ctx, swapClient, ensClient, bzzconfig, nil) } //register within the ethereum node if err := stack.Register(boot); err != nil { diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index 4b53e785d645..2d9553a7bd02 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -23,7 +23,6 @@ import ( "errors" "fmt" "math/big" - "strconv" "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" @@ -77,13 +76,19 @@ type rpcBlock struct { UncleHashes []common.Hash `json:"uncles"` } -func (ec *Client) BlockNumber(ctx context.Context) (uint64, error) { - var number string - err := ec.c.CallContext(ctx, &number, "eth_blockNumber") +func (ec *Client) BlockNumber(ctx context.Context) (big.Int, error) { + var numberstr string + number := &big.Int{} + err := ec.c.CallContext(ctx, &numberstr, "eth_blockNumber") if err != nil { - return 0, err + return *number, err + } + var ok bool + number, ok = number.SetString(numberstr, 10) + if !ok { + err = errors.New("Failed to parse bigint") } - return strconv.ParseUint(number, 10, 64) + return *number, err } func (ec *Client) getBlock(ctx context.Context, method string, args ...interface{}) (*types.Block, error) { diff --git a/swarm/api/api.go b/swarm/api/api.go index 0e11028ec435..1393fa440635 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -365,13 +365,13 @@ func (self *Api) BuildDirectoryTree(mhash string, nameresolver bool) (key storag } // Look up mutable resource updates at specific periods and versions -func (self *Api) DbLookup(key storage.Key, name string, period uint32, version uint32) (storage.Key, io.ReadSeeker, int, error) { +func (self *Api) DbLookup(key storage.Key, name string, period uint32, version uint32) (storage.Key, []byte, error) { var err error if version != 0 { if period == 0 { currentblocknumber, err := self.resource.GetBlock() if err != nil { - return nil, nil, 0, fmt.Errorf("Could not determine latest block: %v", err) + return nil, nil, fmt.Errorf("Could not determine latest block: %v", err) } period = self.resource.BlockToPeriod(name, currentblocknumber) } @@ -382,13 +382,13 @@ func (self *Api) DbLookup(key storage.Key, name string, period uint32, version u _, err = self.resource.LookupLatestByName(name, true) } if err != nil { - return nil, nil, 0, err + return nil, nil, err } key, data, err := self.resource.GetContent(name) if err != nil { - return nil, nil, 0, err + return nil, nil, err } - return key, bytes.NewReader(data), len(data), nil + return key, data, nil } func (self *Api) DbCreate(name string, frequency uint64) (err error) { diff --git a/swarm/api/config.go b/swarm/api/config.go index 31281f9bff35..93b386cc11a8 100644 --- a/swarm/api/config.go +++ b/swarm/api/config.go @@ -110,8 +110,8 @@ func (self *Config) Init(prvKey *ecdsa.PrivateKey) { self.PublicKey = pubkeyhex self.BzzKey = keyhex - self.Swap.Init(self.Contract, prvKey) - //self.SyncParams.Init(self.Path) - //self.HiveParams.Init(self.Path) + if self.SwapEnabled { + self.Swap.Init(self.Contract, prvKey) + } self.StoreParams.Init(self.Path) } diff --git a/swarm/api/config_test.go b/swarm/api/config_test.go index 993388686bb7..5a5f176c0b14 100644 --- a/swarm/api/config_test.go +++ b/swarm/api/config_test.go @@ -49,12 +49,9 @@ func TestConfig(t *testing.T) { if one.PublicKey == "" { t.Fatal("Expected PublicKey to be set") } - - //the Init function should append subdirs to the given path - if one.Swap.PayProfile.Beneficiary == (common.Address{}) { + if one.Swap.PayProfile.Beneficiary == (common.Address{}) && one.SwapEnabled { t.Fatal("Failed to correctly initialize SwapParams") } - if one.StoreParams.ChunkDbPath == one.Path { t.Fatal("Failed to correctly initialize StoreParams") } diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 26c59c313e6e..2a8046f1c70d 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -295,13 +295,12 @@ func (s *Server) HandlePostDb(w http.ResponseWriter, r *Request) { if r.ContentLength == 0 { frequency, err := strconv.ParseUint(r.uri.Path, 10, 64) if err != nil { - w.WriteHeader(http.StatusBadRequest) - http.ServeContent(w, &r.Request, "", time.Now(), bytes.NewReader([]byte(err.Error()))) + s.BadRequest(w, r, fmt.Sprintf("Cannot parse frequency parameter: %v", err)) return } err = s.api.DbCreate(r.uri.Addr, frequency) if err != nil { - w.WriteHeader(http.StatusInternalServerError) + s.Error(w, r, fmt.Errorf("Resource creation failed: %v", err)) return } } else { @@ -324,8 +323,8 @@ func (s *Server) HandlePostDb(w http.ResponseWriter, r *Request) { // bzz-db[-[immutable|-raw]]:// - get latest update // bzz-db[-[immutable|-raw]]:/// - get latest update on period n // bzz-db[-[immutable|-raw]]://// - get update version m of period n +// = ens name or hash func (s *Server) HandleGetDb(w http.ResponseWriter, r *Request) { - w.Header().Set("Content-Type", "application/octet-stream") rootKey, err := s.api.Resolve(r.uri) if err != nil { @@ -340,34 +339,39 @@ func (s *Server) HandleGetDb(w http.ResponseWriter, r *Request) { var updateKey storage.Key var period uint64 var version uint64 - var data io.ReadSeeker + var data []byte var dataLength int now := time.Now() switch len(params) { case 0: - updateKey, data, dataLength, err = s.api.DbLookup(rootKey, r.uri.Addr, 0, 0) - break + updateKey, data, err = s.api.DbLookup(rootKey, r.uri.Addr, 0, 0) case 2: version, err = strconv.ParseUint(params[1], 10, 32) if err != nil { break } + updateKey, data, err = s.api.DbLookup(rootKey, r.uri.Addr, uint32(period), uint32(version)) case 1: + version, err = strconv.ParseUint(params[1], 10, 32) + if err != nil { + break + } period, err = strconv.ParseUint(params[0], 10, 32) if err != nil { break } - updateKey, data, dataLength, err = s.api.DbLookup(rootKey, r.uri.Addr, uint32(period), uint32(version)) - break + updateKey, data, err = s.api.DbLookup(rootKey, r.uri.Addr, uint32(period), uint32(version)) default: - w.WriteHeader(http.StatusBadRequest) - err = fmt.Errorf("params 0-2") + s.BadRequest(w, r, fmt.Sprintf("Invalid mutable resource request")) + return } if err != nil { - w.WriteHeader(http.StatusInternalServerError) + s.Error(w, r, fmt.Errorf("Mutable resource lookup failed: %v", err)) return } if !r.uri.DbRaw() { + w.Header().Set("Content-Type", "application/octet-stream") + } else { entry := api.ManifestEntry{ Hash: rootKey.Hex(), Path: updateKey.Hex(), @@ -376,9 +380,9 @@ func (s *Server) HandleGetDb(w http.ResponseWriter, r *Request) { ModTime: now, Status: http.StatusOK, } - mode := (6 << 2) | (4 << 1) | 4 + mode := 0644 if s.api.DbIsValidated() { - mode |= 2 << 1 + mode |= (2 << 3) | 2 } entry.Mode = int64(mode) manifest := api.Manifest{ @@ -388,12 +392,13 @@ func (s *Server) HandleGetDb(w http.ResponseWriter, r *Request) { } manifestJson, err := json.Marshal(manifest) if err != nil { - w.WriteHeader(http.StatusInternalServerError) + s.Error(w, r, fmt.Errorf("Could not convert manifest to json: %v", err)) return } - data = bytes.NewReader(manifestJson) + w.Header().Set("Content-Type", api.DbManifestType) + data = []byte(manifestJson) } - http.ServeContent(w, &r.Request, "", now, data) + http.ServeContent(w, &r.Request, "", now, bytes.NewReader(data)) } // HandleGet handles a GET request to diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index e2165f1d61d0..42c9ce3f569f 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -47,46 +47,53 @@ func TestBzzGetDb(t *testing.T) { keybytes := make([]byte, common.HashLength) // nearest we get to source of info _, err := rand.Read(keybytes) if err != nil { - fmt.Printf("err: %v\n", err) - return + t.Fatal(err) } url := fmt.Sprintf("%s/bzz-db:/%s/42", srv.URL, fmt.Sprintf("%x", keybytes)) resp, err := http.Post(url, "application/octet-stream", nil) if err != nil { - fmt.Printf("err: %v\n", err) - return + t.Fatal(err) } b, err := ioutil.ReadAll(resp.Body) - fmt.Printf("Create: %s : %s\n", resp.Status, b) + if err != nil { + t.Fatal(err) + } + log.Debug("Create", "status", resp.Status, "body", b) - url = fmt.Sprintf("%s/bzz-db:/%s", srv.URL, fmt.Sprintf("%x", keybytes)) + url = fmt.Sprintf("%s/bzz-db:/%x", srv.URL, keybytes) data := []byte("foo") resp, err = http.Post(url, "application/octet-stream", bytes.NewReader(data)) if err != nil { - fmt.Printf("err: %v\n", err) - return + t.Fatal(err) } b, err = ioutil.ReadAll(resp.Body) - fmt.Printf("Update: %s : %s\n", resp.Status, b) + if err != nil { + t.Fatal(err) + } + log.Debug("Update", "status", resp.Status, "body", b) url = fmt.Sprintf("%s/bzz-db-raw:/%s", srv.URL, fmt.Sprintf("%x", keybytes)) resp, err = http.Get(url) if err != nil { - fmt.Printf("err: %v\n", err) - return + t.Fatal(err) } b, err = ioutil.ReadAll(resp.Body) - fmt.Printf("Get: %s : %s\n", resp.Status, b) + if err != nil { + t.Fatal(err) + } + log.Debug("Get raw", "status", resp.Status, "body", b) url = fmt.Sprintf("%s/bzz-db:/%s", srv.URL, fmt.Sprintf("%x", keybytes)) resp, err = http.Get(url) if err != nil { - fmt.Printf("err: %v\n", err) - return + t.Fatal(err) } b, err = ioutil.ReadAll(resp.Body) - fmt.Printf("Get: %s : %s\n", resp.Status, b) + if err != nil { + t.Fatal(err) + } + log.Debug("Get manifest", "status", resp.Status, "body", b) } diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 23e1a6336b31..c51a85105bf3 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -4,6 +4,7 @@ import ( "context" "encoding/binary" "fmt" + "math/big" "path/filepath" "sync" "time" @@ -12,7 +13,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" ) @@ -58,6 +58,10 @@ type ResourceValidator interface { sign(common.Hash) (Signature, error) // SignFunc } +type ethApi interface { + BlockNumber(context.Context) (big.Int, error) +} + // Mutable resource is an entity which allows updates to a resource // without resorting to ENS on each update. // The update scheme is built on swarm chunks with chunk keys following @@ -119,8 +123,10 @@ type ResourceValidator interface { // TODO: Include modtime in chunk data + signature type ResourceHandler struct { ChunkStore + ctx context.Context + cancelFunc func() validator ResourceValidator - ethClient *ethclient.Client + ethClient ethApi resources map[string]*resource hashLock sync.Mutex resourceLock sync.RWMutex @@ -134,7 +140,7 @@ type ResourceHandler struct { // Create or open resource update chunk store // // If validator is nil, signature and access validation will be deactivated -func NewResourceHandler(datadir string, cloudStore CloudStore, ethClient *ethclient.Client, validator ResourceValidator) (*ResourceHandler, error) { +func NewResourceHandler(datadir string, cloudStore CloudStore, ethClient ethApi, validator ResourceValidator) (*ResourceHandler, error) { hashfunc := MakeHashFunc(SHA3Hash) @@ -602,17 +608,11 @@ func (self *ResourceHandler) Close() { } func (self *ResourceHandler) GetBlock() (uint64, error) { - return self.ethClient.BlockNumber(self.ctx) - // get the block height and convert to uint64 - // var currentblock string - // err := self.rpcClient.Call(¤tblock, "eth_blockNumber") - // if err != nil { - // return 0, err - // } - // if currentblock == "0x0" { - // return 0, nil - // } - // return strconv.ParseUint(currentblock, 10, 64) + bigblocknumber, err := self.ethClient.BlockNumber(self.ctx) + if err != nil { + return 0, err + } + return bigblocknumber.Uint64(), nil } // Calculate the period index (aka major version number) from a given block number @@ -776,10 +776,7 @@ func isSafeName(name string) bool { if err != nil { return false } - if validname != name { - return false - } - return true + return validname == name } // convenience for creating signature hashes of update data diff --git a/swarm/storage/resource_test.go b/swarm/storage/resource_test.go index 2da9b7704aed..dd96f0b3d753 100644 --- a/swarm/storage/resource_test.go +++ b/swarm/storage/resource_test.go @@ -2,6 +2,7 @@ package storage import ( "bytes" + "context" "crypto/ecdsa" "crypto/rand" "encoding/binary" @@ -9,8 +10,6 @@ import ( "io/ioutil" "math/big" "os" - "path/filepath" - "strconv" "strings" "testing" "time" @@ -22,9 +21,7 @@ import ( "github.com/ethereum/go-ethereum/contracts/ens/contract" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/rpc" ) var ( @@ -50,7 +47,7 @@ func init() { // so we use this wrapper to fake returning the block count type fakeBackend struct { *backends.SimulatedBackend - blocknumber uint64 + blocknumber int64 } func (f *fakeBackend) Commit() { @@ -60,13 +57,10 @@ func (f *fakeBackend) Commit() { f.blocknumber++ } -// for faking the rpc service, since we don't need the whole node stack -type FakeRPC struct { - backend *fakeBackend -} - -func (r *FakeRPC) BlockNumber() (string, error) { - return strconv.FormatUint(r.backend.blocknumber, 10), nil +func (f *fakeBackend) BlockNumber(context context.Context) (big.Int, error) { + f.blocknumber++ + biggie := big.NewInt(f.blocknumber) + return *biggie, nil } // check that signature address matches update signer address @@ -84,8 +78,9 @@ func TestResourceReverse(t *testing.T) { // set up rpc and create resourcehandler rh, _, _, teardownTest, err := setupTest(nil, newTestValidator(signer.signContent)) if err != nil { - teardownTest(t, err) + t.Fatal(err) } + defer teardownTest() // generate a hash for block 4200 version 1 key := rh.resourceHash(period, version, rh.nameHash(safeName)) @@ -94,14 +89,14 @@ func TestResourceReverse(t *testing.T) { data := make([]byte, 8) _, err = rand.Read(data) if err != nil { - teardownTest(t, err) + t.Fatal(err) } testHasher.Reset() testHasher.Write(data) digest := rh.keyDataHash(key, data) sig, err := rh.validator.sign(digest) if err != nil { - teardownTest(t, err) + t.Fatal(err) } chunk := newUpdateChunk(key, &sig, period, version, safeName, data) @@ -111,31 +106,30 @@ func TestResourceReverse(t *testing.T) { checkdigest := rh.keyDataHash(chunk.Key, checkdata) recoveredaddress, err := getAddressFromDataSig(checkdigest, *checksig) if err != nil { - teardownTest(t, fmt.Errorf("Retrieve address from signature fail: %v", err)) + t.Fatalf("Retrieve address from signature fail: %v", err) } originaladdress := crypto.PubkeyToAddress(signer.privKey.PublicKey) // check that the metadata retrieved from the chunk matches what we gave it if recoveredaddress != originaladdress { - teardownTest(t, fmt.Errorf("addresses dont match: %x != %x", originaladdress, recoveredaddress)) + t.Fatalf("addresses dont match: %x != %x", originaladdress, recoveredaddress) } if !bytes.Equal(key[:], chunk.Key[:]) { - teardownTest(t, fmt.Errorf("Expected chunk key '%x', was '%x'", key, chunk.Key)) + t.Fatalf("Expected chunk key '%x', was '%x'", key, chunk.Key) } if period != checkperiod { - teardownTest(t, fmt.Errorf("Expected period '%d', was '%d'", period, checkperiod)) + t.Fatalf("Expected period '%d', was '%d'", period, checkperiod) } if version != checkversion { - teardownTest(t, fmt.Errorf("Expected version '%d', was '%d'", version, checkversion)) + t.Fatalf("Expected version '%d', was '%d'", version, checkversion) } if safeName != checkname { - teardownTest(t, fmt.Errorf("Expected name '%s', was '%s'", safeName, checkname)) + t.Fatalf("Expected name '%s', was '%s'", safeName, checkname) } if !bytes.Equal(data, checkdata) { - teardownTest(t, fmt.Errorf("Expectedn data '%x', was '%x'", data, checkdata)) + t.Fatalf("Expectedn data '%x', was '%x'", data, checkdata) } - teardownTest(t, nil) } // make updates and retrieve them based on periods and versions @@ -143,34 +137,35 @@ func TestResourceHandler(t *testing.T) { // make fake backend, set up rpc and create resourcehandler backend := &fakeBackend{ - blocknumber: startBlock, + blocknumber: int64(startBlock), } rh, datadir, _, teardownTest, err := setupTest(backend, nil) if err != nil { - teardownTest(t, err) + t.Fatal(err) } + defer teardownTest() // create a new resource _, err = rh.NewResource(safeName, resourceFrequency) if err != nil { - teardownTest(t, err) + t.Fatal(err) } // check that the new resource is stored correctly namehash := rh.nameHash(safeName) chunk, err := rh.ChunkStore.(*resourceChunkStore).localStore.(*LocalStore).memStore.Get(Key(namehash[:])) if err != nil { - teardownTest(t, err) + t.Fatal(err) } else if len(chunk.SData) < 16 { - teardownTest(t, fmt.Errorf("chunk data must be minimum 16 bytes, is %d", len(chunk.SData))) + t.Fatalf("chunk data must be minimum 16 bytes, is %d", len(chunk.SData)) } startblocknumber := binary.LittleEndian.Uint64(chunk.SData[:8]) chunkfrequency := binary.LittleEndian.Uint64(chunk.SData[8:]) - if startblocknumber != backend.blocknumber { - teardownTest(t, fmt.Errorf("stored block number %d does not match provided block number %d", startblocknumber, backend.blocknumber)) + if startblocknumber != uint64(backend.blocknumber) { + t.Fatalf("stored block number %d does not match provided block number %d", startblocknumber, backend.blocknumber) } if chunkfrequency != resourceFrequency { - teardownTest(t, fmt.Errorf("stored frequency %d does not match provided frequency %d", chunkfrequency, resourceFrequency)) + t.Fatalf("stored frequency %d does not match provided frequency %d", chunkfrequency, resourceFrequency) } // update halfway to first period @@ -179,7 +174,7 @@ func TestResourceHandler(t *testing.T) { data := []byte("blinky") resourcekey["blinky"], err = rh.Update(safeName, data) if err != nil { - teardownTest(t, err) + t.Fatal(err) } // update on first period @@ -187,7 +182,7 @@ func TestResourceHandler(t *testing.T) { data = []byte("pinky") resourcekey["pinky"], err = rh.Update(safeName, data) if err != nil { - teardownTest(t, err) + t.Fatal(err) } // update on second period @@ -195,7 +190,7 @@ func TestResourceHandler(t *testing.T) { data = []byte("inky") resourcekey["inky"], err = rh.Update(safeName, data) if err != nil { - teardownTest(t, err) + t.Fatal(err) } // update just after second period @@ -203,7 +198,7 @@ func TestResourceHandler(t *testing.T) { data = []byte("clyde") resourcekey["clyde"], err = rh.Update(safeName, data) if err != nil { - teardownTest(t, err) + t.Fatal(err) } time.Sleep(time.Second) rh.Close() @@ -215,43 +210,42 @@ func TestResourceHandler(t *testing.T) { rh2, err := NewResourceHandler(datadir, &testCloudStore{}, rh.ethClient, nil) _, err = rh2.LookupLatestByName(safeName, true) if err != nil { - teardownTest(t, err) + t.Fatal(err) } // last update should be "clyde", version two, blockheight startblocknumber + (resourcefrequency * 3) if !bytes.Equal(rh2.resources[safeName].data, []byte("clyde")) { - teardownTest(t, fmt.Errorf("resource data was %v, expected %v", rh2.resources[safeName].data, []byte("clyde"))) + t.Fatalf("resource data was %v, expected %v", rh2.resources[safeName].data, []byte("clyde")) } if rh2.resources[safeName].version != 2 { - teardownTest(t, fmt.Errorf("resource version was %d, expected 2", rh2.resources[safeName].version)) + t.Fatalf("resource version was %d, expected 2", rh2.resources[safeName].version) } if rh2.resources[safeName].lastPeriod != 3 { - teardownTest(t, fmt.Errorf("resource period was %d, expected 3", rh2.resources[safeName].lastPeriod)) + t.Fatalf("resource period was %d, expected 3", rh2.resources[safeName].lastPeriod) } log.Debug("Latest lookup", "period", rh2.resources[safeName].lastPeriod, "version", rh2.resources[safeName].version, "data", rh2.resources[safeName].data) // specific block, latest version rsrc, err := rh2.LookupHistoricalByName(safeName, 3, true) if err != nil { - teardownTest(t, err) + t.Fatal(err) } // check data if !bytes.Equal(rsrc.data, []byte("clyde")) { - teardownTest(t, fmt.Errorf("resource data (historical) was %v, expected %v", rh2.resources[domainName].data, []byte("clyde"))) + t.Fatalf("resource data (historical) was %v, expected %v", rh2.resources[domainName].data, []byte("clyde")) } log.Debug("Historical lookup", "period", rh2.resources[safeName].lastPeriod, "version", rh2.resources[safeName].version, "data", rh2.resources[safeName].data) // specific block, specific version rsrc, err = rh2.LookupVersionByName(safeName, 3, 1, true) if err != nil { - teardownTest(t, err) + t.Fatal(err) } // check data if !bytes.Equal(rsrc.data, []byte("inky")) { - teardownTest(t, fmt.Errorf("resource data (historical) was %v, expected %v", rh2.resources[domainName].data, []byte("inky"))) + t.Fatalf("resource data (historical) was %v, expected %v", rh2.resources[domainName].data, []byte("inky")) } log.Debug("Specific version lookup", "period", rh2.resources[safeName].lastPeriod, "version", rh2.resources[safeName].version, "data", rh2.resources[safeName].data) - teardownTest(t, nil) } @@ -283,34 +277,33 @@ func TestResourceENSOwner(t *testing.T) { // set up rpc and create resourcehandler with ENS sim backend rh, _, _, teardownTest, err := setupTest(contractbackend, validator) if err != nil { - teardownTest(t, err) + t.Fatal(err) } + defer teardownTest() // create new resource when we are owner = ok _, err = rh.NewResource(safeName, resourceFrequency) if err != nil { - teardownTest(t, fmt.Errorf("Create resource fail: %v", err)) + t.Fatalf("Create resource fail: %v", err) } data := []byte("foo") // update resource when we are owner = ok _, err = rh.Update(safeName, data) if err != nil { - teardownTest(t, fmt.Errorf("Update resource fail: %v", err)) + t.Fatalf("Update resource fail: %v", err) } // update resource when we are owner = ok signertwo, err := newTestSigner() if err != nil { - teardownTest(t, err) + t.Fatal(err) } rh.validator.(*ENSValidator).signFunc = signertwo.signContent _, err = rh.Update(safeName, data) if err == nil { - teardownTest(t, fmt.Errorf("Expected resource update fail due to owner mismatch")) + t.Fatalf("Expected resource update fail due to owner mismatch") } - - teardownTest(t, nil) } // fast-forward blockheight @@ -321,7 +314,7 @@ func fwdBlocks(count int, backend *fakeBackend) { } // create rpc and resourcehandler -func setupTest(contractbackend bind.ContractBackend, validator ResourceValidator) (rh *ResourceHandler, datadir string, signer *testSigner, teardown func(*testing.T, error), err error) { +func setupTest(backend ethApi, validator ResourceValidator) (rh *ResourceHandler, datadir string, signer *testSigner, teardown func(), err error) { var fsClean func() var rpcClean func() @@ -337,55 +330,18 @@ func setupTest(contractbackend bind.ContractBackend, validator ResourceValidator // temp datadir datadir, err = ioutil.TempDir("", "rh") if err != nil { - return + return nil, "", nil, nil, err } fsClean = func() { os.RemoveAll(datadir) } - // starting the whole stack just to get blocknumbers is too cumbersome - // so we fake the rpc server to get blocknumbers for testing - ipcpath := filepath.Join(datadir, "test.ipc") - ipcl, err := rpc.CreateIPCListener(ipcpath) - if err != nil { - return - } - rpcserver := rpc.NewServer() - var fake *fakeBackend - if contractbackend != nil { - fake = contractbackend.(*fakeBackend) - } - rpcserver.RegisterName("eth", &FakeRPC{ - backend: fake, - }) - go func() { - rpcserver.ServeListener(ipcl) - }() - rpcClean = func() { - rpcserver.Stop() - } - - // connect to fake rpc - rpcClient, err := rpc.Dial(ipcpath) - if err != nil { - return - } - - ethClient := ethclient.NewClient(rpcClient) - - rh, err = NewResourceHandler(datadir, &testCloudStore{}, ethClient, validator) - teardown = func(t *testing.T, err error) { - cleanF() - if err != nil { - t.Fatal(err) - } - } - - return + rh, err = NewResourceHandler(datadir, &testCloudStore{}, backend, validator) + return rh, datadir, signer, cleanF, nil } // Set up simulated ENS backend for use with ENSResourceHandler tests -func setupENS(addr common.Address, transactOpts *bind.TransactOpts, sub string, top string) (common.Address, bind.ContractBackend, error) { +func setupENS(addr common.Address, transactOpts *bind.TransactOpts, sub string, top string) (common.Address, *fakeBackend, error) { // create the domain hash values to pass to the ENS contract methods var tophash [32]byte diff --git a/swarm/swarm.go b/swarm/swarm.go index 766c210d3d3d..c65056794753 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -54,15 +54,13 @@ type Swarm struct { storage storage.ChunkStore // internal access to storage, common interface to cloud storage backends dpa *storage.DPA // distributed preimage archive, the local API to the storage with document level storage/retrieval support //depo network.StorageHandler // remote request handler, interface between bzz protocol and the storage - cloud storage.CloudStore // procurement, cloud storage backend (can multi-cloud) - bzz *network.Bzz // the logistic manager - backend chequebook.Backend // simple blockchain Backend - privateKey *ecdsa.PrivateKey - corsString string - swapEnabled bool - lstore *storage.LocalStore // local store, needs to store for releasing resources after node stopped - sfs *fuse.SwarmFS // need this to cleanup all the active mounts on node exit - ps *pss.Pss + cloud storage.CloudStore // procurement, cloud storage backend (can multi-cloud) + bzz *network.Bzz // the logistic manager + backend chequebook.Backend // simple blockchain Backend + privateKey *ecdsa.PrivateKey + lstore *storage.LocalStore // local store, needs to store for releasing resources after node stopped + sfs *fuse.SwarmFS // need this to cleanup all the active mounts on node exit + ps *pss.Pss } type SwarmAPI struct { @@ -83,7 +81,7 @@ func (self *Swarm) API() *SwarmAPI { // implements node.Service // If mockStore is not nil, it will be used as the storage for chunk data. // MockStore should be used only for testing. -func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *ethclient.Client, config *api.Config, swapEnabled, syncEnabled bool, cors string, pssEnabled bool, resourceEnabled bool, mockStore *mock.NodeStore) (self *Swarm, err error) { +func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *ethclient.Client, config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err error) { if bytes.Equal(common.FromHex(config.PublicKey), storage.ZeroKey) { return nil, fmt.Errorf("empty public key") @@ -93,11 +91,9 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e } self = &Swarm{ - config: config, - swapEnabled: swapEnabled, - backend: backend, - privateKey: config.Swap.PrivateKey(), - corsString: cors, + config: config, + backend: backend, + privateKey: config.Swap.PrivateKey(), } log.Debug(fmt.Sprintf("Setting up Swarm service components")) @@ -139,7 +135,7 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e log.Debug(fmt.Sprintf("-> Content Store API")) // Pss = postal service over swarm (devp2p over bzz) - if pssEnabled { + if self.config.PssEnabled { pssparams := pss.NewPssParams(self.privateKey) self.ps = pss.NewPss(to, self.dpa, pssparams) if pss.IsActiveHandshake { @@ -162,7 +158,7 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e var resourceHandler *storage.ResourceHandler // if use resource updates - if resourceEnabled { + if self.config.ResourceEnabled { var resourceValidator storage.ResourceValidator if self.dns != nil { resourceValidator, err = storage.NewENSValidator(config.EnsRoot, ensClient, transactOpts, storage.NewGenericResourceSigner(self.privateKey)) @@ -204,7 +200,7 @@ func (self *Swarm) Start(srv *p2p.Server) error { log.Warn("Updated bzz local addr", "oaddr", fmt.Sprintf("%x", newaddr.OAddr), "uaddr", fmt.Sprintf("%x", newaddr.UAddr)) // set chequebook - if self.swapEnabled { + if self.config.SwapEnabled { ctx := context.Background() // The initial setup has no deadline. err := self.SetChequebook(ctx) if err != nil { @@ -237,14 +233,14 @@ func (self *Swarm) Start(srv *p2p.Server) error { addr := net.JoinHostPort(self.config.ListenAddr, self.config.Port) go httpapi.StartHttpServer(self.api, &httpapi.ServerConfig{ Addr: addr, - CorsString: self.corsString, + CorsString: self.config.Cors, }) } log.Debug(fmt.Sprintf("Swarm http proxy started on port: %v", self.config.Port)) - if self.corsString != "" { - log.Debug(fmt.Sprintf("Swarm http proxy started with corsdomain: %v", self.corsString)) + if self.config.Cors != "" { + log.Debug(fmt.Sprintf("Swarm http proxy started with corsdomain: %v", self.config.Cors)) } return nil diff --git a/swarm/testutil/http.go b/swarm/testutil/http.go index 4cec63593612..355610185054 100644 --- a/swarm/testutil/http.go +++ b/swarm/testutil/http.go @@ -17,21 +17,29 @@ package testutil import ( - "crypto/ecdsa" + "context" "io/ioutil" + "math/big" "net/http/httptest" "os" - "path/filepath" "strconv" "testing" - "github.com/ethereum/go-ethereum/ethclient" - "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm/api" httpapi "github.com/ethereum/go-ethereum/swarm/api/http" "github.com/ethereum/go-ethereum/swarm/storage" ) +type fakeBackend struct { + blocknumber int64 +} + +func (f *fakeBackend) BlockNumber(ctx context.Context) (big.Int, error) { + f.blocknumber++ + biggie := big.NewInt(f.blocknumber) + return *biggie, nil +} + func NewTestSwarmServer(t *testing.T) *TestSwarmServer { dir, err := ioutil.TempDir("", "swarm-storage-test") if err != nil { @@ -60,28 +68,8 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer { if err != nil { t.Fatal(err) } - ipcPath := filepath.Join(resourceDir, "test.ipc") - ipcl, err := rpc.CreateIPCListener(ipcPath) - if err != nil { - t.Fatal(err) - } - rpcServer := rpc.NewServer() - rpcServer.RegisterName("eth", &FakeRPC{}) - go func() { - rpcServer.ServeListener(ipcl) - }() - rpcClean := func() { - rpcServer.Stop() - } - - // connect to fake rpc - rpcClient, err := rpc.Dial(ipcPath) - if err != nil { - t.Fatal(err) - } - ethClient := ethclient.NewClient(rpcClient) - rh, err := storage.NewResourceHandler(resourceDir, &testCloudStore{}, ethClient, nil) + rh, err := storage.NewResourceHandler(resourceDir, &testCloudStore{}, &fakeBackend{}, nil) if err != nil { t.Fatal(err) } @@ -94,8 +82,9 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer { dir: dir, hasher: storage.MakeHashFunc(storage.SHA3Hash)(), cleanup: func() { + srv.Close() rh.Close() - rpcClean() + dpa.Stop() os.RemoveAll(dir) os.RemoveAll(resourceDir) }, @@ -104,17 +93,14 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer { type TestSwarmServer struct { *httptest.Server - hasher storage.SwarmHash - privatekey *ecdsa.PrivateKey - Dpa *storage.DPA - dir string - cleanup func() + hasher storage.SwarmHash + Dpa *storage.DPA + dir string + cleanup func() } func (t *TestSwarmServer) Close() { - t.Server.Close() - t.Dpa.Stop() - os.RemoveAll(t.dir) + t.cleanup() } type testCloudStore struct { From 11c3b4cae16c749ee9ad894dc38b196525cb9cf3 Mon Sep 17 00:00:00 2001 From: lash Date: Mon, 22 Jan 2018 05:08:45 +0100 Subject: [PATCH 08/25] swarm/api: Add contenttype contingent resolve manifest -> rsrc --- swarm/api/api.go | 2 +- swarm/api/http/server.go | 80 ++++++++++++++++++++++++++--------- swarm/api/http/server_test.go | 37 +++++++++++----- swarm/api/manifest.go | 4 +- 4 files changed, 90 insertions(+), 33 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index 1393fa440635..2a3de5b5ca8e 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -365,7 +365,7 @@ func (self *Api) BuildDirectoryTree(mhash string, nameresolver bool) (key storag } // Look up mutable resource updates at specific periods and versions -func (self *Api) DbLookup(key storage.Key, name string, period uint32, version uint32) (storage.Key, []byte, error) { +func (self *Api) DbLookup(name string, period uint32, version uint32) (storage.Key, []byte, error) { var err error if version != 0 { if period == 0 { diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 2a8046f1c70d..49103f5f8541 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -292,7 +292,8 @@ func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) { } func (s *Server) HandlePostDb(w http.ResponseWriter, r *Request) { - if r.ContentLength == 0 { + var outdata string + if r.uri.Path != "" { frequency, err := strconv.ParseUint(r.uri.Path, 10, 64) if err != nil { s.BadRequest(w, r, fmt.Sprintf("Cannot parse frequency parameter: %v", err)) @@ -303,20 +304,50 @@ func (s *Server) HandlePostDb(w http.ResponseWriter, r *Request) { s.Error(w, r, fmt.Errorf("Resource creation failed: %v", err)) return } - } else { - data, err := ioutil.ReadAll(r.Body) + manifestKey, err := s.api.NewManifest() if err != nil { - w.WriteHeader(http.StatusInternalServerError) + s.Error(w, r, fmt.Errorf("create manifest err: %v", err)) return } - _, _, _, err = s.api.DbUpdate(r.uri.Addr, data) + newKey, err := s.updateManifest(manifestKey, func(mw *api.ManifestWriter) error { + key, err := mw.AddEntry(bytes.NewReader([]byte(r.uri.Addr)), &api.ManifestEntry{ + Path: r.uri.Addr, + ContentType: api.ResourceContentType, + Mode: 0644, + Size: int64(len(r.uri.Addr)), + ModTime: time.Now(), + }) + if err != nil { + return err + } + s.logDebug("resource manifest for for %s stored", key.Log()) + return nil + }) if err != nil { - w.WriteHeader(http.StatusUnauthorized) - http.ServeContent(w, &r.Request, "", time.Now(), bytes.NewReader([]byte(err.Error()))) + s.Error(w, r, fmt.Errorf("update manifest err: %v", err)) return } + log.Debug("manifests", "new", newKey, "old", manifestKey) + outdata = fmt.Sprintf("%s", newKey) } + + data, err := ioutil.ReadAll(r.Body) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + _, _, _, err = s.api.DbUpdate(r.uri.Addr, data) + if err != nil { + w.Header().Add("Status", fmt.Sprintf("%d", http.StatusUnauthorized)) + http.ServeContent(w, &r.Request, "", time.Now(), bytes.NewReader([]byte(err.Error()))) + return + } + w.WriteHeader(http.StatusOK) + if outdata != "" { + w.Header().Set("Content-type", "text/plain") + fmt.Fprintf(w, outdata) + } } // Retrieve mutable resource updates: @@ -325,13 +356,10 @@ func (s *Server) HandlePostDb(w http.ResponseWriter, r *Request) { // bzz-db[-[immutable|-raw]]://// - get update version m of period n // = ens name or hash func (s *Server) HandleGetDb(w http.ResponseWriter, r *Request) { + s.handleGetDb(w, r, r.uri.Addr) +} - rootKey, err := s.api.Resolve(r.uri) - if err != nil { - s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) - return - } - +func (s *Server) handleGetDb(w http.ResponseWriter, r *Request, name string) { var params []string if len(r.uri.Path) > 0 { params = strings.Split(r.uri.Path, "/") @@ -341,16 +369,18 @@ func (s *Server) HandleGetDb(w http.ResponseWriter, r *Request) { var version uint64 var data []byte var dataLength int + var err error now := time.Now() + log.Debug("handlegetdb", "name", name) switch len(params) { case 0: - updateKey, data, err = s.api.DbLookup(rootKey, r.uri.Addr, 0, 0) + updateKey, data, err = s.api.DbLookup(name, 0, 0) case 2: version, err = strconv.ParseUint(params[1], 10, 32) if err != nil { break } - updateKey, data, err = s.api.DbLookup(rootKey, r.uri.Addr, uint32(period), uint32(version)) + updateKey, data, err = s.api.DbLookup(name, uint32(period), uint32(version)) case 1: version, err = strconv.ParseUint(params[1], 10, 32) if err != nil { @@ -360,7 +390,7 @@ func (s *Server) HandleGetDb(w http.ResponseWriter, r *Request) { if err != nil { break } - updateKey, data, err = s.api.DbLookup(rootKey, r.uri.Addr, uint32(period), uint32(version)) + updateKey, data, err = s.api.DbLookup(name, uint32(period), uint32(version)) default: s.BadRequest(w, r, fmt.Sprintf("Invalid mutable resource request")) return @@ -373,9 +403,9 @@ func (s *Server) HandleGetDb(w http.ResponseWriter, r *Request) { w.Header().Set("Content-Type", "application/octet-stream") } else { entry := api.ManifestEntry{ - Hash: rootKey.Hex(), + Hash: name, Path: updateKey.Hex(), - ContentType: api.DbManifestType, + ContentType: api.ManifestType, Size: int64(dataLength), ModTime: now, Status: http.StatusOK, @@ -395,7 +425,7 @@ func (s *Server) HandleGetDb(w http.ResponseWriter, r *Request) { s.Error(w, r, fmt.Errorf("Could not convert manifest to json: %v", err)) return } - w.Header().Set("Content-Type", api.DbManifestType) + w.Header().Set("Content-Type", api.ManifestType) data = []byte(manifestJson) } http.ServeContent(w, &r.Request, "", now, bytes.NewReader(data)) @@ -461,6 +491,17 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { switch { case r.uri.Raw(): + m := &api.Manifest{} + sz, _ := reader.Size(nil) + b := make([]byte, sz) + reader.Read(b) + err = json.Unmarshal(b, m) + if len(m.Entries) > 0 { + if m.Entries[0].ContentType == api.ResourceContentType { + s.handleGetDb(w, r, m.Entries[0].Path) + return + } + } // allow the request to overwrite the content type using a query // parameter contentType := "application/octet-stream" @@ -468,7 +509,6 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { contentType = typ } w.Header().Set("Content-Type", contentType) - http.ServeContent(w, &r.Request, "", time.Now(), reader) case r.uri.Hash(): w.Header().Set("Content-Type", "text/plain") diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 42c9ce3f569f..cd7d1a0c9b5f 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -45,21 +45,27 @@ func TestBzzGetDb(t *testing.T) { defer srv.Close() keybytes := make([]byte, common.HashLength) // nearest we get to source of info - _, err := rand.Read(keybytes) + copy(keybytes, []byte{42}) + + databytes := make([]byte, 42) + _, err := rand.Read(databytes) if err != nil { t.Fatal(err) } - url := fmt.Sprintf("%s/bzz-db:/%s/42", srv.URL, fmt.Sprintf("%x", keybytes)) - resp, err := http.Post(url, "application/octet-stream", nil) + url := fmt.Sprintf("%s/bzz-db:/%x/42", srv.URL, keybytes) + resp, err := http.Post(url, "application/octet-stream", bytes.NewReader(databytes)) if err != nil { t.Fatal(err) } - b, err := ioutil.ReadAll(resp.Body) + manifesthash, err := ioutil.ReadAll(resp.Body) if err != nil { t.Fatal(err) } - log.Debug("Create", "status", resp.Status, "body", b) + if resp.StatusCode != http.StatusOK { + t.Fatalf("err %s", resp.Status) + } + log.Debug("Create", "status", resp.Status, "body", manifesthash) url = fmt.Sprintf("%s/bzz-db:/%x", srv.URL, keybytes) data := []byte("foo") @@ -67,13 +73,24 @@ func TestBzzGetDb(t *testing.T) { if err != nil { t.Fatal(err) } + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + log.Debug("Update", "status", resp.Status) + + url = fmt.Sprintf("%s/bzz-raw:/%s", srv.URL, manifesthash) + resp, err = http.Get(url) + if err != nil { + t.Fatal(err) + } b, err = ioutil.ReadAll(resp.Body) if err != nil { t.Fatal(err) } - log.Debug("Update", "status", resp.Status, "body", b) + log.Debug("Manifest", "status", resp.Status, "body", fmt.Sprintf("%s", b)) - url = fmt.Sprintf("%s/bzz-db-raw:/%s", srv.URL, fmt.Sprintf("%x", keybytes)) + url = fmt.Sprintf("%s/bzz-db-raw:/%x", srv.URL, keybytes) resp, err = http.Get(url) if err != nil { t.Fatal(err) @@ -82,9 +99,9 @@ func TestBzzGetDb(t *testing.T) { if err != nil { t.Fatal(err) } - log.Debug("Get raw", "status", resp.Status, "body", b) + log.Debug("Get raw", "status", resp.Status) - url = fmt.Sprintf("%s/bzz-db:/%s", srv.URL, fmt.Sprintf("%x", keybytes)) + url = fmt.Sprintf("%s/bzz-db:/%x", srv.URL, keybytes) resp, err = http.Get(url) if err != nil { t.Fatal(err) @@ -93,7 +110,7 @@ func TestBzzGetDb(t *testing.T) { if err != nil { t.Fatal(err) } - log.Debug("Get manifest", "status", resp.Status, "body", b) + log.Debug("Get manifest", "status", resp.Status) } diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index 46b55b3b7a17..fde086b7ac5e 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -33,8 +33,8 @@ import ( ) const ( - ManifestType = "application/bzz-manifest+json" - DbManifestType = "application/bzz-db-manifest+json" + ManifestType = "application/bzz-manifest+json" + ResourceContentType = "application/bzz-resource" ) // Manifest represents a swarm manifest From 80f539c21715a15c943edb2079fc9f13ab560db4 Mon Sep 17 00:00:00 2001 From: lash Date: Mon, 22 Jan 2018 05:13:28 +0100 Subject: [PATCH 09/25] swarm/api: Rename Db -> Resource --- swarm/api/api.go | 10 ++--- swarm/api/http/server.go | 71 +++++++++++------------------------ swarm/api/http/server_test.go | 21 +++-------- swarm/api/uri.go | 10 ++--- 4 files changed, 34 insertions(+), 78 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index 2a3de5b5ca8e..0c8d9d1ea78b 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -365,7 +365,7 @@ func (self *Api) BuildDirectoryTree(mhash string, nameresolver bool) (key storag } // Look up mutable resource updates at specific periods and versions -func (self *Api) DbLookup(name string, period uint32, version uint32) (storage.Key, []byte, error) { +func (self *Api) ResourceLookup(name string, period uint32, version uint32) (storage.Key, []byte, error) { var err error if version != 0 { if period == 0 { @@ -391,22 +391,22 @@ func (self *Api) DbLookup(name string, period uint32, version uint32) (storage.K return key, data, nil } -func (self *Api) DbCreate(name string, frequency uint64) (err error) { +func (self *Api) ResourceCreate(name string, frequency uint64) (err error) { _, err = self.resource.NewResource(name, frequency) return err } -func (self *Api) DbUpdate(name string, data []byte) (storage.Key, uint32, uint32, error) { +func (self *Api) ResourceUpdate(name string, data []byte) (storage.Key, uint32, uint32, error) { key, err := self.resource.Update(name, data) period, _ := self.resource.GetLastPeriod(name) version, _ := self.resource.GetVersion(name) return key, period, version, err } -func (self *Api) DbHashSize() int { +func (self *Api) ResourceHashSize() int { return self.resource.HashSize() } -func (self *Api) DbIsValidated() bool { +func (self *Api) ResourceIsValidated() bool { return self.resource.IsValidated() } diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 49103f5f8541..dc4025896cd6 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -291,7 +291,7 @@ func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) { fmt.Fprint(w, newKey) } -func (s *Server) HandlePostDb(w http.ResponseWriter, r *Request) { +func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) { var outdata string if r.uri.Path != "" { frequency, err := strconv.ParseUint(r.uri.Path, 10, 64) @@ -299,7 +299,7 @@ func (s *Server) HandlePostDb(w http.ResponseWriter, r *Request) { s.BadRequest(w, r, fmt.Sprintf("Cannot parse frequency parameter: %v", err)) return } - err = s.api.DbCreate(r.uri.Addr, frequency) + err = s.api.ResourceCreate(r.uri.Addr, frequency) if err != nil { s.Error(w, r, fmt.Errorf("Resource creation failed: %v", err)) return @@ -333,13 +333,12 @@ func (s *Server) HandlePostDb(w http.ResponseWriter, r *Request) { data, err := ioutil.ReadAll(r.Body) if err != nil { - w.WriteHeader(http.StatusInternalServerError) + s.Error(w, r, err) return } - _, _, _, err = s.api.DbUpdate(r.uri.Addr, data) + _, _, _, err = s.api.ResourceUpdate(r.uri.Addr, data) if err != nil { - w.Header().Add("Status", fmt.Sprintf("%d", http.StatusUnauthorized)) - http.ServeContent(w, &r.Request, "", time.Now(), bytes.NewReader([]byte(err.Error()))) + w.WriteHeader(http.StatusUnauthorized) return } @@ -351,15 +350,15 @@ func (s *Server) HandlePostDb(w http.ResponseWriter, r *Request) { } // Retrieve mutable resource updates: -// bzz-db[-[immutable|-raw]]:// - get latest update -// bzz-db[-[immutable|-raw]]:/// - get latest update on period n -// bzz-db[-[immutable|-raw]]://// - get update version m of period n +// bzz-resource:// - get latest update +// bzz-resource:/// - get latest update on period n +// bzz-resource://// - get update version m of period n // = ens name or hash -func (s *Server) HandleGetDb(w http.ResponseWriter, r *Request) { - s.handleGetDb(w, r, r.uri.Addr) +func (s *Server) HandleGetResource(w http.ResponseWriter, r *Request) { + s.handleGetResource(w, r, r.uri.Addr) } -func (s *Server) handleGetDb(w http.ResponseWriter, r *Request, name string) { +func (s *Server) handleGetResource(w http.ResponseWriter, r *Request, name string) { var params []string if len(r.uri.Path) > 0 { params = strings.Split(r.uri.Path, "/") @@ -368,19 +367,18 @@ func (s *Server) handleGetDb(w http.ResponseWriter, r *Request, name string) { var period uint64 var version uint64 var data []byte - var dataLength int var err error now := time.Now() log.Debug("handlegetdb", "name", name) switch len(params) { case 0: - updateKey, data, err = s.api.DbLookup(name, 0, 0) + updateKey, data, err = s.api.ResourceLookup(name, 0, 0) case 2: version, err = strconv.ParseUint(params[1], 10, 32) if err != nil { break } - updateKey, data, err = s.api.DbLookup(name, uint32(period), uint32(version)) + updateKey, data, err = s.api.ResourceLookup(name, uint32(period), uint32(version)) case 1: version, err = strconv.ParseUint(params[1], 10, 32) if err != nil { @@ -390,7 +388,7 @@ func (s *Server) handleGetDb(w http.ResponseWriter, r *Request, name string) { if err != nil { break } - updateKey, data, err = s.api.DbLookup(name, uint32(period), uint32(version)) + updateKey, data, err = s.api.ResourceLookup(name, uint32(period), uint32(version)) default: s.BadRequest(w, r, fmt.Sprintf("Invalid mutable resource request")) return @@ -399,35 +397,8 @@ func (s *Server) handleGetDb(w http.ResponseWriter, r *Request, name string) { s.Error(w, r, fmt.Errorf("Mutable resource lookup failed: %v", err)) return } - if !r.uri.DbRaw() { - w.Header().Set("Content-Type", "application/octet-stream") - } else { - entry := api.ManifestEntry{ - Hash: name, - Path: updateKey.Hex(), - ContentType: api.ManifestType, - Size: int64(dataLength), - ModTime: now, - Status: http.StatusOK, - } - mode := 0644 - if s.api.DbIsValidated() { - mode |= (2 << 3) | 2 - } - entry.Mode = int64(mode) - manifest := api.Manifest{ - Entries: []api.ManifestEntry{ - entry, - }, - } - manifestJson, err := json.Marshal(manifest) - if err != nil { - s.Error(w, r, fmt.Errorf("Could not convert manifest to json: %v", err)) - return - } - w.Header().Set("Content-Type", api.ManifestType) - data = []byte(manifestJson) - } + log.Debug("Found update", "key", updateKey) + w.Header().Set("Content-Type", "application/octet-stream") http.ServeContent(w, &r.Request, "", now, bytes.NewReader(data)) } @@ -498,7 +469,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { err = json.Unmarshal(b, m) if len(m.Entries) > 0 { if m.Entries[0].ContentType == api.ResourceContentType { - s.handleGetDb(w, r, m.Entries[0].Path) + s.handleGetResource(w, r, m.Entries[0].Path) return } } @@ -755,8 +726,8 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { case "POST": if uri.Raw() || uri.DeprecatedRaw() { s.HandlePostRaw(w, req) - } else if uri.Db() { - s.HandlePostDb(w, req) + } else if uri.Resource() { + s.HandlePostResource(w, req) } else { s.HandlePostFiles(w, req) } @@ -783,8 +754,8 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { case "GET": - if uri.Db() || uri.DbRaw() { - s.HandleGetDb(w, req) + if uri.Resource() { + s.HandleGetResource(w, req) return } diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index cd7d1a0c9b5f..c79922a7f252 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -53,7 +53,7 @@ func TestBzzGetDb(t *testing.T) { t.Fatal(err) } - url := fmt.Sprintf("%s/bzz-db:/%x/42", srv.URL, keybytes) + url := fmt.Sprintf("%s/bzz-resource:/%x/42", srv.URL, keybytes) resp, err := http.Post(url, "application/octet-stream", bytes.NewReader(databytes)) if err != nil { t.Fatal(err) @@ -67,7 +67,7 @@ func TestBzzGetDb(t *testing.T) { } log.Debug("Create", "status", resp.Status, "body", manifesthash) - url = fmt.Sprintf("%s/bzz-db:/%x", srv.URL, keybytes) + url = fmt.Sprintf("%s/bzz-resource:/%x", srv.URL, keybytes) data := []byte("foo") resp, err = http.Post(url, "application/octet-stream", bytes.NewReader(data)) if err != nil { @@ -88,9 +88,9 @@ func TestBzzGetDb(t *testing.T) { if err != nil { t.Fatal(err) } - log.Debug("Manifest", "status", resp.Status, "body", fmt.Sprintf("%s", b)) + log.Debug("Get raw", "status", resp.Status, "body", fmt.Sprintf("%s", b)) - url = fmt.Sprintf("%s/bzz-db-raw:/%x", srv.URL, keybytes) + url = fmt.Sprintf("%s/bzz-resource:/%x", srv.URL, keybytes) resp, err = http.Get(url) if err != nil { t.Fatal(err) @@ -99,18 +99,7 @@ func TestBzzGetDb(t *testing.T) { if err != nil { t.Fatal(err) } - log.Debug("Get raw", "status", resp.Status) - - url = fmt.Sprintf("%s/bzz-db:/%x", srv.URL, keybytes) - resp, err = http.Get(url) - if err != nil { - t.Fatal(err) - } - b, err = ioutil.ReadAll(resp.Body) - if err != nil { - t.Fatal(err) - } - log.Debug("Get manifest", "status", resp.Status) + log.Debug("Get resource", "status", resp.Status, "data", b) } diff --git a/swarm/api/uri.go b/swarm/api/uri.go index c7a929f0f2c2..009bc016fd58 100644 --- a/swarm/api/uri.go +++ b/swarm/api/uri.go @@ -69,7 +69,7 @@ func Parse(rawuri string) (*URI, error) { // check the scheme is valid switch uri.Scheme { - case "bzz", "bzz-raw", "bzz-immutable", "bzz-list", "bzz-hash", "bzzr", "bzzi", "bzz-db", "bzz-db-raw": + case "bzz", "bzz-raw", "bzz-immutable", "bzz-list", "bzz-hash", "bzzr", "bzzi", "bzz-resource": default: return nil, fmt.Errorf("unknown scheme %q", u.Scheme) } @@ -92,12 +92,8 @@ func Parse(rawuri string) (*URI, error) { return uri, nil } -func (u *URI) Db() bool { - return u.Scheme == "bzz-db" -} - -func (u *URI) DbRaw() bool { - return u.Scheme == "bzz-db-raw" +func (u *URI) Resource() bool { + return u.Scheme == "bzz-resource" } func (u *URI) Raw() bool { From aff40c4e8c8661456711ea34ba2e7359ff71a71c Mon Sep 17 00:00:00 2001 From: lash Date: Mon, 22 Jan 2018 18:19:51 +0100 Subject: [PATCH 10/25] swarm/storage, ethclient: Move signature to end of chunk data ethclient: Omit string conversion detour --- ethclient/ethclient.go | 11 +----- swarm/storage/resource.go | 71 ++++++++++++++++++++++----------------- 2 files changed, 42 insertions(+), 40 deletions(-) diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index 2d9553a7bd02..f41de1b42c60 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -77,17 +77,8 @@ type rpcBlock struct { } func (ec *Client) BlockNumber(ctx context.Context) (big.Int, error) { - var numberstr string number := &big.Int{} - err := ec.c.CallContext(ctx, &numberstr, "eth_blockNumber") - if err != nil { - return *number, err - } - var ok bool - number, ok = number.SetString(numberstr, 10) - if !ok { - err = errors.New("Failed to parse bigint") - } + err := ec.c.CallContext(ctx, &number, "eth_blockNumber") return *number, err } diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index c51a85105bf3..6a90f2bdfa6e 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -476,20 +476,12 @@ func (self *ResourceHandler) updateResourceIndex(rsrc *resource, chunk *Chunk) ( func (self *ResourceHandler) parseUpdate(chunkdata []byte) (*Signature, uint32, uint32, string, []byte, error) { var err error cursor := 0 - var signature *Signature - // omit signatures if we have no validator - var sigoffset int - if self.validator != nil { - signature = &Signature{} - copy(signature[:], chunkdata[:signatureLength]) - sigoffset = signatureLength - cursor = sigoffset - } - headerlength := binary.LittleEndian.Uint16(chunkdata[cursor : cursor+2]) - if int(headerlength+2) > len(chunkdata) { - err = fmt.Errorf("Reported header length %d longer than actual data length %d", headerlength, len(chunkdata)) - return nil, 0, 0, "", nil, err + cursor += 2 + datalength := binary.LittleEndian.Uint16(chunkdata[cursor : cursor+2]) + if int(headerlength+datalength+4) > len(chunkdata) { + err = fmt.Errorf("Reported headerlength %d + datalength %d longer than actual chunk data length %d", headerlength, datalength, len(chunkdata)) + return } var period uint32 @@ -501,12 +493,21 @@ func (self *ResourceHandler) parseUpdate(chunkdata []byte) (*Signature, uint32, cursor += 4 version = binary.LittleEndian.Uint32(chunkdata[cursor : cursor+4]) cursor += 4 - namelength := int(headerlength) - cursor + sigoffset + 2 + namelength := int(headerlength) - cursor + 4 name = string(chunkdata[cursor : cursor+namelength]) cursor += namelength - data = make([]byte, len(chunkdata)-cursor) - copy(data, chunkdata[cursor:]) - return signature, period, version, name, data, err + intdatalength := int(datalength) + data = make([]byte, intdatalength) + copy(data, chunkdata[cursor:cursor+intdatalength]) + + // omit signatures if we have no validator + if self.validator != nil { + cursor += intdatalength + signature = &Signature{} + copy(signature[:], chunkdata[cursor:cursor+signatureLength]) + } + + return } // Adds an actual data update @@ -517,9 +518,9 @@ func (self *ResourceHandler) parseUpdate(chunkdata []byte) (*Signature, uint32, // A resource update cannot span chunks, and thus has max length 4096 func (self *ResourceHandler) Update(name string, data []byte) (Key, error) { - var sigoffset int + var signaturelength int if self.validator != nil { - sigoffset = signatureLength + signaturelength = signatureLength } // get the cached information @@ -532,7 +533,7 @@ func (self *ResourceHandler) Update(name string, data []byte) (Key, error) { } // an update can be only one chunk long - datalimit := self.chunkSize() - int64(sigoffset-len(name)-8) + datalimit := self.chunkSize() - int64(signaturelength-len(name)-4-4-2-2) if int64(len(data)) > datalimit { return nil, fmt.Errorf("Data overflow: %d / %d bytes", len(data), datalimit) } @@ -672,27 +673,30 @@ func getAddressFromDataSig(datahash common.Hash, signature Signature) (common.Ad func newUpdateChunk(key Key, signature *Signature, period uint32, version uint32, name string, data []byte) *Chunk { // no signatures if no validator - var sigoffset int + var signaturelength int if signature != nil { - sigoffset = signatureLength + signaturelength = signatureLength } // prepend version and period to allow reverse lookups - headerlength := uint16(len(name) + 4 + 4) + headerlength := len(name) + 4 + 4 + + // also prepend datalength + datalength := len(data) chunk := NewChunk(key, nil) - chunk.SData = make([]byte, sigoffset+int(headerlength)+2+len(data)) + chunk.SData = make([]byte, 4+signaturelength+headerlength+datalength) + // data header length does NOT include the header length prefix bytes themselves cursor := 0 - if signature != nil { - copy(chunk.SData, (*signature)[:]) - cursor += signatureLength - } + binary.LittleEndian.PutUint16(chunk.SData[cursor:], uint16(headerlength)) + cursor += 2 - // data header length does NOT include the header length prefix bytes themselves - binary.LittleEndian.PutUint16(chunk.SData[cursor:], headerlength) + // data length + binary.LittleEndian.PutUint16(chunk.SData[cursor:], uint16(datalength)) cursor += 2 + // header = period + version + name binary.LittleEndian.PutUint32(chunk.SData[cursor:], period) cursor += 4 @@ -703,8 +707,15 @@ func newUpdateChunk(key Key, signature *Signature, period uint32, version uint32 copy(chunk.SData[cursor:], namebytes) cursor += len(namebytes) + // add the data copy(chunk.SData[cursor:], data) + // if signature is present it's the last item in the chunk data + if signature != nil { + cursor += datalength + copy(chunk.SData[cursor:], signature[:]) + } + chunk.Size = int64(len(chunk.SData)) return chunk } From 703051b1aa398a4d5aade97fab780e95c1894556 Mon Sep 17 00:00:00 2001 From: lash Date: Mon, 22 Jan 2018 19:10:32 +0100 Subject: [PATCH 11/25] swarm: Cleanup after rebase on swarm-mutableresources-extsign --- swarm/api/http/server_test.go | 2 +- swarm/storage/resource.go | 11 +++++------ swarm/testutil/http.go | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index c79922a7f252..2837decfdefc 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -40,7 +40,7 @@ func init() { log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) } -func TestBzzGetDb(t *testing.T) { +func TestBzzResource(t *testing.T) { srv := testutil.NewTestSwarmServer(t) defer srv.Close() diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 6a90f2bdfa6e..bf4404634cf9 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -19,7 +19,7 @@ import ( const ( signatureLength = 65 indexSize = 16 - dbDirName = "resource" + DbDirName = "resource" chunkSize = 4096 // temporary until we implement DPA in the resourcehandler defaultStoreTimeout = 4000 * time.Millisecond ) @@ -133,8 +133,6 @@ type ResourceHandler struct { hasher SwarmHash nameHash nameHashFunc storeTimeout time.Duration - ctx context.Context - cancelFunc func() } // Create or open resource update chunk store @@ -157,7 +155,7 @@ func NewResourceHandler(datadir string, cloudStore CloudStore, ethClient ethApi, ctx, cancel := context.WithCancel(context.Background()) rh := &ResourceHandler{ ChunkStore: newResourceChunkStore(path, hashfunc, localStore, cloudStore), - rpcClient: rpcClient, + ethClient: ethClient, resources: make(map[string]*resource), hasher: hashfunc(), validator: validator, @@ -481,7 +479,7 @@ func (self *ResourceHandler) parseUpdate(chunkdata []byte) (*Signature, uint32, datalength := binary.LittleEndian.Uint16(chunkdata[cursor : cursor+2]) if int(headerlength+datalength+4) > len(chunkdata) { err = fmt.Errorf("Reported headerlength %d + datalength %d longer than actual chunk data length %d", headerlength, datalength, len(chunkdata)) - return + return nil, 0, 0, "", nil, err } var period uint32 @@ -501,13 +499,14 @@ func (self *ResourceHandler) parseUpdate(chunkdata []byte) (*Signature, uint32, copy(data, chunkdata[cursor:cursor+intdatalength]) // omit signatures if we have no validator + var signature *Signature if self.validator != nil { cursor += intdatalength signature = &Signature{} copy(signature[:], chunkdata[cursor:cursor+signatureLength]) } - return + return signature, period, version, name, data, nil } // Adds an actual data update diff --git a/swarm/testutil/http.go b/swarm/testutil/http.go index 355610185054..4a6a68e43bfb 100644 --- a/swarm/testutil/http.go +++ b/swarm/testutil/http.go @@ -51,7 +51,7 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer { CacheCapacity: 5000, Radius: 0, } - localStore, err := storage.NewLocalStore(storage.MakeHashFunc(storage.SHA3Hash), storeparams) + localStore, err := storage.NewLocalStore(storage.MakeHashFunc(storage.SHA3Hash), storeparams, nil) if err != nil { os.RemoveAll(dir) t.Fatal(err) From c2e0be67c899df018dbbab826de93b33c88947e6 Mon Sep 17 00:00:00 2001 From: lash Date: Mon, 22 Jan 2018 19:27:24 +0100 Subject: [PATCH 12/25] swarm/storage, swarm/api: Change noparam fmt.Errorf -> errors.New --- swarm/api/http/server.go | 2 +- swarm/storage/resource.go | 19 ++++++++++--------- swarm/storage/resource_ens.go | 4 ++-- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index dc4025896cd6..0b22b9d8da4e 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -447,7 +447,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { return api.SkipManifest }) if entry == nil { - s.NotFound(w, r, fmt.Errorf("Manifest entry could not be loaded")) + s.NotFound(w, r, errors.New("Manifest entry could not be loaded")) return } key = storage.Key(common.Hex2Bytes(entry.Hash)) diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index bf4404634cf9..80251c841ef7 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -3,6 +3,7 @@ package storage import ( "context" "encoding/binary" + "errors" "fmt" "math/big" "path/filepath" @@ -192,7 +193,7 @@ func (self *ResourceHandler) HashSize() int { func (self *ResourceHandler) GetContent(name string) (Key, []byte, error) { rsrc := self.getResource(name) if rsrc == nil || !rsrc.isSynced() { - return nil, nil, fmt.Errorf("Resource does not exist or is not synced") + return nil, nil, errors.New("Resource does not exist or is not synced") } return rsrc.lastKey, rsrc.data, nil } @@ -201,7 +202,7 @@ func (self *ResourceHandler) GetLastPeriod(name string) (uint32, error) { rsrc := self.getResource(name) if rsrc == nil || !rsrc.isSynced() { - return 0, fmt.Errorf("Resource does not exist or is not synced") + return 0, errors.New("Resource does not exist or is not synced") } return rsrc.lastPeriod, nil } @@ -209,7 +210,7 @@ func (self *ResourceHandler) GetLastPeriod(name string) (uint32, error) { func (self *ResourceHandler) GetVersion(name string) (uint32, error) { rsrc := self.getResource(name) if rsrc == nil || !rsrc.isSynced() { - return 0, fmt.Errorf("Resource does not exist or is not synced") + return 0, errors.New("Resource does not exist or is not synced") } return rsrc.version, nil } @@ -228,7 +229,7 @@ func (self *ResourceHandler) NewResource(name string, frequency uint64) (*resour // frequency 0 is invalid if frequency == 0 { - return nil, fmt.Errorf("Frequency cannot be 0") + return nil, errors.New("Frequency cannot be 0") } if !isSafeName(name) { @@ -360,7 +361,7 @@ func (self *ResourceHandler) LookupLatest(nameHash common.Hash, name string, ref func (self *ResourceHandler) lookup(rsrc *resource, period uint32, version uint32, refresh bool) (*resource, error) { if period == 0 { - return nil, fmt.Errorf("period must be >0") + return nil, errors.New("period must be >0") } // start from the last possible block period, and iterate previous ones until we find a match @@ -396,7 +397,7 @@ func (self *ResourceHandler) lookup(rsrc *resource, period uint32, version uint3 log.Trace("rsrc update not found, checking previous period", "period", period, "key", key) period-- } - return nil, fmt.Errorf("no updates found") + return nil, errors.New("no updates found") } // load existing mutable resource into resource struct @@ -525,10 +526,10 @@ func (self *ResourceHandler) Update(name string, data []byte) (Key, error) { // get the cached information rsrc := self.getResource(name) if rsrc == nil { - return nil, fmt.Errorf("Resource object not in index") + return nil, errors.New("Resource object not in index") } if !rsrc.isSynced() { - return nil, fmt.Errorf("Resource object not in sync") + return nil, errors.New("Resource object not in sync") } // an update can be only one chunk long @@ -747,7 +748,7 @@ func (r *resourceChunkStore) Get(key Key) (*Chunk, error) { t := time.NewTimer(time.Second * 1) select { case <-t.C: - return nil, fmt.Errorf("timeout") + return nil, errors.New("timeout") case <-chunk.C: log.Trace("Received resource update chunk", "peer", chunk.Req.Source) } diff --git a/swarm/storage/resource_ens.go b/swarm/storage/resource_ens.go index df008efa175f..33163bc97aa5 100644 --- a/swarm/storage/resource_ens.go +++ b/swarm/storage/resource_ens.go @@ -1,7 +1,7 @@ package storage import ( - "fmt" + "errors" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" @@ -15,7 +15,7 @@ type baseValidator struct { func (b *baseValidator) sign(datahash common.Hash) (signature Signature, err error) { if b.signFunc == nil { - return signature, fmt.Errorf("No signature function") + return signature, errors.New("No signature function") } return b.signFunc(datahash) } From 400b7d6d1c44047d2a18065cc12680478851f618 Mon Sep 17 00:00:00 2001 From: lash Date: Mon, 22 Jan 2018 19:54:22 +0100 Subject: [PATCH 13/25] swarm/api/http: Test result data --- swarm/api/http/server_test.go | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 2837decfdefc..e75bd04bcf88 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -23,37 +23,35 @@ import ( "fmt" "io/ioutil" "net/http" - "os" "strings" "sync" "testing" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/api" swarm "github.com/ethereum/go-ethereum/swarm/api/client" "github.com/ethereum/go-ethereum/swarm/storage" "github.com/ethereum/go-ethereum/swarm/testutil" ) -func init() { - log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) -} - +// \TODO if create -> get -> update -> get, the last get with return 1.1 because 1.2 retrieve is still pending func TestBzzResource(t *testing.T) { srv := testutil.NewTestSwarmServer(t) defer srv.Close() - keybytes := make([]byte, common.HashLength) // nearest we get to source of info + // our mutable resource "name" + keybytes := make([]byte, common.HashLength) copy(keybytes, []byte{42}) - databytes := make([]byte, 42) + // data of update 1 + databytes := make([]byte, 666) _, err := rand.Read(databytes) if err != nil { t.Fatal(err) } - url := fmt.Sprintf("%s/bzz-resource:/%x/42", srv.URL, keybytes) + // creates resource and sets update 1 + url := fmt.Sprintf("%s/bzz-resource:/%x/13", srv.URL, keybytes) resp, err := http.Post(url, "application/octet-stream", bytes.NewReader(databytes)) if err != nil { t.Fatal(err) @@ -65,31 +63,31 @@ func TestBzzResource(t *testing.T) { if resp.StatusCode != http.StatusOK { t.Fatalf("err %s", resp.Status) } - log.Debug("Create", "status", resp.Status, "body", manifesthash) + // update 2 url = fmt.Sprintf("%s/bzz-resource:/%x", srv.URL, keybytes) data := []byte("foo") resp, err = http.Post(url, "application/octet-stream", bytes.NewReader(data)) if err != nil { t.Fatal(err) + } else if resp.StatusCode != http.StatusOK { + t.Fatalf("Update returned %d", resp.Status) } - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - t.Fatal(err) - } - log.Debug("Update", "status", resp.Status) + // get latest update (1.2) through swarm manifest url = fmt.Sprintf("%s/bzz-raw:/%s", srv.URL, manifesthash) resp, err = http.Get(url) if err != nil { t.Fatal(err) } - b, err = ioutil.ReadAll(resp.Body) + b, err := ioutil.ReadAll(resp.Body) if err != nil { t.Fatal(err) + } else if !bytes.Equal(data, b) { + t.Fatalf("Expected body '%x', got '%x'", data, b) } - log.Debug("Get raw", "status", resp.Status, "body", fmt.Sprintf("%s", b)) + // get latest update (1.2) through resource directly url = fmt.Sprintf("%s/bzz-resource:/%x", srv.URL, keybytes) resp, err = http.Get(url) if err != nil { @@ -98,8 +96,9 @@ func TestBzzResource(t *testing.T) { b, err = ioutil.ReadAll(resp.Body) if err != nil { t.Fatal(err) + } else if !bytes.Equal(data, b) { + t.Fatalf("Expected body '%x', got '%x'", data, b) } - log.Debug("Get resource", "status", resp.Status, "data", b) } From 0455925ebf1b902aaba51f83b91b1c1653edc623 Mon Sep 17 00:00:00 2001 From: lash Date: Mon, 22 Jan 2018 23:48:21 +0100 Subject: [PATCH 14/25] swarm: Amend comments from @lmars PR 204 second review, part I --- swarm/api/api.go | 6 +----- swarm/api/http/server.go | 28 ++++++++++++++++------------ swarm/api/http/server_test.go | 13 +++++++++---- swarm/storage/resource.go | 17 ++++++++--------- swarm/storage/resource_test.go | 7 +++++-- swarm/testutil/http.go | 7 +++++-- 6 files changed, 44 insertions(+), 34 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index 0c8d9d1ea78b..5197d7a4bd37 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -384,11 +384,7 @@ func (self *Api) ResourceLookup(name string, period uint32, version uint32) (sto if err != nil { return nil, nil, err } - key, data, err := self.resource.GetContent(name) - if err != nil { - return nil, nil, err - } - return key, data, nil + return self.resource.GetContent(name) } func (self *Api) ResourceCreate(name string, frequency uint64) (err error) { diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 0b22b9d8da4e..c84d47dac496 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -378,19 +378,19 @@ func (s *Server) handleGetResource(w http.ResponseWriter, r *Request, name strin if err != nil { break } - updateKey, data, err = s.api.ResourceLookup(name, uint32(period), uint32(version)) - case 1: - version, err = strconv.ParseUint(params[1], 10, 32) + period, err = strconv.ParseUint(params[0], 10, 32) if err != nil { break } + updateKey, data, err = s.api.ResourceLookup(name, uint32(period), uint32(version)) + case 1: period, err = strconv.ParseUint(params[0], 10, 32) if err != nil { break } updateKey, data, err = s.api.ResourceLookup(name, uint32(period), uint32(version)) default: - s.BadRequest(w, r, fmt.Sprintf("Invalid mutable resource request")) + s.BadRequest(w, r, "Invalid mutable resource request") return } if err != nil { @@ -463,14 +463,18 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { switch { case r.uri.Raw(): m := &api.Manifest{} - sz, _ := reader.Size(nil) - b := make([]byte, sz) - reader.Read(b) - err = json.Unmarshal(b, m) - if len(m.Entries) > 0 { - if m.Entries[0].ContentType == api.ResourceContentType { - s.handleGetResource(w, r, m.Entries[0].Path) - return + sz, err := reader.Size(nil) + if err == nil { + b := make([]byte, sz) + reader.Read(b) + err = json.Unmarshal(b, m) + if err == nil { + if len(m.Entries) > 0 { + if m.Entries[0].ContentType == api.ResourceContentType { + s.handleGetResource(w, r, m.Entries[0].Path) + return + } + } } } // allow the request to overwrite the content type using a query diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index e75bd04bcf88..73036de203d2 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -55,14 +55,14 @@ func TestBzzResource(t *testing.T) { resp, err := http.Post(url, "application/octet-stream", bytes.NewReader(databytes)) if err != nil { t.Fatal(err) + } else if resp.StatusCode != http.StatusOK { + t.Fatalf("err %s", resp.Status) } manifesthash, err := ioutil.ReadAll(resp.Body) if err != nil { t.Fatal(err) } - if resp.StatusCode != http.StatusOK { - t.Fatalf("err %s", resp.Status) - } + resp.Body.Close() // update 2 url = fmt.Sprintf("%s/bzz-resource:/%x", srv.URL, keybytes) @@ -79,6 +79,8 @@ func TestBzzResource(t *testing.T) { resp, err = http.Get(url) if err != nil { t.Fatal(err) + } else if resp.StatusCode != http.StatusOK { + t.Fatalf("err %s", resp.Status) } b, err := ioutil.ReadAll(resp.Body) if err != nil { @@ -86,12 +88,15 @@ func TestBzzResource(t *testing.T) { } else if !bytes.Equal(data, b) { t.Fatalf("Expected body '%x', got '%x'", data, b) } + resp.Body.Close() // get latest update (1.2) through resource directly url = fmt.Sprintf("%s/bzz-resource:/%x", srv.URL, keybytes) resp, err = http.Get(url) if err != nil { t.Fatal(err) + } else if resp.StatusCode != http.StatusOK { + t.Fatalf("err %s", resp.Status) } b, err = ioutil.ReadAll(resp.Body) if err != nil { @@ -99,7 +104,7 @@ func TestBzzResource(t *testing.T) { } else if !bytes.Equal(data, b) { t.Fatalf("Expected body '%x', got '%x'", data, b) } - + resp.Body.Close() } func TestBzzGetPath(t *testing.T) { diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 80251c841ef7..62e0dc545622 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -13,6 +13,7 @@ import ( "golang.org/x/net/idna" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" ) @@ -60,7 +61,7 @@ type ResourceValidator interface { } type ethApi interface { - BlockNumber(context.Context) (big.Int, error) + HeaderByNumber(context.Context, *big.Int) (*types.Header, error) } // Mutable resource is an entity which allows updates to a resource @@ -124,7 +125,7 @@ type ethApi interface { // TODO: Include modtime in chunk data + signature type ResourceHandler struct { ChunkStore - ctx context.Context + ctx context.Context // base for new contexts passed to storage layer and ethapi, to ensure teardown when Close() is called cancelFunc func() validator ResourceValidator ethClient ethApi @@ -609,11 +610,13 @@ func (self *ResourceHandler) Close() { } func (self *ResourceHandler) GetBlock() (uint64, error) { - bigblocknumber, err := self.ethClient.BlockNumber(self.ctx) + ctx, cancel := context.WithCancel(self.ctx) + defer cancel() + blockheader, err := self.ethClient.HeaderByNumber(ctx, nil) if err != nil { return 0, err } - return bigblocknumber.Uint64(), nil + return blockheader.Number.Uint64(), nil } // Calculate the period index (aka major version number) from a given block number @@ -771,11 +774,7 @@ func getNextPeriod(start uint64, current uint64, frequency uint64) uint32 { } func ToSafeName(name string) (string, error) { - validname, err := idna.ToASCII(name) - if err != nil { - return "", err - } - return validname, nil + return idna.ToASCII(name) } // check that name identifiers contain valid bytes diff --git a/swarm/storage/resource_test.go b/swarm/storage/resource_test.go index dd96f0b3d753..e6a00d88f40b 100644 --- a/swarm/storage/resource_test.go +++ b/swarm/storage/resource_test.go @@ -20,6 +20,7 @@ import ( "github.com/ethereum/go-ethereum/contracts/ens" "github.com/ethereum/go-ethereum/contracts/ens/contract" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" ) @@ -57,10 +58,12 @@ func (f *fakeBackend) Commit() { f.blocknumber++ } -func (f *fakeBackend) BlockNumber(context context.Context) (big.Int, error) { +func (f *fakeBackend) HeaderByNumber(context context.Context, bigblock *big.Int) (*types.Header, error) { f.blocknumber++ biggie := big.NewInt(f.blocknumber) - return *biggie, nil + return &types.Header{ + Number: biggie, + }, nil } // check that signature address matches update signer address diff --git a/swarm/testutil/http.go b/swarm/testutil/http.go index 4a6a68e43bfb..cf4b045c6dc2 100644 --- a/swarm/testutil/http.go +++ b/swarm/testutil/http.go @@ -25,6 +25,7 @@ import ( "strconv" "testing" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/swarm/api" httpapi "github.com/ethereum/go-ethereum/swarm/api/http" "github.com/ethereum/go-ethereum/swarm/storage" @@ -34,10 +35,12 @@ type fakeBackend struct { blocknumber int64 } -func (f *fakeBackend) BlockNumber(ctx context.Context) (big.Int, error) { +func (f *fakeBackend) HeaderByNumber(context context.Context, bigblock *big.Int) (*types.Header, error) { f.blocknumber++ biggie := big.NewInt(f.blocknumber) - return *biggie, nil + return &types.Header{ + Number: biggie, + }, nil } func NewTestSwarmServer(t *testing.T) *TestSwarmServer { From a32681cba4868d65af78484001f69095dbdbae6e Mon Sep 17 00:00:00 2001 From: lash Date: Tue, 23 Jan 2018 02:04:57 +0100 Subject: [PATCH 15/25] swarm/storage: Correct channel for waiting on chunk put --- swarm/api/http/server_test.go | 24 +++++++++++++++++++++++- swarm/storage/resource.go | 4 +++- swarm/testutil/http.go | 10 ---------- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 73036de203d2..664e56ca42a1 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -23,17 +23,23 @@ import ( "fmt" "io/ioutil" "net/http" + "os" "strings" "sync" "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/api" swarm "github.com/ethereum/go-ethereum/swarm/api/client" "github.com/ethereum/go-ethereum/swarm/storage" "github.com/ethereum/go-ethereum/swarm/testutil" ) +func init() { + log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) +} + // \TODO if create -> get -> update -> get, the last get with return 1.1 because 1.2 retrieve is still pending func TestBzzResource(t *testing.T) { srv := testutil.NewTestSwarmServer(t) @@ -64,6 +70,22 @@ func TestBzzResource(t *testing.T) { } resp.Body.Close() + // get latest update (1.1) through resource directly + url = fmt.Sprintf("%s/bzz-resource:/%x", srv.URL, keybytes) + resp, err = http.Get(url) + if err != nil { + t.Fatal(err) + } else if resp.StatusCode != http.StatusOK { + t.Fatalf("err %s", resp.Status) + } + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } else if !bytes.Equal(databytes, b) { + t.Fatalf("Expected body '%x', got '%x'", databytes, b) + } + resp.Body.Close() + // update 2 url = fmt.Sprintf("%s/bzz-resource:/%x", srv.URL, keybytes) data := []byte("foo") @@ -82,7 +104,7 @@ func TestBzzResource(t *testing.T) { } else if resp.StatusCode != http.StatusOK { t.Fatalf("err %s", resp.Status) } - b, err := ioutil.ReadAll(resp.Body) + b, err = ioutil.ReadAll(resp.Body) if err != nil { t.Fatal(err) } else if !bytes.Equal(data, b) { diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 62e0dc545622..c7b675f10abf 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -752,14 +752,16 @@ func (r *resourceChunkStore) Get(key Key) (*Chunk, error) { select { case <-t.C: return nil, errors.New("timeout") - case <-chunk.C: + case <-chunk.Req.C: log.Trace("Received resource update chunk", "peer", chunk.Req.Source) } return chunk, nil } func (r *resourceChunkStore) Put(chunk *Chunk) { + chunk.wg = &sync.WaitGroup{} r.netStore.Put(chunk) + chunk.wg.Wait() } func (r *resourceChunkStore) Close() { diff --git a/swarm/testutil/http.go b/swarm/testutil/http.go index cf4b045c6dc2..afbd5d6103d4 100644 --- a/swarm/testutil/http.go +++ b/swarm/testutil/http.go @@ -22,7 +22,6 @@ import ( "math/big" "net/http/httptest" "os" - "strconv" "testing" "github.com/ethereum/go-ethereum/core/types" @@ -117,12 +116,3 @@ func (c *testCloudStore) Deliver(*storage.Chunk) { func (c *testCloudStore) Retrieve(*storage.Chunk) { } - -// for faking the rpc service, since we don't need the whole node stack -type FakeRPC struct { - blocknumber uint64 -} - -func (r *FakeRPC) BlockNumber() (string, error) { - return strconv.FormatUint(r.blocknumber, 10), nil -} From 94303aa1be77f34d4ecb7b8acc899df34d187026 Mon Sep 17 00:00:00 2001 From: lash Date: Tue, 23 Jan 2018 02:50:10 +0100 Subject: [PATCH 16/25] swarm/storage: Implement resourcehandler hashers as sync.Pool --- swarm/storage/resource.go | 51 ++++++++++++++++++++-------------- swarm/storage/resource_test.go | 3 ++ 2 files changed, 33 insertions(+), 21 deletions(-) diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index c7b675f10abf..2ec3e5327c58 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -24,6 +24,7 @@ const ( DbDirName = "resource" chunkSize = 4096 // temporary until we implement DPA in the resourcehandler defaultStoreTimeout = 4000 * time.Millisecond + hasherCount = 8 ) type Signature [signatureLength]byte @@ -130,9 +131,8 @@ type ResourceHandler struct { validator ResourceValidator ethClient ethApi resources map[string]*resource - hashLock sync.Mutex + hashPool sync.Pool resourceLock sync.RWMutex - hasher SwarmHash nameHash nameHashFunc storeTimeout time.Duration } @@ -159,25 +159,34 @@ func NewResourceHandler(datadir string, cloudStore CloudStore, ethClient ethApi, ChunkStore: newResourceChunkStore(path, hashfunc, localStore, cloudStore), ethClient: ethClient, resources: make(map[string]*resource), - hasher: hashfunc(), validator: validator, storeTimeout: defaultStoreTimeout, ctx: ctx, cancelFunc: cancel, + hashPool: sync.Pool{ + New: func() interface{} { + return MakeHashFunc(SHA3Hash)() + }, + }, } if rh.validator != nil { rh.nameHash = rh.validator.nameHash } else { rh.nameHash = func(name string) common.Hash { - rh.hashLock.Lock() - defer rh.hashLock.Unlock() - rh.hasher.Reset() - rh.hasher.Write([]byte(name)) - return common.BytesToHash(rh.hasher.Sum(nil)) + hasher := rh.hashPool.Get().(SwarmHash) + defer rh.hashPool.Put(hasher) + hasher.Reset() + hasher.Write([]byte(name)) + return common.BytesToHash(hasher.Sum(nil)) } } + for i := 0; i < hasherCount; i++ { + hashfunc := MakeHashFunc(SHA3Hash)() + rh.hashPool.Put(hashfunc) + } + return rh, nil } @@ -645,16 +654,16 @@ func (self *ResourceHandler) setResource(name string, rsrc *resource) { // used for chunk keys func (self *ResourceHandler) resourceHash(period uint32, version uint32, namehash common.Hash) Key { // format is: hash(period|version|namehash) - self.hashLock.Lock() - defer self.hashLock.Unlock() - self.hasher.Reset() + hasher := self.hashPool.Get().(SwarmHash) + defer self.hashPool.Put(hasher) + hasher.Reset() b := make([]byte, 4) binary.LittleEndian.PutUint32(b, period) - self.hasher.Write(b) + hasher.Write(b) binary.LittleEndian.PutUint32(b, version) - self.hasher.Write(b) - self.hasher.Write(namehash[:]) - return self.hasher.Sum(nil) + hasher.Write(b) + hasher.Write(namehash[:]) + return hasher.Sum(nil) } func (self *ResourceHandler) hasUpdate(name string, period uint32) bool { @@ -793,10 +802,10 @@ func isSafeName(name string) bool { // convenience for creating signature hashes of update data func (self *ResourceHandler) keyDataHash(key Key, data []byte) common.Hash { - self.hashLock.Lock() - defer self.hashLock.Unlock() - self.hasher.Reset() - self.hasher.Write(key[:]) - self.hasher.Write(data) - return common.BytesToHash(self.hasher.Sum(nil)) + hasher := self.hashPool.Get().(SwarmHash) + defer self.hashPool.Put(hasher) + hasher.Reset() + hasher.Write(key[:]) + hasher.Write(data) + return common.BytesToHash(hasher.Sum(nil)) } diff --git a/swarm/storage/resource_test.go b/swarm/storage/resource_test.go index e6a00d88f40b..537eb35d0958 100644 --- a/swarm/storage/resource_test.go +++ b/swarm/storage/resource_test.go @@ -106,6 +106,9 @@ func TestResourceReverse(t *testing.T) { // check that we can recover the owner account from the update chunk's signature checksig, checkperiod, checkversion, checkname, checkdata, err := rh.parseUpdate(chunk.SData) + if err != nil { + t.Fatal(err) + } checkdigest := rh.keyDataHash(chunk.Key, checkdata) recoveredaddress, err := getAddressFromDataSig(checkdigest, *checksig) if err != nil { From edacd4b7c3600ed7b1888532268d701653132909 Mon Sep 17 00:00:00 2001 From: lash Date: Tue, 23 Jan 2018 03:19:00 +0100 Subject: [PATCH 17/25] swarm/api: Remove faulty manifest handling + add rsrc create keycheck --- swarm/api/api.go | 7 +++-- swarm/api/http/server.go | 53 ++++++----------------------------- swarm/api/http/server_test.go | 25 +++++------------ swarm/storage/resource.go | 8 +++++- swarm/testutil/http.go | 4 +-- 5 files changed, 28 insertions(+), 69 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index 5197d7a4bd37..7abe5295c160 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -387,9 +387,10 @@ func (self *Api) ResourceLookup(name string, period uint32, version uint32) (sto return self.resource.GetContent(name) } -func (self *Api) ResourceCreate(name string, frequency uint64) (err error) { - _, err = self.resource.NewResource(name, frequency) - return err +func (self *Api) ResourceCreate(name string, frequency uint64) (storage.Key, error) { + rsrc, err := self.resource.NewResource(name, frequency) + h := rsrc.NameHash() + return storage.Key(h[:]), err } func (self *Api) ResourceUpdate(name string, data []byte) (storage.Key, uint32, uint32, error) { diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index c84d47dac496..db81d89c598a 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -299,36 +299,12 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) { s.BadRequest(w, r, fmt.Sprintf("Cannot parse frequency parameter: %v", err)) return } - err = s.api.ResourceCreate(r.uri.Addr, frequency) + key, err := s.api.ResourceCreate(r.uri.Addr, frequency) if err != nil { s.Error(w, r, fmt.Errorf("Resource creation failed: %v", err)) return } - manifestKey, err := s.api.NewManifest() - if err != nil { - s.Error(w, r, fmt.Errorf("create manifest err: %v", err)) - return - } - newKey, err := s.updateManifest(manifestKey, func(mw *api.ManifestWriter) error { - key, err := mw.AddEntry(bytes.NewReader([]byte(r.uri.Addr)), &api.ManifestEntry{ - Path: r.uri.Addr, - ContentType: api.ResourceContentType, - Mode: 0644, - Size: int64(len(r.uri.Addr)), - ModTime: time.Now(), - }) - if err != nil { - return err - } - s.logDebug("resource manifest for for %s stored", key.Log()) - return nil - }) - if err != nil { - s.Error(w, r, fmt.Errorf("update manifest err: %v", err)) - return - } - log.Debug("manifests", "new", newKey, "old", manifestKey) - outdata = fmt.Sprintf("%s", newKey) + outdata = key.Hex() } data, err := ioutil.ReadAll(r.Body) @@ -338,15 +314,17 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) { } _, _, _, err = s.api.ResourceUpdate(r.uri.Addr, data) if err != nil { - w.WriteHeader(http.StatusUnauthorized) + s.Error(w, r, fmt.Errorf("Update resource failed: %v", err)) return } - w.WriteHeader(http.StatusOK) if outdata != "" { - w.Header().Set("Content-type", "text/plain") - fmt.Fprintf(w, outdata) + w.Header().Add("Content-type", "text/plain") + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, outdata) + return } + w.WriteHeader(http.StatusOK) } // Retrieve mutable resource updates: @@ -462,21 +440,6 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { switch { case r.uri.Raw(): - m := &api.Manifest{} - sz, err := reader.Size(nil) - if err == nil { - b := make([]byte, sz) - reader.Read(b) - err = json.Unmarshal(b, m) - if err == nil { - if len(m.Entries) > 0 { - if m.Entries[0].ContentType == api.ResourceContentType { - s.handleGetResource(w, r, m.Entries[0].Path) - return - } - } - } - } // allow the request to overwrite the content type using a query // parameter contentType := "application/octet-stream" diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 664e56ca42a1..1a01108977a2 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -48,6 +48,9 @@ func TestBzzResource(t *testing.T) { // our mutable resource "name" keybytes := make([]byte, common.HashLength) copy(keybytes, []byte{42}) + srv.Hasher.Reset() + srv.Hasher.Write([]byte(fmt.Sprintf("%x", keybytes))) + keybyteshash := fmt.Sprintf("%x", srv.Hasher.Sum(nil)) // data of update 1 databytes := make([]byte, 666) @@ -64,9 +67,11 @@ func TestBzzResource(t *testing.T) { } else if resp.StatusCode != http.StatusOK { t.Fatalf("err %s", resp.Status) } - manifesthash, err := ioutil.ReadAll(resp.Body) + b, err := ioutil.ReadAll(resp.Body) if err != nil { t.Fatal(err) + } else if !bytes.Equal(b, []byte(keybyteshash)) { + t.Fatalf("resource update hash mismatch, expected '%s' got '%s'", keybyteshash, b) } resp.Body.Close() @@ -78,7 +83,7 @@ func TestBzzResource(t *testing.T) { } else if resp.StatusCode != http.StatusOK { t.Fatalf("err %s", resp.Status) } - b, err := ioutil.ReadAll(resp.Body) + b, err = ioutil.ReadAll(resp.Body) if err != nil { t.Fatal(err) } else if !bytes.Equal(databytes, b) { @@ -96,22 +101,6 @@ func TestBzzResource(t *testing.T) { t.Fatalf("Update returned %d", resp.Status) } - // get latest update (1.2) through swarm manifest - url = fmt.Sprintf("%s/bzz-raw:/%s", srv.URL, manifesthash) - resp, err = http.Get(url) - if err != nil { - t.Fatal(err) - } else if resp.StatusCode != http.StatusOK { - t.Fatalf("err %s", resp.Status) - } - b, err = ioutil.ReadAll(resp.Body) - if err != nil { - t.Fatal(err) - } else if !bytes.Equal(data, b) { - t.Fatalf("Expected body '%x', got '%x'", data, b) - } - resp.Body.Close() - // get latest update (1.2) through resource directly url = fmt.Sprintf("%s/bzz-resource:/%x", srv.URL, keybytes) resp, err = http.Get(url) diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 2ec3e5327c58..39b4fafa6f19 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -52,6 +52,10 @@ func (self *resource) isSynced() bool { return !self.updated.IsZero() } +func (self *resource) NameHash() common.Hash { + return self.nameHash +} + // Implement to activate validation of resource updates // Specifically signing data and verification of signatures type ResourceValidator interface { @@ -178,7 +182,9 @@ func NewResourceHandler(datadir string, cloudStore CloudStore, ethClient ethApi, defer rh.hashPool.Put(hasher) hasher.Reset() hasher.Write([]byte(name)) - return common.BytesToHash(hasher.Sum(nil)) + hashval := common.BytesToHash(hasher.Sum(nil)) + log.Debug("generic namehasher", "name", name, "hash", hashval) + return hashval } } diff --git a/swarm/testutil/http.go b/swarm/testutil/http.go index afbd5d6103d4..77e3386fd217 100644 --- a/swarm/testutil/http.go +++ b/swarm/testutil/http.go @@ -82,7 +82,7 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer { Server: srv, Dpa: dpa, dir: dir, - hasher: storage.MakeHashFunc(storage.SHA3Hash)(), + Hasher: storage.MakeHashFunc(storage.SHA3Hash)(), cleanup: func() { srv.Close() rh.Close() @@ -95,7 +95,7 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer { type TestSwarmServer struct { *httptest.Server - hasher storage.SwarmHash + Hasher storage.SwarmHash Dpa *storage.DPA dir string cleanup func() From 690522b09d613eaa507ec2f5ca076bd2515bb0b0 Mon Sep 17 00:00:00 2001 From: lash Date: Tue, 23 Jan 2018 14:57:04 +0100 Subject: [PATCH 18/25] swarm/api: Amend @gbalint comments PR 204 + args dep test loglvl --- swarm/api/api.go | 5 ++++- swarm/api/http/server_test.go | 21 ++++++++++++++------- swarm/storage/resource.go | 2 +- swarm/storage/resource_test.go | 7 ++++++- 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index 7abe5295c160..5a222dddc944 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -389,8 +389,11 @@ func (self *Api) ResourceLookup(name string, period uint32, version uint32) (sto func (self *Api) ResourceCreate(name string, frequency uint64) (storage.Key, error) { rsrc, err := self.resource.NewResource(name, frequency) + if err != nil { + return nil, err + } h := rsrc.NameHash() - return storage.Key(h[:]), err + return storage.Key(h[:]), nil } func (self *Api) ResourceUpdate(name string, data []byte) (storage.Key, uint32, uint32, error) { diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 1a01108977a2..ea6f9155d161 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -64,13 +64,15 @@ func TestBzzResource(t *testing.T) { resp, err := http.Post(url, "application/octet-stream", bytes.NewReader(databytes)) if err != nil { t.Fatal(err) - } else if resp.StatusCode != http.StatusOK { + } + if resp.StatusCode != http.StatusOK { t.Fatalf("err %s", resp.Status) } b, err := ioutil.ReadAll(resp.Body) if err != nil { t.Fatal(err) - } else if !bytes.Equal(b, []byte(keybyteshash)) { + } + if !bytes.Equal(b, []byte(keybyteshash)) { t.Fatalf("resource update hash mismatch, expected '%s' got '%s'", keybyteshash, b) } resp.Body.Close() @@ -80,13 +82,15 @@ func TestBzzResource(t *testing.T) { resp, err = http.Get(url) if err != nil { t.Fatal(err) - } else if resp.StatusCode != http.StatusOK { + } + if resp.StatusCode != http.StatusOK { t.Fatalf("err %s", resp.Status) } b, err = ioutil.ReadAll(resp.Body) if err != nil { t.Fatal(err) - } else if !bytes.Equal(databytes, b) { + } + if !bytes.Equal(databytes, b) { t.Fatalf("Expected body '%x', got '%x'", databytes, b) } resp.Body.Close() @@ -97,7 +101,8 @@ func TestBzzResource(t *testing.T) { resp, err = http.Post(url, "application/octet-stream", bytes.NewReader(data)) if err != nil { t.Fatal(err) - } else if resp.StatusCode != http.StatusOK { + } + if resp.StatusCode != http.StatusOK { t.Fatalf("Update returned %d", resp.Status) } @@ -106,13 +111,15 @@ func TestBzzResource(t *testing.T) { resp, err = http.Get(url) if err != nil { t.Fatal(err) - } else if resp.StatusCode != http.StatusOK { + } + if resp.StatusCode != http.StatusOK { t.Fatalf("err %s", resp.Status) } b, err = ioutil.ReadAll(resp.Body) if err != nil { t.Fatal(err) - } else if !bytes.Equal(data, b) { + } + if !bytes.Equal(data, b) { t.Fatalf("Expected body '%x', got '%x'", data, b) } resp.Body.Close() diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 39b4fafa6f19..7bd437e92464 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -703,7 +703,7 @@ func newUpdateChunk(key Key, signature *Signature, period uint32, version uint32 datalength := len(data) chunk := NewChunk(key, nil) - chunk.SData = make([]byte, 4+signaturelength+headerlength+datalength) + chunk.SData = make([]byte, 4+signaturelength+headerlength+datalength) // initial 4 are uint16 length descriptors for headerlength and datalength // data header length does NOT include the header length prefix bytes themselves cursor := 0 diff --git a/swarm/storage/resource_test.go b/swarm/storage/resource_test.go index 537eb35d0958..e84cdb7b9302 100644 --- a/swarm/storage/resource_test.go +++ b/swarm/storage/resource_test.go @@ -6,6 +6,7 @@ import ( "crypto/ecdsa" "crypto/rand" "encoding/binary" + "flag" "fmt" "io/ioutil" "math/big" @@ -37,7 +38,11 @@ var ( func init() { var err error - log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) + verbose := flag.Bool("v", false, "verbose") + flag.Parse() + if *verbose { + log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) + } safeName, err = ToSafeName(domainName) if err != nil { panic(err) From 88ed2d4b2d13550d01016cdc6803bfa80083a43b Mon Sep 17 00:00:00 2001 From: lash Date: Tue, 23 Jan 2018 17:18:46 +0100 Subject: [PATCH 19/25] swarm, ethclient: Add version test for http api Clear redundant addition of BlockNumber in ethclient --- ethclient/ethclient.go | 6 ----- swarm/api/http/server_test.go | 50 ++++++++++++++++++++++++++++------- swarm/storage/resource.go | 3 +-- 3 files changed, 41 insertions(+), 18 deletions(-) diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index f41de1b42c60..87a912901af5 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -76,12 +76,6 @@ type rpcBlock struct { UncleHashes []common.Hash `json:"uncles"` } -func (ec *Client) BlockNumber(ctx context.Context) (big.Int, error) { - number := &big.Int{} - err := ec.c.CallContext(ctx, &number, "eth_blockNumber") - return *number, err -} - func (ec *Client) getBlock(ctx context.Context, method string, args ...interface{}) (*types.Block, error) { var raw json.RawMessage err := ec.c.CallContext(ctx, &raw, method, args...) diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index ea6f9155d161..57d139ac98aa 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -23,24 +23,17 @@ import ( "fmt" "io/ioutil" "net/http" - "os" "strings" "sync" "testing" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/api" swarm "github.com/ethereum/go-ethereum/swarm/api/client" "github.com/ethereum/go-ethereum/swarm/storage" "github.com/ethereum/go-ethereum/swarm/testutil" ) -func init() { - log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) -} - -// \TODO if create -> get -> update -> get, the last get with return 1.1 because 1.2 retrieve is still pending func TestBzzResource(t *testing.T) { srv := testutil.NewTestSwarmServer(t) defer srv.Close() @@ -65,6 +58,7 @@ func TestBzzResource(t *testing.T) { if err != nil { t.Fatal(err) } + defer resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Fatalf("err %s", resp.Status) } @@ -75,7 +69,6 @@ func TestBzzResource(t *testing.T) { if !bytes.Equal(b, []byte(keybyteshash)) { t.Fatalf("resource update hash mismatch, expected '%s' got '%s'", keybyteshash, b) } - resp.Body.Close() // get latest update (1.1) through resource directly url = fmt.Sprintf("%s/bzz-resource:/%x", srv.URL, keybytes) @@ -83,6 +76,7 @@ func TestBzzResource(t *testing.T) { if err != nil { t.Fatal(err) } + defer resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Fatalf("err %s", resp.Status) } @@ -93,7 +87,6 @@ func TestBzzResource(t *testing.T) { if !bytes.Equal(databytes, b) { t.Fatalf("Expected body '%x', got '%x'", databytes, b) } - resp.Body.Close() // update 2 url = fmt.Sprintf("%s/bzz-resource:/%x", srv.URL, keybytes) @@ -102,6 +95,7 @@ func TestBzzResource(t *testing.T) { if err != nil { t.Fatal(err) } + defer resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Fatalf("Update returned %d", resp.Status) } @@ -112,6 +106,25 @@ func TestBzzResource(t *testing.T) { if err != nil { t.Fatal(err) } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("err %s", resp.Status) + } + b, err = ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(data, b) { + t.Fatalf("Expected body '%x', got '%x'", data, b) + } + + // get latest update (1.2) with specified period + url = fmt.Sprintf("%s/bzz-resource:/%x/1", srv.URL, keybytes) + resp, err = http.Get(url) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Fatalf("err %s", resp.Status) } @@ -122,7 +135,24 @@ func TestBzzResource(t *testing.T) { if !bytes.Equal(data, b) { t.Fatalf("Expected body '%x', got '%x'", data, b) } - resp.Body.Close() + + // get first update (1.1) with specified period and version + url = fmt.Sprintf("%s/bzz-resource:/%x/1/1", srv.URL, keybytes) + resp, err = http.Get(url) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("err %s", resp.Status) + } + b, err = ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(databytes, b) { + t.Fatalf("Expected body '%x', got '%x'", databytes, b) + } } func TestBzzGetPath(t *testing.T) { diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 7bd437e92464..21e84667432f 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -625,8 +625,7 @@ func (self *ResourceHandler) Close() { } func (self *ResourceHandler) GetBlock() (uint64, error) { - ctx, cancel := context.WithCancel(self.ctx) - defer cancel() + ctx, _ := context.WithCancel(self.ctx) blockheader, err := self.ethClient.HeaderByNumber(ctx, nil) if err != nil { return 0, err From 4a7b5b862a1dfc02afacfc9625ae3df3b09719a5 Mon Sep 17 00:00:00 2001 From: lash Date: Tue, 23 Jan 2018 21:53:14 +0100 Subject: [PATCH 20/25] swarm/api, swarm/storage: Delinting --- swarm/api/http/server_test.go | 3 ++- swarm/storage/resource.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 57d139ac98aa..9a80405ffc20 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -69,6 +69,7 @@ func TestBzzResource(t *testing.T) { if !bytes.Equal(b, []byte(keybyteshash)) { t.Fatalf("resource update hash mismatch, expected '%s' got '%s'", keybyteshash, b) } + t.Logf("creatreturn %v / %v", keybyteshash, b) // get latest update (1.1) through resource directly url = fmt.Sprintf("%s/bzz-resource:/%x", srv.URL, keybytes) @@ -97,7 +98,7 @@ func TestBzzResource(t *testing.T) { } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - t.Fatalf("Update returned %d", resp.Status) + t.Fatalf("Update returned %s", resp.Status) } // get latest update (1.2) through resource directly diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 21e84667432f..7bd437e92464 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -625,7 +625,8 @@ func (self *ResourceHandler) Close() { } func (self *ResourceHandler) GetBlock() (uint64, error) { - ctx, _ := context.WithCancel(self.ctx) + ctx, cancel := context.WithCancel(self.ctx) + defer cancel() blockheader, err := self.ethClient.HeaderByNumber(ctx, nil) if err != nil { return 0, err From add89714670cffea184814158863d2210820cdf7 Mon Sep 17 00:00:00 2001 From: lash Date: Wed, 24 Jan 2018 02:44:50 +0100 Subject: [PATCH 21/25] swarm/api, swarm/storage: Fullstack contexts --- swarm/api/api.go | 19 +++++++++-------- swarm/api/http/server.go | 10 ++++----- swarm/storage/resource.go | 38 ++++++++++++++-------------------- swarm/storage/resource_test.go | 26 +++++++++++++---------- 4 files changed, 45 insertions(+), 48 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index 5a222dddc944..572b150da225 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -17,6 +17,7 @@ package api import ( + "context" "fmt" "io" "net/http" @@ -365,21 +366,21 @@ func (self *Api) BuildDirectoryTree(mhash string, nameresolver bool) (key storag } // Look up mutable resource updates at specific periods and versions -func (self *Api) ResourceLookup(name string, period uint32, version uint32) (storage.Key, []byte, error) { +func (self *Api) ResourceLookup(ctx context.Context, name string, period uint32, version uint32) (storage.Key, []byte, error) { var err error if version != 0 { if period == 0 { - currentblocknumber, err := self.resource.GetBlock() + currentblocknumber, err := self.resource.GetBlock(ctx) if err != nil { return nil, nil, fmt.Errorf("Could not determine latest block: %v", err) } period = self.resource.BlockToPeriod(name, currentblocknumber) } - _, err = self.resource.LookupVersionByName(name, period, version, true) + _, err = self.resource.LookupVersionByName(ctx, name, period, version, true) } else if period != 0 { - _, err = self.resource.LookupHistoricalByName(name, period, true) + _, err = self.resource.LookupHistoricalByName(ctx, name, period, true) } else { - _, err = self.resource.LookupLatestByName(name, true) + _, err = self.resource.LookupLatestByName(ctx, name, true) } if err != nil { return nil, nil, err @@ -387,8 +388,8 @@ func (self *Api) ResourceLookup(name string, period uint32, version uint32) (sto return self.resource.GetContent(name) } -func (self *Api) ResourceCreate(name string, frequency uint64) (storage.Key, error) { - rsrc, err := self.resource.NewResource(name, frequency) +func (self *Api) ResourceCreate(ctx context.Context, name string, frequency uint64) (storage.Key, error) { + rsrc, err := self.resource.NewResource(ctx, name, frequency) if err != nil { return nil, err } @@ -396,8 +397,8 @@ func (self *Api) ResourceCreate(name string, frequency uint64) (storage.Key, err return storage.Key(h[:]), nil } -func (self *Api) ResourceUpdate(name string, data []byte) (storage.Key, uint32, uint32, error) { - key, err := self.resource.Update(name, data) +func (self *Api) ResourceUpdate(ctx context.Context, name string, data []byte) (storage.Key, uint32, uint32, error) { + key, err := self.resource.Update(ctx, name, data) period, _ := self.resource.GetLastPeriod(name) version, _ := self.resource.GetVersion(name) return key, period, version, err diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index db81d89c598a..a71edfdd23d8 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -299,7 +299,7 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) { s.BadRequest(w, r, fmt.Sprintf("Cannot parse frequency parameter: %v", err)) return } - key, err := s.api.ResourceCreate(r.uri.Addr, frequency) + key, err := s.api.ResourceCreate(r.Context(), r.uri.Addr, frequency) if err != nil { s.Error(w, r, fmt.Errorf("Resource creation failed: %v", err)) return @@ -312,7 +312,7 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) { s.Error(w, r, err) return } - _, _, _, err = s.api.ResourceUpdate(r.uri.Addr, data) + _, _, _, err = s.api.ResourceUpdate(r.Context(), r.uri.Addr, data) if err != nil { s.Error(w, r, fmt.Errorf("Update resource failed: %v", err)) return @@ -350,7 +350,7 @@ func (s *Server) handleGetResource(w http.ResponseWriter, r *Request, name strin log.Debug("handlegetdb", "name", name) switch len(params) { case 0: - updateKey, data, err = s.api.ResourceLookup(name, 0, 0) + updateKey, data, err = s.api.ResourceLookup(r.Context(), name, 0, 0) case 2: version, err = strconv.ParseUint(params[1], 10, 32) if err != nil { @@ -360,13 +360,13 @@ func (s *Server) handleGetResource(w http.ResponseWriter, r *Request, name strin if err != nil { break } - updateKey, data, err = s.api.ResourceLookup(name, uint32(period), uint32(version)) + updateKey, data, err = s.api.ResourceLookup(r.Context(), name, uint32(period), uint32(version)) case 1: period, err = strconv.ParseUint(params[0], 10, 32) if err != nil { break } - updateKey, data, err = s.api.ResourceLookup(name, uint32(period), uint32(version)) + updateKey, data, err = s.api.ResourceLookup(r.Context(), name, uint32(period), uint32(version)) default: s.BadRequest(w, r, "Invalid mutable resource request") return diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 7bd437e92464..81a7621a6321 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -130,8 +130,6 @@ type ethApi interface { // TODO: Include modtime in chunk data + signature type ResourceHandler struct { ChunkStore - ctx context.Context // base for new contexts passed to storage layer and ethapi, to ensure teardown when Close() is called - cancelFunc func() validator ResourceValidator ethClient ethApi resources map[string]*resource @@ -158,15 +156,12 @@ func NewResourceHandler(datadir string, cloudStore CloudStore, ethClient ethApi, DbStore: dbStore, } - ctx, cancel := context.WithCancel(context.Background()) rh := &ResourceHandler{ ChunkStore: newResourceChunkStore(path, hashfunc, localStore, cloudStore), ethClient: ethClient, resources: make(map[string]*resource), validator: validator, storeTimeout: defaultStoreTimeout, - ctx: ctx, - cancelFunc: cancel, hashPool: sync.Pool{ New: func() interface{} { return MakeHashFunc(SHA3Hash)() @@ -241,7 +236,7 @@ func (self *ResourceHandler) chunkSize() int64 { // The signature data should match the hash of the idna-converted name by the validator's namehash function, NOT the raw name bytes. // // The start block of the resource update will be the actual current block height of the connected network. -func (self *ResourceHandler) NewResource(name string, frequency uint64) (*resource, error) { +func (self *ResourceHandler) NewResource(ctx context.Context, name string, frequency uint64) (*resource, error) { // frequency 0 is invalid if frequency == 0 { @@ -272,7 +267,7 @@ func (self *ResourceHandler) NewResource(name string, frequency uint64) (*resour } // get our blockheight at this time - currentblock, err := self.GetBlock() + currentblock, err := self.GetBlock(ctx) if err != nil { return nil, err } @@ -312,11 +307,11 @@ func (self *ResourceHandler) NewResource(name string, frequency uint64) (*resour // update root data was retrieved externally, it typically doesn't) // // -func (self *ResourceHandler) LookupVersionByName(name string, period uint32, version uint32, refresh bool) (*resource, error) { - return self.LookupVersion(self.nameHash(name), name, period, version, refresh) +func (self *ResourceHandler) LookupVersionByName(ctx context.Context, name string, period uint32, version uint32, refresh bool) (*resource, error) { + return self.LookupVersion(ctx, self.nameHash(name), name, period, version, refresh) } -func (self *ResourceHandler) LookupVersion(nameHash common.Hash, name string, period uint32, version uint32, refresh bool) (*resource, error) { +func (self *ResourceHandler) LookupVersion(ctx context.Context, nameHash common.Hash, name string, period uint32, version uint32, refresh bool) (*resource, error) { rsrc, err := self.loadResource(nameHash, name, refresh) if err != nil { return nil, err @@ -332,11 +327,11 @@ func (self *ResourceHandler) LookupVersion(nameHash common.Hash, name string, pe // and returned. // // See also (*ResourceHandler).LookupVersion -func (self *ResourceHandler) LookupHistoricalByName(name string, period uint32, refresh bool) (*resource, error) { - return self.LookupHistorical(self.nameHash(name), name, period, refresh) +func (self *ResourceHandler) LookupHistoricalByName(ctx context.Context, name string, period uint32, refresh bool) (*resource, error) { + return self.LookupHistorical(ctx, self.nameHash(name), name, period, refresh) } -func (self *ResourceHandler) LookupHistorical(nameHash common.Hash, name string, period uint32, refresh bool) (*resource, error) { +func (self *ResourceHandler) LookupHistorical(ctx context.Context, nameHash common.Hash, name string, period uint32, refresh bool) (*resource, error) { rsrc, err := self.loadResource(nameHash, name, refresh) if err != nil { return nil, err @@ -354,18 +349,18 @@ func (self *ResourceHandler) LookupHistorical(nameHash common.Hash, name string, // Version iteration is done as in (*ResourceHandler).LookupHistorical // // See also (*ResourceHandler).LookupHistorical -func (self *ResourceHandler) LookupLatestByName(name string, refresh bool) (*resource, error) { - return self.LookupLatest(self.nameHash(name), name, refresh) +func (self *ResourceHandler) LookupLatestByName(ctx context.Context, name string, refresh bool) (*resource, error) { + return self.LookupLatest(ctx, self.nameHash(name), name, refresh) } -func (self *ResourceHandler) LookupLatest(nameHash common.Hash, name string, refresh bool) (*resource, error) { +func (self *ResourceHandler) LookupLatest(ctx context.Context, nameHash common.Hash, name string, refresh bool) (*resource, error) { // get our blockheight at this time and the next block of the update period rsrc, err := self.loadResource(nameHash, name, refresh) if err != nil { return nil, err } - currentblock, err := self.GetBlock() + currentblock, err := self.GetBlock(ctx) if err != nil { return nil, err } @@ -532,7 +527,7 @@ func (self *ResourceHandler) parseUpdate(chunkdata []byte) (*Signature, uint32, // It is the caller's responsibility to make sure that this data is not stale. // // A resource update cannot span chunks, and thus has max length 4096 -func (self *ResourceHandler) Update(name string, data []byte) (Key, error) { +func (self *ResourceHandler) Update(ctx context.Context, name string, data []byte) (Key, error) { var signaturelength int if self.validator != nil { @@ -555,7 +550,7 @@ func (self *ResourceHandler) Update(name string, data []byte) (Key, error) { } // get our blockheight at this time and the next block of the update period - currentblock, err := self.GetBlock() + currentblock, err := self.GetBlock(ctx) if err != nil { return nil, err } @@ -620,13 +615,10 @@ func (self *ResourceHandler) Update(name string, data []byte) (Key, error) { // Closes the datastore. // Always call this at shutdown to avoid data corruption. func (self *ResourceHandler) Close() { - self.cancelFunc() self.ChunkStore.Close() } -func (self *ResourceHandler) GetBlock() (uint64, error) { - ctx, cancel := context.WithCancel(self.ctx) - defer cancel() +func (self *ResourceHandler) GetBlock(ctx context.Context) (uint64, error) { blockheader, err := self.ethClient.HeaderByNumber(ctx, nil) if err != nil { return 0, err diff --git a/swarm/storage/resource_test.go b/swarm/storage/resource_test.go index e84cdb7b9302..376b8397aed1 100644 --- a/swarm/storage/resource_test.go +++ b/swarm/storage/resource_test.go @@ -157,7 +157,9 @@ func TestResourceHandler(t *testing.T) { defer teardownTest() // create a new resource - _, err = rh.NewResource(safeName, resourceFrequency) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + _, err = rh.NewResource(ctx, safeName, resourceFrequency) if err != nil { t.Fatal(err) } @@ -183,7 +185,7 @@ func TestResourceHandler(t *testing.T) { resourcekey := make(map[string]Key) fwdBlocks(int(resourceFrequency/2), backend) data := []byte("blinky") - resourcekey["blinky"], err = rh.Update(safeName, data) + resourcekey["blinky"], err = rh.Update(ctx, safeName, data) if err != nil { t.Fatal(err) } @@ -191,7 +193,7 @@ func TestResourceHandler(t *testing.T) { // update on first period fwdBlocks(int(resourceFrequency/2), backend) data = []byte("pinky") - resourcekey["pinky"], err = rh.Update(safeName, data) + resourcekey["pinky"], err = rh.Update(ctx, safeName, data) if err != nil { t.Fatal(err) } @@ -199,7 +201,7 @@ func TestResourceHandler(t *testing.T) { // update on second period fwdBlocks(int(resourceFrequency), backend) data = []byte("inky") - resourcekey["inky"], err = rh.Update(safeName, data) + resourcekey["inky"], err = rh.Update(ctx, safeName, data) if err != nil { t.Fatal(err) } @@ -207,7 +209,7 @@ func TestResourceHandler(t *testing.T) { // update just after second period fwdBlocks(1, backend) data = []byte("clyde") - resourcekey["clyde"], err = rh.Update(safeName, data) + resourcekey["clyde"], err = rh.Update(ctx, safeName, data) if err != nil { t.Fatal(err) } @@ -219,7 +221,7 @@ func TestResourceHandler(t *testing.T) { fwdBlocks(int(resourceFrequency*2)-1, backend) rh2, err := NewResourceHandler(datadir, &testCloudStore{}, rh.ethClient, nil) - _, err = rh2.LookupLatestByName(safeName, true) + _, err = rh2.LookupLatestByName(ctx, safeName, true) if err != nil { t.Fatal(err) } @@ -237,7 +239,7 @@ func TestResourceHandler(t *testing.T) { log.Debug("Latest lookup", "period", rh2.resources[safeName].lastPeriod, "version", rh2.resources[safeName].version, "data", rh2.resources[safeName].data) // specific block, latest version - rsrc, err := rh2.LookupHistoricalByName(safeName, 3, true) + rsrc, err := rh2.LookupHistoricalByName(ctx, safeName, 3, true) if err != nil { t.Fatal(err) } @@ -248,7 +250,7 @@ func TestResourceHandler(t *testing.T) { log.Debug("Historical lookup", "period", rh2.resources[safeName].lastPeriod, "version", rh2.resources[safeName].version, "data", rh2.resources[safeName].data) // specific block, specific version - rsrc, err = rh2.LookupVersionByName(safeName, 3, 1, true) + rsrc, err = rh2.LookupVersionByName(ctx, safeName, 3, 1, true) if err != nil { t.Fatal(err) } @@ -293,14 +295,16 @@ func TestResourceENSOwner(t *testing.T) { defer teardownTest() // create new resource when we are owner = ok - _, err = rh.NewResource(safeName, resourceFrequency) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + _, err = rh.NewResource(ctx, safeName, resourceFrequency) if err != nil { t.Fatalf("Create resource fail: %v", err) } data := []byte("foo") // update resource when we are owner = ok - _, err = rh.Update(safeName, data) + _, err = rh.Update(ctx, safeName, data) if err != nil { t.Fatalf("Update resource fail: %v", err) } @@ -311,7 +315,7 @@ func TestResourceENSOwner(t *testing.T) { t.Fatal(err) } rh.validator.(*ENSValidator).signFunc = signertwo.signContent - _, err = rh.Update(safeName, data) + _, err = rh.Update(ctx, safeName, data) if err == nil { t.Fatalf("Expected resource update fail due to owner mismatch") } From 8d7bf3f3ef6b20d38ff12ab8731b0e9009fce2bf Mon Sep 17 00:00:00 2001 From: lash Date: Wed, 24 Jan 2018 19:12:06 +0100 Subject: [PATCH 22/25] swarm/fuse: Update swarm/api:Api.NewAPI constructor --- swarm/fuse/swarmfs_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarm/fuse/swarmfs_test.go b/swarm/fuse/swarmfs_test.go index 42af36345f6f..f0d64c3538a3 100644 --- a/swarm/fuse/swarmfs_test.go +++ b/swarm/fuse/swarmfs_test.go @@ -812,7 +812,7 @@ func TestFUSE(t *testing.T) { if err != nil { t.Fatal(err) } - ta := &testAPI{api: api.NewApi(dpa, nil)} + ta := &testAPI{api: api.NewApi(dpa, nil, nil)} dpa.Start() defer dpa.Stop() From de7454263d6490835549daeefa856c7ee10eb8ab Mon Sep 17 00:00:00 2001 From: lash Date: Wed, 24 Jan 2018 21:18:20 +0100 Subject: [PATCH 23/25] swarm: Add missing privatekey from swarm config w/o swap --- swarm/api/config.go | 10 ++++++++++ swarm/swarm.go | 6 ++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/swarm/api/config.go b/swarm/api/config.go index 93b386cc11a8..14906176e992 100644 --- a/swarm/api/config.go +++ b/swarm/api/config.go @@ -63,6 +63,7 @@ type Config struct { Cors string BzzAccount string BootNodes string + privateKey *ecdsa.PrivateKey } //create a default config with all parameters to set to defaults @@ -113,5 +114,14 @@ func (self *Config) Init(prvKey *ecdsa.PrivateKey) { if self.SwapEnabled { self.Swap.Init(self.Contract, prvKey) } + self.privateKey = prvKey self.StoreParams.Init(self.Path) } + +func (self *Config) ShiftPrivateKey() (privKey *ecdsa.PrivateKey) { + if self.privateKey != nil { + privKey = self.privateKey + self.privateKey = nil + } + return privKey +} diff --git a/swarm/swarm.go b/swarm/swarm.go index c65056794753..93c63eb68d18 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -22,6 +22,7 @@ import ( "crypto/ecdsa" "fmt" "net" + "os" "path/filepath" "github.com/ethereum/go-ethereum/accounts/abi/bind" @@ -93,7 +94,7 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e self = &Swarm{ config: config, backend: backend, - privateKey: config.Swap.PrivateKey(), + privateKey: config.ShiftPrivateKey(), } log.Debug(fmt.Sprintf("Setting up Swarm service components")) @@ -143,7 +144,8 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e } } - // set up high level api + // SET UP high level api + fmt.Fprintf(os.Stderr, "privkey %v\nswap %v\nswapendabled %v\n", self.privateKey, config.Swap, config.SwapEnabled) transactOpts := bind.NewKeyedTransactor(self.privateKey) if ensClient == nil { From f38859d78aff176d0750534297de2a62962b3353 Mon Sep 17 00:00:00 2001 From: lash Date: Thu, 25 Jan 2018 01:01:26 +0100 Subject: [PATCH 24/25] swarm/storage: Delint --- swarm/storage/resource.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 81a7621a6321..03235fa65a66 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -665,10 +665,7 @@ func (self *ResourceHandler) resourceHash(period uint32, version uint32, namehas } func (self *ResourceHandler) hasUpdate(name string, period uint32) bool { - if self.resources[name].lastPeriod == period { - return true - } - return false + return self.resources[name].lastPeriod == period } func getAddressFromDataSig(datahash common.Hash, signature Signature) (common.Address, error) { From ef5eeb6320b1b898008382b3f8c88a77230daf64 Mon Sep 17 00:00:00 2001 From: lash Date: Thu, 25 Jan 2018 14:43:08 +0100 Subject: [PATCH 25/25] swarm: Remove privkey log entry --- swarm/swarm.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/swarm/swarm.go b/swarm/swarm.go index 93c63eb68d18..a95bc176f756 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -144,8 +144,7 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e } } - // SET UP high level api - fmt.Fprintf(os.Stderr, "privkey %v\nswap %v\nswapendabled %v\n", self.privateKey, config.Swap, config.SwapEnabled) + // set up high level api transactOpts := bind.NewKeyedTransactor(self.privateKey) if ensClient == nil {