Skip to content

Commit c9a3303

Browse files
authored
Merge pull request #2 from textileio/sander/bps
Bucket Pinning Service
2 parents ddb3eab + 5c8d30b commit c9a3303

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

89 files changed

+6797
-1217
lines changed

.bingo/.gitignore

-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
# But not these files:
66
!.gitignore
77
!*.mod
8-
!*.sum
98
!README.md
109
!Variables.mk
1110
!variables.env

.bingo/Variables.mk

+6-6
Original file line numberDiff line numberDiff line change
@@ -20,35 +20,35 @@ BUF := $(GOBIN)/buf-v0.20.5
2020
$(BUF): .bingo/buf.mod
2121
@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.
2222
@echo "(re)installing $(GOBIN)/buf-v0.20.5"
23-
@cd .bingo && $(GO) build -modfile=buf.mod -o=$(GOBIN)/buf-v0.20.5 "github.com/bufbuild/buf/cmd/buf"
23+
@cd .bingo && $(GO) build -mod=mod -modfile=buf.mod -o=$(GOBIN)/buf-v0.20.5 "github.com/bufbuild/buf/cmd/buf"
2424

2525
GOMPLATE := $(GOBIN)/gomplate-v3.8.0
2626
$(GOMPLATE): .bingo/gomplate.mod
2727
@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.
2828
@echo "(re)installing $(GOBIN)/gomplate-v3.8.0"
29-
@cd .bingo && $(GO) build -modfile=gomplate.mod -o=$(GOBIN)/gomplate-v3.8.0 "github.com/hairyhenderson/gomplate/v3/cmd/gomplate"
29+
@cd .bingo && $(GO) build -mod=mod -modfile=gomplate.mod -o=$(GOBIN)/gomplate-v3.8.0 "github.com/hairyhenderson/gomplate/v3/cmd/gomplate"
3030

3131
GOVVV := $(GOBIN)/govvv-v0.3.0
3232
$(GOVVV): .bingo/govvv.mod
3333
@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.
3434
@echo "(re)installing $(GOBIN)/govvv-v0.3.0"
35-
@cd .bingo && $(GO) build -modfile=govvv.mod -o=$(GOBIN)/govvv-v0.3.0 "github.com/ahmetb/govvv"
35+
@cd .bingo && $(GO) build -mod=mod -modfile=govvv.mod -o=$(GOBIN)/govvv-v0.3.0 "github.com/ahmetb/govvv"
3636

3737
GOX := $(GOBIN)/gox-v1.0.1
3838
$(GOX): .bingo/gox.mod
3939
@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.
4040
@echo "(re)installing $(GOBIN)/gox-v1.0.1"
41-
@cd .bingo && $(GO) build -modfile=gox.mod -o=$(GOBIN)/gox-v1.0.1 "github.com/mitchellh/gox"
41+
@cd .bingo && $(GO) build -mod=mod -modfile=gox.mod -o=$(GOBIN)/gox-v1.0.1 "github.com/mitchellh/gox"
4242

4343
PROTOC_GEN_BUF_CHECK_BREAKING := $(GOBIN)/protoc-gen-buf-check-breaking-v0.20.5
4444
$(PROTOC_GEN_BUF_CHECK_BREAKING): .bingo/protoc-gen-buf-check-breaking.mod
4545
@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.
4646
@echo "(re)installing $(GOBIN)/protoc-gen-buf-check-breaking-v0.20.5"
47-
@cd .bingo && $(GO) build -modfile=protoc-gen-buf-check-breaking.mod -o=$(GOBIN)/protoc-gen-buf-check-breaking-v0.20.5 "github.com/bufbuild/buf/cmd/protoc-gen-buf-check-breaking"
47+
@cd .bingo && $(GO) build -mod=mod -modfile=protoc-gen-buf-check-breaking.mod -o=$(GOBIN)/protoc-gen-buf-check-breaking-v0.20.5 "github.com/bufbuild/buf/cmd/protoc-gen-buf-check-breaking"
4848

4949
PROTOC_GEN_BUF_CHECK_LINT := $(GOBIN)/protoc-gen-buf-check-lint-v0.20.5
5050
$(PROTOC_GEN_BUF_CHECK_LINT): .bingo/protoc-gen-buf-check-lint.mod
5151
@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.
5252
@echo "(re)installing $(GOBIN)/protoc-gen-buf-check-lint-v0.20.5"
53-
@cd .bingo && $(GO) build -modfile=protoc-gen-buf-check-lint.mod -o=$(GOBIN)/protoc-gen-buf-check-lint-v0.20.5 "github.com/bufbuild/buf/cmd/protoc-gen-buf-check-lint"
53+
@cd .bingo && $(GO) build -mod=mod -modfile=protoc-gen-buf-check-lint.mod -o=$(GOBIN)/protoc-gen-buf-check-lint-v0.20.5 "github.com/bufbuild/buf/cmd/protoc-gen-buf-check-lint"
5454

.bingo/buf.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT
22

3-
go 1.16
3+
go 1.14
44

55
require github.com/bufbuild/buf v0.20.5 // cmd/buf

.bingo/go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
module _ // Fake go.mod auto-created by 'bingo' for go -moddir compatibility with non-Go projects. Commit this file, together with other .mod files.
22

3-
go 1.16
3+
go 1.14

.bingo/gomplate.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT
22

3-
go 1.16
3+
go 1.14
44

55
require github.com/hairyhenderson/gomplate/v3 v3.8.0 // cmd/gomplate

.bingo/govvv.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT
22

3-
go 1.16
3+
go 1.14
44

55
require github.com/ahmetb/govvv v0.3.0

.bingo/govvv.sum

-2
This file was deleted.

.bingo/gox.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT
22

3-
go 1.16
3+
go 1.14
44

55
require github.com/mitchellh/gox v1.0.1
+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT
22

3-
go 1.16
3+
go 1.14
44

55
require github.com/bufbuild/buf v0.20.5 // cmd/protoc-gen-buf-check-breaking

.bingo/protoc-gen-buf-check-lint.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT
22

3-
go 1.16
3+
go 1.14
44

55
require github.com/bufbuild/buf v0.20.5 // cmd/protoc-gen-buf-check-lint

.github/workflows/build.yml

+2-8
Original file line numberDiff line numberDiff line change
@@ -24,23 +24,17 @@ jobs:
2424
buck:
2525
name: Buck CLI
2626
runs-on: ubuntu-latest
27+
container: golang:1.16.0-buster
2728
steps:
28-
- name: setup
29-
uses: actions/setup-go@v1
30-
with:
31-
go-version: 1.16
3229
- name: checkout
3330
uses: actions/checkout@v1
3431
- name: build
3532
run: make build-buck
3633
buckd:
3734
name: Buck Daemon
3835
runs-on: ubuntu-latest
36+
container: golang:1.16.0-buster
3937
steps:
40-
- name: setup
41-
uses: actions/setup-go@v1
42-
with:
43-
go-version: 1.16
4438
- name: checkout
4539
uses: actions/checkout@v1
4640
- name: build

.github/workflows/release.yml

+1-4
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,8 @@ jobs:
66
release-platform-builds:
77
name: Release Builds
88
runs-on: ubuntu-latest
9+
container: golang:1.16.0-buster
910
steps:
10-
- name: Set up Go
11-
uses: actions/setup-go@v1
12-
with:
13-
go-version: 1.16
1411
- name: Check out code
1512
uses: actions/checkout@v1
1613
- name: Cache dependencies

README.md

+2
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@
1010
1111
Join us on our [public Slack channel](https://slack.textile.io/) for news, discussions, and status updates. [Check out our blog](https://medium.com/textileio) for the latest posts and announcements.
1212

13+
### WARNING: This repo is pointing to a [feature branch](https://github.com/textileio/go-threads/pull/498) of `go-threads` that handles identities as Decentralized Identifiers (DIDs) and should be considered alpha. The [textile repo](https://github.com/textileio/textile) contains the non-DID-based bucket implementation currently compatible with [Textile's public hub](https://cloud.hub.textile.io/#/access). DID-based buckets will be integrated into the hub mid 2021. In the meantime, you can still use this repo for standalone bucket peers.
14+
1315
## Table of Contents
1416

1517
- [Security](#security)

access.go

+94-17
Original file line numberDiff line numberDiff line change
@@ -14,28 +14,48 @@ import (
1414
core "github.com/textileio/go-threads/core/thread"
1515
)
1616

17+
// PushPathAccessRoles pushes new access roles to bucket paths.
18+
// Access roles are keyed by did.DID.
19+
// Roles are inherited by path children.
1720
func (b *Buckets) PushPathAccessRoles(
1821
ctx context.Context,
1922
thread core.ID,
20-
key, pth string,
21-
roles map[did.DID]collection.Role,
23+
key string,
2224
identity did.Token,
25+
root path.Resolved,
26+
pth string,
27+
roles map[did.DID]collection.Role,
2328
) (int64, *Bucket, error) {
24-
lk := b.locks.Get(lock(key))
25-
lk.Acquire()
26-
defer lk.Release()
29+
txn, err := b.NewTxn(thread, key, identity)
30+
if err != nil {
31+
return 0, nil, err
32+
}
33+
defer txn.Close()
34+
return txn.PushPathAccessRoles(ctx, root, pth, roles)
35+
}
2736

37+
// PushPathAccessRoles is Txn based PushPathInfo.
38+
func (t *Txn) PushPathAccessRoles(
39+
ctx context.Context,
40+
root path.Resolved,
41+
pth string,
42+
roles map[did.DID]collection.Role,
43+
) (int64, *Bucket, error) {
2844
pth, err := parsePath(pth)
2945
if err != nil {
3046
return 0, nil, err
3147
}
3248

33-
instance, bpth, err := b.getBucketAndPath(ctx, thread, key, pth, identity)
49+
instance, bpth, err := t.b.getBucketAndPath(ctx, t.thread, t.key, t.identity, pth)
3450
if err != nil {
3551
return 0, nil, err
3652
}
53+
if root != nil && root.String() != instance.Path {
54+
return 0, nil, ErrNonFastForward
55+
}
56+
3757
linkKey := instance.GetLinkEncryptionKey()
38-
pathNode, err := dag.GetNodeAtPath(ctx, b.ipfs, bpth, linkKey)
58+
pathNode, err := dag.GetNodeAtPath(ctx, t.b.ipfs, bpth, linkKey)
3959
if err != nil {
4060
return 0, nil, err
4161
}
@@ -86,7 +106,7 @@ func (b *Buckets) PushPathAccessRoles(
86106
return 0, nil, err
87107
}
88108
}
89-
if err := b.c.Verify(ctx, thread, instance, collection.WithIdentity(identity)); err != nil {
109+
if err := t.b.c.Verify(ctx, t.thread, instance, collection.WithIdentity(t.identity)); err != nil {
90110
return 0, nil, err
91111
}
92112

@@ -97,7 +117,7 @@ func (b *Buckets) PushPathAccessRoles(
97117
}
98118
nmap, err := dag.EncryptDag(
99119
ctx,
100-
b.ipfs,
120+
t.b.ipfs,
101121
pathNode,
102122
pth,
103123
linkKey,
@@ -117,41 +137,46 @@ func (b *Buckets) PushPathAccessRoles(
117137
}
118138
pn := nmap[pathNode.Cid()].Node
119139
var dirPath path.Resolved
120-
ctx, dirPath, err = dag.InsertNodeAtPath(ctx, b.ipfs, pn, path.Join(path.New(instance.Path), pth), linkKey)
140+
ctx, dirPath, err = dag.InsertNodeAtPath(ctx, t.b.ipfs, pn, path.Join(path.New(instance.Path), pth), linkKey)
121141
if err != nil {
122142
return 0, nil, err
123143
}
124-
ctx, err = dag.AddAndPinNodes(ctx, b.ipfs, nodes)
144+
ctx, err = dag.AddAndPinNodes(ctx, t.b.ipfs, nodes)
125145
if err != nil {
126146
return 0, nil, err
127147
}
128148
instance.Path = dirPath.String()
129149
}
130150

131-
if err := b.c.Save(ctx, thread, instance, collection.WithIdentity(identity)); err != nil {
151+
if err := t.b.c.Save(ctx, t.thread, instance, collection.WithIdentity(t.identity)); err != nil {
132152
return 0, nil, err
133153
}
134154
}
135155

136-
log.Debugf("pushed access roles for %s in %s", pth, key)
137-
return dag.GetPinnedBytes(ctx), instanceToBucket(thread, instance), nil
156+
log.Debugf("pushed access roles for %s in %s", pth, t.key)
157+
return dag.GetPinnedBytes(ctx), instanceToBucket(t.thread, instance), nil
138158
}
139159

160+
// PullPathAccessRoles pulls access roles for a bucket path.
140161
func (b *Buckets) PullPathAccessRoles(
141162
ctx context.Context,
142163
thread core.ID,
143-
key, pth string,
164+
key string,
144165
identity did.Token,
166+
pth string,
145167
) (map[did.DID]collection.Role, error) {
168+
if err := thread.Validate(); err != nil {
169+
return nil, fmt.Errorf("invalid thread id: %v", err)
170+
}
146171
pth, err := parsePath(pth)
147172
if err != nil {
148173
return nil, err
149174
}
150-
instance, bpth, err := b.getBucketAndPath(ctx, thread, key, pth, identity)
175+
instance, bpth, err := b.getBucketAndPath(ctx, thread, key, identity, pth)
151176
if err != nil {
152177
return nil, err
153178
}
154-
if _, err := dag.GetNodeAtPath(ctx, b.ipfs, bpth, instance.GetLinkEncryptionKey()); err != nil {
179+
if _, err = dag.GetNodeAtPath(ctx, b.ipfs, bpth, instance.GetLinkEncryptionKey()); err != nil {
155180
return nil, fmt.Errorf("could not resolve path: %s", pth)
156181
}
157182
md, _, ok := instance.GetMetadataForPath(pth, false)
@@ -162,3 +187,55 @@ func (b *Buckets) PullPathAccessRoles(
162187
log.Debugf("pulled access roles for %s in %s", pth, key)
163188
return md.Roles, nil
164189
}
190+
191+
// IsReadablePath returns whether or not a path is readable by an identity.
192+
func (b *Buckets) IsReadablePath(
193+
ctx context.Context,
194+
thread core.ID,
195+
key string,
196+
identity did.Token,
197+
pth string,
198+
) (bool, error) {
199+
if err := thread.Validate(); err != nil {
200+
return false, fmt.Errorf("invalid thread id: %v", err)
201+
}
202+
pth, err := parsePath(pth)
203+
if err != nil {
204+
return false, err
205+
}
206+
instance, err := b.c.GetSafe(ctx, thread, key, collection.WithIdentity(identity))
207+
if err != nil {
208+
return false, err
209+
}
210+
_, doc, err := core.Validate(identity, nil)
211+
if err != nil {
212+
return false, err
213+
}
214+
return instance.IsReadablePath(pth, doc.ID), nil
215+
}
216+
217+
// IsWritablePath returns whether or not a path is writable by an identity.
218+
func (b *Buckets) IsWritablePath(
219+
ctx context.Context,
220+
thread core.ID,
221+
key string,
222+
identity did.Token,
223+
pth string,
224+
) (bool, error) {
225+
if err := thread.Validate(); err != nil {
226+
return false, fmt.Errorf("invalid thread id: %v", err)
227+
}
228+
pth, err := parsePath(pth)
229+
if err != nil {
230+
return false, err
231+
}
232+
instance, err := b.c.GetSafe(ctx, thread, key, collection.WithIdentity(identity))
233+
if err != nil {
234+
return false, err
235+
}
236+
_, doc, err := core.Validate(identity, nil)
237+
if err != nil {
238+
return false, err
239+
}
240+
return instance.IsWritablePath(pth, doc.ID), nil
241+
}

api/apitest/apitest.go

+10-8
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ import (
1919
"github.com/textileio/go-buckets"
2020
"github.com/textileio/go-buckets/api/common"
2121
"github.com/textileio/go-buckets/ipns"
22-
"github.com/textileio/go-buckets/util"
2322
dbc "github.com/textileio/go-threads/api/client"
2423
"github.com/textileio/go-threads/core/did"
2524
tdb "github.com/textileio/go-threads/db"
@@ -30,25 +29,26 @@ import (
3029
func NewService(t *testing.T) (listenAddr string, host did.DID) {
3130
err := tutil.SetLogLevels(map[string]logging.LogLevel{
3231
"buckets": logging.LevelDebug,
33-
"buckets-api": logging.LevelDebug,
34-
"buckets-ipns": logging.LevelDebug,
35-
"buckets-dns": logging.LevelDebug,
32+
"buckets/api": logging.LevelDebug,
33+
"buckets/ipns": logging.LevelDebug,
34+
"buckets/dns": logging.LevelDebug,
3635
})
3736
require.NoError(t, err)
3837

3938
threadsAddr := GetThreadsApiAddr()
4039
net, err := nc.NewClient(threadsAddr, common.GetClientRPCOpts(threadsAddr)...)
4140
require.NoError(t, err)
4241

43-
// @todo: Fix me
42+
// @todo: Use service description to build client
4443
doc, err := net.GetServices(context.Background())
4544
require.NoError(t, err)
4645

4746
db, err := dbc.NewClient(threadsAddr, common.GetClientRPCOpts(threadsAddr)...)
4847
require.NoError(t, err)
4948
ipfs, err := httpapi.NewApi(GetIPFSApiMultiAddr())
5049
require.NoError(t, err)
51-
ipnsm, err := ipns.NewManager(tdb.NewTxMapDatastore(), ipfs)
50+
ipnsms := tdb.NewTxMapDatastore()
51+
ipnsm, err := ipns.NewManager(ipnsms, ipfs)
5252
require.NoError(t, err)
5353
lib, err := buckets.NewBuckets(net, db, ipfs, ipnsm, nil)
5454
require.NoError(t, err)
@@ -64,6 +64,8 @@ func NewService(t *testing.T) (listenAddr string, host did.DID) {
6464
server.Stop()
6565
require.NoError(t, lib.Close())
6666
require.NoError(t, ipnsm.Close())
67+
require.NoError(t, ipnsms.Close())
68+
require.NoError(t, db.Close())
6769
require.NoError(t, net.Close())
6870
})
6971

@@ -83,9 +85,9 @@ func GetThreadsApiAddr() string {
8385
func GetIPFSApiMultiAddr() ma.Multiaddr {
8486
env := os.Getenv("IPFS_API_MULTIADDR")
8587
if env != "" {
86-
return util.MustParseAddr(env)
88+
return tutil.MustParseAddr(env)
8789
}
88-
return util.MustParseAddr("/ip4/127.0.0.1/tcp/5012")
90+
return tutil.MustParseAddr("/ip4/127.0.0.1/tcp/5012")
8991
}
9092

9193
// StartServices starts an ipfs and threads node for tests.

0 commit comments

Comments
 (0)