From 003fa6f84e2c17cf10fd2325cc454aa6bba4c271 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 29 Feb 2024 22:51:23 +0100 Subject: [PATCH 1/5] chore: fix names to reduce confusion blocked item was mislabeled --- test/cli/content_blocking_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/cli/content_blocking_test.go b/test/cli/content_blocking_test.go index ddb7c951e88..ea61445f27a 100644 --- a/test/cli/content_blocking_test.go +++ b/test/cli/content_blocking_test.go @@ -34,8 +34,8 @@ func TestContentBlocking(t *testing.T) { node := h.NewNode().Init("--empty-repo", "--profile=test") // Create CIDs we use in test - h.WriteFile("blocked-dir/subdir/indirectly-blocked-file.txt", "indirectly blocked file content") - parentDirCID := node.IPFS("add", "--raw-leaves", "-Q", "-r", filepath.Join(h.Dir, "blocked-dir")).Stdout.Trimmed() + h.WriteFile("parent-dir/blocked-subdir/indirectly-blocked-file.txt", "indirectly blocked file content") + parentDirCID := node.IPFS("add", "--raw-leaves", "-Q", "-r", filepath.Join(h.Dir, "parent-dir")).Stdout.Trimmed() h.WriteFile("directly-blocked-file.txt", "directly blocked file content") blockedCID := node.IPFS("add", "--raw-leaves", "-Q", filepath.Join(h.Dir, "directly-blocked-file.txt")).Stdout.Trimmed() @@ -50,7 +50,7 @@ func TestContentBlocking(t *testing.T) { "//8526ba05eec55e28f8db5974cc891d0d92c8af69d386fc6464f1e9f372caf549\n" + // Legacy CID double-hash block: sha256(bafkqahtcnrxwg23fmqqgi33vmjwgk2dbonuca3dfm5qwg6jamnuwicq/) "//e5b7d2ce2594e2e09901596d8e1f29fa249b74c8c9e32ea01eda5111e4d33f07\n" + // Legacy Path double-hash block: sha256(bafyaagyscufaqalqaacauaqiaejao43vmjygc5didacauaqiae/subpath) "/ipfs/" + blockedCID + "\n" + // block specific CID - "/ipfs/" + parentDirCID + "/subdir*\n" + // block only specific subpath + "/ipfs/" + parentDirCID + "/blocked-subdir*\n" + // block only specific subpath "/ipns/blocked-cid.example.com\n" + "/ipns/blocked-dnslink.example.com\n") @@ -109,7 +109,7 @@ func TestContentBlocking(t *testing.T) { }, { name: "indirectly blocked file (on a blocked subpath)", - path: "/ipfs/" + parentDirCID + "/subdir/indirectly-blocked-file.txt", + path: "/ipfs/" + parentDirCID + "/blocked-subdir/indirectly-blocked-file.txt", }, { name: "/ipns path that resolves to a blocked CID", From 92c97b10cfb324f036011d3aeef0db2e62e3fe66 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 1 Mar 2024 00:28:59 +0100 Subject: [PATCH 2/5] chore: more explicit subpath test --- test/cli/content_blocking_test.go | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/test/cli/content_blocking_test.go b/test/cli/content_blocking_test.go index ea61445f27a..7491ef405d3 100644 --- a/test/cli/content_blocking_test.go +++ b/test/cli/content_blocking_test.go @@ -35,7 +35,9 @@ func TestContentBlocking(t *testing.T) { // Create CIDs we use in test h.WriteFile("parent-dir/blocked-subdir/indirectly-blocked-file.txt", "indirectly blocked file content") - parentDirCID := node.IPFS("add", "--raw-leaves", "-Q", "-r", filepath.Join(h.Dir, "parent-dir")).Stdout.Trimmed() + allowedParentDirCID := node.IPFS("add", "--raw-leaves", "-Q", "-r", "--pin=false", filepath.Join(h.Dir, "parent-dir")).Stdout.Trimmed() + blockedSubDirCID := node.IPFS("add", "--raw-leaves", "-Q", "-r", "--pin=false", filepath.Join(h.Dir, "parent-dir", "blocked-subdir")).Stdout.Trimmed() + node.IPFS("block", "rm", blockedSubDirCID) h.WriteFile("directly-blocked-file.txt", "directly blocked file content") blockedCID := node.IPFS("add", "--raw-leaves", "-Q", filepath.Join(h.Dir, "directly-blocked-file.txt")).Stdout.Trimmed() @@ -50,7 +52,7 @@ func TestContentBlocking(t *testing.T) { "//8526ba05eec55e28f8db5974cc891d0d92c8af69d386fc6464f1e9f372caf549\n" + // Legacy CID double-hash block: sha256(bafkqahtcnrxwg23fmqqgi33vmjwgk2dbonuca3dfm5qwg6jamnuwicq/) "//e5b7d2ce2594e2e09901596d8e1f29fa249b74c8c9e32ea01eda5111e4d33f07\n" + // Legacy Path double-hash block: sha256(bafyaagyscufaqalqaacauaqiaejao43vmjygc5didacauaqiae/subpath) "/ipfs/" + blockedCID + "\n" + // block specific CID - "/ipfs/" + parentDirCID + "/blocked-subdir*\n" + // block only specific subpath + "/ipfs/" + allowedParentDirCID + "/blocked-subdir*\n" + // block only specific subpath "/ipns/blocked-cid.example.com\n" + "/ipns/blocked-dnslink.example.com\n") @@ -94,7 +96,7 @@ func TestContentBlocking(t *testing.T) { // Confirm parent of blocked subpath is not blocked t.Run("Gateway Allows parent Path that is not blocked", func(t *testing.T) { t.Parallel() - resp := client.Get("/ipfs/" + parentDirCID) + resp := client.Get("/ipfs/" + allowedParentDirCID) assert.Equal(t, http.StatusOK, resp.StatusCode) }) @@ -104,12 +106,12 @@ func TestContentBlocking(t *testing.T) { path string }{ { - name: "directly blocked CID", + name: "directly blocked file CID", path: "/ipfs/" + blockedCID, }, { name: "indirectly blocked file (on a blocked subpath)", - path: "/ipfs/" + parentDirCID + "/blocked-subdir/indirectly-blocked-file.txt", + path: "/ipfs/" + allowedParentDirCID + "/blocked-subdir/indirectly-blocked-file.txt", }, { name: "/ipns path that resolves to a blocked CID", @@ -161,9 +163,14 @@ func TestContentBlocking(t *testing.T) { t.Run(cliTestName, func(t *testing.T) { t.Parallel() args := append(cmd, testCase.path) - errMsg := node.RunIPFS(args...).Stderr.Trimmed() - if !strings.Contains(errMsg, expectedMsg) { - t.Errorf("Expected STDERR error message %q, but got: %q", expectedMsg, errMsg) + cmd := node.RunIPFS(args...) + stdout := cmd.Stdout.Trimmed() + stderr := cmd.Stderr.Trimmed() + if !strings.Contains(stderr, expectedMsg) { + t.Errorf("Expected STDERR error message %q, but got: %q", expectedMsg, stderr) + if stdout != "" { + t.Errorf("Expected STDOUT to be empty, but got: %q", stdout) + } } }) } From 453e37793b430c72cf4fef80ad19d3a7434c9f72 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 1 Mar 2024 00:40:34 +0100 Subject: [PATCH 3/5] test: gw over libp2p car blocking --- test/cli/content_blocking_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/cli/content_blocking_test.go b/test/cli/content_blocking_test.go index 7491ef405d3..a29913f6851 100644 --- a/test/cli/content_blocking_test.go +++ b/test/cli/content_blocking_test.go @@ -306,5 +306,17 @@ func TestContentBlocking(t *testing.T) { assert.NotEqual(t, string(body), "directly blocked file content") assert.Contains(t, string(body), blockedMsg, bodyExpl) }) + + t.Run("Denies Blocked CID as CAR", func(t *testing.T) { + t.Parallel() + resp, err := libp2pClient.Get(fmt.Sprintf("/ipfs/%s?format=car", blockedCID)) + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl) + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + assert.NotContains(t, string(body), "directly blocked file content") + assert.Contains(t, string(body), blockedMsg, bodyExpl) + }) }) } From 5cb6d20b7043d31a9dfaa0543f3055ff51850d1f Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 1 Mar 2024 00:57:24 +0100 Subject: [PATCH 4/5] test: car output without blocked subpath --- test/cli/content_blocking_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/cli/content_blocking_test.go b/test/cli/content_blocking_test.go index a29913f6851..61d07242f7e 100644 --- a/test/cli/content_blocking_test.go +++ b/test/cli/content_blocking_test.go @@ -12,7 +12,9 @@ import ( "strings" "testing" + "github.com/ipfs/go-cid" "github.com/ipfs/kubo/test/cli/harness" + carstore "github.com/ipld/go-car/v2/blockstore" "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p/core/peer" libp2phttp "github.com/libp2p/go-libp2p/p2p/http" @@ -100,6 +102,19 @@ func TestContentBlocking(t *testing.T) { assert.Equal(t, http.StatusOK, resp.StatusCode) }) + // Confirm CAR responses skip blocked subpaths + t.Run("Gateway returns CAR without blocked subpath", func(t *testing.T) { + resp := client.Get("/ipfs/" + allowedParentDirCID + "/subdir?format=car") + assert.Equal(t, http.StatusOK, resp.StatusCode) + + bs, err := carstore.NewReadOnly(strings.NewReader(resp.Body), nil) + assert.NoError(t, err) + + has, err := bs.Has(context.Background(), cid.MustParse(blockedSubDirCID)) + assert.NoError(t, err) + assert.False(t, has) + }) + // Ok, now the full list of test cases we want to cover in both CLI and Gateway testCases := []struct { name string From 92acc244e53546ec47bdcf93b4f5c4dbeeafc2dd Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 1 Mar 2024 01:31:09 +0100 Subject: [PATCH 5/5] test: denylist on ?format=car --- test/cli/content_blocking_test.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/test/cli/content_blocking_test.go b/test/cli/content_blocking_test.go index 61d07242f7e..6598354d16e 100644 --- a/test/cli/content_blocking_test.go +++ b/test/cli/content_blocking_test.go @@ -115,6 +115,34 @@ func TestContentBlocking(t *testing.T) { assert.False(t, has) }) + /* TODO: this was already broken in 0.26, but we should fix it + t.Run("Gateway returns CAR without directly blocked CID", func(t *testing.T) { + allowedDirWithDirectlyBlockedCID := node.IPFS("add", "--raw-leaves", "-Q", "-rw", filepath.Join(h.Dir, "directly-blocked-file.txt")).Stdout.Trimmed() + resp := client.Get("/ipfs/" + allowedDirWithDirectlyBlockedCID + "?format=car") + assert.Equal(t, http.StatusOK, resp.StatusCode) + + bs, err := carstore.NewReadOnly(strings.NewReader(resp.Body), nil) + assert.NoError(t, err) + + has, err := bs.Has(context.Background(), cid.MustParse(blockedCID)) + assert.NoError(t, err) + assert.False(t, has, "Returned CAR should not include blockedCID") + }) + */ + + // Confirm CAR responses skip blocked subpaths + t.Run("Gateway returns CAR without blocked subpath", func(t *testing.T) { + resp := client.Get("/ipfs/" + allowedParentDirCID + "/subdir?format=car") + assert.Equal(t, http.StatusOK, resp.StatusCode) + + bs, err := carstore.NewReadOnly(strings.NewReader(resp.Body), nil) + assert.NoError(t, err) + + has, err := bs.Has(context.Background(), cid.MustParse(blockedSubDirCID)) + assert.NoError(t, err) + assert.False(t, has, "Returned CAR should not include blockedSubDirCID") + }) + // Ok, now the full list of test cases we want to cover in both CLI and Gateway testCases := []struct { name string