Improve support for IPNS double-hashed entries

Fixes #40. This ensures that double-hashed IPNS entries both in modern and legacy
forms are correctly supported, including both:
  * /ipns/<domain>
  * /ipns/key

Before, only the modern form with /ipns/<domain> was correctly supported for
double-hashed entries.

Testcases have been added for all forms.

denylist.go

@@ -639,6 +639,7 @@ func (dl *Denylist) IsIPNSPathBlocked(name, subpath string) StatusResponse {
 	c, err := cid.Decode(key)
 	if err == nil {
 		key = c.Hash().B58String()
+		//
 	} else if !strings.ContainsRune(key, '.') {
 		// not a CID. It must be a ipns-dnslink name if it does not
 		// contain ".", maybe they got replaced by "-"
@@ -657,17 +658,68 @@ func (dl *Denylist) IsIPNSPathBlocked(name, subpath string) StatusResponse {
 		}
 	}
 
-	// Double-hash blocking
+	// Double-hash blocking, works by double-hashing "/ipns/<name>/<path>"
+	// Legacy double-hashes for dnslink will hash "domain.com/" (trailing
+	// slash) or "<cidV1b32>/" for ipns-key blocking
+
+	// Let's start with legacy
+	sha256blocks := dl.DoubleHashBlocksDB[multihash.SHA2_256]
+	if sha256blocks != nil {
+		legacyKey := name + "/" + subpath
+		if c.Defined() { // we parsed a CID before
+			legacyCid := cid.NewCidV1(c.Prefix().Codec, c.Hash()).String()
+			legacyKey = legacyCid + "/" + subpath
+		}
+		// now double-hash that key
+		doubleLegacy, err := multihash.Sum([]byte(legacyKey), multihash.SHA2_256, -1)
+		if err != nil {
+			logger.Error(err)
+			return StatusResponse{
+				Path:     p,
+				Status:   StatusErrored,
+				Filename: dl.Filename,
+				Error:    err,
+			}
+		}
+		// now perform the lookup using the B58 string which we use as
+		// key.
+		b58legacy := doubleLegacy.B58String()
+		logger.Debugf("IsIPNSPathBlocked load IPNS doublehash (legacy): %d %s", multihash.SHA2_256, b58legacy)
+		entries, _ := sha256blocks.Load(b58legacy)
+		status, entry := entries.CheckPathStatus("")
+		if status != StatusNotFound { // Hit!
+			return StatusResponse{
+				Path:     p,
+				Status:   status,
+				Filename: dl.Filename,
+				Entry:    entry,
+			}
+		}
+	}
+
+	// Modern double-hash approach
 	for mhCode, blocks := range dl.DoubleHashBlocksDB {
-		double, err := multihash.Sum([]byte(p.String()), mhCode, -1)
+		key := p.String()
+		if c.Defined() { // the ipns path is a CID The
+			// b58-encoded-multihash extracted from an IPNS name
+			// when the IPNS is a CID.
+			key = c.Hash().B58String()
+			if len(subpath) > 0 {
+				key += "/" + subpath
+			}
+		}
+
+		// A base58btc-encoded multihash corresponding to the
+		// above.
+		double, err := multihash.Sum([]byte(key), mhCode, -1)
 		if err != nil {
 			// Usually this means an unsupported hash function was
 			// registered.
 			logger.Error(err)
 			continue
 		}
 		b58 := double.B58String()
-		logger.Debugf("IsPathBlocked load IPNS doublehash: %s", b58)
+		logger.Debugf("IsIPNSPathBlocked load IPNS doublehash: %d %s", mhCode, b58)
 		entries, _ := blocks.Load(b58)
 		status, entry := entries.CheckPathStatus("")
 		if status != StatusNotFound { // Hit!
@@ -774,7 +826,10 @@ func (dl *Denylist) isIPFSIPLDPathBlocked(cidStr, subpath, protocol string) Stat
 	}
 
 	prefix := c.Prefix()
-	for mhCode, blocks := range dl.DoubleHashBlocksDB {
+
+	// Checks for legacy doublehash blocking, which only uses SHA2_256
+	sha256blocks := dl.DoubleHashBlocksDB[multihash.SHA2_256]
+	if sha256blocks != nil {
 		// <cidv1base32>/<path>
 		// TODO: we should be able to disable this part with an Option
 		// or a hint for denylists not using it.
@@ -783,18 +838,21 @@ func (dl *Denylist) isIPFSIPLDPathBlocked(cidStr, subpath, protocol string) Stat
 		// badbits appends / on empty subpath. and hashes that
 		// https://github.com/protocol/badbits.dwebops.pub/blob/main/badbits-lambda/helpers.py#L17
 		v1b32path += "/" + subpath
-		doubleLegacy, err := multihash.Sum([]byte(v1b32path), mhCode, -1)
+		doubleLegacy, err := multihash.Sum([]byte(v1b32path), multihash.SHA2_256, -1)
 		if err != nil {
-			// Usually this means an unsupported hash function was
-			// registered.
 			logger.Error(err)
-			continue
+			return StatusResponse{
+				Path:     p,
+				Status:   StatusErrored,
+				Filename: dl.Filename,
+				Error:    err,
+			}
 		}
 
 		// encode as b58 which is the key we use for the BlocksDB.
 		b58 := doubleLegacy.B58String()
-		logger.Debugf("IsIPFFSIPLDPathBlocked load IPFS doublehash (legacy): %d %s", mhCode, b58)
-		entries, _ := blocks.Load(b58)
+		logger.Debugf("IsIPFFSIPLDPathBlocked load IPFS doublehash (legacy): %d %s", multihash.SHA2_256, b58)
+		entries, _ := sha256blocks.Load(b58)
 		status, entry := entries.CheckPathStatus("")
 		if status != StatusNotFound { // Hit!
 			return StatusResponse{
@@ -804,7 +862,11 @@ func (dl *Denylist) isIPFSIPLDPathBlocked(cidStr, subpath, protocol string) Stat
 				Entry:    entry,
 			}
 		}
+	}
 
+	// Otherwise just check normal double-hashing of multihash
+	// for all double-hashing functions used.
+	for mhCode, blocks := range dl.DoubleHashBlocksDB {
 		// <cidv0>/<path>
 		v0path := c.Hash().B58String()
 		if subpath != "" {
@@ -817,7 +879,7 @@ func (dl *Denylist) isIPFSIPLDPathBlocked(cidStr, subpath, protocol string) Stat
 			logger.Error(err)
 			continue
 		}
-		b58 = double.B58String()
+		b58 := double.B58String()
 		logger.Debugf("IsPathBlocked load IPFS doublehash: %d %s", mhCode, b58)
 		entries, _ = blocks.Load(b58)
 		status, entry = entries.CheckPathStatus("")
@@ -911,15 +973,15 @@ func (dl *Denylist) IsCidBlocked(c cid.Cid) StatusResponse {
 	}
 
 	// Now check if a double-hash covers this CID
-
+	// Legacy double-hashing support.
 	// convert cid to v1 base32
 	// the double-hash using multhash sha2-256
 	// then check that
 	sha256blocks := dl.DoubleHashBlocksDB[multihash.SHA2_256]
 	if sha256blocks != nil {
 		prefix := c.Prefix()
 		b32 := cid.NewCidV1(prefix.Codec, c.Hash()).String() + "/" // yes, needed
-		logger.Debug("IsCidBlocked cidv1b32 ", b32)
+		logger.Debug("IsCidBlocked cidv1b32 (legacy) ", b32)
 		double, err := multihash.Sum([]byte(b32), multihash.SHA2_256, -1)
 		if err != nil {
 			logger.Error(err)
@@ -931,7 +993,7 @@ func (dl *Denylist) IsCidBlocked(c cid.Cid) StatusResponse {
 			}
 		}
 		b58 := double.B58String()
-		logger.Debugf("IsCidBlocked load sha256 doublehash: %s", b58)
+		logger.Debugf("IsCidBlocked load sha256 doublehash (legacy): %s", b58)
 		entries, _ := sha256blocks.Load(b58)
 		status, entry := entries.CheckPathStatus("")
 		if status != StatusNotFound { // Hit!

subscription.go

@@ -99,7 +99,7 @@ func (s *HTTPSubscriber) downloadAndAppend() error {
 	rangeHeader := fmt.Sprintf("bytes=%d-", localFileSize)
 	req.Header.Set("Range", rangeHeader)
 
-	logger.Debug("%s: requesting bytes from %d: %s", s.localFile, localFileSize, req.URL)
+	logger.Debugf("%s: requesting bytes from %d: %s", s.localFile, localFileSize, req.URL)
 
 	resp, err := http.DefaultClient.Do(req)
 	if err != nil {

tester/test.deny

@@ -53,6 +53,16 @@ author: "@hsanjuan"
 # sha256(bafybeiefwqslmf6zyyrxodaxx4vwqircuxpza5ri45ws3y5a62ypxti42e/)
 # blocks only this cid
 //d9d295bde21f422d471a90f2a37ec53049fdf3e5fa3ee2e8f20e10003da429e7
+# Legacy IPNS hash block
+# /ipns/k51qzi5uqu5dixwsch9wpd9rolqby1m0uqj5hhxwtxal0dwltastfmh01dlniq
+# becomes CIDv1 bafzaajaiaejca3vrvdzmu4qntwa2pn6apsd4ug5k63ckdyhnd3g6vdvgvujdw62s
+# then sha256(bafzaajaiaejca3vrvdzmu4qntwa2pn6apsd4ug5k63ckdyhnd3g6vdvgvujdw62s/)
+# should block /ipns/<key> in any format
+//6ef262a67f2c7caa9722b0fe46aced2f1559c749eab2bcf2f2701f43f802e900
+# Legacy DNSLink hash block
+# sha256(very-bad-example.eth/)
+//fb5a70b1aade810d21e8195a0da05f40ebd099e4b4d6bf088dc604e4fcf34263
+
 
 # rule11
 # Legacy Path double-hash block
@@ -63,7 +73,7 @@ author: "@hsanjuan"
 //3f8b9febd851873b3774b937cce126910699ceac56e72e64b866f8e258d09572
 
 # rule12
-# Double hash CID block
+# Double hash CID
 # base58btc-sha256-multihash(QmVTF1yEejXd9iMgoRTFDxBv7HAz9kuZcQNBzHrceuK9HR)
 # This should block bafybeidjwik6im54nrpfg7osdvmx7zojl5oaxqel5cmsz46iuelwf5acja
 # and its CIDv0 form QmVTF1yEejXd9iMgoRTFDxBv7HAz9kuZcQNBzHrceuK9HR
@@ -81,6 +91,25 @@ author: "@hsanjuan"
 //gW813G35CnLsy7gRYYHuf63hrz71U1xoLFDVeV7actx6oX
 
 # rule14
+# Double hash IPNS blocks
+# /ipns/my.domain.com
+# becomes
+//QmX6zeaAb8mC285YbXe4ac7LmX3jvrHBiFrw8kNnCM1bk4
+# /ipns/my.domain2.com/path
+# should only block the specific path
+# and becomes
+//QmcwfDuZYijxztHVR7w71WtdDaETsrGUU3ARWbNiYD3Hhp
+# /ipns/k51qzi5uqu5dixwsch9wpd9rolqby1m0uqj5hhxwtxal0dwltastfmh01dabcd
+# is multihash 12D3KooWHGU91cJWKofZoHQ3hkVgs2c6WAJGxVCjMivwej9ufyuN
+# and hashed again becomes
+//QmdGtYErkPjfpUPjVMUypoX8XTE9PUTBBF6KE9AwWaBs1e
+# /ipns/k51qzi5uqu5dixwsch9wpd9rolqby1m0uqj5hhxwtxal0dwltastfmh01d1234/mypath
+# equivalent to /ipns/bafzaajaiaejca3vrvdzmu4qntwa2pn6apsd4ug5k63ckdyhnd3g6vdvgvujczuoq/mypath
+# is multihash 12D3KooWHGU91cJWKofZoHQ3hkVgs2c6WAJGxVCjMivwej9udmWo/mypath
+# and hashed again becomes
+//QmQouNeftXbxGRt4U8s838diCuMPPHCfpHTR5w8bRREs9d
+
+# rule15
 # These are known cids corresponding to empty blocks/folders
 # Even if they appear here, they should not be blocked
 # empty unixfs directory

tester/tester.go

@@ -139,7 +139,7 @@ func (s *Suite) testCID() error {
 		cid.MustParse("QmesfgDQ3q6prBy2Kg2gKbW4MAGuWiRP2DVuGA5MZSERLo"),
 	}
 
-	// rule14
+	// rule15
 	allowCids := []cid.Cid{
 		cid.MustParse("QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn"),
 		cid.MustParse("bafyaabakaieac"),
@@ -158,7 +158,7 @@ func (s *Suite) testCID() error {
 
 	for _, c := range allowCids {
 		if s.b.IsCidBlocked(c) {
-			return fmt.Errorf("testCID: %s should NOT be blocked (rule14)", c)
+			return fmt.Errorf("testCID: %s should NOT be blocked (rule15)", c)
 		}
 	}
 	return nil
@@ -338,6 +338,15 @@ func (s *Suite) testDoubleHashLegacy() error {
 	if !s.b.IsCidBlocked(c) {
 		return fmt.Errorf("%s: cid %s should be blocked (rule10)", n, c)
 	}
+	rule10 := []string{
+		"/ipns/k51qzi5uqu5dixwsch9wpd9rolqby1m0uqj5hhxwtxal0dwltastfmh01dlniq",
+		"/ipns/bafzaajaiaejca3vrvdzmu4qntwa2pn6apsd4ug5k63ckdyhnd3g6vdvgvujdw62s",
+		"/ipns/very-bad-example.eth",
+		"/ipns/very-bad-example.eth/",
+	}
+	if err := s.testPaths(rule10, n, "rule10", false); err != nil {
+		return err
+	}
 
 	//rule 11 (and 10)
 	rule11 := []string{
@@ -396,5 +405,26 @@ func (s *Suite) testDoubleHash() error {
 		return err
 	}
 
+	// rule14
+	rule14 := []string{
+		"/ipns/my.domain.com",
+		"/ipns/my.domain2.com/path",
+		"/ipns/k51qzi5uqu5dixwsch9wpd9rolqby1m0uqj5hhxwtxal0dwltastfmh01dabcd",
+		"/ipns/k51qzi5uqu5dixwsch9wpd9rolqby1m0uqj5hhxwtxal0dwltastfmh01d1234/mypath",
+		"/ipns/bafzaajaiaejca3vrvdzmu4qntwa2pn6apsd4ug5k63ckdyhnd3g6vdvgvujczuoq/mypath",
+	}
+	rule14allowed := []string{
+		"/ipns/my.domain2.com/path2",
+		"/ipns/k51qzi5uqu5dixwsch9wpd9rolqby1m0uqj5hhxwtxal0dwltastfmh01d1234/mypath2",
+	}
+
+	if err := s.testPaths(rule14, n, "rule14", false); err != nil {
+		return err
+	}
+
+	if err := s.testPaths(rule14allowed, n, "rule14", true); err != nil {
+		return err
+	}
+
 	return nil
 }