feat: preload walk using phase state, call preloader once per link

Author: rvagg

linking/preload/preload.go

@@ -3,10 +3,15 @@ package preload
 import (
 	"context"
 
-	"github.com/ipld/go-ipld-prime"
 	"github.com/ipld/go-ipld-prime/datamodel"
 )
 
+// Loader is a function that will be called with a link discovered in a preload
+// pass of a traversal. A preload pass can be used to collect all links in each
+// block prior to traversal of that block, allowing for parallel (background)
+// loading of blocks in anticipation of eventual actual load during traversal.
+type Loader func(PreloadContext, Link)
+
 // PreloadContext carries information about the current state of a traversal
 // where a set of links that may be preloaded were encountered.
 type PreloadContext struct {
@@ -26,10 +31,10 @@ type PreloadContext struct {
 	ParentNode datamodel.Node
 }
 
+// Link provides the link encountered during a preload pass, the node it was
+// encountered on, and the segment of the path that led to the link.
 type Link struct {
 	Segment  datamodel.PathSegment
 	LinkNode datamodel.Node
-	Link     ipld.Link
+	Link     datamodel.Link
 }
-
-type Loader func(PreloadContext, []Link)

traversal/fns.go

@@ -49,7 +49,7 @@ type Config struct {
 	LinkTargetNodePrototypeChooser LinkTargetNodePrototypeChooser // Chooser for Node implementations to produce during automatic link traversal.
 	LinkVisitOnlyOnce              bool                           // By default, we visit across links wherever we see them again, even if we've visited them before, because the reason for visiting might be different than it was before since we got to it via a different path.  If set to true, track links we've seen before in Progress.SeenLinks and do not visit them again.  Note that sufficiently complex selectors may require valid revisiting of some links, so setting this to true can change behavior noticably and should be done with care.
 	StartAtPath                    datamodel.Path                 // If set, causes a traversal to skip forward until passing this path, and only then begins calling visit functions.  Block loads will also be skipped wherever possible.
-	Preloader                      preload.Loader                 // use breadth first look aheads to preload links
+	Preloader                      preload.Loader                 // Receives a list of links within each block prior to traversal.	This can be used to load blocks in parallel, or even to load blocks in a different order than the traversal would otherwise do.
 }
 
 type Budget struct {