Skip to content

Commit 5d6b7cf

Browse files
rvaggrvagg
committed
fix: recurse preloader at block level
recursing in walk_* breaks preloading at lists and maps, so we have to do it higher
1 parent 15d4253 commit 5d6b7cf

File tree

2 files changed

+125
-147
lines changed

2 files changed

+125
-147
lines changed

‎traversal/walk.go

+123-146
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ import (
99
"github.com/ipld/go-ipld-prime/traversal/selector"
1010
)
1111

12+
type loadLinkFn func(prog Progress, ps datamodel.PathSegment, v datamodel.Node, parent datamodel.Node) (datamodel.Node, error)
13+
1214
// WalkLocal walks a tree of Nodes, visiting each of them,
1315
// and calling the given VisitFn on all of them;
1416
// it does not traverse any links.
@@ -79,7 +81,7 @@ func WalkTransforming(n datamodel.Node, s selector.Selector, fn TransformFn) (da
7981
// and thus continued nested uses of Walk and Focus will see the fully contextualized Path.
8082
func (prog Progress) WalkMatching(n datamodel.Node, s selector.Selector, fn VisitFn) error {
8183
prog.init()
82-
return prog.walkAdv(n, s, func(prog Progress, n datamodel.Node, tr VisitReason) error {
84+
return prog.walkAdv(nil, n, s, func(prog Progress, n datamodel.Node, tr VisitReason) error {
8385
if tr != VisitReason_SelectionMatch {
8486
return nil
8587
}
@@ -143,10 +145,10 @@ func (prog Progress) WalkLocal(n datamodel.Node, fn VisitFn) error {
143145
// An AdvVisitFn is used instead of a VisitFn, so that the reason can be provided.
144146
func (prog Progress) WalkAdv(n datamodel.Node, s selector.Selector, fn AdvVisitFn) error {
145147
prog.init()
146-
return prog.walkAdv(n, s, fn)
148+
return prog.walkAdv(nil, n, s, fn)
147149
}
148150

149-
func (prog Progress) walkAdv(n datamodel.Node, s selector.Selector, fn AdvVisitFn) error {
151+
func (prog Progress) walkAdv(loadLink loadLinkFn, n datamodel.Node, s selector.Selector, fn AdvVisitFn) error {
150152
// Check the budget!
151153
if prog.Budget != nil {
152154
if prog.Budget.NodeBudget <= 0 {
@@ -195,25 +197,67 @@ func (prog Progress) walkAdv(n datamodel.Node, s selector.Selector, fn AdvVisitF
195197
}
196198
}
197199
}
200+
198201
// If we're handling scalars (e.g. not maps and lists) we can return now.
199-
nk := n.Kind()
200-
switch nk {
201-
case datamodel.Kind_Map, datamodel.Kind_List: // continue
202-
default:
202+
if n.Kind() != datamodel.Kind_Map && n.Kind() != datamodel.Kind_List {
203203
return nil
204204
}
205+
205206
// For maps and lists: recurse (in one of two ways, depending on if the selector also states specific interests).
206207
attn := s.Interests()
207-
if attn == nil {
208-
return prog.walkAdv_iterateAll(n, s, fn)
209-
}
210-
if len(attn) == 0 {
208+
if attn != nil && len(attn) == 0 {
211209
return nil
212210
}
213-
return prog.walkAdv_iterateSelective(n, attn, s, fn)
211+
212+
if loadLink == nil {
213+
loadLink = loadLinkActual
214+
if prog.Cfg.Preloader != nil {
215+
preProg := prog
216+
// TODO: handle preProg.Budget .. somehow
217+
// TODO: handle preProg.SeenLinks .. somehow
218+
preloadLinks := make([]preload.Link, 0)
219+
preProg.SeenLinks = make(map[datamodel.Link]struct{})
220+
loadLinkCollect := func(prog Progress, ps datamodel.PathSegment, v datamodel.Node, parent datamodel.Node) (datamodel.Node, error) {
221+
lnk, err := v.AsLink()
222+
if err != nil {
223+
return nil, err
224+
}
225+
preloadLinks = append(preloadLinks, preload.Link{
226+
Segment: ps,
227+
LinkNode: v,
228+
Link: lnk,
229+
})
230+
return nil, SkipMe{}
231+
}
232+
233+
// traversal preload
234+
noopVisitor := func(prog Progress, n datamodel.Node, reason VisitReason) error { return nil }
235+
if attn != nil {
236+
if err := preProg.walkAdv_iterateSelective(loadLinkCollect, n, attn, s, noopVisitor); err != nil {
237+
return err
238+
}
239+
} else {
240+
if err := preProg.walkAdv_iterateAll(loadLinkCollect, n, s, noopVisitor); err != nil {
241+
return err
242+
}
243+
}
244+
245+
prog.Cfg.Preloader(preload.PreloadContext{
246+
Ctx: prog.Cfg.Ctx,
247+
BasePath: prog.Path,
248+
ParentNode: n,
249+
}, preloadLinks)
250+
}
251+
}
252+
253+
// traversal actual
254+
if attn != nil {
255+
return prog.walkAdv_iterateSelective(loadLink, n, attn, s, fn)
256+
}
257+
return prog.walkAdv_iterateAll(loadLink, n, s, fn)
214258
}
215259

216-
func (prog Progress) iterateAll(n datamodel.Node, visit func(datamodel.PathSegment, datamodel.Node, bool) error) error {
260+
func (prog Progress) walkAdv_iterateAll(loadLink loadLinkFn, n datamodel.Node, s selector.Selector, fn AdvVisitFn) error {
217261
var reachedStartAtPath bool
218262
for itr := selector.NewSegmentIterator(n); !itr.Done(); {
219263
if reachedStartAtPath {
@@ -231,81 +275,46 @@ func (prog Progress) iterateAll(n datamodel.Node, visit func(datamodel.PathSegme
231275
continue
232276
}
233277
}
234-
err = visit(ps, v, prog.PastStartAtPath)
235-
if err != nil {
236-
return err
237-
}
238-
}
239-
return nil
240-
}
241-
242-
func (prog Progress) walkAdv_iterateAll(n datamodel.Node, s selector.Selector, fn AdvVisitFn) error {
243-
nextSelectors := make([]selector.Selector, 0, n.Length())
244-
preloadLinks := make([]preload.Link, 0, n.Length())
245-
err := prog.iterateAll(n, func(ps datamodel.PathSegment, v datamodel.Node, pastStartAtPath bool) error {
246278
sNext, err := s.Explore(n, ps)
247279
if err != nil {
248280
return err
249281
}
250-
nextSelectors = append(nextSelectors, sNext)
251-
if sNext != nil && v.Kind() == datamodel.Kind_Link {
252-
lnk, _ := v.AsLink()
253-
preloadLinks = append(preloadLinks, preload.Link{
254-
Segment: ps,
255-
LinkNode: v,
256-
Link: lnk,
257-
})
258-
}
259-
return nil
260-
})
261-
if err != nil {
262-
return err
263-
}
264-
if prog.Cfg.Preloader != nil && len(preloadLinks) > 0 {
265-
prog.Cfg.Preloader(preload.PreloadContext{
266-
Ctx: prog.Cfg.Ctx,
267-
BasePath: prog.Path,
268-
ParentNode: n,
269-
}, preloadLinks)
270-
}
271-
272-
index := 0
273-
return prog.iterateAll(n, func(ps datamodel.PathSegment, v datamodel.Node, pastStartAtPath bool) error {
274-
var err error
275-
sNext := nextSelectors[index]
276-
index++
277-
if sNext == nil {
278-
return nil
279-
}
280-
progNext := prog
281-
if pastStartAtPath {
282-
progNext.PastStartAtPath = true
283-
}
284-
progNext.Path = prog.Path.AppendSegment(ps)
285-
if v.Kind() == datamodel.Kind_Link {
286-
lnk, _ := v.AsLink()
287-
if prog.Cfg.LinkVisitOnlyOnce {
288-
if _, seen := prog.SeenLinks[lnk]; seen {
289-
return nil
282+
if sNext != nil {
283+
progNext := prog
284+
progNext.Path = prog.Path.AppendSegment(ps)
285+
if v.Kind() == datamodel.Kind_Link {
286+
lnk, _ := v.AsLink()
287+
if prog.Cfg.LinkVisitOnlyOnce {
288+
if _, seen := prog.SeenLinks[lnk]; seen {
289+
continue
290+
}
291+
prog.SeenLinks[lnk] = struct{}{}
290292
}
291-
prog.SeenLinks[lnk] = struct{}{}
292-
}
293-
progNext.LastBlock.Path = progNext.Path
294-
progNext.LastBlock.Link = lnk
295-
v, err = progNext.loadLink(v, n)
296-
if err != nil {
297-
if _, ok := err.(SkipMe); ok {
298-
return nil
293+
progNext.LastBlock.Path = progNext.Path
294+
progNext.LastBlock.Link = lnk
295+
v, err = loadLink(progNext, ps, v, n)
296+
if err != nil {
297+
if _, ok := err.(SkipMe); ok {
298+
continue
299+
}
300+
return err
301+
}
302+
err = progNext.walkAdv(nil, v, sNext, fn)
303+
if err != nil {
304+
return err
305+
}
306+
} else {
307+
err = progNext.walkAdv(loadLink, v, sNext, fn)
308+
if err != nil {
309+
return err
299310
}
300-
return err
301311
}
302312
}
303-
304-
return progNext.walkAdv(v, sNext, fn)
305-
})
313+
}
314+
return nil
306315
}
307316

308-
func (prog Progress) iterateSelective(n datamodel.Node, attn []datamodel.PathSegment, visit func(ps datamodel.PathSegment, v datamodel.Node) error) error {
317+
func (prog Progress) walkAdv_iterateSelective(loadLink loadLinkFn, n datamodel.Node, attn []datamodel.PathSegment, s selector.Selector, fn AdvVisitFn) error {
309318
var reachedStartAtPath bool
310319
for _, ps := range attn {
311320
if prog.Path.Len() < prog.Cfg.StartAtPath.Len() {
@@ -320,78 +329,46 @@ func (prog Progress) iterateSelective(n datamodel.Node, attn []datamodel.PathSeg
320329
if err != nil {
321330
continue
322331
}
323-
err = visit(ps, v)
324-
if err != nil {
325-
return err
326-
}
327-
}
328-
return nil
329-
}
330-
331-
func (prog Progress) walkAdv_iterateSelective(n datamodel.Node, attn []datamodel.PathSegment, s selector.Selector, fn AdvVisitFn) error {
332-
nextSelectors := make([]selector.Selector, 0, len(attn))
333-
preloadLinks := make([]preload.Link, 0, len(attn))
334-
err := prog.iterateSelective(n, attn, func(ps datamodel.PathSegment, v datamodel.Node) error {
335332
sNext, err := s.Explore(n, ps)
336333
if err != nil {
337334
return err
338335
}
339-
nextSelectors = append(nextSelectors, sNext)
340-
if sNext != nil && v.Kind() == datamodel.Kind_Link {
341-
lnk, _ := v.AsLink()
342-
preloadLinks = append(preloadLinks, preload.Link{
343-
Segment: ps,
344-
LinkNode: v,
345-
Link: lnk,
346-
})
347-
}
348-
return nil
349-
})
350-
if err != nil {
351-
return err
352-
}
353-
if prog.Cfg.Preloader != nil && len(preloadLinks) > 0 {
354-
prog.Cfg.Preloader(preload.PreloadContext{
355-
Ctx: prog.Cfg.Ctx,
356-
BasePath: prog.Path,
357-
ParentNode: n,
358-
}, preloadLinks)
359-
}
360-
361-
index := 0
362-
return prog.iterateSelective(n, attn, func(ps datamodel.PathSegment, v datamodel.Node) error {
363-
var err error
364-
sNext := nextSelectors[index]
365-
index++
366-
if sNext == nil {
367-
return nil
368-
}
369-
progNext := prog
370-
progNext.Path = prog.Path.AppendSegment(ps)
371-
if v.Kind() == datamodel.Kind_Link {
372-
lnk, _ := v.AsLink()
373-
if prog.Cfg.LinkVisitOnlyOnce {
374-
if _, seen := prog.SeenLinks[lnk]; seen {
375-
return nil
336+
if sNext != nil {
337+
progNext := prog
338+
progNext.Path = prog.Path.AppendSegment(ps)
339+
if v.Kind() == datamodel.Kind_Link {
340+
lnk, _ := v.AsLink()
341+
if prog.Cfg.LinkVisitOnlyOnce {
342+
if _, seen := prog.SeenLinks[lnk]; seen {
343+
continue
344+
}
345+
prog.SeenLinks[lnk] = struct{}{}
376346
}
377-
prog.SeenLinks[lnk] = struct{}{}
378-
}
379-
progNext.LastBlock.Path = progNext.Path
380-
progNext.LastBlock.Link = lnk
381-
v, err = progNext.loadLink(v, n)
382-
if err != nil {
383-
if _, ok := err.(SkipMe); ok {
384-
return nil
347+
progNext.LastBlock.Path = progNext.Path
348+
progNext.LastBlock.Link = lnk
349+
v, err = loadLink(progNext, ps, v, n)
350+
if err != nil {
351+
if _, ok := err.(SkipMe); ok {
352+
continue
353+
}
354+
return err
355+
}
356+
err = progNext.walkAdv(nil, v, sNext, fn)
357+
if err != nil {
358+
return err
359+
}
360+
} else {
361+
err = progNext.walkAdv(loadLink, v, sNext, fn)
362+
if err != nil {
363+
return err
385364
}
386-
return err
387365
}
388366
}
389-
390-
return progNext.walkAdv(v, sNext, fn)
391-
})
367+
}
368+
return nil
392369
}
393370

394-
func (prog Progress) loadLink(v datamodel.Node, parent datamodel.Node) (datamodel.Node, error) {
371+
func loadLinkActual(prog Progress, ps datamodel.PathSegment, v datamodel.Node, parent datamodel.Node) (datamodel.Node, error) {
395372
lnk, err := v.AsLink()
396373
if err != nil {
397374
return nil, err
@@ -496,9 +473,9 @@ func (prog Progress) walkTransforming(n datamodel.Node, s selector.Selector, fn
496473
nk := n.Kind()
497474
switch nk {
498475
case datamodel.Kind_List:
499-
return prog.walk_transform_iterateList(n, s, fn, s.Interests())
476+
return prog.walk_transform_iterateList(loadLinkActual, n, s, fn, s.Interests())
500477
case datamodel.Kind_Map:
501-
return prog.walk_transform_iterateMap(n, s, fn, s.Interests())
478+
return prog.walk_transform_iterateMap(loadLinkActual, n, s, fn, s.Interests())
502479
default:
503480
return n, nil
504481
}
@@ -513,7 +490,7 @@ func contains(interest []datamodel.PathSegment, candidate datamodel.PathSegment)
513490
return false
514491
}
515492

516-
func (prog Progress) walk_transform_iterateList(n datamodel.Node, s selector.Selector, fn TransformFn, attn []datamodel.PathSegment) (datamodel.Node, error) {
493+
func (prog Progress) walk_transform_iterateList(loadLink loadLinkFn, n datamodel.Node, s selector.Selector, fn TransformFn, attn []datamodel.PathSegment) (datamodel.Node, error) {
517494
bldr := n.Prototype().NewBuilder()
518495
lstBldr, err := bldr.BeginList(n.Length())
519496
if err != nil {
@@ -542,7 +519,7 @@ func (prog Progress) walk_transform_iterateList(n datamodel.Node, s selector.Sel
542519
}
543520
progNext.LastBlock.Path = progNext.Path
544521
progNext.LastBlock.Link = lnk
545-
v, err = progNext.loadLink(v, n)
522+
v, err = loadLink(progNext, ps, v, n)
546523
if err != nil {
547524
if _, ok := err.(SkipMe); ok {
548525
continue
@@ -575,7 +552,7 @@ func (prog Progress) walk_transform_iterateList(n datamodel.Node, s selector.Sel
575552
return bldr.Build(), nil
576553
}
577554

578-
func (prog Progress) walk_transform_iterateMap(n datamodel.Node, s selector.Selector, fn TransformFn, attn []datamodel.PathSegment) (datamodel.Node, error) {
555+
func (prog Progress) walk_transform_iterateMap(loadLink loadLinkFn, n datamodel.Node, s selector.Selector, fn TransformFn, attn []datamodel.PathSegment) (datamodel.Node, error) {
579556
bldr := n.Prototype().NewBuilder()
580557
mapBldr, err := bldr.BeginMap(n.Length())
581558
if err != nil {
@@ -609,7 +586,7 @@ func (prog Progress) walk_transform_iterateMap(n datamodel.Node, s selector.Sele
609586
}
610587
progNext.LastBlock.Path = progNext.Path
611588
progNext.LastBlock.Link = lnk
612-
v, err = progNext.loadLink(v, n)
589+
v, err = loadLink(progNext, ps, v, n)
613590
if err != nil {
614591
if _, ok := err.(SkipMe); ok {
615592
continue

‎traversal/walk_test.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -465,7 +465,7 @@ func TestWalkBlockLoadOrder(t *testing.T) {
465465
leafAlphaLnk,
466466
leafAlphaLnk,
467467
}
468-
// same thing but for middleListNode
468+
// same thing but just for middleListNode
469469
middleListNodeLinks := []datamodel.Link{
470470
middleListNodeLnk,
471471
leafAlphaLnk,
@@ -491,6 +491,7 @@ func TestWalkBlockLoadOrder(t *testing.T) {
491491
var count int
492492
lsys := cidlink.DefaultLinkSystem()
493493
lsys.StorageReadOpener = func(lc linking.LinkContext, l datamodel.Link) (io.Reader, error) {
494+
t.Logf("load %d: %v (%s) <> %v (%s) - %s", count, expected[count].String(), linkNames[expected[count]], l.String(), linkNames[l], lc.LinkPath)
494495
qt.Check(t, l.String(), qt.Equals, expected[count].String())
495496
log.Printf("%v (%v) %s<> %v (%v)\n", l, linkNames[l], strings.Repeat(" ", 17-len(linkNames[l])), expected[count], linkNames[expected[count]])
496497
count++

0 commit comments

Comments
 (0)