refactor: refactor OCI store to fully support Predecessors() and Resolve() (#385)

1. Support discovering predecessors that are not tagged
2. Support resolving a manifest by its digest
3. Support resolving a blob by its digest

Resolves: #342
Signed-off-by: Lixia (Sylvia) Lei <lixlei@microsoft.com>

Author: Wwwsylvia

content/descriptor.go

@@ -18,16 +18,14 @@ package content
 import (
 	"github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"oras.land/oras-go/v2/internal/descriptor"
 )
 
-// defaultMediaType is the media type used when no media type is specified.
-const defaultMediaType string = "application/octet-stream"
-
 // NewDescriptorFromBytes returns a descriptor, given the content and media type.
 // If no media type is specified, "application/octet-stream" will be used.
 func NewDescriptorFromBytes(mediaType string, content []byte) ocispec.Descriptor {
 	if mediaType == "" {
-		mediaType = defaultMediaType
+		mediaType = descriptor.DefaultMediaType
 	}
 	return ocispec.Descriptor{
 		MediaType: mediaType,

content/descriptor_test.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"oras.land/oras-go/v2/internal/descriptor"
 )
 
 func TestGenerateDescriptor(t *testing.T) {
@@ -56,7 +57,7 @@ func TestGenerateDescriptor(t *testing.T) {
 			name: "missing media type",
 			args: args{contentBar, ""},
 			want: ocispec.Descriptor{
-				MediaType: defaultMediaType,
+				MediaType: descriptor.DefaultMediaType,
 				Digest:    digest.FromBytes(contentBar),
 				Size:      int64(len(contentBar))},
 		},

content/oci/oci.go

@@ -20,16 +20,19 @@ package oci
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"sync"
 
 	specs "github.com/opencontainers/image-spec/specs-go"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"oras.land/oras-go/v2/content"
 	"oras.land/oras-go/v2/errdef"
+	"oras.land/oras-go/v2/internal/descriptor"
 	"oras.land/oras-go/v2/internal/graph"
 	"oras.land/oras-go/v2/internal/resolver"
 )
@@ -53,11 +56,12 @@ type Store struct {
 	AutoSaveIndex bool
 	root          string
 	indexPath     string
+	index         *ocispec.Index
+	indexLock     sync.Mutex
 
-	storage  content.Storage
-	resolver *resolver.Memory
-	graph    *graph.Memory
-	index    *ocispec.Index
+	storage     content.Storage
+	tagResolver *resolver.Memory
+	graph       *graph.Memory
 }
 
 // New creates a new OCI store with context.Background().
@@ -72,19 +76,17 @@ func NewWithContext(ctx context.Context, root string) (*Store, error) {
 		root:          root,
 		indexPath:     filepath.Join(root, ociImageIndexFile),
 		storage:       NewStorage(root),
-		resolver:      resolver.NewMemory(),
+		tagResolver:   resolver.NewMemory(),
 		graph:         graph.NewMemory(),
 	}
 
 	if err := ensureDir(root); err != nil {
 		return nil, err
 	}
-
 	if err := store.ensureOCILayoutFile(); err != nil {
 		return nil, err
 	}
-
-	if err := store.loadIndex(ctx); err != nil {
+	if err := store.loadIndexFile(ctx); err != nil {
 		return nil, err
 	}
 
@@ -101,8 +103,14 @@ func (s *Store) Push(ctx context.Context, expected ocispec.Descriptor, reader io
 	if err := s.storage.Push(ctx, expected, reader); err != nil {
 		return err
 	}
-
-	return s.graph.Index(ctx, s.storage, expected)
+	if err := s.graph.Index(ctx, s.storage, expected); err != nil {
+		return err
+	}
+	if descriptor.IsManifest(expected) {
+		// tag by digest
+		return s.tag(ctx, expected, expected.Digest.String())
+	}
+	return nil
 }
 
 // Exists returns true if the described content exists.
@@ -111,8 +119,7 @@ func (s *Store) Exists(ctx context.Context, target ocispec.Descriptor) (bool, er
 }
 
 // Tag tags a descriptor with a reference string.
-// A reference should be either a valid tag (e.g. "latest"),
-// or a digest matching the descriptor (e.g. "@sha256:abc123").
+// reference should be a valid tag (e.g. "latest").
 // Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc2/image-layout.md#indexjson-file
 func (s *Store) Tag(ctx context.Context, desc ocispec.Descriptor, reference string) error {
 	if err := validateReference(reference); err != nil {
@@ -131,11 +138,21 @@ func (s *Store) Tag(ctx context.Context, desc ocispec.Descriptor, reference stri
 		desc.Annotations = map[string]string{}
 	}
 	desc.Annotations[ocispec.AnnotationRefName] = reference
+	return s.tag(ctx, desc, reference)
+}
 
-	if err := s.resolver.Tag(ctx, desc, reference); err != nil {
+// tag tags a descriptor with a reference string.
+func (s *Store) tag(ctx context.Context, desc ocispec.Descriptor, reference string) error {
+	dgst := desc.Digest.String()
+	if reference != dgst {
+		// mark desc for deduplication in SaveIndex()
+		if err := s.tagResolver.Tag(ctx, desc, dgst); err != nil {
+			return err
+		}
+	}
+	if err := s.tagResolver.Tag(ctx, desc, reference); err != nil {
 		return err
 	}
-
 	if s.AutoSaveIndex {
 		return s.SaveIndex()
 	}
@@ -148,7 +165,16 @@ func (s *Store) Resolve(ctx context.Context, reference string) (ocispec.Descript
 		return ocispec.Descriptor{}, errdef.ErrMissingReference
 	}
 
-	return s.resolver.Resolve(ctx, reference)
+	// attempt resolving manifest
+	desc, err := s.tagResolver.Resolve(ctx, reference)
+	if err != nil {
+		if errors.Is(err, errdef.ErrNotFound) {
+			// attempt resolving blob
+			return resolveBlob(os.DirFS(s.root), reference)
+		}
+		return ocispec.Descriptor{}, err
+	}
+	return descriptor.Plain(desc), nil
 }
 
 // Predecessors returns the nodes directly pointing to the current node.
@@ -179,20 +205,16 @@ func (s *Store) ensureOCILayoutFile() error {
 	}
 	defer layoutFile.Close()
 
-	var layout *ocispec.ImageLayout
+	var layout ocispec.ImageLayout
 	err = json.NewDecoder(layoutFile).Decode(&layout)
 	if err != nil {
 		return fmt.Errorf("failed to decode OCI layout file: %w", err)
 	}
-	if layout.Version != ocispec.ImageLayoutVersion {
-		return errdef.ErrUnsupportedVersion
-	}
-
-	return nil
+	return validateOCILayout(&layout)
 }
 
-// loadIndex reads the index.json from the file system.
-func (s *Store) loadIndex(ctx context.Context) error {
+// loadIndexFile reads index.json from the file system.
+func (s *Store) loadIndexFile(ctx context.Context) error {
 	indexFile, err := os.Open(s.indexPath)
 	if err != nil {
 		if !os.IsNotExist(err) {
@@ -207,24 +229,12 @@ func (s *Store) loadIndex(ctx context.Context) error {
 	}
 	defer indexFile.Close()
 
-	if err := json.NewDecoder(indexFile).Decode(&s.index); err != nil {
+	var index ocispec.Index
+	if err := json.NewDecoder(indexFile).Decode(&index); err != nil {
 		return fmt.Errorf("failed to decode index file: %w", err)
 	}
-
-	for _, desc := range s.index.Manifests {
-		if ref := desc.Annotations[ocispec.AnnotationRefName]; ref != "" {
-			if err = s.resolver.Tag(ctx, desc, ref); err != nil {
-				return err
-			}
-		}
-
-		// traverse the whole DAG and index predecessors for all the nodes.
-		if err := s.graph.IndexAll(ctx, s.storage, desc); err != nil {
-			return err
-		}
-	}
-
-	return nil
+	s.index = &index
+	return loadIndex(ctx, s.index, s.storage, s.tagResolver, s.graph)
 }
 
 // SaveIndex writes the `index.json` file to the file system.
@@ -233,10 +243,16 @@ func (s *Store) loadIndex(ctx context.Context) error {
 // If AutoSaveIndex is set to false, it's the caller's responsibility
 // to manually call this method when needed.
 func (s *Store) SaveIndex() error {
-	// first need to update the index.
+	s.indexLock.Lock()
+	defer s.indexLock.Unlock()
+
 	var manifests []ocispec.Descriptor
-	refMap := s.resolver.Map()
-	for _, desc := range refMap {
+	refMap := s.tagResolver.Map()
+	for ref, desc := range refMap {
+		if ref == desc.Digest.String() && desc.Annotations[ocispec.AnnotationRefName] != "" {
+			// skip saving desc if ref is a digest and desc is tagged
+			continue
+		}
 		manifests = append(manifests, desc)
 	}