Skip to content

Commit 79e13bc

Browse files
shizhMSFTshizhMSFT
authored
feat: skip foreign layers on oras.Copy (#380)
Resolves #319 Related to #377 Signed-off-by: Shiwei Zhang <shizh@microsoft.com>
1 parent 8500e54 commit 79e13bc

File tree

2 files changed

+211
-3
lines changed

2 files changed

+211
-3
lines changed

‎copy.go

+15-3
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,7 @@ func copyGraph(ctx context.Context, src content.ReadOnlyStorage, dst content.Sto
217217
if err != nil {
218218
return err
219219
}
220+
successors = removeForeignLayers(successors)
220221

221222
// handle leaf nodes
222223
if len(successors) == 0 {
@@ -259,9 +260,6 @@ func copyGraph(ctx context.Context, src content.ReadOnlyStorage, dst content.Sto
259260
func doCopyNode(ctx context.Context, src content.ReadOnlyStorage, dst content.Storage, desc ocispec.Descriptor) error {
260261
rc, err := src.Fetch(ctx, desc)
261262
if err != nil {
262-
if descriptor.IsForeignLayer(desc) && errors.Is(err, errdef.ErrNotFound) {
263-
return fmt.Errorf("the artifact with foreign layer %s is not supported: %w", desc.Digest, err)
264-
}
265263
return err
266264
}
267265
defer rc.Close()
@@ -406,3 +404,17 @@ func prepareCopy(ctx context.Context, dst Target, dstRef string, proxy *cas.Prox
406404

407405
return nil
408406
}
407+
408+
// removeForeignLayers in-place removes all foreign layers in the given slice.
409+
func removeForeignLayers(descs []ocispec.Descriptor) []ocispec.Descriptor {
410+
var j int
411+
for i, desc := range descs {
412+
if !descriptor.IsForeignLayer(desc) {
413+
if i != j {
414+
descs[j] = desc
415+
}
416+
j++
417+
}
418+
}
419+
return descs[:j]
420+
}

‎copy_test.go

+196
Original file line numberDiff line numberDiff line change
@@ -1509,5 +1509,201 @@ func TestCopyGraph_WithConcurrencyLimit(t *testing.T) {
15091509
}
15101510
}
15111511
}
1512+
}
1513+
1514+
func TestCopyGraph_ForeignLayers(t *testing.T) {
1515+
src := cas.NewMemory()
1516+
dst := cas.NewMemory()
1517+
1518+
// generate test content
1519+
var blobs [][]byte
1520+
var descs []ocispec.Descriptor
1521+
appendBlob := func(mediaType string, blob []byte) {
1522+
desc := ocispec.Descriptor{
1523+
MediaType: mediaType,
1524+
Digest: digest.FromBytes(blob),
1525+
Size: int64(len(blob)),
1526+
}
1527+
if mediaType == ocispec.MediaTypeImageLayerNonDistributable {
1528+
desc.URLs = append(desc.URLs, "http://127.0.0.1/dummy")
1529+
blob = nil
1530+
}
1531+
descs = append(descs, desc)
1532+
blobs = append(blobs, blob)
1533+
}
1534+
generateManifest := func(config ocispec.Descriptor, layers ...ocispec.Descriptor) {
1535+
manifest := ocispec.Manifest{
1536+
Config: config,
1537+
Layers: layers,
1538+
}
1539+
manifestJSON, err := json.Marshal(manifest)
1540+
if err != nil {
1541+
t.Fatal(err)
1542+
}
1543+
appendBlob(ocispec.MediaTypeImageManifest, manifestJSON)
1544+
}
1545+
1546+
appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0
1547+
appendBlob(ocispec.MediaTypeImageLayerNonDistributable, []byte("hello")) // Blob 1
1548+
appendBlob(ocispec.MediaTypeImageLayer, []byte("foo")) // Blob 2
1549+
appendBlob(ocispec.MediaTypeImageLayer, []byte("bar")) // Blob 3
1550+
generateManifest(descs[0], descs[1:4]...) // Blob 4
1551+
1552+
ctx := context.Background()
1553+
for i := range blobs {
1554+
if blobs[i] == nil {
1555+
continue
1556+
}
1557+
err := src.Push(ctx, descs[i], bytes.NewReader(blobs[i]))
1558+
if err != nil {
1559+
t.Fatalf("failed to push test content to src: %d: %v", i, err)
1560+
}
1561+
}
1562+
1563+
// test copy
1564+
srcTracker := &storageTracker{Storage: src}
1565+
dstTracker := &storageTracker{Storage: dst}
1566+
root := descs[len(descs)-1]
1567+
if err := oras.CopyGraph(ctx, srcTracker, dstTracker, root, oras.CopyGraphOptions{}); err != nil {
1568+
t.Fatalf("CopyGraph() error = %v, wantErr %v", err, false)
1569+
}
1570+
1571+
// verify contents
1572+
contents := dst.Map()
1573+
if got, want := len(contents), len(blobs)-1; got != want {
1574+
t.Errorf("len(dst) = %v, wantErr %v", got, want)
1575+
}
1576+
for i := range blobs {
1577+
if blobs[i] == nil {
1578+
continue
1579+
}
1580+
got, err := content.FetchAll(ctx, dst, descs[i])
1581+
if err != nil {
1582+
t.Errorf("content[%d] error = %v, wantErr %v", i, err, false)
1583+
continue
1584+
}
1585+
if want := blobs[i]; !bytes.Equal(got, want) {
1586+
t.Errorf("content[%d] = %v, want %v", i, got, want)
1587+
}
1588+
}
1589+
1590+
// verify API counts
1591+
if got, want := srcTracker.fetch, int64(len(blobs)-1); got != want {
1592+
t.Errorf("count(src.Fetch()) = %v, want %v", got, want)
1593+
}
1594+
if got, want := srcTracker.push, int64(0); got != want {
1595+
t.Errorf("count(src.Push()) = %v, want %v", got, want)
1596+
}
1597+
if got, want := srcTracker.exists, int64(0); got != want {
1598+
t.Errorf("count(src.Exists()) = %v, want %v", got, want)
1599+
}
1600+
if got, want := dstTracker.fetch, int64(0); got != want {
1601+
t.Errorf("count(dst.Fetch()) = %v, want %v", got, want)
1602+
}
1603+
if got, want := dstTracker.push, int64(len(blobs)-1); got != want {
1604+
t.Errorf("count(dst.Push()) = %v, want %v", got, want)
1605+
}
1606+
if got, want := dstTracker.exists, int64(len(blobs)-1); got != want {
1607+
t.Errorf("count(dst.Exists()) = %v, want %v", got, want)
1608+
}
1609+
}
1610+
1611+
func TestCopyGraph_ForeignLayers_Mixed(t *testing.T) {
1612+
src := cas.NewMemory()
1613+
dst := cas.NewMemory()
1614+
1615+
// generate test content
1616+
var blobs [][]byte
1617+
var descs []ocispec.Descriptor
1618+
appendBlob := func(mediaType string, blob []byte) {
1619+
desc := ocispec.Descriptor{
1620+
MediaType: mediaType,
1621+
Digest: digest.FromBytes(blob),
1622+
Size: int64(len(blob)),
1623+
}
1624+
if mediaType == ocispec.MediaTypeImageLayerNonDistributable {
1625+
desc.URLs = append(desc.URLs, "http://127.0.0.1/dummy")
1626+
blob = nil
1627+
}
1628+
descs = append(descs, desc)
1629+
blobs = append(blobs, blob)
1630+
}
1631+
generateManifest := func(config ocispec.Descriptor, layers ...ocispec.Descriptor) {
1632+
manifest := ocispec.Manifest{
1633+
Config: config,
1634+
Layers: layers,
1635+
}
1636+
manifestJSON, err := json.Marshal(manifest)
1637+
if err != nil {
1638+
t.Fatal(err)
1639+
}
1640+
appendBlob(ocispec.MediaTypeImageManifest, manifestJSON)
1641+
}
1642+
1643+
appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0
1644+
appendBlob(ocispec.MediaTypeImageLayerNonDistributable, []byte("hello")) // Blob 1
1645+
appendBlob(ocispec.MediaTypeImageLayer, []byte("hello")) // Blob 2
1646+
appendBlob(ocispec.MediaTypeImageLayer, []byte("foo")) // Blob 3
1647+
appendBlob(ocispec.MediaTypeImageLayer, []byte("bar")) // Blob 4
1648+
generateManifest(descs[0], descs[1:5]...) // Blob 5
1649+
1650+
ctx := context.Background()
1651+
for i := range blobs {
1652+
if blobs[i] == nil {
1653+
continue
1654+
}
1655+
err := src.Push(ctx, descs[i], bytes.NewReader(blobs[i]))
1656+
if err != nil {
1657+
t.Fatalf("failed to push test content to src: %d: %v", i, err)
1658+
}
1659+
}
15121660

1661+
// test copy
1662+
srcTracker := &storageTracker{Storage: src}
1663+
dstTracker := &storageTracker{Storage: dst}
1664+
root := descs[len(descs)-1]
1665+
if err := oras.CopyGraph(ctx, srcTracker, dstTracker, root, oras.CopyGraphOptions{
1666+
Concurrency: 1,
1667+
}); err != nil {
1668+
t.Fatalf("CopyGraph() error = %v, wantErr %v", err, false)
1669+
}
1670+
1671+
// verify contents
1672+
contents := dst.Map()
1673+
if got, want := len(contents), len(blobs)-1; got != want {
1674+
t.Errorf("len(dst) = %v, wantErr %v", got, want)
1675+
}
1676+
for i := range blobs {
1677+
if blobs[i] == nil {
1678+
continue
1679+
}
1680+
got, err := content.FetchAll(ctx, dst, descs[i])
1681+
if err != nil {
1682+
t.Errorf("content[%d] error = %v, wantErr %v", i, err, false)
1683+
continue
1684+
}
1685+
if want := blobs[i]; !bytes.Equal(got, want) {
1686+
t.Errorf("content[%d] = %v, want %v", i, got, want)
1687+
}
1688+
}
1689+
1690+
// verify API counts
1691+
if got, want := srcTracker.fetch, int64(len(blobs)-1); got != want {
1692+
t.Errorf("count(src.Fetch()) = %v, want %v", got, want)
1693+
}
1694+
if got, want := srcTracker.push, int64(0); got != want {
1695+
t.Errorf("count(src.Push()) = %v, want %v", got, want)
1696+
}
1697+
if got, want := srcTracker.exists, int64(0); got != want {
1698+
t.Errorf("count(src.Exists()) = %v, want %v", got, want)
1699+
}
1700+
if got, want := dstTracker.fetch, int64(0); got != want {
1701+
t.Errorf("count(dst.Fetch()) = %v, want %v", got, want)
1702+
}
1703+
if got, want := dstTracker.push, int64(len(blobs)-1); got != want {
1704+
t.Errorf("count(dst.Push()) = %v, want %v", got, want)
1705+
}
1706+
if got, want := dstTracker.exists, int64(len(blobs)-1); got != want {
1707+
t.Errorf("count(dst.Exists()) = %v, want %v", got, want)
1708+
}
15131709
}

0 commit comments

Comments
 (0)