Skip to content

Commit

Permalink
use ociblob instead of ocilocalblob for oci v2 (#1768)
Browse files Browse the repository at this point in the history
* remove compatible attribute for v2, so that v2 generate resources with ociblob instead of depricated ocilocalblob

* add e2e test

* add e2e test

* add module config test

* add module config test

* fix github action

* extend e2e test
drop gotest.tools

* fix lint

* test repoURL

* fix TEST_REPOSITORY_URL location

* gofmt

* align name
  • Loading branch information
ruanxin authored Sep 21, 2023
1 parent a4af874 commit b0f945c
Show file tree
Hide file tree
Showing 14 changed files with 232 additions and 48 deletions.
77 changes: 77 additions & 0 deletions .github/workflows/test-e2e-create-module.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
name: TestSuite E2E - create module

on:
push:
branches: [ "main" ]
pull_request:
branches: [ "main" ]
jobs:
e2e:
strategy:
matrix:
e2e-test: [ "create_module_kubebuilder_project", "create_module_module_config"]
name: "Run E2E tests"
runs-on: ubuntu-latest
env:
K3D_VERSION: v5.4.7
MODULE_TEMPLATE_VERSION: v1.0.0
OCI_REPOSITORY_URL: http://k3d-oci.localhost:5001
steps:
- name: Checkout Kyma CLI
uses: actions/checkout@v3
- name: Set up Go
uses: actions/setup-go@v4
with:
go-version-file: 'go.mod'
cache-dependency-path: 'go.sum'
- name: Build Kyma CLI
run: |
make resolve validate build-linux
chmod +x ./bin/kyma-linux
ls -la ./bin
mv ./bin/kyma-linux /usr/local/bin/kyma
timeout-minutes: 10
- name: Checkout template-operator
uses: actions/checkout@v3
with:
repository: kyma-project/template-operator
path: ./template-operator/
- name: export template-operator URL
run: |
cd ./template-operator
echo "TEST_REPOSITORY_URL=$(git remote get-url origin)" >> "$GITHUB_ENV"
- name: Set up k3d
run: wget -qO - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | TAG=$K3D_VERSION bash
- name: Provision OCI Registry
run: |
k3d registry create oci.localhost --port 5001
- name: Run create module with kubebuilder-project
if: ${{ matrix.e2e-test == 'create_module_kubebuilder_project' }}
run: |
kyma alpha create module \
--name kyma-project.io/module/template-operator \
--path ./template-operator \
--registry http://k3d-oci.localhost:5001 \
--insecure \
--kubebuilder-project \
--version $MODULE_TEMPLATE_VERSION -v \
--output /tmp/kubebuilder-template.yaml
echo "MODULE_TEMPLATE_PATH=/tmp/kubebuilder-template.yaml" >> "$GITHUB_ENV"
- name: Run create module with module-config
if: ${{ matrix.e2e-test == 'create_module_module_config' }}
run: |
cd ./template-operator
make build-manifests
kyma alpha create module \
--name kyma-project.io/module/template-operator \
--path . \
--registry http://k3d-oci.localhost:5001 \
--insecure \
--module-config-file ./module-config.yaml \
--version $MODULE_TEMPLATE_VERSION -v \
--output /tmp/module-config-template.yaml
echo "MODULE_TEMPLATE_PATH=/tmp/module-config-template.yaml" >> "$GITHUB_ENV"
- name: Verify module template
run: |
echo $MODULE_TEMPLATE_PATH
make -C tests/e2e ${{ matrix.e2e-test }}
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: Smoke Tests
name: TestSuite Smoke

on:
push:
Expand Down
1 change: 0 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@ require (
go.uber.org/zap v1.25.0
golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1
gopkg.in/yaml.v3 v3.0.1
gotest.tools v2.2.0+incompatible
helm.sh/helm/v3 v3.12.3
istio.io/client-go v1.19.0-alpha.1.0.20230825174843-7a8fbe28cb5d
k8s.io/klog/v2 v2.100.1
Expand Down
10 changes: 5 additions & 5 deletions pkg/docker/docker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@ import (
"github.com/docker/docker/api/types/mount"
"github.com/docker/go-connections/nat"
"github.com/kyma-project/cli/pkg/docker/mocks"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"gotest.tools/assert"
)

func Test_SplitDockerDomain(t *testing.T) {
Expand Down Expand Up @@ -66,20 +66,20 @@ func genConfigFile() *dockerConfigFile.ConfigFile {

func Test_Resolve_happy_path(t *testing.T) {
tmpHome, err := os.MkdirTemp("/tmp", "config-test")
assert.NilError(t, err)
assert.Nil(t, err)
defer os.RemoveAll(tmpHome)

configFile := genConfigFile()
b, err := json.Marshal(configFile)
assert.NilError(t, err)
assert.Nil(t, err)
tmpFile := fmt.Sprintf("%s/config.json", tmpHome)
err = os.WriteFile(tmpFile, b, 0600)
assert.NilError(t, err)
assert.Nil(t, err)

os.Setenv("DOCKER_CONFIG", tmpHome)

dockerCFG, err := resolve("example.com")
assert.NilError(t, err)
assert.Nil(t, err)
assert.Equal(t, dockerCFG.Username, "user")
assert.Equal(t, dockerCFG.Password, "pass")
}
Expand Down
20 changes: 5 additions & 15 deletions pkg/module/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,14 @@ import (
"os"
"strings"

"github.com/kyma-project/cli/pkg/module/gitsource"
"github.com/mandelsoft/vfs/pkg/projectionfs"
"github.com/mandelsoft/vfs/pkg/vfs"
"github.com/open-component-model/ocm/pkg/common/accessobj"
"github.com/open-component-model/ocm/pkg/contexts/ocm/attrs/compatattr"
ocm "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc"
v1 "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc/meta/v1"
compdescv2 "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc/versions/v2"
"github.com/open-component-model/ocm/pkg/contexts/ocm/cpi"
"github.com/open-component-model/ocm/pkg/contexts/ocm/repositories/comparch"

"github.com/kyma-project/cli/pkg/module/gitsource"
)

type Source interface {
Expand All @@ -41,9 +38,6 @@ func CreateArchive(fs vfs.FileSystem, path, gitRemote string, def *Definition, i
}

ctx := cpi.DefaultContext()
if err := compatattr.Set(ctx, def.SchemaVersion == compdescv2.SchemaVersion); err != nil {
return nil, fmt.Errorf("could not set compatibility attribute for v2: %w", err)
}

archive, err := comparch.New(
ctx,
Expand All @@ -63,14 +57,10 @@ func CreateArchive(fs vfs.FileSystem, path, gitRemote string, def *Definition, i
return nil, err
}

if compatattr.Get(ctx) {
cd.Provider = v1.Provider{Name: "internal"}
} else {
cd.Provider = v1.Provider{Name: "kyma-project.io", Labels: v1.Labels{*builtByCLI}}
}
cd.Provider = v1.Provider{Name: "kyma-project.io", Labels: v1.Labels{*builtByCLI}}

if isTargetDirAGitRepo {
if err := addSources(ctx, cd, def, gitRemote); err != nil {
if err := addSources(cd, def, gitRemote); err != nil {
return nil, err
}
}
Expand All @@ -87,14 +77,14 @@ func CreateArchive(fs vfs.FileSystem, path, gitRemote string, def *Definition, i
}

// addSources adds the sources to the component descriptor. If the def.Source is a git repository
func addSources(ctx cpi.Context, cd *ocm.ComponentDescriptor, def *Definition, gitRemote string) error {
func addSources(cd *ocm.ComponentDescriptor, def *Definition, gitRemote string) error {
if strings.HasSuffix(def.Source, ".git") {
gitSource := gitsource.NewGitSource()
var err error
if def.Repo, err = gitSource.DetermineRepositoryURL(gitRemote, def.Repo, def.Source); err != nil {
return err
}
src, err := gitSource.FetchSource(ctx, def.Source, def.Repo, def.Version)
src, err := gitSource.FetchSource(def.Source, def.Repo, def.Version)

if err != nil {
return err
Expand Down
9 changes: 2 additions & 7 deletions pkg/module/gitsource/git.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,22 +8,17 @@ import (

"github.com/go-git/go-git/v5"
"github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/github"
"github.com/open-component-model/ocm/pkg/contexts/ocm/attrs/compatattr"
ocm "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc"
ocmv1 "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc/meta/v1"
"github.com/open-component-model/ocm/pkg/contexts/ocm/cpi"
)

func (g GitSource) FetchSource(ctx cpi.Context, path, repo, version string) (*ocm.Source, error) {
func (g GitSource) FetchSource(path, repo, version string) (*ocm.Source, error) {
ref, commit, err := g.getGitInfo(path)
if err != nil {
return nil, err
}

sourceType := "git"
if !compatattr.Get(ctx) {
sourceType = github.CONSUMER_TYPE
}
sourceType := github.CONSUMER_TYPE

label, err := ocmv1.NewLabel(refLabel, ref, ocmv1.WithVersion(ocmVersion))
if err != nil {
Expand Down
5 changes: 2 additions & 3 deletions pkg/module/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,15 @@ import (
"github.com/open-component-model/ocm/pkg/contexts/oci"
"github.com/open-component-model/ocm/pkg/contexts/oci/repositories/ocireg"
"github.com/open-component-model/ocm/pkg/contexts/ocm"
"github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc"
"github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc/versions/ocm.software/v3alpha1"
// nolint:revive
_ "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc/versions"
"github.com/open-component-model/ocm/pkg/contexts/ocm/cpi"
"github.com/open-component-model/ocm/pkg/contexts/ocm/repositories/comparch"
"github.com/open-component-model/ocm/pkg/contexts/ocm/repositories/genericocireg"
)

// nolint:gochecknoinits
func init() {
compdesc.RegisterScheme(&v3alpha1.DescriptorVersion{})
ocm.DefaultContext().RepositoryTypes().Register(ocireg.Type, genericocireg.NewRepositoryType(oci.DefaultContext()))
ocm.DefaultContext().RepositoryTypes().Register(
ocireg.LegacyType, genericocireg.NewRepositoryType(oci.DefaultContext()),
Expand Down
5 changes: 2 additions & 3 deletions pkg/module/layer.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,9 @@ import (
)

const (
configLayerName = "config"
rawManifestLayerName = "raw-manifest"
RawManifestLayerName = "raw-manifest"
typeHelmChart = "helm-chart"
typeYaml = "yaml"
TypeYaml = "yaml"
)

// Layer encapsulates all necessary data to create an OCI layer
Expand Down
10 changes: 3 additions & 7 deletions pkg/module/remote.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ import (
"github.com/open-component-model/ocm/pkg/contexts/credentials/repositories/dockerconfig"
oci "github.com/open-component-model/ocm/pkg/contexts/oci/repositories/ocireg"
"github.com/open-component-model/ocm/pkg/contexts/ocm"
"github.com/open-component-model/ocm/pkg/contexts/ocm/attrs/compatattr"
"github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc"
"github.com/open-component-model/ocm/pkg/contexts/ocm/cpi"
"github.com/open-component-model/ocm/pkg/contexts/ocm/repositories/comparch"
Expand Down Expand Up @@ -42,12 +41,8 @@ type Remote struct {

func (r *Remote) GetRepository(ctx cpi.Context) (cpi.Repository, error) {
creds := r.getCredentials(ctx)
var repoType string
if compatattr.Get(ctx) {
repoType = oci.LegacyType
} else {
repoType = oci.Type
}
repoType := oci.Type

url := NoSchemeURL(r.Registry)
if r.Insecure {
url = fmt.Sprintf("http://%s", url)
Expand All @@ -57,6 +52,7 @@ func (r *Remote) GetRepository(ctx cpi.Context) (cpi.Repository, error) {
ObjectVersionedType: runtime.NewVersionedObjectType(repoType),
BaseURL: url,
}

genericSpec := genericocireg.NewRepositorySpec(
ociRepoSpec, &ocireg.ComponentRepositoryMeta{
ComponentNameMapping: ocireg.ComponentNameMapping(r.NameMapping),
Expand Down
8 changes: 4 additions & 4 deletions pkg/module/resources.go
Original file line number Diff line number Diff line change
Expand Up @@ -160,8 +160,8 @@ func Inspect(def *Definition, log *zap.SugaredLogger) error {

// generated raw manifest -> layer 1
def.Layers = append(def.Layers, Layer{
name: rawManifestLayerName,
resourceType: typeYaml,
name: RawManifestLayerName,
resourceType: TypeYaml,
path: def.SingleManifestPath,
})

Expand Down Expand Up @@ -252,8 +252,8 @@ func inspectProject(def *Definition, p *kubebuilder.Project, layers []Layer, s s
def.Repo = p.Repo
def.DefaultCR = cr
def.Layers = append(def.Layers, Layer{
name: rawManifestLayerName,
resourceType: typeYaml,
name: RawManifestLayerName,
resourceType: TypeYaml,
path: renderedManifestPath,
})
def.Layers = append(def.Layers, layers...)
Expand Down
5 changes: 3 additions & 2 deletions pkg/module/template_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,19 @@ package module

import (
"bytes"
"github.com/kyma-project/lifecycle-manager/api/v1beta2"
"reflect"
"testing"
"text/template"

"github.com/kyma-project/lifecycle-manager/api/v1beta2"

"github.com/mandelsoft/vfs/pkg/osfs"
"github.com/open-component-model/ocm/pkg/common/accessio"
"github.com/open-component-model/ocm/pkg/common/accessobj"
"github.com/open-component-model/ocm/pkg/contexts/ocm"
"github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc"
"github.com/open-component-model/ocm/pkg/contexts/ocm/repositories/ctf"
"gotest.tools/assert"
"github.com/stretchr/testify/assert"
"sigs.k8s.io/yaml"
)

Expand Down
40 changes: 40 additions & 0 deletions tests/e2e/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
ifeq (,$(shell go env GOBIN))
GOBIN=$(shell go env GOPATH)/bin
else
GOBIN=$(shell go env GOBIN)
endif

# Setting SHELL to bash allows bash commands to be executed by recipes.
# This is a requirement for 'setup-envtest.sh' in the test target.
# Options are set to exit when a recipe line exits non-zero or a piped command fails.
SHELL = /usr/bin/env bash -o pipefail
.SHELLFLAGS = -ec


##@ General

# The help target prints out all targets with their descriptions organized
# beneath their categories. The categories are represented by '##@' and the
# target descriptions by '##'. The awk commands is responsible for reading the
# entire set of makefiles included in this invocation, looking for lines of the
# file as xyz: ## something, and then pretty-format the target and help. Then,
# if there's a line with ##@ something, that gets pretty-printed as a category.
# More info on the usage of ANSI control characters for terminal formatting:
# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters
# More info on the awk command:
# http://linuxcommand.org/lc3_adv_awk.php

.PHONY: help
help: ## Display this help.
@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)

##@ E2E Tests

.PHONY: create_module_kubebuilder_project
create_module_kubebuilder_project:
go test --tags=create_module_kubebuilder_project

.PHONY: create_module_module_config
create_module_module_config:
go test --tags=create_module_module_config
Loading

0 comments on commit b0f945c

Please sign in to comment.