From 256fe3d38fd2ba7b1c600cb7e592d44f19660cc6 Mon Sep 17 00:00:00 2001 From: Tomas Tomecek Date: Tue, 10 Oct 2017 14:26:12 +0200 Subject: [PATCH 1/9] add tests Signed-off-by: Tomas Tomecek --- .travis.yml | 10 ++++++ Makefile | 3 ++ tests/test_container.py | 80 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+) create mode 100644 tests/test_container.py diff --git a/.travis.yml b/.travis.yml index d9741e5..ed84c5f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,21 @@ +language: python +python: + - "2.7" + - "3.5" + - "3.6" sudo: required services: - docker +before_install: + - sudo apt-get -y install acl netcat + - pip install docker six xattr + - pip install git+https://github.com/fedora-modularity/conu script: - hooks/pre_build # Docker Hub hack - sudo cp -av ./Dockerfile.template ./Dockerfile - make build + - make test env: - DG_BINARY="docker run -v $(pwd):/var/dgdir slavek/distgen" notifications: diff --git a/Makefile b/Makefile index bf20968..f5e5e0b 100644 --- a/Makefile +++ b/Makefile @@ -45,6 +45,9 @@ run: enumerate-tools: docker run -it -v ${PWD}:/src -e TOOLS_PACKAGES=$(shell $(DG_EXEC) --template="{{spec.packages|join(\",\")}}") --rm $(REPOSITORY) /src/enumerate-tools.py +test: + IMAGE_NAME=$(REPOSITORY) pytest + clean: rm Dockerfile || : rm root/README.md || : diff --git a/tests/test_container.py b/tests/test_container.py new file mode 100644 index 0000000..6b6606d --- /dev/null +++ b/tests/test_container.py @@ -0,0 +1,80 @@ +#!/usr/bin/python3 + +import logging +import os + +import conu + +import pytest + + +IMAGE = os.environ.get("IMAGE_NAME", "docker.io/modularitycontainers/tools") +TAG = os.environ.get("IMAGE_TAG", "latest") + + +class TestContainer: + image = None + container = None + backend = None + + @classmethod + def setup_class(cls): + cls.backend = conu.DockerBackend(logging_level=logging.DEBUG) + cls.image = cls.backend.ImageClass(IMAGE, tag=TAG) + c = conu.DockerRunBuilder(command=["sleep", "infinity"]) + c.options += [ + "--net", "host", + "--pid=host", + "--ipc", "host", + "-it", + "--privileged", + "-v", "/run:/run", + "-v", "/:/host", + "-v", "/var/log:/var/log", + ] + machine_id_path = "/etc/machine-id" + if os.path.exists(machine_id_path): + c.options += [ + "-v", "%s:%s" % (machine_id_path, machine_id_path) + ] + localtime_path = "/etc/localtime" + if os.path.exists(localtime_path): + c.options += [ + "-v", "%s:%s" % (localtime_path, localtime_path) + ] + + cls.container = cls.image.run_via_binary(c) + + def test_ethtool(self): + # with self.container.mount() as fs: + # networks_devices = os.listdir(fs.p("/sys/class/net")) + networks_devices = ["lo"] + for device in networks_devices: + self.container.execute(["ethtool", device]) + with pytest.raises(conu.ConuException): + self.container.execute(["ethtool", "quantum-teleport"]) + + def test_netstat(self): + self.container.execute(["netstat"]) + + def test_ss(self): + self.container.execute(["ss"]) + + def test_pstack(self): + self.container.execute(["pstack", "1"]) + + def test_nstat(self): + self.container.execute(["nstat"]) + + def test_numastat(self): + self.container.execute(["numastat"]) + + def test_pmap(self): + self.container.execute(["pmap", "1"]) + + def test_strace(self): + self.container.execute(["strace", "-V"]) + + +if __name__ == '__main__': + pytest.main() From 7d611bdcafdf19f1c9a4b3c022ec508f181bbfc1 Mon Sep 17 00:00:00 2001 From: Tomas Tomecek Date: Tue, 17 Oct 2017 13:47:35 +0200 Subject: [PATCH 2/9] wip - test inside a container Signed-off-by: Tomas Tomecek --- Dockerfile.tests | 2 ++ Makefile | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Dockerfile.tests diff --git a/Dockerfile.tests b/Dockerfile.tests new file mode 100644 index 0000000..35d8ac6 --- /dev/null +++ b/Dockerfile.tests @@ -0,0 +1,2 @@ +FROM docker.io/modularitycontainers/conu +COPY ./tests /tests diff --git a/Makefile b/Makefile index f5e5e0b..bedc5cf 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ enumerate-tools: docker run -it -v ${PWD}:/src -e TOOLS_PACKAGES=$(shell $(DG_EXEC) --template="{{spec.packages|join(\",\")}}") --rm $(REPOSITORY) /src/enumerate-tools.py test: - IMAGE_NAME=$(REPOSITORY) pytest + docker run --privileged -v /var/run/docker.sock:/var/run/docker.sock -v ${PWD}/tests/:/tests -e IMAGE_NAME=$(REPOSITORY) modularitycontainers/conu clean: rm Dockerfile || : From bd1b8391d7f332c812ad3c8d6db5c5bd95be2f18 Mon Sep 17 00:00:00 2001 From: Tomas Tomecek Date: Wed, 29 Nov 2017 14:54:48 +0100 Subject: [PATCH 3/9] run tests inside CCCP Signed-off-by: Tomas Tomecek --- Makefile | 2 +- cccp.yml | 2 +- requirements.sh | 21 +++++++++++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) create mode 100755 requirements.sh diff --git a/Makefile b/Makefile index bedc5cf..04869f9 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ enumerate-tools: docker run -it -v ${PWD}:/src -e TOOLS_PACKAGES=$(shell $(DG_EXEC) --template="{{spec.packages|join(\",\")}}") --rm $(REPOSITORY) /src/enumerate-tools.py test: - docker run --privileged -v /var/run/docker.sock:/var/run/docker.sock -v ${PWD}/tests/:/tests -e IMAGE_NAME=$(REPOSITORY) modularitycontainers/conu + docker run --privileged -v /var/run/docker.sock:/var/run/docker.sock -v ${PWD}/tests/:/tests -e IMAGE_NAME=$(REPOSITORY) docker.io/modularitycontainers/conu clean: rm Dockerfile || : diff --git a/cccp.yml b/cccp.yml index 594a2c5..81e0d3b 100644 --- a/cccp.yml +++ b/cccp.yml @@ -19,7 +19,7 @@ test-skip : True # This is path of the script that initiates the user defined tests. It must be able to # return an exit code. -test-script : null +test-script : ./requirements.sh && pytest -vv ./tests/ # This is the path of custom build script. build-script : null diff --git a/requirements.sh b/requirements.sh new file mode 100755 index 0000000..6dcc7fa --- /dev/null +++ b/requirements.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e + +if grep "CentOS Linux 7" /etc/os-release >/dev/null; then + cat >/etc/yum.repos.d/virt.repo < Date: Thu, 15 Feb 2018 09:58:42 +0100 Subject: [PATCH 4/9] standard test invocation Signed-off-by: Tomas Tomecek --- .dockerignore | 2 + .gitignore | 2 + Makefile | 8 +- tests/Makefile | 14 +++ tests/ansible.cfg | 3 + tests/integration/runtest.sh | 3 + tests/integration/test_container.py | 79 ++++++++++++++ tests/inventory | 1 + tests/test_container.py | 80 --------------- tests/tests.yml | 153 ++++++++++++++++++++++++++++ 10 files changed, 262 insertions(+), 83 deletions(-) create mode 100644 .dockerignore create mode 100644 tests/Makefile create mode 100644 tests/ansible.cfg create mode 100644 tests/integration/runtest.sh create mode 100644 tests/integration/test_container.py create mode 100644 tests/inventory delete mode 100644 tests/test_container.py create mode 100644 tests/tests.yml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..b8eaa7f --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +./Fedora-Cloud-Base-27-1.6.x86_64.qcow2 +./tests diff --git a/.gitignore b/.gitignore index a201d44..f20c192 100644 --- a/.gitignore +++ b/.gitignore @@ -2,5 +2,7 @@ Dockerfile help/help.md.* root/ +Fedora-Cloud-Base-27-1.6.x86_64.qcow2 + *.py[co] *.swp diff --git a/Makefile b/Makefile index 04869f9..7425371 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build run default enumerate-tools upstream fedora-downstream source +.PHONY: build run default enumerate-tools upstream fedora-downstream source test check DISTRO := fedora-27-x86_64 VARIANT := upstream @@ -45,8 +45,10 @@ run: enumerate-tools: docker run -it -v ${PWD}:/src -e TOOLS_PACKAGES=$(shell $(DG_EXEC) --template="{{spec.packages|join(\",\")}}") --rm $(REPOSITORY) /src/enumerate-tools.py -test: - docker run --privileged -v /var/run/docker.sock:/var/run/docker.sock -v ${PWD}/tests/:/tests -e IMAGE_NAME=$(REPOSITORY) docker.io/modularitycontainers/conu +check: test + +test: build + make -C tests/ check-local IMAGE_NAME=$(REPOSITORY) clean: rm Dockerfile || : diff --git a/tests/Makefile b/tests/Makefile new file mode 100644 index 0000000..07489a4 --- /dev/null +++ b/tests/Makefile @@ -0,0 +1,14 @@ +.PHONY: check-local check-in-vm + +IMAGE_NAME := "" +VM_IMAGE := ../Fedora-Cloud-Base-27-1.6.x86_64.qcow2 + +check-local: + ansible-playbook -e subject=$(IMAGE_NAME) -v ./tests.yml + +check-in-vm: $(VM_IMAGE) + ansible-playbook -e setup=true -e vm_image=$(VM_IMAGE) ./tests.yml + +$(VM_IMAGE): + curl -o $(VM_IMAGE) -s https://download.fedoraproject.org/fedora/releases/27/CloudImages/x86_64/images/$(VM_IMAGE) + diff --git a/tests/ansible.cfg b/tests/ansible.cfg new file mode 100644 index 0000000..26b8557 --- /dev/null +++ b/tests/ansible.cfg @@ -0,0 +1,3 @@ +[defaults] +inventory = ./inventory +retry_files_enabled = false diff --git a/tests/integration/runtest.sh b/tests/integration/runtest.sh new file mode 100644 index 0000000..3a56327 --- /dev/null +++ b/tests/integration/runtest.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +pytest-3 -vv ./test_container.py diff --git a/tests/integration/test_container.py b/tests/integration/test_container.py new file mode 100644 index 0000000..4818355 --- /dev/null +++ b/tests/integration/test_container.py @@ -0,0 +1,79 @@ +#!/usr/bin/python3 + +import logging +import os + +import conu + +import pytest + + +IMAGE = os.environ.get("IMAGE_NAME", "docker.io/modularitycontainers/tools") +TAG = os.environ.get("IMAGE_TAG", "latest") + + +@pytest.fixture(scope="module") +def container(request): + with conu.DockerBackend(logging_level=logging.DEBUG) as backend: + im = backend.ImageClass(IMAGE, tag=TAG) + b = conu.DockerRunBuilder(command=["sleep", "infinity"]) + b.options += [ + "--net", "host", + "--pid=host", + "--ipc", "host", + "-it", + "--privileged", + "-v", "/run:/run", + "-v", "/:/host", + "-v", "/var/log:/var/log", + ] + machine_id_path = "/etc/machine-id" + if os.path.exists(machine_id_path): + b.options += [ + "-v", "%s:%s" % (machine_id_path, machine_id_path) + ] + localtime_path = "/etc/localtime" + if os.path.exists(localtime_path): + b.options += [ + "-v", "%s:%s" % (localtime_path, localtime_path) + ] + container = im.run_via_binary(b) + yield container + container.stop() + container.delete() + + +class TestContainer: + def test_ethtool(self, container): + # with self.container.mount() as fs: + # networks_devices = os.listdir(fs.p("/sys/class/net")) + networks_devices = ["lo"] + for device in networks_devices: + container.execute(["ethtool", device]) + with pytest.raises(conu.ConuException): + container.execute(["ethtool", "quantum-teleport"]) + + def test_netstat(self, container): + container.execute(["netstat"]) + + def test_ss(self, container): + container.execute(["ss"]) + + def test_pstack(self, container): + container.execute(["pstack", "1"]) + + def test_nstat(self, container): + container.execute(["nstat"]) + + def test_numastat(self, container): + container.execute(["numastat"]) + + def test_pmap(self, container): + container.execute(["pmap", "1"]) + + def test_strace(self, container): + container.execute(["strace", "-V"]) + + +if __name__ == '__main__': + pytest.main() diff --git a/tests/inventory b/tests/inventory new file mode 100644 index 0000000..2302eda --- /dev/null +++ b/tests/inventory @@ -0,0 +1 @@ +localhost ansible_connection=local diff --git a/tests/test_container.py b/tests/test_container.py deleted file mode 100644 index 6b6606d..0000000 --- a/tests/test_container.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/python3 - -import logging -import os - -import conu - -import pytest - - -IMAGE = os.environ.get("IMAGE_NAME", "docker.io/modularitycontainers/tools") -TAG = os.environ.get("IMAGE_TAG", "latest") - - -class TestContainer: - image = None - container = None - backend = None - - @classmethod - def setup_class(cls): - cls.backend = conu.DockerBackend(logging_level=logging.DEBUG) - cls.image = cls.backend.ImageClass(IMAGE, tag=TAG) - c = conu.DockerRunBuilder(command=["sleep", "infinity"]) - c.options += [ - "--net", "host", - "--pid=host", - "--ipc", "host", - "-it", - "--privileged", - "-v", "/run:/run", - "-v", "/:/host", - "-v", "/var/log:/var/log", - ] - machine_id_path = "/etc/machine-id" - if os.path.exists(machine_id_path): - c.options += [ - "-v", "%s:%s" % (machine_id_path, machine_id_path) - ] - localtime_path = "/etc/localtime" - if os.path.exists(localtime_path): - c.options += [ - "-v", "%s:%s" % (localtime_path, localtime_path) - ] - - cls.container = cls.image.run_via_binary(c) - - def test_ethtool(self): - # with self.container.mount() as fs: - # networks_devices = os.listdir(fs.p("/sys/class/net")) - networks_devices = ["lo"] - for device in networks_devices: - self.container.execute(["ethtool", device]) - with pytest.raises(conu.ConuException): - self.container.execute(["ethtool", "quantum-teleport"]) - - def test_netstat(self): - self.container.execute(["netstat"]) - - def test_ss(self): - self.container.execute(["ss"]) - - def test_pstack(self): - self.container.execute(["pstack", "1"]) - - def test_nstat(self): - self.container.execute(["nstat"]) - - def test_numastat(self): - self.container.execute(["numastat"]) - - def test_pmap(self): - self.container.execute(["pmap", "1"]) - - def test_strace(self): - self.container.execute(["strace", "-V"]) - - -if __name__ == '__main__': - pytest.main() diff --git a/tests/tests.yml b/tests/tests.yml new file mode 100644 index 0000000..bbe7088 --- /dev/null +++ b/tests/tests.yml @@ -0,0 +1,153 @@ +# support use cases: +# * testing an image built locally in current environment (set pull variable to false) +# * testing an image built locally inside a VM (supply vm_image variable) +# * testing an image present in registry in current environment (set pull to true) +# * testing an image present in registry inside a VM (set pull and vm_image variables) +--- +- hosts: localhost + vars: + # path to the VM image to use + # "" means to run in current environment + vm_image: "" + + # inventory name of the VM + vm_name: "mayfly" + + # which python interpreter should be used by ansible (e.g. Fedora doesn't have /usr/bin/python) + vm_python_interpreter: "/usr/bin/python3" + + # run on localhost by default + target_host: "localhost" + + # don't pull the test subject by default + pull: false + + # don't set up the environment by default (instal and start container runtime) + setup: false + + # path to the script which will provision the VM + # vm_provisioning_script: /usr/share/ansible/inventory/standard-inventory-qcow2 + vm_provisioning_script: /home/tt/g/standard-test-roles/inventory/standard-inventory-qcow2 + + required_packages: + - python3-conu + - python3-pytest + tests: + - integration + + # our test subject + subject: "" + + artifacts_path: "{{ playbook_dir }}/artifacts/" + + tasks: + + - name: provision VM if needed + block: + - name: create lock file to synchronize on the VM + tempfile: + prefix: inventory-cloud + suffix: .lock + state: file + register: tmp_lock_file + - name: provision the VM + command: "{{ vm_provisioning_script }} {{ vm_image }}" + register: vm_provision_data + environment: + LOCK_ON_FILE: "{{ tmp_lock_file.path }}" + # TEST_DEBUG: 1 + - name: prepare inventory data for add_host + set_fact: + inventory_data: '{{ (vm_provision_data.stdout | from_json)._meta.hostvars[vm_image] }}' + # `add_host: "{{ inventory_data }}"` is unsupported :< + - add_host: + name: "{{ vm_name }}" + ansible_ssh_common_args: "{{ inventory_data.ansible_ssh_common_args }}" + ansible_ssh_host: "{{ inventory_data.ansible_ssh_host }}" + ansible_ssh_pass: "{{ inventory_data.ansible_ssh_pass }}" + ansible_ssh_port: "{{ inventory_data.ansible_ssh_port }}" + ansible_ssh_private_key_file: "{{ inventory_data.ansible_ssh_private_key_file }}" + ansible_ssh_user: "{{ inventory_data.ansible_ssh_user }}" + ansible_python_interpreter: "{% if vm_python_interpreter != '' %}{{ vm_python_interpreter }}{% else %}/usr/bin/python2{% endif %}" + - name: set target environment + set_fact: + target_host: "{{ vm_name }}" + - name: Gather facts in the provisioned system + setup: {} + delegate_to: "{{ target_host }}" + delegate_facts: true + when: vm_image != "" + + - name: prepare the environment to run tests + block: + - name: Install the container engine + package: + name: docker + state: present + become: true + - name: Start the container engine + systemd: + name: docker + state: started + become: true + delegate_to: "{{ target_host }}" + when: setup + + - name: Pull the test subject (=container image) + command: docker pull {{ subject }} + delegate_to: "{{ target_host }}" + when: pull + + - name: Copy test subject from host inside the VM + block: + # FIXME: make this configurable + - name: Create temporary directory for the image + tempfile: + state: directory + register: tmp + delegate_to: localhost + - name: Save the image to a file + command: 'docker save -o {{ tmp.path + "/image.tar.gz" }} {{ subject }}' + delegate_to: localhost + # - command: sleep 99999 + # delegate_to: localhost + - name: Copy the image from host to the target + # synchronize is so unreliable + copy: + src: '{{ tmp.path + "/image.tar.gz" }}' + dest: '/' + - name: Load the image on the target into dockerd + command: 'docker load -i /image.tar.gz' + - file: + state: absent + path: "{{ tmp.path }}" + delegate_to: localhost + when: not pull and vm_image != "" + delegate_to: "{{ target_host }}" + + - name: Create temp dir to store tests + tempfile: + state: directory + register: tmp_tests + delegate_to: "{{ target_host }}" + + - block: + - file: + state: directory + path: "{{ artifacts_path }}" + - import_role: + name: standard-test-basic + vars: + tenv_workdir: "{{ tmp_tests.path }}" + artifacts: "{{ artifacts_path }}" + delegate_to: "{{ target_host }}" + always: + - name: delete the vm lock file + file: + path: "{{ tmp_lock_file.path }}" + state: absent + when: vm_image != "" + - name: delete the temp dir + file: + path: "{{ tmp_tests.path }}" + state: absent From 6ef1bd80e051002e91215832a8e0af002eafb76e Mon Sep 17 00:00:00 2001 From: Tomas Tomecek Date: Tue, 20 Feb 2018 18:05:02 +0100 Subject: [PATCH 5/9] enforce creating root/ Signed-off-by: Tomas Tomecek --- Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7425371..a7a2686 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,9 @@ RENDERED_DOCKERFILE_MD := ./Dockerfile default: run +root/: + mkdir -p ./root + $(RENDERED_DOCKERFILE_MD): $(SOURCE_DOCKERFILE_MD) $(DG_EXEC) --template $(SOURCE_DOCKERFILE_MD) --output $(RENDERED_DOCKERFILE_MD) @@ -28,7 +31,7 @@ $(RENDERED_HELP_MD): $(SOURCE_HELP_MD) specs/multispec.yaml @# go-md2man -in=${SOURCE_HELP_MD} -out=./root/help.1 $(shell TOOLS_CONTAINER_SKIP_ENUMERATION=false $(DG_EXEC) --template $(SOURCE_HELP_MD) --output $(RENDERED_HELP_MD)) -source: $(RENDERED_HELP_MD) $(RENDERED_README_MD) $(RENDERED_DOCKERFILE_MD) +source: root/ $(RENDERED_HELP_MD) $(RENDERED_README_MD) $(RENDERED_DOCKERFILE_MD) fedora-downstream: make -e source VARIANT="fedora" From 1d574af43240e1a1d23db146fa36dd957e1f6fa7 Mon Sep 17 00:00:00 2001 From: Tomas Tomecek Date: Tue, 20 Feb 2018 18:05:22 +0100 Subject: [PATCH 6/9] standard test invocation: take 2 --- .gitignore | 3 +- Makefile | 9 ++- tests/Makefile | 14 +++-- tests/ansible.cfg | 1 + tests/in-vm.yml | 82 +++++++++++++++++++++++++ tests/local.yml | 61 ++++++++++++++++++ tests/tests.yml | 153 ---------------------------------------------- 7 files changed, 160 insertions(+), 163 deletions(-) create mode 100644 tests/in-vm.yml create mode 100644 tests/local.yml delete mode 100644 tests/tests.yml diff --git a/.gitignore b/.gitignore index f20c192..731fac7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,9 @@ Dockerfile help/help.md.* root/ +tests/artifacts/ -Fedora-Cloud-Base-27-1.6.x86_64.qcow2 +*.qcow2 *.py[co] *.swp diff --git a/Makefile b/Makefile index a7a2686..85369bb 100644 --- a/Makefile +++ b/Makefile @@ -20,13 +20,13 @@ default: run root/: mkdir -p ./root -$(RENDERED_DOCKERFILE_MD): $(SOURCE_DOCKERFILE_MD) +$(RENDERED_DOCKERFILE_MD): $(SOURCE_DOCKERFILE_MD) specs/* $(DG_EXEC) --template $(SOURCE_DOCKERFILE_MD) --output $(RENDERED_DOCKERFILE_MD) -$(RENDERED_README_MD): $(SOURCE_README_MD) +$(RENDERED_README_MD): $(SOURCE_README_MD) specs/* $(DG_EXEC) --template $(SOURCE_README_MD) --output $(RENDERED_README_MD) -$(RENDERED_HELP_MD): $(SOURCE_HELP_MD) specs/multispec.yaml +$(RENDERED_HELP_MD): $(SOURCE_HELP_MD) specs/* @# FIXME: current go-md2man can't convert tables :< @# go-md2man -in=${SOURCE_HELP_MD} -out=./root/help.1 $(shell TOOLS_CONTAINER_SKIP_ENUMERATION=false $(DG_EXEC) --template $(SOURCE_HELP_MD) --output $(RENDERED_HELP_MD)) @@ -53,6 +53,9 @@ check: test test: build make -C tests/ check-local IMAGE_NAME=$(REPOSITORY) +check-in-vm: build + make -C tests/ check-in-vm IMAGE_NAME=$(REPOSITORY) + clean: rm Dockerfile || : rm root/README.md || : diff --git a/tests/Makefile b/tests/Makefile index 07489a4..489711e 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,14 +1,16 @@ .PHONY: check-local check-in-vm IMAGE_NAME := "" -VM_IMAGE := ../Fedora-Cloud-Base-27-1.6.x86_64.qcow2 +VM_IMAGE_NAME := Fedora-Cloud-Base-27-1.6.x86_64.qcow2 +VM_IMAGE_PATH = ../$(VM_IMAGE_NAME) +INVENTORY := /usr/share/ansible/inventory/standard-inventory-qcow2 check-local: - ansible-playbook -e subject=$(IMAGE_NAME) -v ./tests.yml + ansible-playbook -e subject=$(IMAGE_NAME) ./local.yml -check-in-vm: $(VM_IMAGE) - ansible-playbook -e setup=true -e vm_image=$(VM_IMAGE) ./tests.yml +check-in-vm: $(VM_IMAGE_PATH) + TEST_SUBJECTS=$(VM_IMAGE_PATH) ansible-playbook -e ansible_python_interpreter=/usr/bin/python3 -e subject=$(IMAGE_NAME) -i $(INVENTORY) -e setup=true -e vm_image=$(VM_IMAGE) ./in-vm.yml -$(VM_IMAGE): - curl -o $(VM_IMAGE) -s https://download.fedoraproject.org/fedora/releases/27/CloudImages/x86_64/images/$(VM_IMAGE) +$(VM_IMAGE_PATH): + curl -o $(VM_IMAGE_PATH) -s https://download.fedoraproject.org/fedora/releases/27/CloudImages/x86_64/images/$(VM_IMAGE_NAME) diff --git a/tests/ansible.cfg b/tests/ansible.cfg index 26b8557..1cf0c4d 100644 --- a/tests/ansible.cfg +++ b/tests/ansible.cfg @@ -1,3 +1,4 @@ [defaults] inventory = ./inventory retry_files_enabled = false +# roles_path = /roles diff --git a/tests/in-vm.yml b/tests/in-vm.yml new file mode 100644 index 0000000..98f89f4 --- /dev/null +++ b/tests/in-vm.yml @@ -0,0 +1,82 @@ +# support use cases: +# * testing an image built locally in current environment (set pull variable to false) +# * testing an image built locally inside a VM (supply vm_image variable) +# * testing an image present in registry in current environment (set pull to true) +# * testing an image present in registry inside a VM (set pull and vm_image variables) +--- +- name: Integration tests for tools container image executed in current environment + hosts: localhost + vars: + # don't pull the test subject by default + pull: false + + # don't set up the environment by default (instal and start container runtime) + setup: false + + required_packages: + - python3-conu + - python3-pytest + + # tests to be invoked (this is utilized by basic standard test role) + tests: + # test suites = directories, where the tests live + - integration + + # our test subject + subject: "" + + # path where the test artifacts will be stored - logs + artifacts: "{{ playbook_dir }}/artifacts/" + + tasks: + - name: prepare the environment to run tests + block: + - name: Install the container engine + package: + name: docker + state: present + become: true + - name: Start the container engine + systemd: + name: docker + state: started + become: true + when: setup + + - name: Pull the test subject (=container image) + command: docker pull {{ subject }} + when: pull + + - name: Copy test subject from host inside the VM + block: + # FIXME: make this configurable + - name: Create temporary directory for the image + tempfile: + state: directory + register: tmp + - name: Save the image to a file + command: 'docker save -o {{ tmp.path + "/image.tar.gz" }} {{ subject }}' + - name: Copy the image from host to the target + # synchronize is so unreliable + synchronize: + src: '{{ tmp.path + "/image.tar.gz" }}' + dest: '/tmp/' + mode: push + ssh_args: "-o UserKnownHostsFile=/dev/null -i {{ ansible_ssh_private_key_file }}" + - file: + state: absent + path: "{{ tmp.path }}" + when: not pull + delegate_to: localhost + + - block: + - name: Load the image on the target into dockerd + command: 'docker load -i /tmp/image.tar.gz' + - file: + state: absent + path: "/tmp/image.tar.gz" + when: not pull + + - name: Execute the role which performs testing + import_role: + name: standard-test-basic diff --git a/tests/local.yml b/tests/local.yml new file mode 100644 index 0000000..c2c0045 --- /dev/null +++ b/tests/local.yml @@ -0,0 +1,61 @@ +# support use cases: +# * testing an image built locally in current environment (set pull variable to false) +# * testing an image present in registry in current environment (set pull to true) +--- +- name: Integration tests for tools container image executed in current environment + hosts: localhost + vars: + # don't pull the test subject by default + pull: false + + # don't set up the environment by default (install and start container runtime) + setup: false + + required_packages: + - python3-conu + - python3-pytest + + tests: + - integration + + # our test subject + subject: "" + + # path where the test artifacts will be stored - logs + artifacts: "{{ playbook_dir }}/artifacts/" + + tasks: + - name: prepare the environment to run tests + block: + - name: Install the container engine + package: + name: docker + state: present + become: true + - name: Start the container engine + systemd: + name: docker + state: started + become: true + when: setup + + - name: Pull the test subject (=container image) + command: docker pull {{ subject }} + when: pull + + - block: + # should this be configurable? + - name: Create temp dir to store tests + tempfile: + state: directory + register: tmp_tests + - name: Execute the role which performs testing + import_role: + name: standard-test-basic + vars: + tenv_workdir: "{{ tmp_tests.path }}" + always: + - name: delete the temp dir + file: + path: "{{ tmp_tests.path }}" + state: absent diff --git a/tests/tests.yml b/tests/tests.yml deleted file mode 100644 index bbe7088..0000000 --- a/tests/tests.yml +++ /dev/null @@ -1,153 +0,0 @@ -# support use cases: -# * testing an image built locally in current environment (set pull variable to false) -# * testing an image built locally inside a VM (supply vm_image variable) -# * testing an image present in registry in current environment (set pull to true) -# * testing an image present in registry inside a VM (set pull and vm_image variables) ---- -- hosts: localhost - vars: - # path to the VM image to use - # "" means to run in current environment - vm_image: "" - - # inventory name of the VM - vm_name: "mayfly" - - # which python interpreter should be used by ansible (e.g. Fedora doesn't have /usr/bin/python) - vm_python_interpreter: "/usr/bin/python3" - - # run on localhost by default - target_host: "localhost" - - # don't pull the test subject by default - pull: false - - # don't set up the environment by default (instal and start container runtime) - setup: false - - # path to the script which will provision the VM - # vm_provisioning_script: /usr/share/ansible/inventory/standard-inventory-qcow2 - vm_provisioning_script: /home/tt/g/standard-test-roles/inventory/standard-inventory-qcow2 - - required_packages: - - python3-conu - - python3-pytest - tests: - - integration - - # our test subject - subject: "" - - artifacts_path: "{{ playbook_dir }}/artifacts/" - - tasks: - - - name: provision VM if needed - block: - - name: create lock file to synchronize on the VM - tempfile: - prefix: inventory-cloud - suffix: .lock - state: file - register: tmp_lock_file - - name: provision the VM - command: "{{ vm_provisioning_script }} {{ vm_image }}" - register: vm_provision_data - environment: - LOCK_ON_FILE: "{{ tmp_lock_file.path }}" - # TEST_DEBUG: 1 - - name: prepare inventory data for add_host - set_fact: - inventory_data: '{{ (vm_provision_data.stdout | from_json)._meta.hostvars[vm_image] }}' - # `add_host: "{{ inventory_data }}"` is unsupported :< - - add_host: - name: "{{ vm_name }}" - ansible_ssh_common_args: "{{ inventory_data.ansible_ssh_common_args }}" - ansible_ssh_host: "{{ inventory_data.ansible_ssh_host }}" - ansible_ssh_pass: "{{ inventory_data.ansible_ssh_pass }}" - ansible_ssh_port: "{{ inventory_data.ansible_ssh_port }}" - ansible_ssh_private_key_file: "{{ inventory_data.ansible_ssh_private_key_file }}" - ansible_ssh_user: "{{ inventory_data.ansible_ssh_user }}" - ansible_python_interpreter: "{% if vm_python_interpreter != '' %}{{ vm_python_interpreter }}{% else %}/usr/bin/python2{% endif %}" - - name: set target environment - set_fact: - target_host: "{{ vm_name }}" - - name: Gather facts in the provisioned system - setup: {} - delegate_to: "{{ target_host }}" - delegate_facts: true - when: vm_image != "" - - - name: prepare the environment to run tests - block: - - name: Install the container engine - package: - name: docker - state: present - become: true - - name: Start the container engine - systemd: - name: docker - state: started - become: true - delegate_to: "{{ target_host }}" - when: setup - - - name: Pull the test subject (=container image) - command: docker pull {{ subject }} - delegate_to: "{{ target_host }}" - when: pull - - - name: Copy test subject from host inside the VM - block: - # FIXME: make this configurable - - name: Create temporary directory for the image - tempfile: - state: directory - register: tmp - delegate_to: localhost - - name: Save the image to a file - command: 'docker save -o {{ tmp.path + "/image.tar.gz" }} {{ subject }}' - delegate_to: localhost - # - command: sleep 99999 - # delegate_to: localhost - - name: Copy the image from host to the target - # synchronize is so unreliable - copy: - src: '{{ tmp.path + "/image.tar.gz" }}' - dest: '/' - - name: Load the image on the target into dockerd - command: 'docker load -i /image.tar.gz' - - file: - state: absent - path: "{{ tmp.path }}" - delegate_to: localhost - when: not pull and vm_image != "" - delegate_to: "{{ target_host }}" - - - name: Create temp dir to store tests - tempfile: - state: directory - register: tmp_tests - delegate_to: "{{ target_host }}" - - - block: - - file: - state: directory - path: "{{ artifacts_path }}" - - import_role: - name: standard-test-basic - vars: - tenv_workdir: "{{ tmp_tests.path }}" - artifacts: "{{ artifacts_path }}" - delegate_to: "{{ target_host }}" - always: - - name: delete the vm lock file - file: - path: "{{ tmp_lock_file.path }}" - state: absent - when: vm_image != "" - - name: delete the temp dir - file: - path: "{{ tmp_tests.path }}" - state: absent From 739b155f053270e95b9a85b7036e1d8cb4eb715d Mon Sep 17 00:00:00 2001 From: Tomas Tomecek Date: Mon, 26 Feb 2018 11:27:04 +0100 Subject: [PATCH 7/9] make: implement DEBUG_MODE to ease ansible debugging Signed-off-by: Tomas Tomecek --- Makefile | 9 +++++++-- tests/Makefile | 5 +++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 85369bb..2792edd 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,11 @@ DISTRO := fedora-27-x86_64 VARIANT := upstream DG_BINARY ?= dg DG_EXEC = $(DG_BINARY) --max-passes 25 --spec specs/common.yml --multispec specs/multispec.yaml --distro $(DISTRO).yaml --multispec-selector variant=$(VARIANT) +# set to 1 to enable debugging +DEBUG_MODE ?= 0 +ifeq ($(DEBUG_MODE), 1) + ANSIBLE_EXTRA_ARGS := -vv +endif REPOSITORY = $(shell ${DG_EXEC} --template={{spec.repository}}) @@ -51,10 +56,10 @@ enumerate-tools: check: test test: build - make -C tests/ check-local IMAGE_NAME=$(REPOSITORY) + make -C tests/ check-local IMAGE_NAME=$(REPOSITORY) ANSIBLE_EXTRA_ARGS=$(ANSIBLE_EXTRA_ARGS) check-in-vm: build - make -C tests/ check-in-vm IMAGE_NAME=$(REPOSITORY) + make -C tests/ check-in-vm IMAGE_NAME=$(REPOSITORY) ANSIBLE_EXTRA_ARGS=$(ANSIBLE_EXTRA_ARGS) clean: rm Dockerfile || : diff --git a/tests/Makefile b/tests/Makefile index 489711e..efa9c1b 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -4,12 +4,13 @@ IMAGE_NAME := "" VM_IMAGE_NAME := Fedora-Cloud-Base-27-1.6.x86_64.qcow2 VM_IMAGE_PATH = ../$(VM_IMAGE_NAME) INVENTORY := /usr/share/ansible/inventory/standard-inventory-qcow2 +ANSIBLE_EXTRA_ARGS ?= check-local: - ansible-playbook -e subject=$(IMAGE_NAME) ./local.yml + ansible-playbook $(ANSIBLE_EXTRA_ARGS) -e subject=$(IMAGE_NAME) ./local.yml check-in-vm: $(VM_IMAGE_PATH) - TEST_SUBJECTS=$(VM_IMAGE_PATH) ansible-playbook -e ansible_python_interpreter=/usr/bin/python3 -e subject=$(IMAGE_NAME) -i $(INVENTORY) -e setup=true -e vm_image=$(VM_IMAGE) ./in-vm.yml + TEST_SUBJECTS=$(VM_IMAGE_PATH) ansible-playbook $(ANSIBLE_EXTRA_ARGS) -e ansible_python_interpreter=/usr/bin/python3 -e subject=$(IMAGE_NAME) -i $(INVENTORY) -e setup=true -e vm_image=$(VM_IMAGE) ./in-vm.yml $(VM_IMAGE_PATH): curl -o $(VM_IMAGE_PATH) -s https://download.fedoraproject.org/fedora/releases/27/CloudImages/x86_64/images/$(VM_IMAGE_NAME) From f063cb565df03d68a0aa9db47591ebd29d66c3d6 Mon Sep 17 00:00:00 2001 From: Tomas Tomecek Date: Mon, 26 Feb 2018 12:54:23 +0100 Subject: [PATCH 8/9] tests running in travis Signed-off-by: Tomas Tomecek --- .travis.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index ed84c5f..ebc47f3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,22 +1,17 @@ language: python python: - - "2.7" - "3.5" - - "3.6" sudo: required services: - docker before_install: - - sudo apt-get -y install acl netcat - - pip install docker six xattr - - pip install git+https://github.com/fedora-modularity/conu + - sudo apt-get -y install acl python3-xattr python3-jinja2 ansible + - pip install xattr conu distgen script: - hooks/pre_build # Docker Hub hack - sudo cp -av ./Dockerfile.template ./Dockerfile - make build - make test -env: - - DG_BINARY="docker run -v $(pwd):/var/dgdir slavek/distgen" notifications: email: false From 887f26a66317753d222ac0f320eda60e46ddd050 Mon Sep 17 00:00:00 2001 From: Tomas Tomecek Date: Mon, 5 Mar 2018 11:59:30 +0100 Subject: [PATCH 9/9] tests/make: correct obtaining VM image Signed-off-by: Tomas Tomecek --- tests/Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index efa9c1b..32ce255 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -13,5 +13,4 @@ check-in-vm: $(VM_IMAGE_PATH) TEST_SUBJECTS=$(VM_IMAGE_PATH) ansible-playbook $(ANSIBLE_EXTRA_ARGS) -e ansible_python_interpreter=/usr/bin/python3 -e subject=$(IMAGE_NAME) -i $(INVENTORY) -e setup=true -e vm_image=$(VM_IMAGE) ./in-vm.yml $(VM_IMAGE_PATH): - curl -o $(VM_IMAGE_PATH) -s https://download.fedoraproject.org/fedora/releases/27/CloudImages/x86_64/images/$(VM_IMAGE_NAME) - + curl -L -o $(VM_IMAGE_PATH) https://download.fedoraproject.org/pub/fedora/linux/releases/27/CloudImages/x86_64/images/$(VM_IMAGE_NAME)