Getting rid of "centos_swan_image" (#658)

Author: iwankgb

integration_tests/experiments/memcached-sensitivity-profile/experiment_test.go

@@ -138,7 +138,6 @@ func TestExperiment(t *testing.T) {
 		"SWAN_MUTILATE_AGENT_CONNECTIONS":          "1",
 		"SWAN_MUTILATE_AGENT_AFFINITY":             "false",
 		"SWAN_MUTILATE_MASTER_AFFINITY":            "false",
-		"SWAN_KUBERNETES_CONTAINER_IMAGE":          "centos_swan_image",
 	}
 
 	Convey("With environment prepared for experiment", t, func() {

integration_tests/pkg/executor/kubernetes_test.go

@@ -43,7 +43,6 @@ func TestKubernetesExecutor(t *testing.T) {
 	// Pod executuor config.
 	executorConfig := executor.DefaultKubernetesConfig()
 	executorConfig.Address = fmt.Sprintf("http://127.0.0.1:%d", config.KubeAPIPort)
-	executorConfig.ContainerImage = "centos_swan_image"
 
 	// Create kubectl helper for communicate with Kubernetes cluster.
 	kubectl, err := testhelpers.NewKubeClient(executorConfig)

integration_tests/pkg/snap/sessions/docker_test.go

@@ -62,7 +62,6 @@ func TestSnapDockerSession(t *testing.T) {
 
 		// Waiting for Kubernetes Executor.
 		kubernetesConfig := executor.DefaultKubernetesConfig()
-		kubernetesConfig.ContainerImage = "centos_swan_image"
 		kubernetesConfig.Address = fmt.Sprintf("127.0.0.1:%d", config.KubeAPIPort)
 		kubeExecutor, err := executor.NewKubernetes(kubernetesConfig)
 		So(err, ShouldBeNil)

pkg/executor/kubernetes_test.go

@@ -34,7 +34,6 @@ func TestKubernetes(t *testing.T) {
 
 	Convey("After create new pod object", t, func() {
 		config := DefaultKubernetesConfig()
-		config.ContainerImage = "centos_swan_image"
 
 		Convey("with default unspecified resources, expect BestEffort", func() {
 			podExecutor := &k8s{config, nil}

vagrant/README.md

@@ -44,16 +44,7 @@ You will need to build [mutilate](https://github.com/leverich/mutilate) by hand
 sudo ln -svf /opt/swan/bin/* /usr/bin/
 ```
 
-Now you should be able to run an experiment on the Kubernetes cluster that was automatically provisioned. If you want to be able to run without Kuberenetes, then you will need to take a few more steps.
-
-Some of the project dependencies are distributed as [Docker image](https://hub.docker.com/r/intelsdi/swan/). They need to be extracted from the container in order to be used for non-Kubernetes experiments. To do this you need to execute the following commands:
-
-```sh
-make extract_binaries
-sudo chown -R $USER:$USER opt 
-sudo cp -fa opt/swan /opt
-sudo ln -svf /opt/swan/bin/* /usr/bin/
-```
+Now you should be able to run an experiment on the Kubernetes cluster or your virtual machine.
 
 If you want to be able to use [iBench](https://github.com/stanford-mast/iBench) they you will need to compile binaries and make then available in `$PATH` (see mutilate description above). Keep in mind that compiling iBench binaries may require a lot of RAM and can't be done on default Vagrant VM configuration.
 
@@ -65,11 +56,11 @@ Vagrant will allocate 2 CPUs and 4096 MB RAM for the VM by default. You can cons
 
 ## Deeper dive
 
-If you want to learn more about VM configuration and installed packages refer to [provisioning script](provision.sh).
+If you want to learn more about VM configuration and installed packages refer to [provisioning script](provision.sh). It calls three other scripts and their names should be self explanatory.
 
 Note that the `~/.glide` directory from your host will be mounted on the VM to speed up Go dependency management.
 
-The script is responsible for:
+The scripts are responsible for:
 * Installing all the necessary CentOS packages that are needed to build Swan, run experiments and analyse their results.
 * Installing [Snap](http://snap-telemetry.io/) and its plugins that are responsible for gathering experiment results.
 * Installing [Docker](https://www.docker.com/) that allows running experiment on [Kubernetes](https://kubernetes.io) cluster.
@@ -78,6 +69,12 @@ The script is responsible for:
 * Setting up SSH for root.
 * Installing [Go](https://golang.org/).
 
+If you wish to setup experiment environment on another host then you should be able to run [provision_experiment_environment.sh](provision_experiment_environment.sh) on the host. You will need to provide following environmental variables when calling the script:
+* `SWAN_USER` - name of the user that will run experiments.
+* `HOME_DIR` - home directory of `SWAN_USER`
+
+Example call can be found in [the provisioning script](provision.sh).
+
 ### Provisioners
 
 There are two provisioners defined in the [Vagrantfile](Vagrantfile): `aws` and `virtualbox`. Our CI infracture uses the first of them while the second should be used for development. If you try to use `aws` provider on your own it will fail as AMI is not publicly available.

vagrant/provision_ci_environment.sh

@@ -46,32 +46,15 @@ if [ -e "$HOME_DIR/swan_s3_creds/.s3cfg" ]; then
         chown -R ${VAGRANT_USER}:${VAGRANT_USER} ${HOME_DIR}/.glide
     fi
 
-    echo "Synchronize /opt/swan"
-    # For manuall installtion
-    # sudo sh -c "mkdir -p /opt/swan && chown -R $USER:$USER /opt/swan"
-    s3cmd sync s3://swan-artifacts/workloads/swan/ /opt/swan/
 
-    echo "Install binaries from /opt/swan"
-    ln -sv ${SWAN_BIN}/* /bin/
-
-    echo "Download centos_swan_image docker image"
-    s3cmd sync s3://swan-artifacts/workloads/centos_swan_image.tgz /tmp/centos_swan_image.tgz
-
-    
-    echo "Import centos_swan_image to docker"
-    gunzip /tmp/centos_swan_image.tgz 
-    docker image load -i /tmp/centos_swan_image.tar 
-    docker images centos_swan_image
-
-    echo "Check centos_swan_image docker"
-    docker run --rm centos_swan_image memcached -V
-    docker run --rm centos_swan_image mutilate --version
-    docker run --rm centos_swan_image caffe.sh --version
+    echo "Downloading private dependencies"
+    s3cmd cp --recursive s3://swan-artifacts/workloads/swan-private/  s3://swan-artifacts/workloads/swan/
 fi
 echo "--------------------------- Provisioning CI environment done (`date`)"
 
 echo "--------------------------- Post install (`date`)"
-ln -sf $HOME_DIR/go/src/github.com/intelsdi-x/swan $HOME_DIR
+ln -svf $HOME_DIR/go/src/github.com/intelsdi-x/swan $HOME_DIR
+ln -svf /opt/swan/bin/* /bin/
 chown -R $SWAN_USER:$SWAN_USER $HOME_DIR
 chown -R $SWAN_USER:$SWAN_USER /opt/swan
 

vagrant/provision_experiment_environment.sh

@@ -99,8 +99,12 @@ wget --no-verbose https://github.com/intelsdi-x/swan/releases/download/${SWAN_VE
 tar -xzf /tmp/swan.tar.gz -C ${SWAN_BIN}
 
 
-echo "----------------------------- Install Swan Workloads(`date`)"
-sudo docker run -v /opt:/output intelsdi/swan cp -R /opt/swan /output
+echo "----------------------------- Pulling docker image (`date`)"
+docker pull intelsdi/swan
+
+
+echo "----------------------------- Retrieve binares from Docker container (`date`)"
+docker run --rm -v /opt:/output intelsdi/swan cp -R /opt/swan /output
 
 
 echo "----------------------------- Install Kubernetes (`date`)"