Merge pull request #1 from psarna/zpp_kafka

Merge changes from origin into the fork (branch zpp_kafka)

scripts/kafkadev_local/.gitignore

@@ -0,0 +1,3 @@
+*.tfstate
+*.tfstate.backup
+.terraform/

scripts/kafkadev_local/README.md

@@ -0,0 +1,94 @@
+# Kafkadev (local)
+
+Dev/test local deployment of Kafka (and Zookeeper) using Docker.
+
+## Dependencies
+- Docker
+- Terraform v0.12
+
+### Installing Docker
+<pre>
+# Install and start Docker
+<b>
+$ sudo dnf install docker
+$ sudo systemctl start docker
+</b>
+# Add current user to docker group
+<b>
+$ sudo groupadd docker
+$ sudo usermod -aG docker $USER
+$ newgrp docker 
+</b></pre>
+
+### Installing Terraform
+<pre><b>$ wget https://releases.hashicorp.com/terraform/0.12.12/terraform_0.12.12_linux_amd64.zip
+$ unzip terraform_0.12.12_linux_amd64.zip
+$ sudo cp terraform /usr/bin/
+</b></pre>
+
+## Usage
+### Starting cluster
+The following example creates a Kafka cluster with 5 brokers and a configured Zookeeper instance in the `172.13.0.0/16` network. 
+
+The IP addresses of started Kafka brokers are printed as `kafka_addr` and the names of Docker containers are exported as `kafka_name`.
+
+Run the commands in the `kafkadev_local` directory:
+
+<pre>
+<b>$ terraform init</b>
+<b>$ terraform apply</b>
+
+var.kafka_count
+  The number of started Kafka brokers.
+
+  Enter a value: <b>5</b>
+
+var.network_cidr
+  The IPv4 network prefix for started containers, written in CIDR format, e.g. 172.13.0.0/16.
+
+  Enter a value: <b>172.13.0.0/16</b>
+
+[...]
+
+Do you want to perform these actions?
+  Terraform will perform the actions described above.
+  Only 'yes' will be accepted to approve.
+
+  Enter a value: <b>yes</b>
+
+[...]
+
+Apply complete! Resources: 10 added, 0 changed, 0 destroyed.
+
+Outputs:
+<b>
+kafka_addr = [
+  "172.13.0.1",
+  "172.13.0.2",
+  "172.13.0.3",
+  "172.13.0.4",
+  "172.13.0.5",
+]
+kafka_name = [
+  "kafkadev_6b34_kafka1",
+  "kafkadev_6b34_kafka2",
+  "kafkadev_6b34_kafka3",
+  "kafkadev_6b34_kafka4",
+  "kafkadev_6b34_kafka5",
+]</b>
+zookeeper_addr = 172.13.0.6
+zookeeper_name = kafkadev_6b34_zookeeper
+</pre>
+### Stopping cluster
+Use the following command to stop and remove the Kafka cluster:
+
+<pre>
+<b>$ terraform destroy</b>
+[...]
+
+Do you really want to destroy all resources?
+  Terraform will destroy all your managed infrastructure, as shown above.
+  There is no undo. Only 'yes' will be accepted to confirm.
+
+  Enter a value: <b>yes</b>
+</pre>

scripts/kafkadev_local/config.tf

@@ -0,0 +1,17 @@
+provider "docker" {
+  host = "unix:///var/run/docker.sock"
+}
+
+variable "network_cidr" {
+  type        = string
+  description = "The IPv4 network prefix for started containers, written in CIDR format, e.g. 172.13.0.0/16."
+}
+
+variable "kafka_count" {
+  type        = number
+  description = "The number of started Kafka brokers."
+}
+
+resource "random_id" "deployment_id" {
+  byte_length = 2
+}

scripts/kafkadev_local/kafka.tf

@@ -0,0 +1,39 @@
+data "docker_registry_image" "kafka" {
+  name = "wurstmeister/kafka:latest"
+}
+
+resource "docker_image" "kafka" {
+  name          = "docker.io/${data.docker_registry_image.kafka.name}"
+  pull_triggers = [data.docker_registry_image.kafka.sha256_digest]
+  keep_locally  = true
+}
+
+resource "docker_container" "kafka" {
+  count = var.kafka_count
+
+  name  = "kafkadev_${random_id.deployment_id.hex}_kafka${count.index + 1}"
+  image = docker_image.kafka.latest
+
+  env = [
+    "KAFKA_ADVERTISED_PORT=9092",
+    "KAFKA_ADVERTISED_HOST_NAME=${cidrhost(var.network_cidr, count.index + 1)}",
+    "KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://${cidrhost(var.network_cidr, count.index + 1)}:9092",
+    "KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT",
+    "KAFKA_ZOOKEEPER_CONNECT=${docker_container.zookeeper.network_data[0].ip_address}:2181"
+  ]
+
+  networks_advanced {
+    name         = docker_network.network.name
+    ipv4_address = cidrhost(var.network_cidr, count.index + 1)
+  }
+}
+
+output "kafka_addr" {
+  value       = docker_container.kafka[*].network_data[0].ip_address
+  description = "The IP addresses of started Kafka brokers."
+}
+
+output "kafka_name" {
+  value       = docker_container.kafka[*].name
+  description = "The names of Kafka Docker containers."
+}

scripts/kafkadev_local/network.tf

@@ -0,0 +1,9 @@
+resource "docker_network" "network" {
+  name   = "kafkadev_${random_id.deployment_id.hex}_network"
+  driver = "bridge"
+
+  ipam_config {
+    subnet  = var.network_cidr
+    gateway = cidrhost(var.network_cidr, var.kafka_count + 2)
+  }
+}

scripts/kafkadev_local/zookeeper.tf

@@ -0,0 +1,29 @@
+data "docker_registry_image" "zookeeper" {
+  name = "wurstmeister/zookeeper:latest"
+}
+
+resource "docker_image" "zookeeper" {
+  name          = "docker.io/${data.docker_registry_image.zookeeper.name}"
+  pull_triggers = [data.docker_registry_image.zookeeper.sha256_digest]
+  keep_locally  = true
+}
+
+resource "docker_container" "zookeeper" {
+  name  = "kafkadev_${random_id.deployment_id.hex}_zookeeper"
+  image = docker_image.zookeeper.latest
+
+  networks_advanced {
+    name         = docker_network.network.name
+    ipv4_address = cidrhost(var.network_cidr, var.kafka_count + 1)
+  }
+}
+
+output "zookeeper_addr" {
+  value       = docker_container.zookeeper.network_data[0].ip_address
+  description = "The IP address of Zookeeper."
+}
+
+output "zookeeper_name" {
+  value       = docker_container.zookeeper.name
+  description = "The name of Zookeeper Docker containers."
+}