Skip to content

Commit 87c56c3

Browse files
robbiezhangrobbiezhang
committed
Private Commit for Azure Console Shell
Remove SPN secrets from agent node Remove the Kube Dashboard and Heapster Addons Add agentpool label on the agent nodes Use static IP address for system and agentpool1
1 parent cb47749 commit 87c56c3

15 files changed

+225
-218
lines changed

‎parts/kubernetesagentcustomdata.yml

+3-4
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@ write_files:
121121
encoding: gzip
122122
owner: "root"
123123
content: !!binary |
124-
KUBELET_SERVICE_B64_GZIP_STR
124+
KUBELET_SERVICE_AGENT_B64_GZIP_STR
125125

126126
- path: "/opt/azure/containers/kubelet.sh"
127127
permissions: "0755"
@@ -136,7 +136,7 @@ write_files:
136136
encoding: gzip
137137
owner: "root"
138138
content: !!binary |
139-
{{WrapAsVariable "provisionScript"}}
139+
{{WrapAsVariable "agentProvisionScript"}}
140140

141141
runcmd:
142142
- apt-get update
@@ -153,5 +153,4 @@ runcmd:
153153
- apt-get install -y docker-engine
154154
- systemctl restart docker
155155
- mkdir -p /etc/kubernetes/manifests
156-
- usermod -aG docker {{WrapAsVariable "username"}}
157-
156+
- usermod -aG docker {{WrapAsVariable "username"}}

‎parts/kubernetesagentcustomscript.sh

+98
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
#!/bin/bash
2+
3+
###########################################################
4+
# START SECRET DATA - ECHO DISABLED
5+
###########################################################
6+
7+
# Fields for `azure.json`
8+
KUBELET_PRIVATE_KEY="${1}"
9+
NETWORK_POLICY="${2}"
10+
11+
KUBELET_PRIVATE_KEY_PATH="/etc/kubernetes/certs/client.key"
12+
touch "${KUBELET_PRIVATE_KEY_PATH}"
13+
chmod 0644 "${KUBELET_PRIVATE_KEY_PATH}"
14+
chown root:root "${KUBELET_PRIVATE_KEY_PATH}"
15+
echo "${KUBELET_PRIVATE_KEY}" | base64 --decode > "${KUBELET_PRIVATE_KEY_PATH}"
16+
17+
###########################################################
18+
# END OF SECRET DATA
19+
###########################################################
20+
21+
set -x
22+
23+
function ensureDocker() {
24+
systemctl enable docker
25+
systemctl restart docker
26+
dockerStarted=1
27+
for i in {1..600}; do
28+
if ! /usr/bin/docker info; then
29+
echo "status $?"
30+
/bin/systemctl restart docker
31+
else
32+
echo "docker started"
33+
dockerStarted=0
34+
break
35+
fi
36+
sleep 1
37+
done
38+
if [ $dockerStarted -ne 0 ]
39+
then
40+
echo "docker did not start"
41+
exit 1
42+
fi
43+
}
44+
45+
function setAgentPool() {
46+
AGENTPOOL=`hostname | cut -d- -f2`
47+
sed -i "s/^KUBELET_NODE_LABELS=.*/KUBELET_NODE_LABELS=role=agent,agentpool=${AGENTPOOL}/" /etc/default/kubelet
48+
}
49+
50+
function ensureKubelet() {
51+
systemctl enable kubelet
52+
systemctl restart kubelet
53+
}
54+
55+
function setNetworkPlugin () {
56+
sed -i "s/^KUBELET_NETWORK_PLUGIN=.*/KUBELET_NETWORK_PLUGIN=${1}/" /etc/default/kubelet
57+
}
58+
59+
function setDockerOpts () {
60+
sed -i "s#^DOCKER_OPTS=.*#DOCKER_OPTS=${1}#" /etc/default/kubelet
61+
}
62+
63+
function configNetworkPolicy() {
64+
if [[ ! -z "${APISERVER_PRIVATE_KEY}" ]]; then
65+
# on masters
66+
ADDONS="calico-configmap.yaml calico-daemonset.yaml"
67+
ADDONS_PATH=/etc/kubernetes/addons
68+
CALICO_URL="https://github.com/simonswine/calico/raw/master/v2.0/getting-started/kubernetes/installation/hosted/k8s-backend-addon-manager"
69+
if [[ "${NETWORK_POLICY}" = "calico" ]]; then
70+
# download calico yamls
71+
for addon in ${ADDONS}; do
72+
curl -o "${ADDONS_PATH}/${addon}" -sSL --retry 12 --retry-delay 10 "${CALICO_URL}/${addon}"
73+
done
74+
else
75+
# make sure calico yaml are removed
76+
for addon in ${ADDONS}; do
77+
rm -f "${ADDONS_PATH}/${addon}"
78+
done
79+
fi
80+
else
81+
# on agents
82+
if [[ "${NETWORK_POLICY}" = "calico" ]]; then
83+
setNetworkPlugin cni
84+
setDockerOpts " --volume=/etc/cni/:/etc/cni:ro --volume=/opt/cni/:/opt/cni:ro"
85+
else
86+
setNetworkPlugin kubenet
87+
setDockerOpts ""
88+
fi
89+
fi
90+
}
91+
92+
ensureDocker
93+
configNetworkPolicy
94+
setAgentPool
95+
ensureKubelet
96+
97+
echo "Install complete successfully"
98+

‎parts/kubernetesagentkubelet.service

+51
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
[Unit]
2+
Description=Kubelet
3+
Requires=docker.service
4+
After=docker.service
5+
6+
[Service]
7+
Restart=always
8+
EnvironmentFile=/etc/default/kubelet
9+
SuccessExitStatus=143
10+
ExecStartPre=/bin/bash /opt/azure/containers/kubelet.sh
11+
ExecStartPre=/bin/mkdir -p /var/lib/kubelet
12+
ExecStartPre=/bin/bash -c "if [ $(mount | grep \"/var/lib/kubelet\" | wc -l) -le 0 ] ; then /bin/mount --bind /var/lib/kubelet /var/lib/kubelet ; fi"
13+
ExecStartPre=/bin/mount --make-shared /var/lib/kubelet
14+
ExecStartPre=-/sbin/ebtables -t nat --list
15+
ExecStartPre=-/sbin/iptables -t nat --list
16+
ExecStart=/usr/bin/docker run \
17+
--net=host \
18+
--pid=host \
19+
--privileged \
20+
--rm \
21+
--volume=/dev:/dev \
22+
--volume=/sys:/sys:ro \
23+
--volume=/var/run:/var/run:rw \
24+
--volume=/var/lib/docker/:/var/lib/docker:rw \
25+
--volume=/var/lib/kubelet/:/var/lib/kubelet:shared \
26+
--volume=/var/log:/var/log:rw \
27+
--volume=/etc/kubernetes/:/etc/kubernetes:ro \
28+
--volume=/srv/kubernetes/:/srv/kubernetes:ro $DOCKER_OPTS \
29+
${KUBELET_IMAGE} \
30+
/hyperkube kubelet \
31+
--kubeconfig=/var/lib/kubelet/kubeconfig \
32+
--require-kubeconfig \
33+
--pod-infra-container-image="${KUBELET_POD_INFRA_CONTAINER_IMAGE}" \
34+
--address=0.0.0.0 \
35+
--allow-privileged=true \
36+
--enable-server \
37+
--enable-debugging-handlers \
38+
--pod-manifest-path=/etc/kubernetes/manifests \
39+
--cluster-dns=${KUBELET_CLUSTER_DNS} \
40+
--cluster-domain=cluster.local \
41+
--register-schedulable=${KUBELET_REGISTER_SCHEDULABLE} \
42+
--node-labels=${KUBELET_NODE_LABELS} \
43+
--cloud-provider= \
44+
--cloud-config= \
45+
--azure-container-registry-config= \
46+
--hairpin-mode=promiscuous-bridge \
47+
--network-plugin=${KUBELET_NETWORK_PLUGIN} \
48+
--v=2
49+
50+
[Install]
51+
WantedBy=multi-user.target

‎parts/kubernetesagentresourcesvmas.t

+9-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,15 @@
2727
{{if eq $seq 1}}
2828
"primary": true,
2929
{{end}}
30+
{{if eq $.Name "system"}}
31+
"privateIPAddress": "[concat(variables('masterFirstAddrPrefix'), copyIndex(add(50, int(variables('masterFirstAddrOctet4')))))]",
32+
"privateIPAllocationMethod": "Static",
33+
{{else if eq $.Name "agentpool1"}}
34+
"privateIPAddress": "[concat(variables('masterFirstAddrPrefix'), copyIndex(add(100, int(variables('masterFirstAddrOctet4')))))]",
35+
"privateIPAllocationMethod": "Static",
36+
{{else}}
3037
"privateIPAllocationMethod": "Dynamic",
38+
{{end}}
3139
"subnet": {
3240
"id": "[variables('{{$.Name}}VnetSubnetID')]"
3341
}
@@ -205,7 +213,7 @@
205213
"autoUpgradeMinorVersion": true,
206214
"settings": {},
207215
"protectedSettings": {
208-
"commandToExecute": "[concat('/usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh ',variables('tenantID'),' ',variables('subscriptionId'),' ',variables('resourceGroup'),' ',variables('location'),' ',variables('subnetName'),' ',variables('nsgName'),' ',variables('virtualNetworkName'),' ',variables('routeTableName'),' ',variables('primaryAvailablitySetName'),' ',variables('servicePrincipalClientId'),' ',variables('servicePrincipalClientSecret'),' ',variables('clientPrivateKey'),' ',variables('targetEnvironment'),' ',variables('networkPolicy'),' >> /var/log/azure/cluster-provision.log 2>&1 &\" &')]"
216+
"commandToExecute": "[concat('/usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh ',variables('clientPrivateKey'),' ',variables('networkPolicy'),'>> /var/log/azure/agent-provision.log 2>&1 &\" &')]"
209217
}
210218
}
211219
}

‎parts/kubernetesagentvars.t

+1
Original file line numberDiff line numberDiff line change
@@ -18,3 +18,4 @@
1818
"{{.Name}}VnetSubnetID": "[variables('vnetSubnetID')]",
1919
"{{.Name}}SubnetName": "[variables('subnetName')]",
2020
{{end}}
21+
"agentProvisionScript": "{{GetKubernetesAgentB64Provision}}",

‎parts/kubernetesmasteraddons-heapster-deployment.yaml

-70
This file was deleted.

‎parts/kubernetesmasteraddons-heapster-service.yaml

-14
This file was deleted.

‎parts/kubernetesmasteraddons-kube-dns-deployment.yaml

+2-1
Original file line numberDiff line numberDiff line change
@@ -100,4 +100,5 @@ spec:
100100
memory: 50Mi
101101
dnsPolicy: Default
102102
nodeSelector:
103-
beta.kubernetes.io/os: linux
103+
beta.kubernetes.io/os: linux
104+
agentpool: system

‎parts/kubernetesmasteraddons-kubernetes-dashboard-service.yaml

-15
This file was deleted.

‎parts/kubernetesmastercustomdata.yml

-30
Original file line numberDiff line numberDiff line change
@@ -152,34 +152,6 @@ write_files:
152152
content: !!binary |
153153
MASTER_ADDON_KUBE_PROXY_DAEMONSET_B64_GZIP_STR
154154

155-
- path: /etc/kubernetes/addons/kubernetes-dashboard-deployment.yaml
156-
permissions: "0644"
157-
encoding: gzip
158-
owner: "root"
159-
content: !!binary |
160-
MASTER_ADDON_KUBERNETES_DASHBOARD_DEPLOYMENT_B64_GZIP_STR
161-
162-
- path: /etc/kubernetes/addons/kubernetes-dashboard-service.yaml
163-
permissions: "0644"
164-
encoding: gzip
165-
owner: "root"
166-
content: !!binary |
167-
MASTER_ADDON_KUBERNETES_DASHBOARD_SERVICE_B64_GZIP_STR
168-
169-
- path: /etc/kubernetes/addons/kube-heapster-service.yaml
170-
permissions: "0644"
171-
encoding: gzip
172-
owner: "root"
173-
content: !!binary |
174-
MASTER_ADDON_HEAPSTER_SERVICE_B64_GZIP_STR
175-
176-
- path: /etc/kubernetes/addons/kube-heapster-deployment.yaml
177-
permissions: "0644"
178-
encoding: gzip
179-
owner: "root"
180-
content: !!binary |
181-
MASTER_ADDON_HEAPSTER_DEPLOYMENT_B64_GZIP_STR
182-
183155
- path: /etc/kubernetes/addons/default-storage-class.yaml
184156
permissions: "0644"
185157
encoding: gzip
@@ -248,8 +220,6 @@ write_files:
248220
sed -i "s|<kubernetesHyperkubeSpec>|{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/manifests/kube-scheduler.yaml"
249221
sed -i "s|<kubernetesHyperkubeSpec>|{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/addons/kube-proxy-daemonset.yaml"
250222
sed -i "s|<kubernetesKubeDNSSpec>|{{WrapAsVariable "kubernetesKubeDNSSpec"}}|g; s|<kubernetesDNSMasqSpec>|{{WrapAsVariable "kubernetesDNSMasqSpec"}}|g; s|<kubernetesExecHealthzSpec>|{{WrapAsVariable "kubernetesExecHealthzSpec"}}|g" "/etc/kubernetes/addons/kube-dns-deployment.yaml"
251-
sed -i "s|<kubernetesHeapsterSpec>|{{WrapAsVariable "kubernetesHeapsterSpec"}}|g; s|<kubernetesAddonResizerSpec>|{{WrapAsVariable "kubernetesAddonResizerSpec"}}|g" "/etc/kubernetes/addons/kube-heapster-deployment.yaml"
252-
sed -i "s|<kubernetesDashboardSpec>|{{WrapAsVariable "kubernetesDashboardSpec"}}|g" "/etc/kubernetes/addons/kubernetes-dashboard-deployment.yaml"
253223
254224
echo $(curl -f --retry 5 http://169.254.169.254/metadata/v1/InstanceInfo 2>/dev/null | jq -r .FD) > /etc/kubernetes/fd
255225
echo "{{WrapAsVariable "masterVMSize"}}" > /etc/kubernetes/vmsize

‎parts/kubernetesmastercustomscript.sh

-1
Original file line numberDiff line numberDiff line change
@@ -331,6 +331,5 @@ if [[ ! -z "${APISERVER_PRIVATE_KEY}" ]]; then
331331
ensureApiserver
332332
fi
333333

334-
# If APISERVER_PRIVATE_KEY is empty, then we are not on the master
335334
echo "Install complete successfully"
336335

‎parts/kubernetesmastervars.t

+1-1
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@
9191
{{end}}
9292
"nsgName": "[concat(variables('masterVMNamePrefix'), 'nsg')]",
9393
"nsgID": "[resourceId('Microsoft.Network/networkSecurityGroups',variables('nsgName'))]",
94-
"primaryAvailablitySetName": "[concat('{{ (index .AgentPoolProfiles 0).Name }}-availabilitySet-',variables('nameSuffix'))]",
94+
"primaryAvailablitySetName": "[concat('{{ (index .AgentPoolProfiles 1).Name }}-availabilitySet-',variables('nameSuffix'))]",
9595
"masterPublicIPAddressName": "[concat(variables('orchestratorName'), '-master-ip-', variables('masterFqdnPrefix'), '-', variables('nameSuffix'))]",
9696
"masterLbID": "[resourceId('Microsoft.Network/loadBalancers',variables('masterLbName'))]",
9797
"masterLbIPConfigID": "[concat(variables('masterLbID'),'/frontendIPConfigurations/', variables('masterLbIPConfigName'))]",

‎pkg/acsengine/azureconst.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -1110,4 +1110,4 @@ func GetClassicSizeMap() string {
11101110
}
11111111
}
11121112
`
1113-
}
1113+
}

0 commit comments

Comments
 (0)