{"status":{},"spec":{"description":"* [Kubernetes API](https:\/\/@@{Kubernetes_Master.address[0]}@@:6443)","resources":{"client_attrs":{"85fa2262_deployment":{"y":-68,"x":-166},"34875daf_deployment":{"y":-68,"x":-166},"77b729d5_deployment":{"y":104,"x":118},"39cf7283_deployment":{"y":104,"x":118},"2cbb2f6a_deployment":{"y":-68,"x":-166},"d6584f2e_deployment":{"y":104,"x":118}},"service_definition_list":[{"singleton":false,"action_list":[{"description":"System action for creating an application","type":"system","critical":false,"runbook":{"task_definition_list":[{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Master"},"retries":"0","description":"","child_tasks_local_reference_list":[{"kind":"app_task","name":"Dashboard"},{"kind":"app_task","name":"HELM"}],"name":"f33eac3f_dag","attrs":{"edges":[{"from_task_reference":{"kind":"app_task","name":"Dashboard"},"edge_type":"user_defined","type":"","to_task_reference":{"kind":"app_task","name":"HELM"}}],"type":""},"timeout_secs":"0","type":"DAG","variable_list":[]},{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Master"},"retries":"0","description":"","child_tasks_local_reference_list":[],"name":"Dashboard","attrs":{"script":"#!\/bin\/bash\nset -ex\n\nif [ @@{calm_array_index}@@ -ne 0 ];then\n\texit\nfi\nexport PATH=$PATH:\/opt\/bin\n\nsudo mkdir \/etc\/kubernetes\/addons\/dashboard\necho '# Copyright 2015 Google Inc. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http:\/\/www.apache.org\/licenses\/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# Configuration to deploy release version of the Dashboard UI compatible with\n# Kubernetes 1.7.\n#\n# Example usage: kubectl create -f \n\n# ------------------- Dashboard Secret ------------------- #\n\napiVersion: v1\nkind: Secret\nmetadata:\n labels:\n k8s-app: kubernetes-dashboard\n name: kubernetes-dashboard-certs\n namespace: kube-system\ntype: Opaque\n\n---\n# ------------------- Dashboard Service Account ------------------- #\n\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n labels:\n k8s-app: kubernetes-dashboard\n name: kubernetes-dashboard\n namespace: kube-system\n\n---\n# ------------------- Dashboard Role & Role Binding ------------------- #\n\n---\napiVersion: rbac.authorization.k8s.io\/v1beta1\nkind: ClusterRoleBinding\nmetadata:\n name: kubernetes-full-access\n namespace: kube-system\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: cluster-admin\nsubjects:\n- kind: ServiceAccount\n name: kubernetes-dashboard\n namespace: kube-system\n\n---\n# ------------------- Dashboard Deployment ------------------- #\n\nkind: Deployment\napiVersion: apps\/v1beta2\nmetadata:\n labels:\n k8s-app: kubernetes-dashboard\n name: kubernetes-dashboard\n namespace: kube-system\nspec:\n replicas: 1\n revisionHistoryLimit: 10\n selector:\n matchLabels:\n k8s-app: kubernetes-dashboard\n template:\n metadata:\n labels:\n k8s-app: kubernetes-dashboard\n spec:\n containers:\n - name: kubernetes-dashboard\n image: k8s.gcr.io\/kubernetes-dashboard-amd64:v1.8.3\n ports:\n - containerPort: 8443\n protocol: TCP\n args:\n - --auto-generate-certificates\n # Uncomment the following line to manually specify Kubernetes API server Host\n # If not specified, Dashboard will attempt to auto discover the API server and connect\n # to it. Uncomment only if the default does not work.\n # - --apiserver-host=http:\/\/my-address:port\n volumeMounts:\n - name: kubernetes-dashboard-certs\n mountPath: \/certs\n # Create on-disk volume to store exec logs\n - mountPath: \/tmp\n name: tmp-volume\n livenessProbe:\n httpGet:\n scheme: HTTPS\n path: \/\n port: 8443\n initialDelaySeconds: 30\n timeoutSeconds: 30\n volumes:\n - name: kubernetes-dashboard-certs\n secret:\n secretName: kubernetes-dashboard-certs\n - name: tmp-volume\n emptyDir: {}\n serviceAccountName: kubernetes-dashboard\n # Comment the following tolerations if Dashboard must not be deployed on master\n tolerations:\n - key: node-role.kubernetes.io\/master\n effect: NoSchedule\n\n---\n# ------------------- Dashboard Service ------------------- #\n\nkind: Service\napiVersion: v1\nmetadata:\n labels:\n k8s-app: kubernetes-dashboard\n name: kubernetes-dashboard\n namespace: kube-system\nspec:\n type: NodePort\n ports:\n - port: 8443\n nodePort: 30443\n selector:\n k8s-app: kubernetes-dashboard' | sudo tee \/etc\/kubernetes\/addons\/dashboard\/kubernetes-dashboard.yaml\nkubectl create -f \/etc\/kubernetes\/addons\/dashboard\/kubernetes-dashboard.yaml\n","type":"","command_line_args":"","exit_status":[],"script_type":"sh"},"timeout_secs":"0","type":"EXEC","variable_list":[]},{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Master"},"retries":"0","description":"","child_tasks_local_reference_list":[],"name":"HELM","attrs":{"script":"#!\/bin\/bash\nset -ex\n\nif [ @@{calm_array_index}@@ -ne 0 ];then\n\texit\nfi\nexport PATH=$PATH:\/opt\/bin\nsudo mkdir \/etc\/kubernetes\/addons\/helm\necho \"#helm init --service-account helm\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: helm\n namespace: kube-system\n---\napiVersion: rbac.authorization.k8s.io\/v1beta1\nkind: ClusterRoleBinding\nmetadata:\n name: helm\n namespace: kube-system\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: cluster-admin\nsubjects:\n- kind: ServiceAccount\n name: helm\n namespace: kube-system\" | sudo tee \/etc\/kubernetes\/addons\/helm\/helm.yaml\n\nkubectl create -f \/etc\/kubernetes\/addons\/helm\/helm.yaml\nprintf -v no_proxy '%s,' 10.132.249.{1..255}\nexport no_proxy=${no_proxy}localhost\necho $no_proxy\nhttp_proxy=http:\/\/10.132.71.38:1080\/ no_proxy=${no_proxy} helm init --service-account helm\n","type":"","command_line_args":"","exit_status":[],"script_type":"sh"},"timeout_secs":"0","type":"EXEC","variable_list":[]}],"description":"","name":"a9639da1_runbook","main_task_local_reference":{"kind":"app_task","name":"f33eac3f_dag"},"variable_list":[]},"name":"action_create"},{"description":"System action for deleting an application. Deletes created VMs as well","type":"system","critical":false,"runbook":{"task_definition_list":[{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Master"},"retries":"0","description":"","child_tasks_local_reference_list":[],"name":"62a92d34_dag","attrs":{"edges":[],"type":""},"timeout_secs":"0","type":"DAG","variable_list":[]}],"description":"","name":"8c4aa276_runbook","main_task_local_reference":{"kind":"app_task","name":"62a92d34_dag"},"variable_list":[]},"name":"action_delete"},{"description":"System action for starting an application","type":"system","critical":false,"runbook":{"task_definition_list":[{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Master"},"retries":"0","description":"","child_tasks_local_reference_list":[{"kind":"app_task","name":"Start"}],"name":"334b4484_dag","attrs":{"edges":[],"type":""},"timeout_secs":"0","type":"DAG","variable_list":[]},{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Master"},"retries":"0","description":"","child_tasks_local_reference_list":[],"name":"Start","attrs":{"script":"#!\/bin\/bash\nset -ex\nsudo systemctl start etcd docker kubelet\nsudo systemctl enable etcd docker kubelet\n","type":"","command_line_args":"","exit_status":[],"script_type":"sh"},"timeout_secs":"0","type":"EXEC","variable_list":[]}],"description":"","name":"daee30ae_runbook","main_task_local_reference":{"kind":"app_task","name":"334b4484_dag"},"variable_list":[]},"name":"action_start"},{"description":"System action for stopping an application","type":"system","critical":false,"runbook":{"task_definition_list":[{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Master"},"retries":"0","description":"","child_tasks_local_reference_list":[],"name":"9713c3d0_dag","attrs":{"edges":[],"type":""},"timeout_secs":"0","type":"DAG","variable_list":[]}],"description":"","name":"6407a345_runbook","main_task_local_reference":{"kind":"app_task","name":"9713c3d0_dag"},"variable_list":[]},"name":"action_stop"},{"description":"System action for restarting an application","type":"system","critical":false,"runbook":{"task_definition_list":[{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Master"},"retries":"0","description":"","child_tasks_local_reference_list":[],"name":"414912df_dag","attrs":{"edges":[],"type":""},"timeout_secs":"0","type":"DAG","variable_list":[]}],"description":"","name":"d5c8316f_runbook","main_task_local_reference":{"kind":"app_task","name":"414912df_dag"},"variable_list":[]},"name":"action_restart"}],"depends_on_list":[],"name":"Kubernetes_Master","port_list":[],"tier":"","variable_list":[],"description":""},{"singleton":false,"action_list":[{"description":"System action for creating an application","type":"system","critical":false,"runbook":{"task_definition_list":[{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Minion"},"retries":"0","description":"","child_tasks_local_reference_list":[],"name":"4af7c45f_dag","attrs":{"edges":[],"type":""},"timeout_secs":"0","type":"DAG","variable_list":[]}],"description":"","name":"7c6e4932_runbook","main_task_local_reference":{"kind":"app_task","name":"4af7c45f_dag"},"variable_list":[]},"name":"action_create"},{"description":"System action for deleting an application. Deletes created VMs as well","type":"system","critical":false,"runbook":{"task_definition_list":[{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Minion"},"retries":"0","description":"","child_tasks_local_reference_list":[],"name":"1e2c50bb_dag","attrs":{"edges":[],"type":""},"timeout_secs":"0","type":"DAG","variable_list":[]}],"description":"","name":"1d794193_runbook","main_task_local_reference":{"kind":"app_task","name":"1e2c50bb_dag"},"variable_list":[]},"name":"action_delete"},{"description":"System action for starting an application","type":"system","critical":false,"runbook":{"task_definition_list":[{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Minion"},"retries":"0","description":"","child_tasks_local_reference_list":[{"kind":"app_task","name":"Start"}],"name":"970128c7_dag","attrs":{"edges":[],"type":""},"timeout_secs":"0","type":"DAG","variable_list":[]},{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Minion"},"retries":"0","description":"","child_tasks_local_reference_list":[],"name":"Start","attrs":{"script":"#!\/bin\/bash\nset -ex\n\nsudo systemctl start docker kubelet\nsudo systemctl enable docker kubelet","type":"","command_line_args":"","exit_status":[],"script_type":"sh"},"timeout_secs":"0","type":"EXEC","variable_list":[]}],"description":"","name":"884352cf_runbook","main_task_local_reference":{"kind":"app_task","name":"970128c7_dag"},"variable_list":[]},"name":"action_start"},{"description":"System action for stopping an application","type":"system","critical":false,"runbook":{"task_definition_list":[{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Minion"},"retries":"0","description":"","child_tasks_local_reference_list":[],"name":"6c01dc28_dag","attrs":{"edges":[],"type":""},"timeout_secs":"0","type":"DAG","variable_list":[]}],"description":"","name":"0cb172a7_runbook","main_task_local_reference":{"kind":"app_task","name":"6c01dc28_dag"},"variable_list":[]},"name":"action_stop"},{"description":"System action for restarting an application","type":"system","critical":false,"runbook":{"task_definition_list":[{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Minion"},"retries":"0","description":"","child_tasks_local_reference_list":[],"name":"1d66b5be_dag","attrs":{"edges":[],"type":""},"timeout_secs":"0","type":"DAG","variable_list":[]}],"description":"","name":"3a6f587d_runbook","main_task_local_reference":{"kind":"app_task","name":"1d66b5be_dag"},"variable_list":[]},"name":"action_restart"}],"depends_on_list":[],"name":"Kubernetes_Minion","port_list":[],"tier":"","variable_list":[],"description":""}],"substrate_definition_list":[{"description":"","action_list":[],"type":"AHV_VM","name":"AHV_Centos_K8SC","readiness_probe":{"connection_type":"SSH","disable_readiness_probe":false,"address":"@@{platform.status.resources.nic_list[0].ip_endpoint_list[0].ip}@@","delay_secs":"0","connection_port":22,"login_credential_local_reference":{"kind":"app_credential","name":"CENTOS"}},"editables":{"readiness_probe":{"connection_type":true,"connection_port":true,"timeout_secs":true},"create_spec":{"name":true,"resources":{"nic_list":{},"serial_port_list":{},"num_vcpus_per_socket":true,"num_sockets":true,"memory_size_mib":true,"guest_customization":true,"disk_list":{"1":{"disk_size_mib":true},"3":{"disk_size_mib":true},"2":{"disk_size_mib":true}}}}},"os_type":"Linux","create_spec":{"name":"K8SC-@@{calm_array_index}@@-@@{calm_time}@@","resources":{"nic_list":[{"nic_type":"NORMAL_NIC","ip_endpoint_list":[],"network_function_chain_reference":null,"network_function_nic_type":"INGRESS","mac_address":"","subnet_reference":{"kind":"subnet","type":"","name":"","uuid":"ca13c531-07a4-4bd0-8602-25e4c1ce74e8"},"type":""}],"serial_port_list":[],"guest_tools":null,"num_vcpus_per_socket":1,"num_sockets":2,"gpu_list":[],"memory_size_mib":4096,"parent_reference":null,"hardware_clock_timezone":"","guest_customization":{"cloud_init":{"meta_data":"","type":"","user_data":"#cloud-config\nusers:\n - name: centos\n ssh-authorized-keys:\n - @@{INSTANCE_PUBLIC_KEY}@@\n sudo: ['ALL=(ALL) NOPASSWD:ALL']"},"type":"","sysprep":null},"power_state":"ON","type":"","boot_config":{"boot_device":{"type":"","disk_address":{"type":"","device_index":0,"adapter_type":"SCSI"}},"type":"","mac_address":""},"disk_list":[{"data_source_reference":{"kind":"image","type":"","name":"my-img","uuid":"3d0e5238-715a-4a02-97fe-1c2de56e7f8a"},"type":"","disk_size_mib":0,"volume_group_reference":null,"device_properties":{"type":"","disk_address":{"type":"","device_index":0,"adapter_type":"SCSI"},"device_type":"DISK"}},{"data_source_reference":null,"type":"","disk_size_mib":10240,"volume_group_reference":null,"device_properties":{"type":"","disk_address":{"type":"","device_index":1,"adapter_type":"SCSI"},"device_type":"DISK"}},{"data_source_reference":null,"type":"","disk_size_mib":10240,"volume_group_reference":null,"device_properties":{"type":"","disk_address":{"type":"","device_index":2,"adapter_type":"SCSI"},"device_type":"DISK"}},{"data_source_reference":null,"type":"","disk_size_mib":10240,"volume_group_reference":null,"device_properties":{"type":"","disk_address":{"type":"","device_index":3,"adapter_type":"SCSI"},"device_type":"DISK"}}]},"availability_zone_reference":null,"backup_policy":null,"type":"","cluster_reference":null,"categories":""},"variable_list":[]},{"description":"","action_list":[],"type":"AHV_VM","name":"AHV_Centos_K8SM","readiness_probe":{"connection_type":"SSH","disable_readiness_probe":false,"address":"@@{platform.status.resources.nic_list[0].ip_endpoint_list[0].ip}@@","delay_secs":"0","connection_port":22,"login_credential_local_reference":{"kind":"app_credential","name":"CENTOS"}},"editables":{"readiness_probe":{"connection_type":true,"connection_port":true,"timeout_secs":true},"create_spec":{"name":true,"resources":{"nic_list":{},"serial_port_list":{},"num_vcpus_per_socket":true,"num_sockets":true,"memory_size_mib":true,"guest_customization":true,"disk_list":{"1":{"disk_size_mib":true},"3":{"disk_size_mib":true},"2":{"disk_size_mib":true}}}}},"os_type":"Linux","create_spec":{"name":"K8SM-@@{calm_array_index}@@-@@{calm_time}@@","resources":{"nic_list":[{"nic_type":"NORMAL_NIC","ip_endpoint_list":[],"network_function_chain_reference":null,"network_function_nic_type":"INGRESS","mac_address":"","subnet_reference":{"kind":"subnet","type":"","name":"","uuid":"ca13c531-07a4-4bd0-8602-25e4c1ce74e8"},"type":""}],"serial_port_list":[],"guest_tools":null,"num_vcpus_per_socket":1,"num_sockets":4,"gpu_list":[],"memory_size_mib":4096,"parent_reference":null,"hardware_clock_timezone":"","guest_customization":{"cloud_init":{"meta_data":"","type":"","user_data":"#cloud-config\nusers:\n - name: centos\n ssh-authorized-keys:\n - @@{INSTANCE_PUBLIC_KEY}@@\n sudo: ['ALL=(ALL) NOPASSWD:ALL']"},"type":"","sysprep":null},"power_state":"ON","type":"","boot_config":{"boot_device":{"type":"","disk_address":{"type":"","device_index":0,"adapter_type":"SCSI"}},"type":"","mac_address":""},"disk_list":[{"data_source_reference":{"kind":"image","type":"","name":"my-img","uuid":"3d0e5238-715a-4a02-97fe-1c2de56e7f8a"},"type":"","disk_size_mib":0,"volume_group_reference":null,"device_properties":{"type":"","disk_address":{"type":"","device_index":0,"adapter_type":"SCSI"},"device_type":"DISK"}},{"data_source_reference":null,"type":"","disk_size_mib":10240,"volume_group_reference":null,"device_properties":{"type":"","disk_address":{"type":"","device_index":1,"adapter_type":"SCSI"},"device_type":"DISK"}},{"data_source_reference":null,"type":"","disk_size_mib":10240,"volume_group_reference":null,"device_properties":{"type":"","disk_address":{"type":"","device_index":2,"adapter_type":"SCSI"},"device_type":"DISK"}},{"data_source_reference":null,"type":"","disk_size_mib":10240,"volume_group_reference":null,"device_properties":{"type":"","disk_address":{"type":"","device_index":3,"adapter_type":"SCSI"},"device_type":"DISK"}}]},"availability_zone_reference":null,"backup_policy":null,"type":"","cluster_reference":null,"categories":""},"variable_list":[]},{"description":"","action_list":[],"type":"AWS_VM","name":"AWS_Centos_K8SC","readiness_probe":{"connection_type":"SSH","disable_readiness_probe":false,"delay_secs":"0","connection_port":22,"address":"@@{public_ip_address}@@"},"editables":{"readiness_probe":{"connection_type":true,"connection_port":true,"timeout_secs":true},"create_spec":{"name":true,"resources":{"instance_type":true,"instance_profile_name":true,"availability_zone":true,"subnet_id":true,"key_name":true,"region":true,"image_id":true,"security_group_list":true,"block_device_map":{"data_disk_list":{"1":{"size_gb":true},"0":{"size_gb":true},"2":{"size_gb":true}},"root_disk":{"size_gb":true}},"associate_public_ip_address":true,"vpc_id":true,"tag_list":true}}},"os_type":"Linux","create_spec":{"name":"K8SC-@@{calm_array_index}@@-@@{calm_time}@@","availability_zone_reference":null,"backup_policy":null,"type":"PROVISION_AWS_VM","cluster_reference":null,"resources":{"instance_profile_name":"kubernetes-ebs-volumes","availability_zone":"us-east-1a","subnet_id":"subnet-c599a5ef","key_name":"calm-blueprints","region":"us-east-1","instance_initiated_shutdown_behavior":"","user_data":"","image_id":"ami-1a003360","instance_type":"t2.medium","state":"RUNNING","security_group_list":[{"type":"","security_group_id":"sg-3b938f40"}],"block_device_map":{"data_disk_list":[{"size_gb":8,"volume_type":"GP2","device_name":"\/dev\/sdb","iops":1,"snapshot_id":"","type":"","delete_on_termination":true},{"size_gb":8,"volume_type":"GP2","device_name":"\/dev\/sdc","iops":1,"snapshot_id":"","type":"","delete_on_termination":true},{"size_gb":8,"volume_type":"GP2","device_name":"\/dev\/sdd","iops":1,"snapshot_id":"","type":"","delete_on_termination":true}],"type":"","root_disk":{"size_gb":8,"volume_type":"GP2","device_name":"\/dev\/sda1","iops":1,"snapshot_id":"","type":"","delete_on_termination":true}},"private_ip_address":"","vpc_id":"vpc-ffd54d98","tag_list":[{"type":"","key":"kubernetes.io\/cluster\/@@{KUBE_CLUSTER_NAME}@@","value":"true"}],"type":"","account_uuid":"eb08a14a-41f8-045c-d9ab-0e30dc4dc5fb","associate_public_ip_address":true}},"variable_list":[]},{"description":"","action_list":[],"type":"AWS_VM","name":"AWS_Centos_K8SM","readiness_probe":{"connection_type":"SSH","disable_readiness_probe":false,"delay_secs":"0","connection_port":22,"address":"@@{public_ip_address}@@"},"editables":{"readiness_probe":{"connection_type":true,"connection_port":true,"timeout_secs":true},"create_spec":{"name":true,"resources":{"instance_type":true,"instance_profile_name":true,"availability_zone":true,"subnet_id":true,"key_name":true,"region":true,"image_id":true,"security_group_list":true,"block_device_map":{"data_disk_list":{"1":{"size_gb":true},"0":{"size_gb":true},"2":{"size_gb":true}},"root_disk":{"size_gb":true}},"associate_public_ip_address":true,"vpc_id":true,"tag_list":true}}},"os_type":"Linux","create_spec":{"name":"K8SM-@@{calm_array_index}@@-@@{calm_time}@@","availability_zone_reference":null,"backup_policy":null,"type":"PROVISION_AWS_VM","cluster_reference":null,"resources":{"instance_profile_name":"kubernetes-ebs-volumes","availability_zone":"us-east-1a","subnet_id":"subnet-9d1085b1","key_name":"calm-blueprints","region":"us-east-1","instance_initiated_shutdown_behavior":"","user_data":"","image_id":"ami-1a003360","instance_type":"t2.large","state":"RUNNING","security_group_list":[{"type":"","security_group_id":"sg-3b938f40"}],"block_device_map":{"data_disk_list":[{"size_gb":8,"volume_type":"GP2","device_name":"\/dev\/sdb","iops":1,"snapshot_id":"","type":"","delete_on_termination":true},{"size_gb":8,"volume_type":"GP2","device_name":"\/dev\/sdc","iops":1,"snapshot_id":"","type":"","delete_on_termination":true},{"size_gb":8,"volume_type":"GP2","device_name":"\/dev\/sdd","iops":1,"snapshot_id":"","type":"","delete_on_termination":true}],"type":"","root_disk":{"size_gb":8,"volume_type":"GP2","device_name":"\/dev\/sda1","iops":1,"snapshot_id":"","type":"","delete_on_termination":true}},"private_ip_address":"","vpc_id":"vpc-ffd54d98","tag_list":[{"type":"","key":"kubernetes.io\/cluster\/@@{KUBE_CLUSTER_NAME}@@","value":"owned"}],"type":"","account_uuid":"eb08a14a-41f8-045c-d9ab-0e30dc4dc5fb","associate_public_ip_address":true}},"variable_list":[]},{"description":"","action_list":[],"type":"GCP_VM","name":"GCP_Centos_K8SC","readiness_probe":{"connection_type":"SSH","disable_readiness_probe":false,"delay_secs":"0","connection_port":22,"address":"@@{public_ip_address}@@"},"editables":{"readiness_probe":{"connection_type":true,"connection_port":true,"timeout_secs":true},"create_spec":{"resources":{"labels":true,"machineType":true,"name":true,"tags":{"items":true},"disks":{"0":{"initializeParams":{"diskSizeGb":true,"sourceImage":true,"diskType":true},"autoDelete":true}},"blankDisks":{"1":{"disk_type":true,"name":true,"sizeGb":true,"autoDelete":true},"0":{"disk_type":true,"name":true,"sizeGb":true,"autoDelete":true}},"metadata":{"items":true},"sshKeys":{"0":true},"serviceAccounts":{"0":{"scopes":true}},"networkInterfaces":{"0":{"accessConfigs":{"0":{"config_type":true,"name":true}},"network":true,"subnetwork":true}}}}},"os_type":"Linux","create_spec":{"type":"PROVISION_GCP_VM","resources":{"disks":[{"diskEncryptionKey":{"rawKey":"","type":""},"deviceName":"","initializeParams":{"sourceImageEncryptionKey":{"rawKey":"","type":""},"diskName":"","diskType":"https:\/\/www.googleapis.com\/compute\/v1\/projects\/nucalm-devopos\/zones\/us-central1-c\/diskTypes\/pd-ssd","diskSizeGb":-1,"sourceImage":"https:\/\/www.googleapis.com\/compute\/v1\/projects\/nucalm-devopos\/global\/images\/centos-7","type":""},"autoDelete":true,"boot":true,"source":"","mode":"","interface":"","disk_type":"PERSISTENT","type":""}],"machineType":"https:\/\/www.googleapis.com\/compute\/v1\/projects\/nucalm-devopos\/zones\/us-central1-c\/machineTypes\/n1-highcpu-2","description":"","zone":"us-central1-c","tags":{"items":[],"type":"","fingerprint":""},"guestCustomization":{"type":"","sysprep":"","startupScript":""},"labelFingerprint":"","labels":[],"name":"k8sc-@@{calm_array_index}@@-@@{calm_time}@@","account_uuid":"6a843c92-ceb7-336c-5ca4-6bfaef5d1a4b","canIpForward":false,"scheduling":{"automaticRestart":true,"type":"","preemptible":false,"onHostMaintenance":"TERMINATE"},"minCpuPlatform":"","blankDisks":[{"disk_type":"https:\/\/www.googleapis.com\/compute\/v1\/projects\/nucalm-devopos\/zones\/us-central1-c\/diskTypes\/pd-ssd","type":"","name":"k8sc-disk1-@@{calm_array_index}@@-@@{calm_time}@@","sizeGb":10,"autoDelete":true},{"disk_type":"https:\/\/www.googleapis.com\/compute\/v1\/projects\/nucalm-devopos\/zones\/us-central1-c\/diskTypes\/pd-ssd","type":"","name":"k8sc-disk2-@@{calm_array_index}@@-@@{calm_time}@@","sizeGb":10,"autoDelete":true},{"disk_type":"https:\/\/www.googleapis.com\/compute\/v1\/projects\/nucalm-devopos\/zones\/us-central1-c\/diskTypes\/pd-ssd","type":"","name":"k8sc-disk3-@@{calm_array_index}@@-@@{calm_time}@@","sizeGb":10,"autoDelete":true}],"networkInterfaces":[{"aliasIpRanges":[],"network":"https:\/\/www.googleapis.com\/compute\/v1\/projects\/nucalm-devopos\/global\/networks\/default","accessConfigs":[{"config_type":"ONE_TO_ONE_NAT","type":"","name":"k8sc-acn-@@{calm_array_index}@@-@@{calm_time}@@","natIP":""}],"networkIP":"","subnetwork":"https:\/\/www.googleapis.com\/compute\/v1\/projects\/nucalm-devopos\/regions\/us-central1\/subnetworks\/default","type":""}],"type":"","sshKeys":["centos:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDMGAS\/K0Mb\/uwFXwD+hkjguy7VMZk2hpuhwPl9FUZwVBrURf\/i9QMJ5\/paPEZixu8VlRx7Inu4iun7rQfrnfeIYInmBwspXHYiTK3oHJAgZnrAHVEf1p6YaxLINlT1NI5yOAGPRWW6of8rBDBH1ObwU2+wcSx\/1H0uIs3aZNLufr+Rh628ACxAum2Gt8AVRj6ua2BPFyt5VTdclyysAmeh1AiixNgOZXOz6y\/i4TbzpY78I3isuKpxsUeXX6jxEMQol406jHDUF6njEOPIQG2zVZ3QJlTG9OlN+NiyZG9WkZz0VG\/6M8ixxIHHI2dNwUbBFv2HUu+8X9LTLFq2O7KjX9Hp7uZKBAySHA3eKaKHIp2bZuU1bT5PRPkggngX86xg+T+OMNnutbAiMnRJ8+FvD5So+5TIx4b9GgxAxure3x2yRPT9lOiQOB+CVpJPxR0Rn9bOI+wiPnD0kAGvK\/fHT+pqL4PM+hTnJtp9rrCRzIQApBx1263jEcYffhW2epZQRO+he5CMawFJ5TBe08om2AaDJ8GQdrpF6YA3W8DzHbmL3DPVVHdmqPLn10o+LX4gv5SdIIDVGdjKOc1BCnLTRmM28d5+sLDt\/M+kvcQgf0y0yDjMVjGECZkt39hbm4ELMHzZtzYLmHNhBZxRqHeJ7qFTuv1kx88OW3Xc5mbBNQ== centos@nutanix.com\n"],"serviceAccounts":[{"scopes":["https:\/\/www.googleapis.com\/auth\/cloud-platform"],"type":"","email":"108048128720-compute@developer.gserviceaccount.com"}],"metadata":{"items":[],"type":"","fingerprint":""}}},"variable_list":[]},{"description":"","action_list":[],"type":"GCP_VM","name":"GCP_Centos_K8SM","readiness_probe":{"connection_type":"SSH","disable_readiness_probe":false,"delay_secs":"0","connection_port":22,"address":"@@{public_ip_address}@@"},"editables":{"readiness_probe":{"connection_type":true,"connection_port":true,"timeout_secs":true},"create_spec":{"resources":{"labels":true,"machineType":true,"name":true,"tags":{"items":true},"disks":{"0":{"initializeParams":{"diskSizeGb":true,"sourceImage":true,"diskType":true},"autoDelete":true}},"blankDisks":{"1":{"disk_type":true,"name":true,"sizeGb":true,"autoDelete":true},"0":{"disk_type":true,"name":true,"sizeGb":true,"autoDelete":true},"2":{"disk_type":true,"name":true,"sizeGb":true,"autoDelete":true}},"metadata":{"items":true},"sshKeys":{"0":true},"serviceAccounts":{"0":{"scopes":true}},"networkInterfaces":{"0":{"accessConfigs":{"0":{"config_type":true,"name":true}},"network":true,"subnetwork":true}}}}},"os_type":"Linux","create_spec":{"type":"PROVISION_GCP_VM","resources":{"disks":[{"diskEncryptionKey":{"rawKey":"","type":""},"deviceName":"","initializeParams":{"sourceImageEncryptionKey":{"rawKey":"","type":""},"diskName":"","diskType":"https:\/\/www.googleapis.com\/compute\/v1\/projects\/nucalm-devopos\/zones\/us-central1-c\/diskTypes\/pd-ssd","diskSizeGb":-1,"sourceImage":"https:\/\/www.googleapis.com\/compute\/v1\/projects\/nucalm-devopos\/global\/images\/centos-7","type":""},"autoDelete":true,"boot":true,"source":"","mode":"","interface":"","disk_type":"PERSISTENT","type":""}],"machineType":"https:\/\/www.googleapis.com\/compute\/v1\/projects\/nucalm-devopos\/zones\/us-central1-c\/machineTypes\/n1-highcpu-4","description":"","zone":"us-central1-c","tags":{"items":[],"type":"","fingerprint":""},"guestCustomization":{"type":"","sysprep":"","startupScript":""},"labelFingerprint":"","labels":[],"name":"k8sm-@@{calm_array_index}@@-@@{calm_time}@@","account_uuid":"6a843c92-ceb7-336c-5ca4-6bfaef5d1a4b","canIpForward":false,"scheduling":{"automaticRestart":true,"type":"","preemptible":false,"onHostMaintenance":"TERMINATE"},"minCpuPlatform":"","blankDisks":[{"disk_type":"https:\/\/www.googleapis.com\/compute\/v1\/projects\/nucalm-devopos\/zones\/us-central1-c\/diskTypes\/pd-ssd","type":"","name":"k8sm-disk1-@@{calm_array_index}@@-@@{calm_time}@@","sizeGb":10,"autoDelete":true},{"disk_type":"https:\/\/www.googleapis.com\/compute\/v1\/projects\/nucalm-devopos\/zones\/us-central1-c\/diskTypes\/pd-ssd","type":"","name":"k8sm-disk2-@@{calm_array_index}@@-@@{calm_time}@@","sizeGb":10,"autoDelete":true},{"disk_type":"https:\/\/www.googleapis.com\/compute\/v1\/projects\/nucalm-devopos\/zones\/us-central1-c\/diskTypes\/pd-ssd","type":"","name":"k8sm-disk3-@@{calm_array_index}@@-@@{calm_time}@@","sizeGb":10,"autoDelete":true}],"networkInterfaces":[{"aliasIpRanges":[],"network":"https:\/\/www.googleapis.com\/compute\/v1\/projects\/nucalm-devopos\/global\/networks\/default","accessConfigs":[{"config_type":"ONE_TO_ONE_NAT","type":"","name":"k8sm-acm-@@{calm_array_index}@@-@@{calm_time}@@","natIP":""}],"networkIP":"","subnetwork":"https:\/\/www.googleapis.com\/compute\/v1\/projects\/nucalm-devopos\/regions\/us-central1\/subnetworks\/default","type":""}],"type":"","sshKeys":["centos:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDMGAS\/K0Mb\/uwFXwD+hkjguy7VMZk2hpuhwPl9FUZwVBrURf\/i9QMJ5\/paPEZixu8VlRx7Inu4iun7rQfrnfeIYInmBwspXHYiTK3oHJAgZnrAHVEf1p6YaxLINlT1NI5yOAGPRWW6of8rBDBH1ObwU2+wcSx\/1H0uIs3aZNLufr+Rh628ACxAum2Gt8AVRj6ua2BPFyt5VTdclyysAmeh1AiixNgOZXOz6y\/i4TbzpY78I3isuKpxsUeXX6jxEMQol406jHDUF6njEOPIQG2zVZ3QJlTG9OlN+NiyZG9WkZz0VG\/6M8ixxIHHI2dNwUbBFv2HUu+8X9LTLFq2O7KjX9Hp7uZKBAySHA3eKaKHIp2bZuU1bT5PRPkggngX86xg+T+OMNnutbAiMnRJ8+FvD5So+5TIx4b9GgxAxure3x2yRPT9lOiQOB+CVpJPxR0Rn9bOI+wiPnD0kAGvK\/fHT+pqL4PM+hTnJtp9rrCRzIQApBx1263jEcYffhW2epZQRO+he5CMawFJ5TBe08om2AaDJ8GQdrpF6YA3W8DzHbmL3DPVVHdmqPLn10o+LX4gv5SdIIDVGdjKOc1BCnLTRmM28d5+sLDt\/M+kvcQgf0y0yDjMVjGECZkt39hbm4ELMHzZtzYLmHNhBZxRqHeJ7qFTuv1kx88OW3Xc5mbBNQ== centos@nutanix.com\n"],"serviceAccounts":[{"scopes":["https:\/\/www.googleapis.com\/auth\/cloud-platform"],"type":"","email":"108048128720-compute@developer.gserviceaccount.com"}],"metadata":{"items":[],"type":"","fingerprint":""}}},"variable_list":[]}],"credential_definition_list":[{"username":"centos","description":"","type":"KEY","secret":{"attrs":{"is_secret_modified":false,"secret_reference":{}}},"name":"CENTOS","editables":{"username":true,"secret":true}}],"package_definition_list":[{"description":"","action_list":[],"type":"DEB","service_local_reference_list":[{"kind":"app_service","name":"Kubernetes_Master"}],"name":"AHV_Centos_K8SC_Package","version":"","options":{"install_runbook":{"task_definition_list":[{"target_any_local_reference":{"kind":"app_package","name":"AHV_Centos_K8SC_Package"},"retries":"0","description":"","message_list":[],"child_tasks_local_reference_list":[{"kind":"app_task","name":"ETCD Docker Kubelet Install"},{"kind":"app_task","name":"Generate Certs"},{"kind":"app_task","name":"Configure Services"},{"kind":"app_task","name":"Add User Roles"},{"kind":"app_task","name":"Network Configuration"},{"kind":"app_task","name":"DNS Configuration"},{"kind":"app_task","name":"NVP Configuration"}],"name":"dac441af_dag","state":"ACTIVE","attrs":{"edges":[{"from_task_reference":{"kind":"app_task","name":"ETCD Docker Kubelet Install"},"edge_type":"user_defined","type":"","to_task_reference":{"kind":"app_task","name":"Generate Certs"}},{"from_task_reference":{"kind":"app_task","name":"Generate Certs"},"edge_type":"user_defined","type":"","to_task_reference":{"kind":"app_task","name":"Configure Services"}},{"from_task_reference":{"kind":"app_task","name":"Configure Services"},"edge_type":"user_defined","type":"","to_task_reference":{"kind":"app_task","name":"Add User Roles"}},{"from_task_reference":{"kind":"app_task","name":"Add User Roles"},"edge_type":"user_defined","type":"","to_task_reference":{"kind":"app_task","name":"Network Configuration"}},{"from_task_reference":{"kind":"app_task","name":"Network Configuration"},"edge_type":"user_defined","type":"","to_task_reference":{"kind":"app_task","name":"DNS Configuration"}},{"from_task_reference":{"kind":"app_task","name":"DNS Configuration"},"edge_type":"user_defined","type":"","to_task_reference":{"kind":"app_task","name":"NVP Configuration"}}],"type":""},"timeout_secs":"0","type":"DAG","variable_list":[]},{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Master"},"retries":"0","description":"","message_list":[],"child_tasks_local_reference_list":[],"name":"ETCD Docker Kubelet Install","state":"ACTIVE","attrs":{"script":"#!\/bin\/bash\nset -ex\n\nsudo easy_install netaddr\n\nETCD_VERSION=\"v3.2.24\" #defualt v3.2.11\nKUBELET_IMAGE_TAG=\"@@{KUBE_IMAGE_TAG}@@\"\nif [[ \"@@{KUBE_IMAGE_TAG_NEW}@@x\" != \"x\" ]]; then\n\tKUBELET_IMAGE_TAG=\"@@{KUBE_IMAGE_TAG_NEW}@@\"\nfi\nINTERNAL_IP=\"@@{address}@@\"\nCONTROLLER_IPS=\"@@{calm_array_address}@@\"\nMINION_IPS=\"@@{AHV_Centos_K8SM.address}@@\"\nNODE_NAME=\"controller@@{calm_array_index}@@\"\nCLUSTER_SUBNET=\"@@{KUBE_CLUSTER_SUBNET}@@\"\nSERVICE_SUBNET=\"@@{KUBE_SERVICE_SUBNET}@@\"\nKUBE_CLUSTER_DNS=\"@@{KUBE_DNS_IP}@@\"\nDOCKER_VERSION=\"@@{DOCKER_VERSION}@@\"\nETCD_CERT_PATH=\"\/etc\/ssl\/certs\/etcd\"\nKUBE_CERT_PATH=\"\/etc\/kubernetes\/ssl\"\nKUBE_MANIFEST_PATH=\"\/etc\/kubernetes\/manifests\"\nKUBE_CNI_BIN_PATH=\"\/opt\/cni\/bin\"\nKUBE_CNI_CONF_PATH=\"\/etc\/cni\/net.d\"\nVERSION=$(echo \"${KUBELET_IMAGE_TAG}\" | tr \"_\" \" \" | awk '{print $1}')\nMASTER_API_HTTPS=6443\nETCD_SERVER_PORT=2379\nETCD_CLIENT_PORT=2380\nMASTER_API_PORT=8080\nFIRST_IP_SERVICE_SUBNET=$(python -c \"from netaddr import * ; print IPNetwork('${SERVICE_SUBNET}')[1]\")\n\nsudo mkdir -p \/opt\/kube-ssl ${KUBE_CERT_PATH} ${KUBE_CNI_BIN_PATH} ${ETCD_CERT_PATH} ${KUBE_MANIFEST_PATH} ${KUBE_CNI_CONF_PATH}\n\nsudo hostnamectl set-hostname --static ${NODE_NAME}\nsudo yum update -y --quiet\nsudo yum install -y wget iscsi-initiator-utils socat --quiet\n\ncount=0\nfor ip in $(echo \"${CONTROLLER_IPS}\" | tr \",\" \"\\n\"); do\n echo \"${ip} controller${count}\" | sudo tee -a \/etc\/hosts\n CON+=\"controller${count}=https:\/\/${ip}:${ETCD_CLIENT_PORT}\",\n ETCD+=\"https:\/\/${ip}:${ETCD_SERVER_PORT}\",\n CONS_NAMES+=\"controller${count}\",\n count=$((count+1))\ndone\nETCD_ALL_CONTROLLERS=$(echo $CON | sed 's\/,$\/\/')\nETCD_SERVERS=$(echo $ETCD | sed 's\/,$\/\/')\nCONTROLLER_NAMES=$(echo $CONS_NAMES | sed 's\/,$\/\/')\n \ncount=0\nfor ip in $(echo ${MINION_IPS} | tr \",\" \"\\n\"); do\n echo \"${ip} minion${count}\" | sudo tee -a \/etc\/hosts\n MIN_NAMES+=\"minion${count}\",\n count=$((count+1))\ndone\nMINION_NAMES=$(echo $MIN_NAMES | sed 's\/,$\/\/') \n \n#wget -q \"https:\/\/github.com\/coreos\/etcd\/releases\/download\/${ETCD_VERSION}\/etcd-${ETCD_VERSION}-linux-amd64.tar.gz\"\n#wget -q https:\/\/github.com\/containernetworking\/plugins\/releases\/download\/v0.6.0\/cni-plugins-amd64-v0.6.0.tgz\n#wget -q https:\/\/storage.googleapis.com\/kubernetes-release\/release\/${VERSION}\/bin\/linux\/amd64\/kubelet\nchmod +x kubelet\nsudo mv kubelet \/usr\/bin\/kubelet\n\n# -*- Bootstrapping a H\/A etcd cluster.\ntar -xvf etcd-${ETCD_VERSION}-linux-amd64.tar.gz\nsudo mv etcd-${ETCD_VERSION}-linux-amd64\/etcd* \/usr\/bin\/\nrm -rf etcd-${ETCD_VERSION}-linux-amd64*\n\necho \"[Unit]\nDescription=etcd\nDocumentation=https:\/\/github.com\/coreos\n\n[Service]\nExecStart=\/usr\/bin\/etcd \\\\\n --name ${NODE_NAME} \\\\\n --cert-file=${ETCD_CERT_PATH}\/etcd-server.pem \\\\\n --key-file=${ETCD_CERT_PATH}\/etcd-server-key.pem \\\\\n --peer-cert-file=${ETCD_CERT_PATH}\/etcd-peer.pem \\\\\n --peer-key-file=${ETCD_CERT_PATH}\/etcd-peer-key.pem \\\\\n --trusted-ca-file=${ETCD_CERT_PATH}\/etcd-ca.pem \\\\\n --peer-trusted-ca-file=${ETCD_CERT_PATH}\/etcd-ca.pem \\\\\n --peer-client-cert-auth \\\\\n --client-cert-auth \\\\\n --initial-advertise-peer-urls https:\/\/${INTERNAL_IP}:${ETCD_CLIENT_PORT} \\\\\n --listen-peer-urls https:\/\/${INTERNAL_IP}:${ETCD_CLIENT_PORT} \\\\\n --listen-client-urls https:\/\/${INTERNAL_IP}:${ETCD_SERVER_PORT},http:\/\/127.0.0.1:${ETCD_SERVER_PORT} \\\\\n --advertise-client-urls https:\/\/${INTERNAL_IP}:${ETCD_SERVER_PORT} \\\\\n --initial-cluster-token etcd-cluster-0 \\\\\n --initial-cluster ${ETCD_ALL_CONTROLLERS} \\\\\n --initial-cluster-state new \\\\\n --data-dir=\/var\/lib\/etcd \\\\\n --wal-dir=\/var\/lib\/etcd\/wal \\\\\n --max-wals=0\nRestart=on-failure\nRestartSec=5\n\n[Install]\nWantedBy=multi-user.target\" | sudo tee \/etc\/systemd\/system\/etcd.service\n\nsudo yum install -y --quiet yum-utils\nsudo yum-config-manager --add-repo https:\/\/download.docker.com\/linux\/centos\/docker-ce.repo\nsudo yum install -y --quiet --setopt=obsoletes=0 docker-ce-${DOCKER_VERSION} docker-ce-selinux-${DOCKER_VERSION}\n\nsudo sed -i '\/ExecStart=\/c\\\\ExecStart=\/usr\/bin\/dockerd -H tcp:\/\/0.0.0.0:2375 -H unix:\/\/\/var\/run\/docker.sock' \/usr\/lib\/systemd\/system\/docker.service\n\ncp \/usr\/lib\/systemd\/system\/docker.service \/tmp\nsudo sed -i '\/\\[Service\\]\/c\\\\[Service]\\nEnvironment=\\\"HTTP_PROXY=http:\/\/10.132.71.38:1080\/\\\"' \/usr\/lib\/systemd\/system\/docker.service\n\nsudo systemctl enable docker\nsudo usermod -a -G docker $USER\n\nsudo mkdir -p \/etc\/docker\necho '{\n \"storage-driver\": \"overlay\"\n}' | sudo tee \/etc\/docker\/daemon.json\n\necho '{\n \"name\": \"cbr0\",\n \"type\": \"flannel\",\n \"delegate\": {\n \"isDefaultGateway\": true\n }\n}' | sudo tee ${KUBE_CNI_CONF_PATH}\/10-flannel.conf\n\nsudo tar -zxvf cni-plugins-amd64-v0.6.0.tgz -C ${KUBE_CNI_BIN_PATH}\nrm -rf cni-plugins-amd64-v0.6.0.tgz\n\necho \"[Unit]\nDescription=Kubernetes Kubelet\nDocumentation=https:\/\/github.com\/GoogleCloudPlatform\/kubernetes\nAfter=docker.service\nRequires=docker.service\n\n[Service]\nExecStart=\/usr\/bin\/kubelet \\\\\n --allow-privileged=true \\\\\n --anonymous-auth=false \\\\\n --authorization-mode=Webhook \\\\\n --cluster-dns=${KUBE_CLUSTER_DNS} \\\\\n --cluster-domain=cluster.local \\\\\n --container-runtime=docker \\\\\n --enable-custom-metrics \\\\\n --kubeconfig=${KUBE_CERT_PATH}\/${NODE_NAME}.kubeconfig \\\\\n --network-plugin=cni \\\\\n --pod-cidr=${CLUSTER_SUBNET} \\\\\n --register-node=true \\\\\n --runtime-request-timeout=10m \\\\\n --client-ca-file=${KUBE_CERT_PATH}\/ca.pem \\\\\n --tls-cert-file=${KUBE_CERT_PATH}\/${NODE_NAME}.pem \\\\\n --tls-private-key-file=${KUBE_CERT_PATH}\/${NODE_NAME}-key.pem \\\\\n --pod-manifest-path=${KUBE_MANIFEST_PATH} \\\\\n --read-only-port=0 \\\\\n --protect-kernel-defaults=false \\\\\n --make-iptables-util-chains=true \\\\\n --keep-terminated-pod-volumes=false \\\\\n --event-qps=0 \\\\\n --cadvisor-port=0 \\\\\n --runtime-cgroups=\/systemd\/system.slice \\\\\n --kubelet-cgroups=\/systemd\/system.slice \\\\\n --node-labels 'node-role.kubernetes.io\/master=true' \\\\\n --node-labels 'node-role.kubernetes.io\/etcd=true' \\\\\n --register-with-taints=node-role.kubernetes.io\/master=true:NoSchedule \\\\\n --v=2\nRestart=on-failure\nRestartSec=5\n\n[Install]\nWantedBy=multi-user.target\" | sudo tee \/etc\/systemd\/system\/kubelet.service\n\nsudo mkdir -p \/var\/lib\/docker\nsudo yum install -y lvm2 --quiet\nsudo pvcreate \/dev\/sd{b,c,d}\nsudo vgcreate docker \/dev\/sd{b,c,d}\nsleep 3\nsudo lvcreate -l 100%VG -n docker_lvm docker\nsudo mkfs.xfs \/dev\/docker\/docker_lvm\n\necho -e \"\/dev\/docker\/docker_lvm \\t \/var\/lib\/docker \\t xfs \\t defaults \\t 0 0\" | sudo tee -a \/etc\/fstab\nsudo mount -a\necho \"InitiatorName=iqn.1994-05.com.nutanix:k8s-worker\" | sudo tee \/etc\/iscsi\/initiatorname.iscsi\necho 'exclude=docker*' | sudo tee -a \/etc\/yum.conf\n\n#wget -q https:\/\/pkg.cfssl.org\/R1.2\/cfssl_linux-amd64 https:\/\/pkg.cfssl.org\/R1.2\/cfssljson_linux-amd64\n#wget -q https:\/\/storage.googleapis.com\/kubernetes-release\/release\/${VERSION}\/bin\/linux\/amd64\/kubectl\n#wget -q \"https:\/\/storage.googleapis.com\/kubernetes-helm\/helm-v2.8.1-linux-amd64.tar.gz\"\n\ntar -zxvf helm-v2.8.2-linux-amd64.tar.gz\nchmod +x cfssl_linux-amd64 cfssljson_linux-amd64 kubectl linux-amd64\/helm\nsudo mv cfssl_linux-amd64 \/usr\/local\/bin\/cfssl\nsudo mv cfssljson_linux-amd64 \/usr\/local\/bin\/cfssljson\nsudo mv kubectl linux-amd64\/helm \/usr\/local\/bin\/\nrm -rf helm-v2.8.2-linux-amd64.tar.gz linux-amd64","type":"","command_line_args":"","exit_status":[],"script_type":"sh"},"timeout_secs":"0","type":"EXEC","variable_list":[]},{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Master"},"retries":"0","description":"","message_list":[],"child_tasks_local_reference_list":[],"name":"Generate Certs","state":"ACTIVE","attrs":{"script":"#!\/bin\/bash\nset -ex\n\nINTERNAL_IP=\"@@{address}@@\"\nCONTROLLER_IPS=\"@@{calm_array_address}@@\"\nMINION_IPS=\"@@{AHV_Centos_K8SM.address}@@\"\nMASTER_API_HTTPS=6443\nSERVICE_SUBNET=\"@@{KUBE_SERVICE_SUBNET}@@\"\nKUBE_CLUSTER_NAME=\"@@{KUBE_CLUSTER_NAME}@@\"\nFIRST_IP_SERVICE_SUBNET=$(python -c \"from netaddr import * ; print IPNetwork('${SERVICE_SUBNET}')[1]\")\n\ncount=0\nfor ip in $(echo \"${CONTROLLER_IPS}\" | tr \",\" \"\\n\"); do\n CONS_NAMES+=\"controller${count}\",\n count=$((count+1))\ndone\n\nCONTROLLER_NAMES=$(echo $CONS_NAMES | sed 's\/,$\/\/')\n \ncount=0\nfor ip in $(echo ${MINION_IPS} | tr \",\" \"\\n\"); do\n MIN_NAMES+=\"minion${count}\",\n count=$((count+1))\ndone\nMINION_NAMES=$(echo $MIN_NAMES | sed 's\/,$\/\/') \n\nif [ @@{calm_array_index}@@ -ne 0 ];then\n exit\nfi\nsudo chown -R $USER:$USER \/opt\/kube-ssl && cd \/opt\/kube-ssl\necho '{\n \"signing\": {\n \"default\": {\n \"expiry\": \"8760h\"\n },\n \"profiles\": {\n \"server\": {\n \"expiry\": \"8760h\",\n \"usages\": [ \"signing\", \"key encipherment\", \"server auth\", \"client auth\" ]\n },\n \"client\": {\n \"expiry\": \"8760h\",\n \"usages\": [ \"key encipherment\", \"client auth\" ]\n },\n \"client-server\": {\n \"expiry\": \"8760h\",\n \"usages\": [ \"key encipherment\", \"server auth\", \"client auth\" ]\n }\n }\n }\n}' | tee ca-config.json\n\necho '{\n \"CN\": \"etcd-ca\",\n \"key\": {\n \"algo\": \"rsa\",\n \"size\": 2048\n },\n \"names\": [\n {\n \"C\": \"US\",\n \"L\": \"San Jose\",\n \"O\": \"etcd\",\n \"OU\": \"CA\",\n \"ST\": \"California\"\n }\n ]\n}' | tee etcd-ca-csr.json\n\ncfssl gencert -initca etcd-ca-csr.json | cfssljson -bare etcd-ca\n\necho '{\n \"CN\": \"etcd\",\n \"hosts\": [],\n \"key\": {\n \"algo\": \"rsa\",\n \"size\": 2048\n },\n \"names\": [\n {\n \"C\": \"US\",\n \"L\": \"San Jose\",\n \"O\": \"etcd\",\n \"OU\": \"CA\",\n \"ST\": \"California\"\n }\n ]\n}' | tee etcd-csr.json\n\ncfssl gencert -ca=etcd-ca.pem -ca-key=etcd-ca-key.pem -config=ca-config.json -hostname=${CONTROLLER_IPS} -profile=server etcd-csr.json | cfssljson -bare etcd-server\ncfssl gencert -ca=etcd-ca.pem -ca-key=etcd-ca-key.pem -config=ca-config.json -hostname=${CONTROLLER_IPS} -profile=client-server etcd-csr.json | cfssljson -bare etcd-peer\ncfssl gencert -ca=etcd-ca.pem -ca-key=etcd-ca-key.pem -config=ca-config.json -hostname=${CONTROLLER_IPS} -profile=client etcd-csr.json | cfssljson -bare etcd-client\n\necho '{\n \"CN\": \"kube-ca\",\n \"key\": {\n \"algo\": \"rsa\",\n \"size\": 2048\n },\n \"names\": [\n {\n \"C\": \"US\",\n \"L\": \"San Jose\",\n \"O\": \"kube\",\n \"OU\": \"CA\",\n \"ST\": \"California\"\n }\n ]\n}' | tee kube-ca-csr.json\n\ncfssl gencert -initca kube-ca-csr.json | cfssljson -bare ca\n\necho '{\n \"CN\": \"kubernetes\",\n \"key\": {\n \"algo\": \"rsa\",\n \"size\": 2048\n },\n \"names\": [\n {\n \"C\": \"US\",\n \"L\": \"San Jose\",\n \"O\": \"kube\",\n \"OU\": \"Cluster\",\n \"ST\": \"California\"\n }\n ]\n}' | tee kubernetes-csr.json\n\ncfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -hostname=${CONTROLLER_NAMES},${CONTROLLER_IPS},${MINION_NAMES},${MINION_IPS},${FIRST_IP_SERVICE_SUBNET},127.0.0.1,kubernetes.default,kubernetes,kubernetes.default.svc,kubernetes.default.svc.cluster.local -profile=server kubernetes-csr.json | cfssljson -bare kubernetes\n\necho '{\n \"CN\": \"system:kube-controller-manager\",\n \"hosts\": [],\n \"key\": {\n \"algo\": \"rsa\",\n \"size\": 2048\n },\n \"names\": [\n {\n \"C\": \"US\",\n \"L\": \"San Jose\",\n \"O\": \"system:kube-controller-manager\",\n \"OU\": \"Cluster\",\n \"ST\": \"California\"\n }\n ]\n}' | tee kube-controller-manager-csr.json\n\ncfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server kube-controller-manager-csr.json | cfssljson -bare kube-controller-manager\n\necho '{\n \"CN\": \"system:kube-scheduler\",\n \"hosts\": [],\n \"key\": {\n \"algo\": \"rsa\",\n \"size\": 2048\n },\n \"names\": [\n {\n \"C\": \"US\",\n \"L\": \"San Jose\",\n \"O\": \"system:kube-scheduler\",\n \"OU\": \"Cluster\",\n \"ST\": \"California\"\n }\n ]\n}' | tee kube-scheduler-csr.json\n\ncfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server kube-scheduler-csr.json | cfssljson -bare kube-scheduler\n\ncount=0\nfor ip in $(echo ${CONTROLLER_IPS} | tr \",\" \"\\n\"); do\ninstance=\"controller${count}\"\necho \"{\n \\\"CN\\\": \\\"system:node:${instance}\\\",\n \\\"key\\\": {\n \\\"algo\\\": \\\"rsa\\\",\n \\\"size\\\": 2048\n },\n \\\"names\\\": [\n {\n \\\"C\\\": \\\"US\\\",\n \\\"L\\\": \\\"San Jose\\\",\n \\\"O\\\": \\\"system:nodes\\\",\n \\\"OU\\\": \\\"Kubernetes The Hard Way\\\",\n \\\"ST\\\": \\\"California\\\"\n }\n ]\n}\" | tee ${instance}-csr.json\ncfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -hostname=${instance},${ip} -profile=client-server ${instance}-csr.json | cfssljson -bare ${instance}\ncount=$((count+1))\ndone \n\n# -*- Creating kube-proxy certificates\necho '{\n \"CN\": \"system:kube-proxy\",\n \"hosts\": [],\n \"key\": {\n \"algo\": \"rsa\",\n \"size\": 2048\n },\n \"names\": [\n {\n \"C\": \"US\",\n \"L\": \"San Jose\",\n \"O\": \"system:node-proxier\",\n \"OU\": \"Cluster\",\n \"ST\": \"California\"\n }\n ]\n}' | tee kube-proxy-csr.json\n\ncfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client kube-proxy-csr.json | cfssljson -bare kube-proxy\n\necho '{\n \"CN\": \"admin\",\n \"hosts\": [],\n \"key\": {\n \"algo\": \"rsa\",\n \"size\": 2048\n },\n \"names\": [\n {\n \"C\": \"US\",\n \"L\": \"San Jose\",\n \"O\": \"system:masters\",\n \"OU\": \"Cluster\",\n \"ST\": \"California\"\n }\n ]\n}' | tee admin-csr.json\n\ncfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client admin-csr.json | cfssljson -bare admin\n\ncount=0\nfor ip in $(echo ${CONTROLLER_IPS} | tr \",\" \"\\n\"); do\nkubectl config set-cluster ${KUBE_CLUSTER_NAME} --certificate-authority=ca.pem --embed-certs=true --server=https:\/\/${INTERNAL_IP}:${MASTER_API_HTTPS} --kubeconfig=controller${count}.kubeconfig\nkubectl config set-credentials system:node:controller${count} --client-certificate=controller${count}.pem --client-key=controller${count}-key.pem --embed-certs=true --kubeconfig=controller${count}.kubeconfig\nkubectl config set-context default --cluster=${KUBE_CLUSTER_NAME} --user=system:node:controller${count} --kubeconfig=controller${count}.kubeconfig\nkubectl config use-context default --kubeconfig=controller${count}.kubeconfig\ncount=$((count+1))\ndone\n\nkubectl config set-cluster ${KUBE_CLUSTER_NAME} --certificate-authority=ca.pem --embed-certs=true --server=https:\/\/${INTERNAL_IP}:${MASTER_API_HTTPS} --kubeconfig=kube-controller-manager.kubeconfig\nkubectl config set-credentials kube-controller-manager --client-certificate=kube-controller-manager.pem --client-key=kube-controller-manager-key.pem --embed-certs=true --kubeconfig=kube-controller-manager.kubeconfig\nkubectl config set-context default --cluster=${KUBE_CLUSTER_NAME} --user=kube-controller-manager --kubeconfig=kube-controller-manager.kubeconfig\nkubectl config use-context default --kubeconfig=kube-controller-manager.kubeconfig\n\nkubectl config set-cluster ${KUBE_CLUSTER_NAME} --certificate-authority=ca.pem --embed-certs=true --server=https:\/\/${INTERNAL_IP}:${MASTER_API_HTTPS} --kubeconfig=kube-scheduler.kubeconfig\nkubectl config set-credentials kube-scheduler --client-certificate=kube-scheduler.pem --client-key=kube-scheduler-key.pem --embed-certs=true --kubeconfig=kube-scheduler.kubeconfig\nkubectl config set-context default --cluster=${KUBE_CLUSTER_NAME} --user=kube-scheduler --kubeconfig=kube-scheduler.kubeconfig\nkubectl config use-context default --kubeconfig=kube-scheduler.kubeconfig\n\nkubectl config set-cluster ${KUBE_CLUSTER_NAME} --certificate-authority=ca.pem --embed-certs=true --server=https:\/\/${INTERNAL_IP}:${MASTER_API_HTTPS} --kubeconfig=kube-proxy.kubeconfig\nkubectl config set-credentials kube-proxy --client-certificate=kube-proxy.pem --client-key=kube-proxy-key.pem --embed-certs=true --kubeconfig=kube-proxy.kubeconfig\nkubectl config set-context default --cluster=${KUBE_CLUSTER_NAME} --user=kube-proxy --kubeconfig=kube-proxy.kubeconfig\nkubectl config use-context default --kubeconfig=kube-proxy.kubeconfig\n\nENCRYPTION_KEY=$(head -c 32 \/dev\/urandom | base64)\necho \"kind: EncryptionConfig\napiVersion: v1\nresources:\n - resources:\n - secrets\n providers:\n - aescbc:\n keys:\n - name: key1\n secret: ${ENCRYPTION_KEY}\n - identity: {}\" | tee encryption-config.yaml\n\necho \"@@{CENTOS.secret}@@\" | tee ~\/.ssh\/id_rsa\nchmod 400 ~\/.ssh\/id_rsa\n\ncount=0\nfor ip in $(echo ${CONTROLLER_IPS} | tr \",\" \"\\n\"); do\n instance=\"controller${count}\"\n scp -o stricthostkeychecking=no admin*.pem ca*.pem etcd*.pem kubernetes*.pem ${instance}* kube-proxy.kubeconfig kube-controller-manager.kubeconfig kube-scheduler.kubeconfig encryption-config.yaml ${instance}:\ncount=$((count+1))\ndone","type":"","command_line_args":"","exit_status":[],"script_type":"sh"},"timeout_secs":"0","type":"EXEC","variable_list":[]},{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Master"},"retries":"0","description":"","message_list":[],"child_tasks_local_reference_list":[],"name":"Configure Services","state":"ACTIVE","attrs":{"script":"#!\/bin\/bash\nset -ex\n\nKUBELET_IMAGE_TAG=\"@@{KUBE_IMAGE_TAG}@@\"\nif [[ \"@@{KUBE_IMAGE_TAG_NEW}@@x\" != \"x\" ]]; then\n\tKUBELET_IMAGE_TAG=\"@@{KUBE_IMAGE_TAG_NEW}@@\"\nfi\nINTERNAL_IP=\"@@{address}@@\"\nCONTROLLER_IPS=\"@@{calm_array_address}@@\"\nCLUSTER_SUBNET=\"@@{KUBE_CLUSTER_SUBNET}@@\"\nSERVICE_SUBNET=\"@@{KUBE_SERVICE_SUBNET}@@\"\nETCD_CERT_PATH=\"\/etc\/ssl\/certs\/etcd\"\nKUBE_CERT_PATH=\"\/etc\/kubernetes\/ssl\"\nKUBE_MANIFEST_PATH=\"\/etc\/kubernetes\/manifests\"\nMASTER_API_HTTPS=6443\nETCD_SERVER_PORT=2379\nVERSION=$(echo \"${KUBELET_IMAGE_TAG}\" | tr \"_\" \" \" | awk '{print $1}')\nCONTROLLER_COUNT=$(echo \"@@{calm_array_address}@@\" | tr ',' '\\n' | wc -l)\n\nsudo cp ca*.pem etcd-*.pem kubernetes*.pem ${HOSTNAME}* kube-*.kubeconfig encryption-config.yaml ${KUBE_CERT_PATH}\/\nsudo chmod +r ${KUBE_CERT_PATH}\/*\n\nsudo cp etcd-*.pem ${ETCD_CERT_PATH}\/\nsudo chmod +r ${ETCD_CERT_PATH}\/*\n\ncount=0\nfor ip in $(echo \"${CONTROLLER_IPS}\" | tr \",\" \"\\n\"); do\n ETCD+=\"https:\/\/${ip}:${ETCD_SERVER_PORT}\",\n count=$((count+1))\ndone\nETCD_SERVERS=$(echo $ETCD | sed 's\/,$\/\/')\n\necho \"apiVersion: v1\nkind: Pod\nmetadata:\n name: kube-apiserver\n namespace: kube-system\n labels:\n k8s-app: kube-apiserver\nspec:\n hostNetwork: true\n containers:\n - name: kube-apiserver\n image: quay.io\/coreos\/hyperkube:${KUBELET_IMAGE_TAG}\n command:\n - \/hyperkube\n - apiserver\n - --admission-control=Initializers,NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota\n - --advertise-address=${INTERNAL_IP}\n - --allow-privileged=true\n - --anonymous-auth=false\n - --insecure-port=0\n - --secure-port=${MASTER_API_HTTPS}\n - --profiling=false\n - --repair-malformed-updates=false\n - --apiserver-count=${CONTROLLER_COUNT}\n - --audit-log-maxage=30\n - --audit-log-maxbackup=10\n - --audit-log-maxsize=100\n - --audit-log-path=\/var\/lib\/audit.log\n - --authorization-mode=Node,RBAC\n - --bind-address=0.0.0.0\n - --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP\n - --event-ttl=1h\n - --service-account-lookup=true\n - --enable-swagger-ui=true\n - --storage-backend=etcd3\n - --etcd-cafile=${ETCD_CERT_PATH}\/etcd-ca.pem\n - --etcd-certfile=${ETCD_CERT_PATH}\/etcd-client.pem\n - --etcd-keyfile=${ETCD_CERT_PATH}\/etcd-client-key.pem\n - --etcd-servers=${ETCD_SERVERS}\n - --experimental-encryption-provider-config=${KUBE_CERT_PATH}\/encryption-config.yaml\n - --tls-ca-file=${KUBE_CERT_PATH}\/ca.pem\n - --tls-cert-file=${KUBE_CERT_PATH}\/kubernetes.pem\n - --tls-private-key-file=${KUBE_CERT_PATH}\/kubernetes-key.pem\n - --kubelet-client-certificate=${KUBE_CERT_PATH}\/kubernetes.pem\n - --kubelet-client-key=${KUBE_CERT_PATH}\/kubernetes-key.pem\n - --kubelet-https=true\n - --runtime-config=api\/all\n - --service-account-key-file=${KUBE_CERT_PATH}\/kubernetes.pem\n - --service-cluster-ip-range=${SERVICE_SUBNET}\n - --service-node-port-range=30000-32767\n - --client-ca-file=${KUBE_CERT_PATH}\/ca.pem\n - --v=2\n ports:\n - containerPort: ${MASTER_API_HTTPS}\n hostPort: ${MASTER_API_HTTPS}\n name: https\n - containerPort: 8080\n hostPort: 8080\n name: local\n volumeMounts:\n - mountPath: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n readOnly: true\n - mountPath: \/etc\/ssl\/certs\n name: ssl-certs-host\n readOnly: true\n - mountPath: \/etc\/pki\n name: ca-certs-etc-pki\n readOnly: true\n volumes:\n - hostPath:\n path: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n - hostPath:\n path: \/etc\/ssl\/certs\n name: ssl-certs-host\n - hostPath:\n path: \/etc\/pki\n name: ca-certs-etc-pki\" | sudo tee ${KUBE_MANIFEST_PATH}\/kube-apiserver.yaml\n\necho \"apiVersion: v1\nkind: Pod\nmetadata:\n name: kube-proxy\n namespace: kube-system\n labels:\n k8s-app: kube-proxy\nspec:\n hostNetwork: true\n containers:\n - name: kube-proxy\n image: quay.io\/coreos\/hyperkube:${KUBELET_IMAGE_TAG}\n command:\n - \/hyperkube\n - proxy\n - --cluster-cidr=${CLUSTER_SUBNET}\n - --masquerade-all=true\n - --kubeconfig=${KUBE_CERT_PATH}\/kube-proxy.kubeconfig\n - --proxy-mode=iptables\n securityContext:\n privileged: true\n volumeMounts:\n - mountPath: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n readOnly: true\n - mountPath: \/etc\/ssl\/certs\n name: ssl-certs-host\n readOnly: true\n volumes:\n - hostPath:\n path: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n - hostPath:\n path: \/etc\/ssl\/certs\n name: ssl-certs-host\" | sudo tee ${KUBE_MANIFEST_PATH}\/kube-proxy.yaml\n \necho \"apiVersion: v1\nkind: Pod\nmetadata:\n name: kube-controller-manager\n namespace: kube-system\n labels:\n k8s-app: kube-controller-manager\nspec:\n hostNetwork: true\n containers:\n - name: kube-controller-manager\n image: quay.io\/coreos\/hyperkube:${KUBELET_IMAGE_TAG}\n command:\n - \/hyperkube\n - controller-manager\n - --address=0.0.0.0 \n - --allocate-node-cidrs=true \n - --cluster-cidr=${CLUSTER_SUBNET}\n - --cluster-name=kubernetes-prod-cluster \n - --leader-elect=true \n - --kubeconfig=${KUBE_CERT_PATH}\/kube-controller-manager.kubeconfig \n - --service-account-private-key-file=${KUBE_CERT_PATH}\/kubernetes-key.pem \n - --service-cluster-ip-range=${SERVICE_SUBNET}\n - --terminated-pod-gc-threshold=100 \n - --profiling=false \n - --use-service-account-credentials=true \n - --v=2\n livenessProbe:\n httpGet:\n host: 127.0.0.1\n path: \/healthz\n port: 10252\n initialDelaySeconds: 15\n timeoutSeconds: 1\n volumeMounts:\n - mountPath: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n readOnly: true\n - mountPath: \/etc\/ssl\/certs\n name: ssl-certs-host\n readOnly: true\n - mountPath: \/etc\/pki\n name: ca-certs-etc-pki\n readOnly: true\n volumes:\n - hostPath:\n path: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n - hostPath:\n path: \/etc\/ssl\/certs\n name: ssl-certs-host\n - hostPath:\n path: \/etc\/pki\n name: ca-certs-etc-pki\" | sudo tee ${KUBE_MANIFEST_PATH}\/kube-controller-manager.yaml\n \necho \"apiVersion: v1\nkind: Pod\nmetadata:\n name: kube-scheduler\n namespace: kube-system\n labels:\n k8s-app: kube-scheduler\nspec:\n hostNetwork: true\n containers:\n - name: kube-scheduler\n image: quay.io\/coreos\/hyperkube:${KUBELET_IMAGE_TAG}\n command:\n - \/hyperkube\n - scheduler\n - --kubeconfig=${KUBE_CERT_PATH}\/kube-scheduler.kubeconfig\n - --leader-elect=true\n - --profiling=false\n - --v=2\n livenessProbe:\n httpGet:\n host: 127.0.0.1\n path: \/healthz\n port: 10251\n initialDelaySeconds: 15\n timeoutSeconds: 1\n volumeMounts:\n - mountPath: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n readOnly: true\n volumes:\n - hostPath:\n path: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\" | sudo tee ${KUBE_MANIFEST_PATH}\/kube-scheduler.yaml","type":"","command_line_args":"","exit_status":[],"script_type":"sh"},"timeout_secs":"0","type":"EXEC","variable_list":[]},{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Master"},"retries":"0","description":"","message_list":[],"child_tasks_local_reference_list":[],"name":"Add User Roles","state":"ACTIVE","attrs":{"script":"#!\/bin\/bash\nset -ex\n\nMASTER_API_HTTPS=6443\nINTERNAL_IP=\"@@{address}@@\"\nKUBE_CLUSTER_NAME=\"@@{KUBE_CLUSTER_NAME}@@\"\n\nsudo systemctl start etcd docker kubelet iscsid\nsudo systemctl enable etcd docker kubelet iscsid\n\nexport PATH=$PATH:\/opt\/bin\n\nmkdir CA\nmv admin*.pem ca*.pem etcd-*.pem kubernetes*.pem controller* kube-*.kubeconfig encryption-config.yaml CA\/\nif [ @@{calm_array_index}@@ -ne 0 ];then\n exit\nfi\ncp \/opt\/kube-ssl\/admin*.pem CA\/\nCOUNT=0\nwhile [[ $(curl --key CA\/admin-key.pem --cert CA\/admin.pem --cacert CA\/ca.pem https:\/\/${INTERNAL_IP}:${MASTER_API_HTTPS}\/healthz) != \"ok\" ]] ; do\n echo \"sleep for 5 secs\"\n sleep 25\n COUNT=$(($COUNT+1))\n if [[ $COUNT -eq 50 ]]; then\n \techo \"Error: creating cluster\"\n exit 1\n fi\ndone\n\nkubectl config set-cluster ${KUBE_CLUSTER_NAME} --certificate-authority=$HOME\/CA\/ca.pem --embed-certs=true --server=https:\/\/${INTERNAL_IP}:${MASTER_API_HTTPS}\nkubectl config set-credentials admin --client-certificate=$HOME\/CA\/admin.pem --client-key=$HOME\/CA\/admin-key.pem\nkubectl config set-context ${KUBE_CLUSTER_NAME} --cluster=${KUBE_CLUSTER_NAME} --user=admin\nkubectl config use-context ${KUBE_CLUSTER_NAME}\n\ncat <\ntype: nutanix\/abs\n---\nkind: StorageClass\napiVersion: storage.k8s.io\/v1\nmetadata:\n name: silver \n namespace: kube-system\n annotations:\n storageclass.beta.kubernetes.io\/is-default-class: \"true\"\nprovisioner: nutanix\/abs\nparameters:\n prismEndPoint: @@{PE_CLUSTER_IP}@@:9440\n dataServiceEndPoint: @@{PE_DATA_SERVICE_IP}@@:3260\n secretName: ntnx-secret\n storageContainer: @@{PE_CONTAINER_NAME}@@\n fsType: ext4\n chapAuthEnabled: \"false\"' | sudo tee \/etc\/kubernetes\/addons\/volume_plugin\/nutanix-provisioner.yaml\n\nsudo sed -i \"s\/\/${NTNX_SECRET}\/\" \/etc\/kubernetes\/addons\/volume_plugin\/nutanix-provisioner.yaml\nkubectl create -f \/etc\/kubernetes\/addons\/volume_plugin\/nutanix-provisioner.yaml\n\necho \"apiVersion: v1\nkind: Secret\nmetadata:\n name: ntnx-secret\n namespace: default\ndata:\n key: \ntype: nutanix\/abs\" | sudo tee -a \/etc\/kubernetes\/addons\/volume_plugin\/ntnx-secret.yaml\nsudo sed -i \"s\/\/${NTNX_SECRET}\/\" \/etc\/kubernetes\/addons\/volume_plugin\/ntnx-secret.yaml\nkubectl create -f \/etc\/kubernetes\/addons\/volume_plugin\/ntnx-secret.yaml\n","type":"","command_line_args":"","exit_status":[],"script_type":"sh"},"timeout_secs":"0","type":"EXEC","variable_list":[]}],"description":"","name":"8e760b73_runbook","state":"ACTIVE","main_task_local_reference":{"kind":"app_task","name":"dac441af_dag"},"message_list":[],"variable_list":[]},"type":"","uninstall_runbook":{"task_definition_list":[{"target_any_local_reference":{"kind":"app_package","name":"AHV_Centos_K8SC_Package"},"retries":"0","description":"","message_list":[],"child_tasks_local_reference_list":[],"name":"6324786a_dag","state":"ACTIVE","attrs":{"edges":[],"type":""},"timeout_secs":"0","type":"DAG","variable_list":[]}],"description":"","name":"6522ac7a_runbook","state":"ACTIVE","main_task_local_reference":{"kind":"app_task","name":"6324786a_dag"},"message_list":[],"variable_list":[]}},"variable_list":[]},{"description":"","action_list":[],"type":"DEB","service_local_reference_list":[{"kind":"app_service","name":"Kubernetes_Minion"}],"name":"AHV_Centos_K8SM_Package","version":"","options":{"install_runbook":{"task_definition_list":[{"target_any_local_reference":{"kind":"app_package","name":"AHV_Centos_K8SM_Package"},"retries":"0","description":"","message_list":[],"child_tasks_local_reference_list":[{"kind":"app_task","name":"Docker Kubelet Install"},{"kind":"app_task","name":"GetCerts"}],"name":"c1566b3d_dag","state":"ACTIVE","attrs":{"edges":[{"from_task_reference":{"kind":"app_task","name":"Docker Kubelet Install"},"edge_type":"user_defined","type":"","to_task_reference":{"kind":"app_task","name":"GetCerts"}}],"type":""},"timeout_secs":"0","type":"DAG","variable_list":[]},{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Minion"},"retries":"0","description":"","message_list":[],"child_tasks_local_reference_list":[],"name":"Docker Kubelet Install","state":"ACTIVE","attrs":{"script":"#!\/bin\/bash\nset -ex\n\nKUBELET_IMAGE_TAG=\"@@{KUBE_IMAGE_TAG}@@\"\nif [[ \"@@{KUBE_IMAGE_TAG_NEW}@@x\" != \"x\" ]]; then\n\tKUBELET_IMAGE_TAG=\"@@{KUBE_IMAGE_TAG_NEW}@@\"\nfi\nINTERNAL_IP=\"@@{address}@@\"\nCONTROLLER_IPS=\"@@{AHV_Centos_K8SC.address}@@\"\nNODE_NAME=\"minion@@{calm_array_index}@@\"\nCLUSTER_SUBNET=\"@@{KUBE_CLUSTER_SUBNET}@@\"\nSERVICE_SUBNET=\"@@{KUBE_SERVICE_SUBNET}@@\"\nKUBE_CLUSTER_DNS=\"@@{KUBE_DNS_IP}@@\"\nDOCKER_VERSION=\"@@{DOCKER_VERSION}@@\"\nKUBE_CERT_PATH=\"\/etc\/kubernetes\/ssl\"\nKUBE_MANIFEST_PATH=\"\/etc\/kubernetes\/manifests\"\nKUBE_CNI_BIN_PATH=\"\/opt\/cni\/bin\"\nKUBE_CNI_CONF_PATH=\"\/etc\/cni\/net.d\"\nETCD_SERVER_PORT=2379\nVERSION=$(echo \"${KUBELET_IMAGE_TAG}\" | tr \"_\" \" \" | awk '{print $1}')\n\nsudo mkdir -p ${KUBE_CERT_PATH} ${KUBE_MANIFEST_PATH} ${KUBE_CNI_CONF_PATH} ${KUBE_CNI_BIN_PATH}\nsudo hostnamectl set-hostname --static ${NODE_NAME}\n\nsudo yum update -y --quiet\nsudo yum install -y wget iscsi-initiator-utils socat --quiet\n\n#wget -q https:\/\/github.com\/containernetworking\/plugins\/releases\/download\/v0.6.0\/cni-plugins-amd64-v0.6.0.tgz\n#wget -q https:\/\/storage.googleapis.com\/kubernetes-release\/release\/${VERSION}\/bin\/linux\/amd64\/kubelet\n#wget -q https:\/\/storage.googleapis.com\/kubernetes-release\/release\/${VERSION}\/bin\/linux\/amd64\/kubectl\n#wget -q https:\/\/pkg.cfssl.org\/R1.2\/cfssl_linux-amd64 https:\/\/pkg.cfssl.org\/R1.2\/cfssljson_linux-amd64\n\nchmod +x kubelet kubectl cfssl_linux-amd64 cfssljson_linux-amd64\nsudo mv kubelet kubectl \/usr\/bin\/\nsudo mv cfssl_linux-amd64 \/usr\/local\/bin\/cfssl\nsudo mv cfssljson_linux-amd64 \/usr\/local\/bin\/cfssljson\n\nsudo yum install -y --quiet yum-utils\nsudo yum-config-manager --add-repo https:\/\/download.docker.com\/linux\/centos\/docker-ce.repo\nsudo yum install -y --quiet --setopt=obsoletes=0 docker-ce-${DOCKER_VERSION} docker-ce-selinux-${DOCKER_VERSION}\n\nsudo sed -i '\/ExecStart=\/c\\\\ExecStart=\/usr\/bin\/dockerd -H tcp:\/\/0.0.0.0:2375 -H unix:\/\/\/var\/run\/docker.sock' \/usr\/lib\/systemd\/system\/docker.service\n\ncp \/usr\/lib\/systemd\/system\/docker.service \/tmp\nsudo sed -i '\/\\[Service\\]\/c\\\\[Service]\\nEnvironment=\\\"HTTP_PROXY=http:\/\/10.132.71.38:1080\/\\\"' \/usr\/lib\/systemd\/system\/docker.service\n\nsudo systemctl enable docker\nsudo usermod -a -G docker $USER\n\nsudo mkdir -p \/etc\/docker\necho '{\n \"storage-driver\": \"overlay\"\n}' | sudo tee \/etc\/docker\/daemon.json\n\necho '{\n \"name\": \"cbr0\",\n \"type\": \"flannel\",\n \"delegate\": {\n \"isDefaultGateway\": true\n }\n}' | sudo tee ${KUBE_CNI_CONF_PATH}\/10-flannel.conf\n\nsudo tar -zxvf cni-plugins-amd64-v0.6.0.tgz -C ${KUBE_CNI_BIN_PATH}\nrm -rf cni-plugins-amd64-v0.6.0.tgz\n\necho \"[Unit]\nDescription=Kubernetes Kubelet\nDocumentation=https:\/\/github.com\/GoogleCloudPlatform\/kubernetes\nAfter=docker.service\nRequires=docker.service\n\n[Service]\nExecStart=\/usr\/bin\/kubelet \\\\\n --allow-privileged=true \\\\\n --anonymous-auth=false \\\\\n --authorization-mode=Webhook \\\\\n --cluster-dns=${KUBE_CLUSTER_DNS} \\\\\n --cluster-domain=cluster.local \\\\\n --container-runtime=docker \\\\\n --enable-custom-metrics \\\\\n --kubeconfig=${KUBE_CERT_PATH}\/${NODE_NAME}.kubeconfig \\\\\n --network-plugin=cni \\\\\n --pod-cidr=${CLUSTER_SUBNET} \\\\\n --register-node=true \\\\\n --runtime-request-timeout=10m \\\\\n --client-ca-file=${KUBE_CERT_PATH}\/ca.pem \\\\\n --tls-cert-file=${KUBE_CERT_PATH}\/${NODE_NAME}.pem \\\\\n --tls-private-key-file=${KUBE_CERT_PATH}\/${NODE_NAME}-key.pem \\\\\n --pod-manifest-path=${KUBE_MANIFEST_PATH} \\\\\n --read-only-port=0 \\\\\n --protect-kernel-defaults=false \\\\\n --make-iptables-util-chains=true \\\\\n --keep-terminated-pod-volumes=false \\\\\n --event-qps=0 \\\\\n --cadvisor-port=0 \\\\\n --runtime-cgroups=\/systemd\/system.slice \\\\\n --kubelet-cgroups=\/systemd\/system.slice \\\\\n --eviction-hard=memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5% \\\\\n --node-labels 'node-role.kubernetes.io\/worker=true' \\\\\n --node-labels 'beta.kubernetes.io\/fluentd-ds-ready=true' \\\\\n --v=2\nRestart=on-failure\nRestartSec=5\n\n[Install]\nWantedBy=multi-user.target\" | sudo tee \/etc\/systemd\/system\/kubelet.service\n\necho \"apiVersion: v1\nkind: Pod\nmetadata:\n name: kube-proxy\n namespace: kube-system\nspec:\n hostNetwork: true\n containers:\n - name: kube-proxy\n image: quay.io\/coreos\/hyperkube:${KUBELET_IMAGE_TAG}\n command:\n - \/hyperkube\n - proxy\n - --cluster-cidr=${CLUSTER_SUBNET}\n - --masquerade-all=true\n - --kubeconfig=${KUBE_CERT_PATH}\/kube-proxy.kubeconfig\n - --proxy-mode=iptables\n securityContext:\n privileged: true\n volumeMounts:\n - mountPath: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n readOnly: true\n - mountPath: \/etc\/ssl\/certs\n name: ssl-certs-host\n readOnly: true\n volumes:\n - hostPath:\n path: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n - hostPath:\n path: \/usr\/share\/ca-certificates\n name: ssl-certs-host\" | sudo tee ${KUBE_MANIFEST_PATH}\/kube-proxy.yaml\n\necho \"InitiatorName=iqn.1994-05.com.nutanix:k8s-worker\" | sudo tee \/etc\/iscsi\/initiatorname.iscsi\n\nsudo mkdir -p \/var\/lib\/docker\nsudo yum install -y lvm2 --quiet\nsudo pvcreate \/dev\/sd{b,c,d}\nsudo vgcreate docker \/dev\/sd{b,c,d}\nsleep 3\nsudo lvcreate -l 100%VG -n docker_lvm docker\nsudo mkfs.xfs \/dev\/docker\/docker_lvm\n\necho -e \"\/dev\/docker\/docker_lvm \\t \/var\/lib\/docker \\t xfs \\t defaults \\t 0 0\" | sudo tee -a \/etc\/fstab\nsudo mount -a\n\necho 'exclude=docker*' | sudo tee -a \/etc\/yum.conf\n\necho \"@@{CENTOS.secret}@@\" | tee ~\/.ssh\/id_rsa\nchmod 400 ~\/.ssh\/id_rsa\n\n#while [ ! -f ${NODE_NAME}.kubeconfig ] ; do echo \"waiting for certs sleeping 5\" && sleep 5; done\n\n#sudo cp *.pem *.kubeconfig ${KUBE_CERT_PATH}\/\n#sudo chmod +r ${KUBE_CERT_PATH}\/*\n\n#sudo systemctl start docker kubelet iscsid\n#sudo systemctl enable docker kubelet iscsid","type":"","command_line_args":"","exit_status":[],"script_type":"sh"},"timeout_secs":"0","type":"EXEC","variable_list":[]},{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Minion"},"retries":"0","description":"","message_list":[],"child_tasks_local_reference_list":[],"name":"GetCerts","state":"ACTIVE","attrs":{"script":"#!\/bin\/bash\nset -ex\n\nKUBE_CLUSTER_NAME=\"@@{KUBE_CLUSTER_NAME}@@\"\nMASTER_IP=\"@@{AHV_Centos_K8SC.address[0]}@@\"\nINSTANCE_IP=\"@@{address}@@\"\ninstance=\"minion@@{calm_array_index}@@\"\nKUBE_CERT_PATH=\"\/etc\/kubernetes\/ssl\"\nMASTER_API_HTTPS=6443\n\nwhile [ ! $(ssh -o stricthostkeychecking=no $MASTER_IP \"ls \/opt\/kube-ssl\/encryption-config.yaml 2>\/dev\/null\") ] ; do echo \"waiting for certs sleeping 5\" && sleep 5; done\n\nscp -o stricthostkeychecking=no ${MASTER_IP}:\/opt\/kube-ssl\/{ca*.pem,kubernetes*.pem,kube-proxy.kubeconfig,ca-config.json} .\n\ninstance=\"minion@@{calm_array_index}@@\"\necho \"{\n \\\"CN\\\": \\\"system:node:${instance}\\\",\n \\\"key\\\": {\n \\\"algo\\\": \\\"rsa\\\",\n \\\"size\\\": 2048\n },\n \\\"names\\\": [\n {\n \\\"C\\\": \\\"US\\\",\n \\\"L\\\": \\\"San Jose\\\",\n \\\"O\\\": \\\"system:nodes\\\",\n \\\"OU\\\": \\\"Kubernetes The Hard Way\\\",\n \\\"ST\\\": \\\"California\\\"\n }\n ]\n}\" | tee ${instance}-csr.json\ncfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -hostname=${instance},${INSTANCE_IP} -profile=client-server ${instance}-csr.json | cfssljson -bare ${instance}\n\nkubectl config set-cluster ${KUBE_CLUSTER_NAME} --certificate-authority=ca.pem --embed-certs=true --server=https:\/\/${MASTER_IP}:${MASTER_API_HTTPS} --kubeconfig=${instance}.kubeconfig\nkubectl config set-credentials system:node:${instance} --client-certificate=${instance}.pem --client-key=${instance}-key.pem --embed-certs=true --kubeconfig=${instance}.kubeconfig\nkubectl config set-context default --cluster=${KUBE_CLUSTER_NAME} --user=system:node:${instance} --kubeconfig=${instance}.kubeconfig\nkubectl config use-context default --kubeconfig=${instance}.kubeconfig\n\nsudo cp *.pem *.kubeconfig ${KUBE_CERT_PATH}\/\nsudo chmod +r ${KUBE_CERT_PATH}\/*\n\nrm -rf ${instance}-csr.json","type":"","command_line_args":"","exit_status":[],"script_type":"sh"},"timeout_secs":"0","type":"EXEC","variable_list":[]}],"description":"","name":"9a6e189b_runbook","state":"ACTIVE","main_task_local_reference":{"kind":"app_task","name":"c1566b3d_dag"},"message_list":[],"variable_list":[]},"type":"","uninstall_runbook":{"task_definition_list":[{"target_any_local_reference":{"kind":"app_package","name":"AHV_Centos_K8SM_Package"},"retries":"0","description":"","message_list":[],"child_tasks_local_reference_list":[{"kind":"app_task","name":"Remove Node"}],"name":"abd3357c_dag","state":"ACTIVE","attrs":{"edges":[],"type":""},"timeout_secs":"0","type":"DAG","variable_list":[]},{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Minion"},"retries":"0","description":"","message_list":[],"child_tasks_local_reference_list":[],"name":"Remove Node","state":"ACTIVE","attrs":{"script":"#!\/bin\/bash\nset -ex\n\nMASTER_IP=\"@@{Kubernetes_Master.address[0]}@@\"\nssh -o stricthostkeychecking=no ${MASTER_IP} \"kubectl drain 'minion@@{calm_array_index}@@' --ignore-daemonsets --delete-local-data --force\"\nsleep 10\nssh -o stricthostkeychecking=no ${MASTER_IP} \"kubectl delete node 'minion@@{calm_array_index}@@'\"","type":"","command_line_args":"","exit_status":[],"script_type":"sh"},"timeout_secs":"0","type":"EXEC","variable_list":[]}],"description":"","name":"3cb17acf_runbook","state":"ACTIVE","main_task_local_reference":{"kind":"app_task","name":"abd3357c_dag"},"message_list":[],"variable_list":[]}},"variable_list":[]},{"description":"","action_list":[],"type":"DEB","service_local_reference_list":[{"kind":"app_service","name":"Kubernetes_Master"}],"name":"AWS_Centos_K8SC_Package","version":"","options":{"install_runbook":{"task_definition_list":[{"target_any_local_reference":{"kind":"app_package","name":"AWS_Centos_K8SC_Package"},"retries":"0","description":"","message_list":[],"child_tasks_local_reference_list":[{"kind":"app_task","name":"ETCD Docker Kubelet Install"},{"kind":"app_task","name":"Generate Certs"},{"kind":"app_task","name":"Configure Services"},{"kind":"app_task","name":"Add User Roles"},{"kind":"app_task","name":"Network Configuration"},{"kind":"app_task","name":"DNS Configuration"},{"kind":"app_task","name":"EBS VolumePlugin"}],"name":"84e097cf_dag","state":"ACTIVE","attrs":{"edges":[{"from_task_reference":{"kind":"app_task","name":"ETCD Docker Kubelet Install"},"edge_type":"user_defined","type":"","to_task_reference":{"kind":"app_task","name":"Generate Certs"}},{"from_task_reference":{"kind":"app_task","name":"Generate Certs"},"edge_type":"user_defined","type":"","to_task_reference":{"kind":"app_task","name":"Configure Services"}},{"from_task_reference":{"kind":"app_task","name":"Configure Services"},"edge_type":"user_defined","type":"","to_task_reference":{"kind":"app_task","name":"Add User Roles"}},{"from_task_reference":{"kind":"app_task","name":"Add User Roles"},"edge_type":"user_defined","type":"","to_task_reference":{"kind":"app_task","name":"Network Configuration"}},{"from_task_reference":{"kind":"app_task","name":"Network Configuration"},"edge_type":"user_defined","type":"","to_task_reference":{"kind":"app_task","name":"DNS Configuration"}},{"from_task_reference":{"kind":"app_task","name":"DNS Configuration"},"edge_type":"user_defined","type":"","to_task_reference":{"kind":"app_task","name":"EBS VolumePlugin"}}],"type":""},"timeout_secs":"0","type":"DAG","variable_list":[]},{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Master"},"retries":"0","description":"","message_list":[],"child_tasks_local_reference_list":[],"name":"ETCD Docker Kubelet Install","state":"ACTIVE","attrs":{"script":"#!\/bin\/bash\nset -ex\n\nsudo easy_install netaddr\n\nETCD_VERSION=\"v3.2.11\"\nKUBELET_IMAGE_TAG=\"@@{KUBE_IMAGE_TAG}@@\"\nif [[ \"@@{KUBE_IMAGE_TAG_NEW}@@x\" != \"x\" ]]; then\n\tKUBELET_IMAGE_TAG=\"@@{KUBE_IMAGE_TAG_NEW}@@\"\nfi\nINTERNAL_IP=\"@@{private_ip_address}@@\"\nCONTROLLER_IPS=\"@@{calm_array_private_ip_address}@@\" # private Ip's\nMINION_IPS=\"@@{AWS_Centos_K8SM.private_ip_address}@@\" # private Ip's\nCONTROLLER_DNS=\"@@{calm_array_private_dns_name}@@\" # private Dns\nMINION_DNS=\"@@{AWS_Centos_K8SM.private_dns_name}@@\" # private Dns\nPUBLIC_CONTROLLER_IPS=\"@@{calm_array_public_ip_address}@@\"\nPUBLIC_MINION_IPS=\"@@{AWS_Centos_K8SM.public_ip_address}@@\"\nPUBLIC_CONTROLLER_DNS=\"@@{calm_array_public_dns_name}@@\"\nPUBLIC_MINION_DNS=\"@@{AWS_Centos_K8SM.public_dns_name}@@\"\nNODE_NAME=\"controller@@{calm_array_index}@@\"\nCLUSTER_SUBNET=\"@@{KUBE_CLUSTER_SUBNET}@@\"\nSERVICE_SUBNET=\"@@{KUBE_SERVICE_SUBNET}@@\"\nKUBE_CLUSTER_DNS=\"@@{KUBE_DNS_IP}@@\"\nDOCKER_VERSION=\"@@{DOCKER_VERSION}@@\"\nETCD_CERT_PATH=\"\/etc\/ssl\/certs\/etcd\"\nKUBE_CERT_PATH=\"\/etc\/kubernetes\/ssl\"\nKUBE_MANIFEST_PATH=\"\/etc\/kubernetes\/manifests\"\nKUBE_CNI_BIN_PATH=\"\/opt\/cni\/bin\"\nKUBE_CNI_CONF_PATH=\"\/etc\/cni\/net.d\"\nVERSION=$(echo \"${KUBELET_IMAGE_TAG}\" | tr \"_\" \" \" | awk '{print $1}')\nMASTER_API_HTTPS=6443\nETCD_SERVER_PORT=2379\nETCD_CLIENT_PORT=2380\nMASTER_API_PORT=8080\nFIRST_IP_SERVICE_SUBNET=$(python -c \"from netaddr import * ; print IPNetwork('${SERVICE_SUBNET}')[1]\")\n\nsudo mkdir -p \/opt\/kube-ssl ${KUBE_CERT_PATH} ${KUBE_CNI_BIN_PATH} ${ETCD_CERT_PATH} ${KUBE_MANIFEST_PATH} ${KUBE_CNI_CONF_PATH}\n\nsudo hostnamectl set-hostname --static ${NODE_NAME}\nsudo yum update -y --quiet\nsudo yum install -y wget iscsi-initiator-utils socat --quiet\n\ncount=0\nfor ip in $(echo \"${CONTROLLER_IPS}\" | tr \",\" \"\\n\"); do\n echo \"${ip} controller${count}\" | sudo tee -a \/etc\/hosts\n CON+=\"controller${count}=https:\/\/${ip}:${ETCD_CLIENT_PORT}\",\n ETCD+=\"https:\/\/${ip}:${ETCD_SERVER_PORT}\",\n CONS_NAMES+=\"controller${count}\",\n count=$((count+1))\ndone\nETCD_ALL_CONTROLLERS=$(echo $CON | sed 's\/,$\/\/')\nETCD_SERVERS=$(echo $ETCD | sed 's\/,$\/\/')\nCONTROLLER_NAMES=$(echo $CONS_NAMES | sed 's\/,$\/\/')\n \ncount=0\nfor ip in $(echo ${MINION_IPS} | tr \",\" \"\\n\"); do\n echo \"${ip} minion${count}\" | sudo tee -a \/etc\/hosts\n MIN_NAMES+=\"minion${count}\",\n count=$((count+1))\ndone\nMINION_NAMES=$(echo $MIN_NAMES | sed 's\/,$\/\/') \n \nwget -q \"https:\/\/github.com\/coreos\/etcd\/releases\/download\/${ETCD_VERSION}\/etcd-${ETCD_VERSION}-linux-amd64.tar.gz\"\nwget -q https:\/\/github.com\/containernetworking\/plugins\/releases\/download\/v0.6.0\/cni-plugins-amd64-v0.6.0.tgz\nwget -q https:\/\/storage.googleapis.com\/kubernetes-release\/release\/${VERSION}\/bin\/linux\/amd64\/kubelet\nchmod +x kubelet\nsudo mv kubelet \/usr\/bin\/kubelet\n\n# -*- Bootstrapping a H\/A etcd cluster.\ntar -xvf etcd-${ETCD_VERSION}-linux-amd64.tar.gz\nsudo mv etcd-${ETCD_VERSION}-linux-amd64\/etcd* \/usr\/bin\/\nrm -rf etcd-${ETCD_VERSION}-linux-amd64*\n\necho \"[Unit]\nDescription=etcd\nDocumentation=https:\/\/github.com\/coreos\n\n[Service]\nExecStart=\/usr\/bin\/etcd \\\\\n --name ${NODE_NAME} \\\\\n --cert-file=${ETCD_CERT_PATH}\/etcd-server.pem \\\\\n --key-file=${ETCD_CERT_PATH}\/etcd-server-key.pem \\\\\n --peer-cert-file=${ETCD_CERT_PATH}\/etcd-peer.pem \\\\\n --peer-key-file=${ETCD_CERT_PATH}\/etcd-peer-key.pem \\\\\n --trusted-ca-file=${ETCD_CERT_PATH}\/etcd-ca.pem \\\\\n --peer-trusted-ca-file=${ETCD_CERT_PATH}\/etcd-ca.pem \\\\\n --peer-client-cert-auth \\\\\n --client-cert-auth \\\\\n --initial-advertise-peer-urls https:\/\/${INTERNAL_IP}:${ETCD_CLIENT_PORT} \\\\\n --listen-peer-urls https:\/\/${INTERNAL_IP}:${ETCD_CLIENT_PORT} \\\\\n --listen-client-urls https:\/\/${INTERNAL_IP}:${ETCD_SERVER_PORT},http:\/\/127.0.0.1:${ETCD_SERVER_PORT} \\\\\n --advertise-client-urls https:\/\/${INTERNAL_IP}:${ETCD_SERVER_PORT} \\\\\n --initial-cluster-token etcd-cluster-0 \\\\\n --initial-cluster ${ETCD_ALL_CONTROLLERS} \\\\\n --initial-cluster-state new \\\\\n --data-dir=\/var\/lib\/etcd \\\\\n --wal-dir=\/var\/lib\/etcd\/wal \\\\\n --max-wals=0\nRestart=on-failure\nRestartSec=5\n\n[Install]\nWantedBy=multi-user.target\" | sudo tee \/etc\/systemd\/system\/etcd.service\n\nsudo yum install -y --quiet yum-utils\nsudo yum-config-manager --add-repo https:\/\/download.docker.com\/linux\/centos\/docker-ce.repo\nsudo yum install -y --quiet --setopt=obsoletes=0 docker-ce-${DOCKER_VERSION} docker-ce-selinux-${DOCKER_VERSION}\n\nsudo sed -i '\/ExecStart=\/c\\\\ExecStart=\/usr\/bin\/dockerd -H tcp:\/\/0.0.0.0:2375 -H unix:\/\/\/var\/run\/docker.sock' \/usr\/lib\/systemd\/system\/docker.service\nsudo systemctl enable docker\nsudo usermod -a -G docker $USER\n\nsudo mkdir -p \/etc\/docker\necho '{\n \"storage-driver\": \"overlay\"\n}' | sudo tee \/etc\/docker\/daemon.json\n\necho '{\n \"name\": \"cbr0\",\n \"type\": \"flannel\",\n \"delegate\": {\n \"isDefaultGateway\": true\n }\n}' | sudo tee ${KUBE_CNI_CONF_PATH}\/10-flannel.conf\n\nsudo tar -zxvf cni-plugins-amd64-v0.6.0.tgz -C ${KUBE_CNI_BIN_PATH}\nrm -rf cni-plugins-amd64-v0.6.0.tgz\n\necho \"[Unit]\nDescription=Kubernetes Kubelet\nDocumentation=https:\/\/github.com\/GoogleCloudPlatform\/kubernetes\nAfter=docker.service\nRequires=docker.service\n\n[Service]\nExecStart=\/usr\/bin\/kubelet \\\\\n --allow-privileged=true \\\\\n --anonymous-auth=false \\\\\n --authorization-mode=Webhook \\\\\n --cluster-dns=${KUBE_CLUSTER_DNS} \\\\\n --cluster-domain=cluster.local \\\\\n --container-runtime=docker \\\\\n --enable-custom-metrics \\\\\n --kubeconfig=${KUBE_CERT_PATH}\/${NODE_NAME}.kubeconfig \\\\\n --network-plugin=cni \\\\\n --pod-cidr=${CLUSTER_SUBNET} \\\\\n --register-node=true \\\\\n --runtime-request-timeout=10m \\\\\n --client-ca-file=${KUBE_CERT_PATH}\/ca.pem \\\\\n --tls-cert-file=${KUBE_CERT_PATH}\/${NODE_NAME}.pem \\\\\n --tls-private-key-file=${KUBE_CERT_PATH}\/${NODE_NAME}-key.pem \\\\\n --pod-manifest-path=${KUBE_MANIFEST_PATH} \\\\\n --read-only-port=0 \\\\\n --protect-kernel-defaults=false \\\\\n --make-iptables-util-chains=true \\\\\n --keep-terminated-pod-volumes=false \\\\\n --event-qps=0 \\\\\n --cadvisor-port=0 \\\\\n --runtime-cgroups=\/systemd\/system.slice \\\\\n --kubelet-cgroups=\/systemd\/system.slice \\\\\n --node-labels 'node-role.kubernetes.io\/master=true' \\\\\n --node-labels 'node-role.kubernetes.io\/etcd=true' \\\\\n --register-with-taints=node-role.kubernetes.io\/master=true:NoSchedule \\\\\n --cloud-provider=aws \\\\\n --v=2\nRestart=on-failure\nRestartSec=5\n\n[Install]\nWantedBy=multi-user.target\" | sudo tee \/etc\/systemd\/system\/kubelet.service\n\nsudo mkdir -p \/var\/lib\/docker\nsudo yum install -y lvm2 --quiet\nsudo pvcreate \/dev\/xvd{b,c,d}\nsudo vgcreate docker \/dev\/xvd{b,c,d}\nsleep 3\nsudo lvcreate -l 100%VG -n docker_lvm docker\nsudo mkfs.xfs \/dev\/docker\/docker_lvm\n\necho -e \"\/dev\/docker\/docker_lvm \\t \/var\/lib\/docker \\t xfs \\t defaults \\t 0 0\" | sudo tee -a \/etc\/fstab\nsudo mount -a\necho 'exclude=docker*' | sudo tee -a \/etc\/yum.conf\n\nwget -q https:\/\/pkg.cfssl.org\/R1.2\/cfssl_linux-amd64 https:\/\/pkg.cfssl.org\/R1.2\/cfssljson_linux-amd64\nwget -q https:\/\/storage.googleapis.com\/kubernetes-release\/release\/${VERSION}\/bin\/linux\/amd64\/kubectl\nwget -q \"https:\/\/storage.googleapis.com\/kubernetes-helm\/helm-v2.8.1-linux-amd64.tar.gz\"\n\ntar -zxvf helm-v2.8.1-linux-amd64.tar.gz\nchmod +x cfssl_linux-amd64 cfssljson_linux-amd64 kubectl linux-amd64\/helm\nsudo mv cfssl_linux-amd64 \/usr\/local\/bin\/cfssl\nsudo mv cfssljson_linux-amd64 \/usr\/local\/bin\/cfssljson\nsudo mv kubectl linux-amd64\/helm \/usr\/local\/bin\/\nrm -rf helm-v2.8.1-linux-amd64.tar.gz linux-amd64","type":"","command_line_args":"","exit_status":[],"script_type":"sh"},"timeout_secs":"0","type":"EXEC","variable_list":[]},{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Master"},"retries":"0","description":"","message_list":[],"child_tasks_local_reference_list":[],"name":"Generate Certs","state":"ACTIVE","attrs":{"script":"#!\/bin\/bash\nset -ex\n\nINTERNAL_IP=\"@@{private_ip_address}@@\"\nCONTROLLER_IPS=\"@@{calm_array_private_ip_address}@@\" # private Ip's\nMINION_IPS=\"@@{AWS_Centos_K8SM.private_ip_address}@@\" # private Ip's\nCONTROLLER_DNS=\"@@{calm_array_private_dns_name}@@\" # private Dns\nMINION_DNS=\"@@{AWS_Centos_K8SM.private_dns_name}@@\" # private Dns\nPUBLIC_CONTROLLER_IPS=\"@@{calm_array_public_ip_address}@@\"\nPUBLIC_MINION_IPS=\"@@{AWS_Centos_K8SM.public_ip_address}@@\"\nPUBLIC_CONTROLLER_DNS=\"@@{calm_array_public_dns_name}@@\"\nPUBLIC_MINION_DNS=\"@@{AWS_Centos_K8SM.public_dns_name}@@\"\nMASTER_API_HTTPS=6443\nSERVICE_SUBNET=\"@@{KUBE_SERVICE_SUBNET}@@\"\nKUBE_CLUSTER_NAME=\"@@{KUBE_CLUSTER_NAME}@@\"\nFIRST_IP_SERVICE_SUBNET=$(python -c \"from netaddr import * ; print IPNetwork('${SERVICE_SUBNET}')[1]\")\n\ncount=0\nfor ip in $(echo \"${CONTROLLER_IPS}\" | tr \",\" \"\\n\"); do\n CONS_NAMES+=\"controller${count}\",\n count=$((count+1))\ndone\nCONTROLLER_NAMES=$(echo $CONS_NAMES | sed 's\/,$\/\/')\n \ncount=0\nfor ip in $(echo ${MINION_IPS} | tr \",\" \"\\n\"); do\n MIN_NAMES+=\"minion${count}\",\n count=$((count+1))\ndone\nMINION_NAMES=$(echo $MIN_NAMES | sed 's\/,$\/\/') \n\nif [ @@{calm_array_index}@@ -ne 0 ];then\n exit\nfi\nsudo chown -R $USER:$USER \/opt\/kube-ssl && cd \/opt\/kube-ssl\necho '{\n \"signing\": {\n \"default\": {\n \"expiry\": \"8760h\"\n },\n \"profiles\": {\n \"server\": {\n \"expiry\": \"8760h\",\n \"usages\": [ \"signing\", \"key encipherment\", \"server auth\", \"client auth\" ]\n },\n \"client\": {\n \"expiry\": \"8760h\",\n \"usages\": [ \"key encipherment\", \"client auth\" ]\n },\n \"client-server\": {\n \"expiry\": \"8760h\",\n \"usages\": [ \"key encipherment\", \"server auth\", \"client auth\" ]\n }\n }\n }\n}' | tee ca-config.json\n\necho '{\n \"CN\": \"etcd-ca\",\n \"key\": {\n \"algo\": \"rsa\",\n \"size\": 2048\n },\n \"names\": [\n {\n \"C\": \"US\",\n \"L\": \"San Jose\",\n \"O\": \"etcd\",\n \"OU\": \"CA\",\n \"ST\": \"California\"\n }\n ]\n}' | tee etcd-ca-csr.json\n\ncfssl gencert -initca etcd-ca-csr.json | cfssljson -bare etcd-ca\n\necho '{\n \"CN\": \"etcd\",\n \"hosts\": [],\n \"key\": {\n \"algo\": \"rsa\",\n \"size\": 2048\n },\n \"names\": [\n {\n \"C\": \"US\",\n \"L\": \"San Jose\",\n \"O\": \"etcd\",\n \"OU\": \"CA\",\n \"ST\": \"California\"\n }\n ]\n}' | tee etcd-csr.json\n\ncfssl gencert -ca=etcd-ca.pem -ca-key=etcd-ca-key.pem -config=ca-config.json -hostname=${CONTROLLER_IPS},${CONTROLLER_DNS},${PUBLIC_CONTROLLER_IPS},${PUBLIC_CONTROLLER_DNS} -profile=server etcd-csr.json | cfssljson -bare etcd-server\ncfssl gencert -ca=etcd-ca.pem -ca-key=etcd-ca-key.pem -config=ca-config.json -hostname=${CONTROLLER_IPS},${CONTROLLER_DNS},${PUBLIC_CONTROLLER_IPS},${PUBLIC_CONTROLLER_DNS} -profile=client-server etcd-csr.json | cfssljson -bare etcd-peer\ncfssl gencert -ca=etcd-ca.pem -ca-key=etcd-ca-key.pem -config=ca-config.json -hostname=${CONTROLLER_IPS},${CONTROLLER_DNS},${PUBLIC_CONTROLLER_IPS},${PUBLIC_CONTROLLER_DNS} -profile=client etcd-csr.json | cfssljson -bare etcd-client\n\necho '{\n \"CN\": \"kube-ca\",\n \"key\": {\n \"algo\": \"rsa\",\n \"size\": 2048\n },\n \"names\": [\n {\n \"C\": \"US\",\n \"L\": \"San Jose\",\n \"O\": \"kube\",\n \"OU\": \"CA\",\n \"ST\": \"California\"\n }\n ]\n}' | tee kube-ca-csr.json\n\ncfssl gencert -initca kube-ca-csr.json | cfssljson -bare ca\n\necho '{\n \"CN\": \"kubernetes\",\n \"key\": {\n \"algo\": \"rsa\",\n \"size\": 2048\n },\n \"names\": [\n {\n \"C\": \"US\",\n \"L\": \"San Jose\",\n \"O\": \"kube\",\n \"OU\": \"Cluster\",\n \"ST\": \"California\"\n }\n ]\n}' | tee kubernetes-csr.json\n\ncfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json \\\n-hostname=${CONTROLLER_NAMES},${CONTROLLER_IPS},${MINION_NAMES},${MINION_IPS},${FIRST_IP_SERVICE_SUBNET},${CONTROLLER_DNS},${MINION_DNS},${PUBLIC_CONTROLLER_IPS},${PUBLIC_MINION_IPS},${PUBLIC_CONTROLLER_DNS},${PUBLIC_MINION_DNS},127.0.0.1,kubernetes.default,kubernetes,kubernetes.default.svc,kubernetes.default.svc.cluster.local \\\n-profile=server kubernetes-csr.json | cfssljson -bare kubernetes\n\necho '{\n \"CN\": \"system:kube-controller-manager\",\n \"hosts\": [],\n \"key\": {\n \"algo\": \"rsa\",\n \"size\": 2048\n },\n \"names\": [\n {\n \"C\": \"US\",\n \"L\": \"San Jose\",\n \"O\": \"system:kube-controller-manager\",\n \"OU\": \"Cluster\",\n \"ST\": \"California\"\n }\n ]\n}' | tee kube-controller-manager-csr.json\n\ncfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server kube-controller-manager-csr.json | cfssljson -bare kube-controller-manager\n\necho '{\n \"CN\": \"system:kube-scheduler\",\n \"hosts\": [],\n \"key\": {\n \"algo\": \"rsa\",\n \"size\": 2048\n },\n \"names\": [\n {\n \"C\": \"US\",\n \"L\": \"San Jose\",\n \"O\": \"system:kube-scheduler\",\n \"OU\": \"Cluster\",\n \"ST\": \"California\"\n }\n ]\n}' | tee kube-scheduler-csr.json\n\ncfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server kube-scheduler-csr.json | cfssljson -bare kube-scheduler\n\ncount=0\nfor name in $(echo ${CONTROLLER_DNS} | tr \",\" \"\\n\"); do\ninstance=\"controller${count}\"\necho \"{\n \\\"CN\\\": \\\"system:node:${name}\\\",\n \\\"key\\\": {\n \\\"algo\\\": \\\"rsa\\\",\n \\\"size\\\": 2048\n },\n \\\"names\\\": [\n {\n \\\"C\\\": \\\"US\\\",\n \\\"L\\\": \\\"San Jose\\\",\n \\\"O\\\": \\\"system:nodes\\\",\n \\\"OU\\\": \\\"Kubernetes The Hard Way\\\",\n \\\"ST\\\": \\\"California\\\"\n }\n ]\n}\" | tee ${instance}-csr.json\ncfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -hostname=${instance},${name} -profile=client-server ${instance}-csr.json | cfssljson -bare ${instance}\ncount=$((count+1))\ndone \n\n#count=0\n#for name in $(echo ${MINION_DNS} | tr \",\" \"\\n\"); do\n#instance=\"minion${count}\"\n#echo \"{\n# \\\"CN\\\": \\\"system:node:${name}\\\",\n# \\\"key\\\": {\n# \\\"algo\\\": \\\"rsa\\\",\n# \\\"size\\\": 2048\n# },\n# \\\"names\\\": [\n# {\n# \\\"C\\\": \\\"US\\\",\n# \\\"L\\\": \\\"San Jose\\\",\n# \\\"O\\\": \\\"system:nodes\\\",\n# \\\"OU\\\": \\\"Kubernetes The Hard Way\\\",\n# \\\"ST\\\": \\\"California\\\"\n# }\n# ]\n#}\" | tee ${instance}-csr.json\n#cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -hostname=${instance},${name} -profile=client-server ${instance}-csr.json | cfssljson -bare ${instance}\n#count=$((count+1))\n#done\n\n# -*- Creating kube-proxy certificates\necho '{\n \"CN\": \"system:kube-proxy\",\n \"hosts\": [],\n \"key\": {\n \"algo\": \"rsa\",\n \"size\": 2048\n },\n \"names\": [\n {\n \"C\": \"US\",\n \"L\": \"San Jose\",\n \"O\": \"system:node-proxier\",\n \"OU\": \"Cluster\",\n \"ST\": \"California\"\n }\n ]\n}' | tee kube-proxy-csr.json\n\ncfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client kube-proxy-csr.json | cfssljson -bare kube-proxy\n\necho '{\n \"CN\": \"admin\",\n \"hosts\": [],\n \"key\": {\n \"algo\": \"rsa\",\n \"size\": 2048\n },\n \"names\": [\n {\n \"C\": \"US\",\n \"L\": \"San Jose\",\n \"O\": \"system:masters\",\n \"OU\": \"Cluster\",\n \"ST\": \"California\"\n }\n ]\n}' | tee admin-csr.json\n\ncfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client admin-csr.json | cfssljson -bare admin\n\ncount=0\nfor name in $(echo ${CONTROLLER_DNS} | tr \",\" \"\\n\"); do\nkubectl config set-cluster ${KUBE_CLUSTER_NAME} --certificate-authority=ca.pem --embed-certs=true --server=https:\/\/${INTERNAL_IP}:${MASTER_API_HTTPS} --kubeconfig=controller${count}.kubeconfig\nkubectl config set-credentials system:node:${name} --client-certificate=controller${count}.pem --client-key=controller${count}-key.pem --embed-certs=true --kubeconfig=controller${count}.kubeconfig\nkubectl config set-context default --cluster=${KUBE_CLUSTER_NAME} --user=system:node:${name} --kubeconfig=controller${count}.kubeconfig\nkubectl config use-context default --kubeconfig=controller${count}.kubeconfig\ncount=$((count+1))\ndone\n\n#count=0\n#for name in $(echo ${MINION_DNS} | tr \",\" \"\\n\"); do\n#kubectl config set-cluster ${KUBE_CLUSTER_NAME} --certificate-authority=ca.pem --embed-certs=true --server=https:\/\/${INTERNAL_IP}:${MASTER_API_HTTPS} --kubeconfig=minion${count}.kubeconfig\n#kubectl config set-credentials system:node:${name} --client-certificate=minion${count}.pem --client-key=minion${count}-key.pem --embed-certs=true --kubeconfig=minion${count}.kubeconfig\n#kubectl config set-context default --cluster=${KUBE_CLUSTER_NAME} --user=system:node:${name} --kubeconfig=minion${count}.kubeconfig\n#kubectl config use-context default --kubeconfig=minion${count}.kubeconfig\n#count=$((count+1))\n#done\n\nkubectl config set-cluster ${KUBE_CLUSTER_NAME} --certificate-authority=ca.pem --embed-certs=true --server=https:\/\/${INTERNAL_IP}:${MASTER_API_HTTPS} --kubeconfig=kube-controller-manager.kubeconfig\nkubectl config set-credentials kube-controller-manager --client-certificate=kube-controller-manager.pem --client-key=kube-controller-manager-key.pem --embed-certs=true --kubeconfig=kube-controller-manager.kubeconfig\nkubectl config set-context default --cluster=${KUBE_CLUSTER_NAME} --user=kube-controller-manager --kubeconfig=kube-controller-manager.kubeconfig\nkubectl config use-context default --kubeconfig=kube-controller-manager.kubeconfig\n\nkubectl config set-cluster ${KUBE_CLUSTER_NAME} --certificate-authority=ca.pem --embed-certs=true --server=https:\/\/${INTERNAL_IP}:${MASTER_API_HTTPS} --kubeconfig=kube-scheduler.kubeconfig\nkubectl config set-credentials kube-scheduler --client-certificate=kube-scheduler.pem --client-key=kube-scheduler-key.pem --embed-certs=true --kubeconfig=kube-scheduler.kubeconfig\nkubectl config set-context default --cluster=${KUBE_CLUSTER_NAME} --user=kube-scheduler --kubeconfig=kube-scheduler.kubeconfig\nkubectl config use-context default --kubeconfig=kube-scheduler.kubeconfig\n\n\nkubectl config set-cluster ${KUBE_CLUSTER_NAME} --certificate-authority=ca.pem --embed-certs=true --server=https:\/\/${INTERNAL_IP}:${MASTER_API_HTTPS} --kubeconfig=kube-proxy.kubeconfig\nkubectl config set-credentials kube-proxy --client-certificate=kube-proxy.pem --client-key=kube-proxy-key.pem --embed-certs=true --kubeconfig=kube-proxy.kubeconfig\nkubectl config set-context default --cluster=${KUBE_CLUSTER_NAME} --user=kube-proxy --kubeconfig=kube-proxy.kubeconfig\nkubectl config use-context default --kubeconfig=kube-proxy.kubeconfig\n\nENCRYPTION_KEY=$(head -c 32 \/dev\/urandom | base64)\necho \"kind: EncryptionConfig\napiVersion: v1\nresources:\n - resources:\n - secrets\n providers:\n - aescbc:\n keys:\n - name: key1\n secret: ${ENCRYPTION_KEY}\n - identity: {}\" | tee encryption-config.yaml\n\necho \"@@{CENTOS.secret}@@\" | tee ~\/.ssh\/id_rsa\nchmod 400 ~\/.ssh\/id_rsa\n\ncount=0\nfor ip in $(echo ${CONTROLLER_IPS} | tr \",\" \"\\n\"); do\n instance=\"controller${count}\"\n scp -o stricthostkeychecking=no admin*.pem ca*.pem etcd*.pem kubernetes*.pem ${instance}* kube-proxy.kubeconfig kube-controller-manager.kubeconfig kube-scheduler.kubeconfig encryption-config.yaml ${instance}:\ncount=$((count+1))\ndone\n\n#count=0\n#for ip in $(echo ${MINION_IPS} | tr \",\" \"\\n\"); do\n# instance=\"minion${count}\"\n# scp -o stricthostkeychecking=no ca*.pem kubernetes*.pem ${instance}* kube-proxy.kubeconfig ${instance}:\n#count=$((count+1))\n#done","type":"","command_line_args":"","exit_status":[],"script_type":"sh"},"timeout_secs":"0","type":"EXEC","variable_list":[]},{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Master"},"retries":"0","description":"","message_list":[],"child_tasks_local_reference_list":[],"name":"Configure Services","state":"ACTIVE","attrs":{"script":"#!\/bin\/bash\nset -ex\n\nKUBELET_IMAGE_TAG=\"@@{KUBE_IMAGE_TAG}@@\"\nif [[ \"@@{KUBE_IMAGE_TAG_NEW}@@x\" != \"x\" ]]; then\n\tKUBELET_IMAGE_TAG=\"@@{KUBE_IMAGE_TAG_NEW}@@\"\nfi\nINTERNAL_IP=\"@@{private_ip_address}@@\"\nCONTROLLER_IPS=\"@@{calm_array_private_ip_address}@@\"\nCLUSTER_SUBNET=\"@@{KUBE_CLUSTER_SUBNET}@@\"\nSERVICE_SUBNET=\"@@{KUBE_SERVICE_SUBNET}@@\"\nETCD_CERT_PATH=\"\/etc\/ssl\/certs\/etcd\"\nKUBE_CERT_PATH=\"\/etc\/kubernetes\/ssl\"\nKUBE_MANIFEST_PATH=\"\/etc\/kubernetes\/manifests\"\nMASTER_API_HTTPS=6443\nETCD_SERVER_PORT=2379\nVERSION=$(echo \"${KUBELET_IMAGE_TAG}\" | tr \"_\" \" \" | awk '{print $1}')\nCONTROLLER_COUNT=$(echo \"@@{calm_array_private_ip_address}@@\" | tr ',' '\\n' | wc -l)\n\nsudo cp ca*.pem etcd-*.pem kubernetes*.pem ${HOSTNAME}* kube-*.kubeconfig encryption-config.yaml ${KUBE_CERT_PATH}\/\nsudo chmod +r ${KUBE_CERT_PATH}\/*\n\nsudo cp etcd-*.pem ${ETCD_CERT_PATH}\/\nsudo chmod +r ${ETCD_CERT_PATH}\/*\n\ncount=0\nfor ip in $(echo \"${CONTROLLER_IPS}\" | tr \",\" \"\\n\"); do\n ETCD+=\"https:\/\/${ip}:${ETCD_SERVER_PORT}\",\n count=$((count+1))\ndone\nETCD_SERVERS=$(echo $ETCD | sed 's\/,$\/\/')\n\necho \"apiVersion: v1\nkind: Pod\nmetadata:\n name: kube-apiserver\n namespace: kube-system\n labels:\n k8s-app: kube-apiserver\nspec:\n hostNetwork: true\n containers:\n - name: kube-apiserver\n image: quay.io\/coreos\/hyperkube:${KUBELET_IMAGE_TAG}\n command:\n - \/hyperkube\n - apiserver\n - --admission-control=Initializers,NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota\n - --advertise-address=${INTERNAL_IP}\n - --allow-privileged=true\n - --anonymous-auth=false\n - --insecure-port=0\n - --secure-port=${MASTER_API_HTTPS}\n - --profiling=false\n - --repair-malformed-updates=false\n - --apiserver-count=${CONTROLLER_COUNT}\n - --audit-log-maxage=30\n - --audit-log-maxbackup=10\n - --audit-log-maxsize=100\n - --audit-log-path=\/var\/lib\/audit.log\n - --authorization-mode=Node,RBAC\n - --bind-address=0.0.0.0\n - --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP\n - --event-ttl=1h\n - --service-account-lookup=true\n - --enable-swagger-ui=true\n - --storage-backend=etcd3\n - --etcd-cafile=${ETCD_CERT_PATH}\/etcd-ca.pem\n - --etcd-certfile=${ETCD_CERT_PATH}\/etcd-client.pem\n - --etcd-keyfile=${ETCD_CERT_PATH}\/etcd-client-key.pem\n - --etcd-servers=${ETCD_SERVERS}\n - --experimental-encryption-provider-config=${KUBE_CERT_PATH}\/encryption-config.yaml\n - --tls-ca-file=${KUBE_CERT_PATH}\/ca.pem\n - --tls-cert-file=${KUBE_CERT_PATH}\/kubernetes.pem\n - --tls-private-key-file=${KUBE_CERT_PATH}\/kubernetes-key.pem\n - --kubelet-client-certificate=${KUBE_CERT_PATH}\/kubernetes.pem\n - --kubelet-client-key=${KUBE_CERT_PATH}\/kubernetes-key.pem\n - --kubelet-https=true\n - --runtime-config=api\/all\n - --service-account-key-file=${KUBE_CERT_PATH}\/kubernetes.pem\n - --service-cluster-ip-range=${SERVICE_SUBNET}\n - --service-node-port-range=30000-32767\n - --client-ca-file=${KUBE_CERT_PATH}\/ca.pem\n - --cloud-provider=aws\n - --v=2\n ports:\n - containerPort: ${MASTER_API_HTTPS}\n hostPort: ${MASTER_API_HTTPS}\n name: https\n - containerPort: 8080\n hostPort: 8080\n name: local\n volumeMounts:\n - mountPath: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n readOnly: true\n - mountPath: \/etc\/ssl\/certs\n name: ssl-certs-host\n readOnly: true\n - mountPath: \/etc\/pki\n name: ca-certs-etc-pki\n readOnly: true\n volumes:\n - hostPath:\n path: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n - hostPath:\n path: \/etc\/ssl\/certs\n name: ssl-certs-host\n - hostPath:\n path: \/etc\/pki\n name: ca-certs-etc-pki\" | sudo tee ${KUBE_MANIFEST_PATH}\/kube-apiserver.yaml\n\necho \"apiVersion: v1\nkind: Pod\nmetadata:\n name: kube-proxy\n namespace: kube-system\n labels:\n k8s-app: kube-proxy\nspec:\n hostNetwork: true\n containers:\n - name: kube-proxy\n image: quay.io\/coreos\/hyperkube:${KUBELET_IMAGE_TAG}\n command:\n - \/hyperkube\n - proxy\n - --cluster-cidr=${CLUSTER_SUBNET}\n - --masquerade-all=true\n - --kubeconfig=${KUBE_CERT_PATH}\/kube-proxy.kubeconfig\n - --proxy-mode=iptables\n securityContext:\n privileged: true\n volumeMounts:\n - mountPath: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n readOnly: true\n - mountPath: \/etc\/ssl\/certs\n name: ssl-certs-host\n readOnly: true\n volumes:\n - hostPath:\n path: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n - hostPath:\n path: \/etc\/ssl\/certs\n name: ssl-certs-host\" | sudo tee ${KUBE_MANIFEST_PATH}\/kube-proxy.yaml\n \necho \"apiVersion: v1\nkind: Pod\nmetadata:\n name: kube-controller-manager\n namespace: kube-system\n labels:\n k8s-app: kube-controller-manager\nspec:\n hostNetwork: true\n containers:\n - name: kube-controller-manager\n image: quay.io\/coreos\/hyperkube:${KUBELET_IMAGE_TAG}\n command:\n - \/hyperkube\n - controller-manager\n - --address=0.0.0.0 \n - --allocate-node-cidrs=true \n - --cluster-cidr=${CLUSTER_SUBNET}\n - --cluster-name=kubernetes-prod-cluster \n - --leader-elect=true \n - --kubeconfig=${KUBE_CERT_PATH}\/kube-controller-manager.kubeconfig \n - --service-account-private-key-file=${KUBE_CERT_PATH}\/kubernetes-key.pem \n - --service-cluster-ip-range=${SERVICE_SUBNET}\n - --terminated-pod-gc-threshold=100 \n - --profiling=false \n - --use-service-account-credentials=true\n - --cloud-provider=aws\n - --v=2\n livenessProbe:\n httpGet:\n host: 127.0.0.1\n path: \/healthz\n port: 10252\n initialDelaySeconds: 15\n timeoutSeconds: 1\n volumeMounts:\n - mountPath: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n readOnly: true\n - mountPath: \/etc\/ssl\/certs\n name: ssl-certs-host\n readOnly: true\n - mountPath: \/etc\/pki\n name: ca-certs-etc-pki\n readOnly: true\n volumes:\n - hostPath:\n path: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n - hostPath:\n path: \/etc\/ssl\/certs\n name: ssl-certs-host\n - hostPath:\n path: \/etc\/pki\n name: ca-certs-etc-pki\" | sudo tee ${KUBE_MANIFEST_PATH}\/kube-controller-manager.yaml\n \necho \"apiVersion: v1\nkind: Pod\nmetadata:\n name: kube-scheduler\n namespace: kube-system\n labels:\n k8s-app: kube-scheduler\nspec:\n hostNetwork: true\n containers:\n - name: kube-scheduler\n image: quay.io\/coreos\/hyperkube:${KUBELET_IMAGE_TAG}\n command:\n - \/hyperkube\n - scheduler\n - --kubeconfig=${KUBE_CERT_PATH}\/kube-scheduler.kubeconfig\n - --leader-elect=true\n - --profiling=false\n - --v=2\n livenessProbe:\n httpGet:\n host: 127.0.0.1\n path: \/healthz\n port: 10251\n initialDelaySeconds: 15\n timeoutSeconds: 1\n volumeMounts:\n - mountPath: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n readOnly: true\n volumes:\n - hostPath:\n path: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\" | sudo tee ${KUBE_MANIFEST_PATH}\/kube-scheduler.yaml","type":"","command_line_args":"","exit_status":[],"script_type":"sh"},"timeout_secs":"0","type":"EXEC","variable_list":[]},{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Master"},"retries":"0","description":"","message_list":[],"child_tasks_local_reference_list":[],"name":"Add User Roles","state":"ACTIVE","attrs":{"script":"#!\/bin\/bash\nset -ex\n\nMASTER_API_HTTPS=6443\nINTERNAL_IP=\"@@{private_ip_address}@@\"\nKUBE_CLUSTER_NAME=\"@@{KUBE_CLUSTER_NAME}@@\"\n\nsudo systemctl start etcd docker kubelet\nsudo systemctl enable etcd docker kubelet\n\nexport PATH=$PATH:\/opt\/bin\n\nmkdir CA\nmv admin*.pem ca*.pem etcd-*.pem kubernetes*.pem controller* kube-*.kubeconfig encryption-config.yaml CA\/\nif [ @@{calm_array_index}@@ -ne 0 ];then\n exit\nfi\ncp \/opt\/kube-ssl\/admin*.pem CA\/\n\nCOUNT=0\nwhile [[ $(curl --key CA\/admin-key.pem --cert CA\/admin.pem --cacert CA\/ca.pem https:\/\/${INTERNAL_IP}:${MASTER_API_HTTPS}\/healthz) != \"ok\" ]] ; do\n echo \"sleep for 5 secs\"\n sleep 5\n COUNT=$(($COUNT+1))\n if [[ $COUNT -eq 50 ]]; then\n \techo \"Error: creating cluster\"\n exit 1\n fi\ndone\n\nkubectl config set-cluster ${KUBE_CLUSTER_NAME} --certificate-authority=$HOME\/CA\/ca.pem --embed-certs=true --server=https:\/\/${INTERNAL_IP}:${MASTER_API_HTTPS}\nkubectl config set-credentials admin --client-certificate=$HOME\/CA\/admin.pem --client-key=$HOME\/CA\/admin-key.pem\nkubectl config set-context ${KUBE_CLUSTER_NAME} --cluster=${KUBE_CLUSTER_NAME} --user=admin\nkubectl config use-context ${KUBE_CLUSTER_NAME}\n\ncat <\/dev\/null\") ] ; do echo \"waiting for certs sleeping 5\" && sleep 5; done\n\nscp -o stricthostkeychecking=no ${MASTER_IP}:\/opt\/kube-ssl\/{ca*.pem,kubernetes*.pem,kube-proxy.kubeconfig,ca-config.json} .\n\ninstance=\"minion@@{calm_array_index}@@\"\necho \"{\n \\\"CN\\\": \\\"system:node:${INSTANCE_NAME}\\\",\n \\\"key\\\": {\n \\\"algo\\\": \\\"rsa\\\",\n \\\"size\\\": 2048\n },\n \\\"names\\\": [\n {\n \\\"C\\\": \\\"US\\\",\n \\\"L\\\": \\\"San Jose\\\",\n \\\"O\\\": \\\"system:nodes\\\",\n \\\"OU\\\": \\\"Kubernetes The Hard Way\\\",\n \\\"ST\\\": \\\"California\\\"\n }\n ]\n}\" | tee ${instance}-csr.json\ncfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -hostname=${instance},${INSTANCE_NAME} -profile=client-server ${instance}-csr.json | cfssljson -bare ${instance}\n\nkubectl config set-cluster ${KUBE_CLUSTER_NAME} --certificate-authority=ca.pem --embed-certs=true --server=https:\/\/${MASTER_IP}:${MASTER_API_HTTPS} --kubeconfig=${instance}.kubeconfig\nkubectl config set-credentials system:node:${INSTANCE_NAME} --client-certificate=${instance}.pem --client-key=${instance}-key.pem --embed-certs=true --kubeconfig=${instance}.kubeconfig\nkubectl config set-context default --cluster=${KUBE_CLUSTER_NAME} --user=system:node:${INSTANCE_NAME} --kubeconfig=${instance}.kubeconfig\nkubectl config use-context default --kubeconfig=${instance}.kubeconfig\n\nsudo cp *.pem *.kubeconfig ${KUBE_CERT_PATH}\/\nsudo chmod +r ${KUBE_CERT_PATH}\/*\n\nrm -rf ${instance}-csr.json","type":"","command_line_args":"","exit_status":[],"script_type":"sh"},"timeout_secs":"0","type":"EXEC","variable_list":[]}],"description":"","name":"dca27b9e_runbook","state":"ACTIVE","main_task_local_reference":{"kind":"app_task","name":"a4ecbecd_dag"},"message_list":[],"variable_list":[]},"type":"","uninstall_runbook":{"task_definition_list":[{"target_any_local_reference":{"kind":"app_package","name":"AWS_Centos_K8SM_Package"},"retries":"0","description":"","message_list":[],"child_tasks_local_reference_list":[{"kind":"app_task","name":"Remove Node"}],"name":"9e5b50bc_dag","state":"ACTIVE","attrs":{"edges":[],"type":""},"timeout_secs":"0","type":"DAG","variable_list":[]},{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Minion"},"retries":"0","description":"","message_list":[],"child_tasks_local_reference_list":[],"name":"Remove Node","state":"ACTIVE","attrs":{"script":"#!\/bin\/bash\nset -ex\n\nMASTER_IP=\"@@{Kubernetes_Master.address[0]}@@\"\nssh -o stricthostkeychecking=no ${MASTER_IP} \"kubectl drain '@@{private_dns_name}@@' --ignore-daemonsets --delete-local-data --force\"\nsleep 10\nssh -o stricthostkeychecking=no ${MASTER_IP} \"kubectl delete node '@@{private_dns_name}@@'\"","type":"","command_line_args":"","exit_status":[],"script_type":"sh"},"timeout_secs":"0","type":"EXEC","variable_list":[]}],"description":"","name":"137c8a73_runbook","state":"ACTIVE","main_task_local_reference":{"kind":"app_task","name":"9e5b50bc_dag"},"message_list":[],"variable_list":[]}},"variable_list":[]},{"description":"","action_list":[],"type":"DEB","service_local_reference_list":[{"kind":"app_service","name":"Kubernetes_Master"}],"name":"GCP_Centos_K8SC_Package","version":"","options":{"install_runbook":{"task_definition_list":[{"target_any_local_reference":{"kind":"app_package","name":"GCP_Centos_K8SC_Package"},"retries":"0","description":"","message_list":[],"child_tasks_local_reference_list":[{"kind":"app_task","name":"ETCD Docker Kubelet Install"},{"kind":"app_task","name":"Generate Certs"},{"kind":"app_task","name":"Configure Services"},{"kind":"app_task","name":"Add User Roles"},{"kind":"app_task","name":"Network Configuration"},{"kind":"app_task","name":"DNS Configuration"},{"kind":"app_task","name":"GCE VolumePlugin"}],"name":"e07e43b0_dag","state":"ACTIVE","attrs":{"edges":[{"from_task_reference":{"kind":"app_task","name":"ETCD Docker Kubelet Install"},"edge_type":"user_defined","type":"","to_task_reference":{"kind":"app_task","name":"Generate Certs"}},{"from_task_reference":{"kind":"app_task","name":"Generate Certs"},"edge_type":"user_defined","type":"","to_task_reference":{"kind":"app_task","name":"Configure Services"}},{"from_task_reference":{"kind":"app_task","name":"Configure Services"},"edge_type":"user_defined","type":"","to_task_reference":{"kind":"app_task","name":"Add User Roles"}},{"from_task_reference":{"kind":"app_task","name":"Add User Roles"},"edge_type":"user_defined","type":"","to_task_reference":{"kind":"app_task","name":"Network Configuration"}},{"from_task_reference":{"kind":"app_task","name":"Network Configuration"},"edge_type":"user_defined","type":"","to_task_reference":{"kind":"app_task","name":"DNS Configuration"}},{"from_task_reference":{"kind":"app_task","name":"DNS Configuration"},"edge_type":"user_defined","type":"","to_task_reference":{"kind":"app_task","name":"GCE VolumePlugin"}}],"type":""},"timeout_secs":"0","type":"DAG","variable_list":[]},{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Master"},"retries":"0","description":"","message_list":[],"child_tasks_local_reference_list":[],"name":"ETCD Docker Kubelet Install","state":"ACTIVE","attrs":{"script":"#!\/bin\/bash\nset -ex\n\nsudo easy_install netaddr\n\nETCD_VERSION=\"v3.2.11\"\nKUBELET_IMAGE_TAG=\"@@{KUBE_IMAGE_TAG}@@\"\nif [[ \"@@{KUBE_IMAGE_TAG_NEW}@@x\" != \"x\" ]]; then\n\tKUBELET_IMAGE_TAG=\"@@{KUBE_IMAGE_TAG_NEW}@@\"\nfi\nINTERNAL_IP=\"@@{private_ip_address}@@\"\nCONTROLLER_IPS=\"@@{calm_array_private_ip_address}@@\" # private Ip's\nMINION_IPS=\"@@{GCP_Centos_K8SM.private_ip_address}@@\" # private Ip's\nPUBLIC_CONTROLLER_IPS=\"@@{calm_array_public_ip_address}@@\"\nPUBLIC_MINION_IPS=\"@@{GCP_Centos_K8SM.public_ip_address}@@\"\nNODE_NAME=\"controller@@{calm_array_index}@@\"\nCLUSTER_SUBNET=\"@@{KUBE_CLUSTER_SUBNET}@@\"\nSERVICE_SUBNET=\"@@{KUBE_SERVICE_SUBNET}@@\"\nKUBE_CLUSTER_DNS=\"@@{KUBE_DNS_IP}@@\"\nDOCKER_VERSION=\"@@{DOCKER_VERSION}@@\"\nETCD_CERT_PATH=\"\/etc\/ssl\/certs\/etcd\"\nKUBE_CERT_PATH=\"\/etc\/kubernetes\/ssl\"\nKUBE_MANIFEST_PATH=\"\/etc\/kubernetes\/manifests\"\nKUBE_CNI_BIN_PATH=\"\/opt\/cni\/bin\"\nKUBE_CNI_CONF_PATH=\"\/etc\/cni\/net.d\"\nVERSION=$(echo \"${KUBELET_IMAGE_TAG}\" | tr \"_\" \" \" | awk '{print $1}')\nMASTER_API_HTTPS=6443\nETCD_SERVER_PORT=2379\nETCD_CLIENT_PORT=2380\nMASTER_API_PORT=8080\nFIRST_IP_SERVICE_SUBNET=$(python -c \"from netaddr import * ; print IPNetwork('${SERVICE_SUBNET}')[1]\")\n\nsudo mkdir -p \/opt\/kube-ssl ${KUBE_CERT_PATH} ${KUBE_CNI_BIN_PATH} ${ETCD_CERT_PATH} ${KUBE_MANIFEST_PATH} ${KUBE_CNI_CONF_PATH}\n\n#sudo hostnamectl set-hostname --static ${NODE_NAME}\nsudo yum update -y --quiet\nsudo yum install -y wget iscsi-initiator-utils socat --quiet\n\ncount=0\nfor ip in $(echo \"${CONTROLLER_IPS}\" | tr \",\" \"\\n\"); do\n echo \"${ip} controller${count}\" | sudo tee -a \/etc\/hosts\n CON+=\"controller${count}=https:\/\/${ip}:${ETCD_CLIENT_PORT}\",\n ETCD+=\"https:\/\/${ip}:${ETCD_SERVER_PORT}\",\n CONS_NAMES+=\"controller${count}\",\n count=$((count+1))\ndone\nETCD_ALL_CONTROLLERS=$(echo $CON | sed 's\/,$\/\/')\nETCD_SERVERS=$(echo $ETCD | sed 's\/,$\/\/')\nCONTROLLER_NAMES=$(echo $CONS_NAMES | sed 's\/,$\/\/')\n \ncount=0\nfor ip in $(echo ${MINION_IPS} | tr \",\" \"\\n\"); do\n echo \"${ip} minion${count}\" | sudo tee -a \/etc\/hosts\n MIN_NAMES+=\"minion${count}\",\n count=$((count+1))\ndone\nMINION_NAMES=$(echo $MIN_NAMES | sed 's\/,$\/\/') \n \nwget -q \"https:\/\/github.com\/coreos\/etcd\/releases\/download\/${ETCD_VERSION}\/etcd-${ETCD_VERSION}-linux-amd64.tar.gz\"\nwget -q https:\/\/github.com\/containernetworking\/plugins\/releases\/download\/v0.6.0\/cni-plugins-amd64-v0.6.0.tgz\nwget -q https:\/\/storage.googleapis.com\/kubernetes-release\/release\/${VERSION}\/bin\/linux\/amd64\/kubelet\nchmod +x kubelet\nsudo mv kubelet \/usr\/bin\/kubelet\n\n# -*- Bootstrapping a H\/A etcd cluster.\ntar -xvf etcd-${ETCD_VERSION}-linux-amd64.tar.gz\nsudo mv etcd-${ETCD_VERSION}-linux-amd64\/etcd* \/usr\/bin\/\nrm -rf etcd-${ETCD_VERSION}-linux-amd64*\n\necho \"[Unit]\nDescription=etcd\nDocumentation=https:\/\/github.com\/coreos\n\n[Service]\nExecStart=\/usr\/bin\/etcd \\\\\n --name ${NODE_NAME} \\\\\n --cert-file=${ETCD_CERT_PATH}\/etcd-server.pem \\\\\n --key-file=${ETCD_CERT_PATH}\/etcd-server-key.pem \\\\\n --peer-cert-file=${ETCD_CERT_PATH}\/etcd-peer.pem \\\\\n --peer-key-file=${ETCD_CERT_PATH}\/etcd-peer-key.pem \\\\\n --trusted-ca-file=${ETCD_CERT_PATH}\/etcd-ca.pem \\\\\n --peer-trusted-ca-file=${ETCD_CERT_PATH}\/etcd-ca.pem \\\\\n --peer-client-cert-auth \\\\\n --client-cert-auth \\\\\n --initial-advertise-peer-urls https:\/\/${INTERNAL_IP}:${ETCD_CLIENT_PORT} \\\\\n --listen-peer-urls https:\/\/${INTERNAL_IP}:${ETCD_CLIENT_PORT} \\\\\n --listen-client-urls https:\/\/${INTERNAL_IP}:${ETCD_SERVER_PORT},http:\/\/127.0.0.1:${ETCD_SERVER_PORT} \\\\\n --advertise-client-urls https:\/\/${INTERNAL_IP}:${ETCD_SERVER_PORT} \\\\\n --initial-cluster-token etcd-cluster-0 \\\\\n --initial-cluster ${ETCD_ALL_CONTROLLERS} \\\\\n --initial-cluster-state new \\\\\n --data-dir=\/var\/lib\/etcd \\\\\n --wal-dir=\/var\/lib\/etcd\/wal \\\\\n --max-wals=0\nRestart=on-failure\nRestartSec=5\n\n[Install]\nWantedBy=multi-user.target\" | sudo tee \/etc\/systemd\/system\/etcd.service\n\nsudo yum install -y --quiet yum-utils\nsudo yum-config-manager --add-repo https:\/\/download.docker.com\/linux\/centos\/docker-ce.repo\nsudo yum install -y --quiet --setopt=obsoletes=0 docker-ce-${DOCKER_VERSION} docker-ce-selinux-${DOCKER_VERSION}\n\nsudo sed -i '\/ExecStart=\/c\\\\ExecStart=\/usr\/bin\/dockerd -H tcp:\/\/0.0.0.0:2375 -H unix:\/\/\/var\/run\/docker.sock' \/usr\/lib\/systemd\/system\/docker.service\nsudo systemctl enable docker\nsudo usermod -a -G docker $USER\n\nsudo mkdir -p \/etc\/docker\necho '{\n \"storage-driver\": \"overlay\"\n}' | sudo tee \/etc\/docker\/daemon.json\n\necho '{\n \"name\": \"cbr0\",\n \"type\": \"flannel\",\n \"delegate\": {\n \"isDefaultGateway\": true\n }\n}' | sudo tee ${KUBE_CNI_CONF_PATH}\/10-flannel.conf\n\nsudo tar -zxvf cni-plugins-amd64-v0.6.0.tgz -C ${KUBE_CNI_BIN_PATH}\nrm -rf cni-plugins-amd64-v0.6.0.tgz\n\necho \"[Unit]\nDescription=Kubernetes Kubelet\nDocumentation=https:\/\/github.com\/GoogleCloudPlatform\/kubernetes\nAfter=docker.service\nRequires=docker.service\n\n[Service]\nExecStart=\/usr\/bin\/kubelet \\\\\n --allow-privileged=true \\\\\n --anonymous-auth=false \\\\\n --authorization-mode=Webhook \\\\\n --cluster-dns=${KUBE_CLUSTER_DNS} \\\\\n --cluster-domain=cluster.local \\\\\n --container-runtime=docker \\\\\n --enable-custom-metrics \\\\\n --kubeconfig=${KUBE_CERT_PATH}\/${NODE_NAME}.kubeconfig \\\\\n --network-plugin=cni \\\\\n --pod-cidr=${CLUSTER_SUBNET} \\\\\n --register-node=true \\\\\n --runtime-request-timeout=10m \\\\\n --client-ca-file=${KUBE_CERT_PATH}\/ca.pem \\\\\n --tls-cert-file=${KUBE_CERT_PATH}\/${NODE_NAME}.pem \\\\\n --tls-private-key-file=${KUBE_CERT_PATH}\/${NODE_NAME}-key.pem \\\\\n --pod-manifest-path=${KUBE_MANIFEST_PATH} \\\\\n --read-only-port=0 \\\\\n --protect-kernel-defaults=false \\\\\n --make-iptables-util-chains=true \\\\\n --keep-terminated-pod-volumes=false \\\\\n --event-qps=0 \\\\\n --cadvisor-port=0 \\\\\n --runtime-cgroups=\/systemd\/system.slice \\\\\n --kubelet-cgroups=\/systemd\/system.slice \\\\\n --node-labels 'node-role.kubernetes.io\/master=true' \\\\\n --node-labels 'node-role.kubernetes.io\/etcd=true' \\\\\n --register-with-taints=node-role.kubernetes.io\/master=true:NoSchedule \\\\\n --cloud-provider=gce \\\\\n --v=2\nRestart=on-failure\nRestartSec=5\n\n[Install]\nWantedBy=multi-user.target\" | sudo tee \/etc\/systemd\/system\/kubelet.service\n\nsudo mkdir -p \/var\/lib\/docker\nsudo yum install -y lvm2 --quiet\nsudo pvcreate \/dev\/sd{b,c,d}\nsudo vgcreate docker \/dev\/sd{b,c,d}\nsleep 3\nsudo lvcreate -l 100%VG -n docker_lvm docker\nsudo mkfs.xfs \/dev\/docker\/docker_lvm\n\necho -e \"\/dev\/docker\/docker_lvm \\t \/var\/lib\/docker \\t xfs \\t defaults \\t 0 0\" | sudo tee -a \/etc\/fstab\nsudo mount -a\necho 'exclude=docker*' | sudo tee -a \/etc\/yum.conf\n\nwget -q https:\/\/pkg.cfssl.org\/R1.2\/cfssl_linux-amd64 https:\/\/pkg.cfssl.org\/R1.2\/cfssljson_linux-amd64\nwget -q https:\/\/storage.googleapis.com\/kubernetes-release\/release\/${VERSION}\/bin\/linux\/amd64\/kubectl\nwget -q \"https:\/\/storage.googleapis.com\/kubernetes-helm\/helm-v2.8.1-linux-amd64.tar.gz\"\n\ntar -zxvf helm-v2.8.1-linux-amd64.tar.gz\nchmod +x cfssl_linux-amd64 cfssljson_linux-amd64 kubectl linux-amd64\/helm\nsudo mv cfssl_linux-amd64 \/usr\/local\/bin\/cfssl\nsudo mv cfssljson_linux-amd64 \/usr\/local\/bin\/cfssljson\nsudo mv kubectl linux-amd64\/helm \/usr\/local\/bin\/\nrm -rf helm-v2.8.1-linux-amd64.tar.gz linux-amd64","type":"","command_line_args":"","exit_status":[],"script_type":"sh"},"timeout_secs":"0","type":"EXEC","variable_list":[]},{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Master"},"retries":"0","description":"","message_list":[],"child_tasks_local_reference_list":[],"name":"Generate Certs","state":"ACTIVE","attrs":{"script":"#!\/bin\/bash\nset -ex\n\nINTERNAL_IP=\"@@{private_ip_address}@@\"\nCONTROLLER_IPS=\"@@{calm_array_private_ip_address}@@\" # private Ip's\nMINION_IPS=\"@@{GCP_Centos_K8SM.private_ip_address}@@\" # private Ip's\nPUBLIC_CONTROLLER_IPS=\"@@{calm_array_public_ip_address}@@\"\nPUBLIC_MINION_IPS=\"@@{GCP_Centos_K8SM.public_ip_address}@@\"\nCONTROLLER_HOSTNAMES=\"@@{calm_array_name}@@\"\nMASTER_API_HTTPS=6443\nSERVICE_SUBNET=\"@@{KUBE_SERVICE_SUBNET}@@\"\nKUBE_CLUSTER_NAME=\"@@{KUBE_CLUSTER_NAME}@@\"\nFIRST_IP_SERVICE_SUBNET=$(python -c \"from netaddr import * ; print IPNetwork('${SERVICE_SUBNET}')[1]\")\n\ncount=0\nfor ip in $(echo \"${CONTROLLER_IPS}\" | tr \",\" \"\\n\"); do\n CONS_NAMES+=\"controller${count}\",\n count=$((count+1))\ndone\nCONTROLLER_NAMES=$(echo $CONS_NAMES | sed 's\/,$\/\/')\n \ncount=0\nfor ip in $(echo ${MINION_IPS} | tr \",\" \"\\n\"); do\n MIN_NAMES+=\"minion${count}\",\n count=$((count+1))\ndone\nMINION_NAMES=$(echo $MIN_NAMES | sed 's\/,$\/\/') \n\nif [ @@{calm_array_index}@@ -ne 0 ];then\n exit\nfi\nsudo chown -R $USER:$USER \/opt\/kube-ssl && cd \/opt\/kube-ssl\necho '{\n \"signing\": {\n \"default\": {\n \"expiry\": \"8760h\"\n },\n \"profiles\": {\n \"server\": {\n \"expiry\": \"8760h\",\n \"usages\": [ \"signing\", \"key encipherment\", \"server auth\", \"client auth\" ]\n },\n \"client\": {\n \"expiry\": \"8760h\",\n \"usages\": [ \"key encipherment\", \"client auth\" ]\n },\n \"client-server\": {\n \"expiry\": \"8760h\",\n \"usages\": [ \"key encipherment\", \"server auth\", \"client auth\" ]\n }\n }\n }\n}' | tee ca-config.json\n\necho '{\n \"CN\": \"etcd-ca\",\n \"key\": {\n \"algo\": \"rsa\",\n \"size\": 2048\n },\n \"names\": [\n {\n \"C\": \"US\",\n \"L\": \"San Jose\",\n \"O\": \"etcd\",\n \"OU\": \"CA\",\n \"ST\": \"California\"\n }\n ]\n}' | tee etcd-ca-csr.json\n\ncfssl gencert -initca etcd-ca-csr.json | cfssljson -bare etcd-ca\n\necho '{\n \"CN\": \"etcd\",\n \"hosts\": [],\n \"key\": {\n \"algo\": \"rsa\",\n \"size\": 2048\n },\n \"names\": [\n {\n \"C\": \"US\",\n \"L\": \"San Jose\",\n \"O\": \"etcd\",\n \"OU\": \"CA\",\n \"ST\": \"California\"\n }\n ]\n}' | tee etcd-csr.json\n\ncfssl gencert -ca=etcd-ca.pem -ca-key=etcd-ca-key.pem -config=ca-config.json -hostname=${CONTROLLER_IPS},${PUBLIC_CONTROLLER_IPS} -profile=server etcd-csr.json | cfssljson -bare etcd-server\ncfssl gencert -ca=etcd-ca.pem -ca-key=etcd-ca-key.pem -config=ca-config.json -hostname=${CONTROLLER_IPS},${PUBLIC_CONTROLLER_IPS} -profile=client-server etcd-csr.json | cfssljson -bare etcd-peer\ncfssl gencert -ca=etcd-ca.pem -ca-key=etcd-ca-key.pem -config=ca-config.json -hostname=${CONTROLLER_IPS},${PUBLIC_CONTROLLER_IPS} -profile=client etcd-csr.json | cfssljson -bare etcd-client\n\necho '{\n \"CN\": \"kube-ca\",\n \"key\": {\n \"algo\": \"rsa\",\n \"size\": 2048\n },\n \"names\": [\n {\n \"C\": \"US\",\n \"L\": \"San Jose\",\n \"O\": \"kube\",\n \"OU\": \"CA\",\n \"ST\": \"California\"\n }\n ]\n}' | tee kube-ca-csr.json\n\ncfssl gencert -initca kube-ca-csr.json | cfssljson -bare ca\n\necho '{\n \"CN\": \"kubernetes\",\n \"key\": {\n \"algo\": \"rsa\",\n \"size\": 2048\n },\n \"names\": [\n {\n \"C\": \"US\",\n \"L\": \"San Jose\",\n \"O\": \"kube\",\n \"OU\": \"Cluster\",\n \"ST\": \"California\"\n }\n ]\n}' | tee kubernetes-csr.json\n\ncfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json \\\n-hostname=${CONTROLLER_NAMES},${CONTROLLER_IPS},${MINION_NAMES},${MINION_IPS},${FIRST_IP_SERVICE_SUBNET},${PUBLIC_CONTROLLER_IPS},${PUBLIC_MINION_IPS},127.0.0.1,kubernetes.default,kubernetes,kubernetes.default.svc,kubernetes.default.svc.cluster.local \\\n-profile=server kubernetes-csr.json | cfssljson -bare kubernetes\n\necho '{\n \"CN\": \"system:kube-controller-manager\",\n \"hosts\": [],\n \"key\": {\n \"algo\": \"rsa\",\n \"size\": 2048\n },\n \"names\": [\n {\n \"C\": \"US\",\n \"L\": \"San Jose\",\n \"O\": \"system:kube-controller-manager\",\n \"OU\": \"Cluster\",\n \"ST\": \"California\"\n }\n ]\n}' | tee kube-controller-manager-csr.json\n\ncfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server kube-controller-manager-csr.json | cfssljson -bare kube-controller-manager\n\necho '{\n \"CN\": \"system:kube-scheduler\",\n \"hosts\": [],\n \"key\": {\n \"algo\": \"rsa\",\n \"size\": 2048\n },\n \"names\": [\n {\n \"C\": \"US\",\n \"L\": \"San Jose\",\n \"O\": \"system:kube-scheduler\",\n \"OU\": \"Cluster\",\n \"ST\": \"California\"\n }\n ]\n}' | tee kube-scheduler-csr.json\n\ncfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server kube-scheduler-csr.json | cfssljson -bare kube-scheduler\n\ncount=0\nfor name in $(echo ${CONTROLLER_HOSTNAMES} | tr \",\" \"\\n\"); do\ninstance=\"controller${count}\"\necho \"{\n \\\"CN\\\": \\\"system:node:${name}\\\",\n \\\"key\\\": {\n \\\"algo\\\": \\\"rsa\\\",\n \\\"size\\\": 2048\n },\n \\\"names\\\": [\n {\n \\\"C\\\": \\\"US\\\",\n \\\"L\\\": \\\"San Jose\\\",\n \\\"O\\\": \\\"system:nodes\\\",\n \\\"OU\\\": \\\"Kubernetes The Hard Way\\\",\n \\\"ST\\\": \\\"California\\\"\n }\n ]\n}\" | tee ${instance}-csr.json\ncfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -hostname=${instance},${name} -profile=client-server ${instance}-csr.json | cfssljson -bare ${instance}\ncount=$((count+1))\ndone \n\n#count=0\n#for ip in $(echo ${MINION_IPS} | tr \",\" \"\\n\"); do\n#instance=\"minion${count}\"\n#echo \"{\n# \\\"CN\\\": \\\"system:node:${instance}\\\",\n# \\\"key\\\": {\n# \\\"algo\\\": \\\"rsa\\\",\n# \\\"size\\\": 2048\n# },\n# \\\"names\\\": [\n# {\n# \\\"C\\\": \\\"US\\\",\n# \\\"L\\\": \\\"San Jose\\\",\n# \\\"O\\\": \\\"system:nodes\\\",\n# \\\"OU\\\": \\\"Kubernetes The Hard Way\\\",\n# \\\"ST\\\": \\\"California\\\"\n# }\n# ]\n#}\" | tee ${instance}-csr.json\n#cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -hostname=${instance},${ip} -profile=client-server ${instance}-csr.json | cfssljson -bare ${instance}\n#count=$((count+1))\n#done\n\n# -*- Creating kube-proxy certificates\necho '{\n \"CN\": \"system:kube-proxy\",\n \"hosts\": [],\n \"key\": {\n \"algo\": \"rsa\",\n \"size\": 2048\n },\n \"names\": [\n {\n \"C\": \"US\",\n \"L\": \"San Jose\",\n \"O\": \"system:node-proxier\",\n \"OU\": \"Cluster\",\n \"ST\": \"California\"\n }\n ]\n}' | tee kube-proxy-csr.json\n\ncfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client kube-proxy-csr.json | cfssljson -bare kube-proxy\n\necho '{\n \"CN\": \"admin\",\n \"hosts\": [],\n \"key\": {\n \"algo\": \"rsa\",\n \"size\": 2048\n },\n \"names\": [\n {\n \"C\": \"US\",\n \"L\": \"San Jose\",\n \"O\": \"system:masters\",\n \"OU\": \"Cluster\",\n \"ST\": \"California\"\n }\n ]\n}' | tee admin-csr.json\n\ncfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client admin-csr.json | cfssljson -bare admin\n\ncount=0\nfor name in $(echo ${CONTROLLER_HOSTNAMES} | tr \",\" \"\\n\"); do\nkubectl config set-cluster ${KUBE_CLUSTER_NAME} --certificate-authority=ca.pem --embed-certs=true --server=https:\/\/${INTERNAL_IP}:${MASTER_API_HTTPS} --kubeconfig=controller${count}.kubeconfig\nkubectl config set-credentials system:node:${name} --client-certificate=controller${count}.pem --client-key=controller${count}-key.pem --embed-certs=true --kubeconfig=controller${count}.kubeconfig\nkubectl config set-context default --cluster=${KUBE_CLUSTER_NAME} --user=system:node:${name} --kubeconfig=controller${count}.kubeconfig\nkubectl config use-context default --kubeconfig=controller${count}.kubeconfig\ncount=$((count+1))\ndone\n\n#count=0\n#for ip in $(echo ${MINION_IPS} | tr \",\" \"\\n\"); do\n#kubectl config set-cluster ${KUBE_CLUSTER_NAME} --certificate-authority=ca.pem --embed-certs=true --server=https:\/\/${INTERNAL_IP}:${MASTER_API_HTTPS} --kubeconfig=minion${count}.kubeconfig\n#kubectl config set-credentials system:node:minion${count} --client-certificate=minion${count}.pem --client-key=minion${count}-key.pem --embed-certs=true --kubeconfig=minion${count}.kubeconfig\n#kubectl config set-context default --cluster=${KUBE_CLUSTER_NAME} --user=system:node:minion${count} --kubeconfig=minion${count}.kubeconfig\n#kubectl config use-context default --kubeconfig=minion${count}.kubeconfig\n#count=$((count+1))\n#done\n\nkubectl config set-cluster ${KUBE_CLUSTER_NAME} --certificate-authority=ca.pem --embed-certs=true --server=https:\/\/${INTERNAL_IP}:${MASTER_API_HTTPS} --kubeconfig=kube-controller-manager.kubeconfig\nkubectl config set-credentials kube-controller-manager --client-certificate=kube-controller-manager.pem --client-key=kube-controller-manager-key.pem --embed-certs=true --kubeconfig=kube-controller-manager.kubeconfig\nkubectl config set-context default --cluster=${KUBE_CLUSTER_NAME} --user=kube-controller-manager --kubeconfig=kube-controller-manager.kubeconfig\nkubectl config use-context default --kubeconfig=kube-controller-manager.kubeconfig\n\nkubectl config set-cluster ${KUBE_CLUSTER_NAME} --certificate-authority=ca.pem --embed-certs=true --server=https:\/\/${INTERNAL_IP}:${MASTER_API_HTTPS} --kubeconfig=kube-scheduler.kubeconfig\nkubectl config set-credentials kube-scheduler --client-certificate=kube-scheduler.pem --client-key=kube-scheduler-key.pem --embed-certs=true --kubeconfig=kube-scheduler.kubeconfig\nkubectl config set-context default --cluster=${KUBE_CLUSTER_NAME} --user=kube-scheduler --kubeconfig=kube-scheduler.kubeconfig\nkubectl config use-context default --kubeconfig=kube-scheduler.kubeconfig\n\n\nkubectl config set-cluster ${KUBE_CLUSTER_NAME} --certificate-authority=ca.pem --embed-certs=true --server=https:\/\/${INTERNAL_IP}:${MASTER_API_HTTPS} --kubeconfig=kube-proxy.kubeconfig\nkubectl config set-credentials kube-proxy --client-certificate=kube-proxy.pem --client-key=kube-proxy-key.pem --embed-certs=true --kubeconfig=kube-proxy.kubeconfig\nkubectl config set-context default --cluster=${KUBE_CLUSTER_NAME} --user=kube-proxy --kubeconfig=kube-proxy.kubeconfig\nkubectl config use-context default --kubeconfig=kube-proxy.kubeconfig\n\nENCRYPTION_KEY=$(head -c 32 \/dev\/urandom | base64)\necho \"kind: EncryptionConfig\napiVersion: v1\nresources:\n - resources:\n - secrets\n providers:\n - aescbc:\n keys:\n - name: key1\n secret: ${ENCRYPTION_KEY}\n - identity: {}\" | tee encryption-config.yaml\n\necho \"@@{CENTOS.secret}@@\" | tee ~\/.ssh\/id_rsa\nchmod 400 ~\/.ssh\/id_rsa\n\ncount=0\nfor ip in $(echo ${CONTROLLER_IPS} | tr \",\" \"\\n\"); do\n instance=\"controller${count}\"\n scp -o stricthostkeychecking=no admin*.pem ca*.pem etcd*.pem kubernetes*.pem ${instance}* kube-proxy.kubeconfig kube-controller-manager.kubeconfig kube-scheduler.kubeconfig encryption-config.yaml ${instance}:\ncount=$((count+1))\ndone\n\n#count=0\n#for ip in $(echo ${MINION_IPS} | tr \",\" \"\\n\"); do\n# instance=\"minion${count}\"\n# scp -o stricthostkeychecking=no ca*.pem kubernetes*.pem ${instance}* kube-proxy.kubeconfig ${instance}:\n#count=$((count+1))\n#done\n","type":"","command_line_args":"","exit_status":[],"script_type":"sh"},"timeout_secs":"0","type":"EXEC","variable_list":[]},{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Master"},"retries":"0","description":"","message_list":[],"child_tasks_local_reference_list":[],"name":"Configure Services","state":"ACTIVE","attrs":{"script":"#!\/bin\/bash\nset -ex\n\nKUBELET_IMAGE_TAG=\"@@{KUBE_IMAGE_TAG}@@\"\nif [[ \"@@{KUBE_IMAGE_TAG_NEW}@@x\" != \"x\" ]]; then\n\tKUBELET_IMAGE_TAG=\"@@{KUBE_IMAGE_TAG_NEW}@@\"\nfi\nINTERNAL_IP=\"@@{private_ip_address}@@\"\nCONTROLLER_IPS=\"@@{calm_array_private_ip_address}@@\"\nCLUSTER_SUBNET=\"@@{KUBE_CLUSTER_SUBNET}@@\"\nSERVICE_SUBNET=\"@@{KUBE_SERVICE_SUBNET}@@\"\nETCD_CERT_PATH=\"\/etc\/ssl\/certs\/etcd\"\nKUBE_CERT_PATH=\"\/etc\/kubernetes\/ssl\"\nKUBE_MANIFEST_PATH=\"\/etc\/kubernetes\/manifests\"\nNODE_NAME=\"controller@@{calm_array_index}@@\"\nMASTER_API_HTTPS=6443\nETCD_SERVER_PORT=2379\nVERSION=$(echo \"${KUBELET_IMAGE_TAG}\" | tr \"_\" \" \" | awk '{print $1}')\nCONTROLLER_COUNT=$(echo \"@@{calm_array_private_ip_address}@@\" | tr ',' '\\n' | wc -l)\n\nsudo cp ca*.pem etcd-*.pem kubernetes*.pem ${NODE_NAME}* kube-*.kubeconfig encryption-config.yaml ${KUBE_CERT_PATH}\/\nsudo chmod +r ${KUBE_CERT_PATH}\/*\n\nsudo cp etcd-*.pem ${ETCD_CERT_PATH}\/\nsudo chmod +r ${ETCD_CERT_PATH}\/*\n\ncount=0\nfor ip in $(echo \"${CONTROLLER_IPS}\" | tr \",\" \"\\n\"); do\n ETCD+=\"https:\/\/${ip}:${ETCD_SERVER_PORT}\",\n count=$((count+1))\ndone\nETCD_SERVERS=$(echo $ETCD | sed 's\/,$\/\/')\n\necho \"apiVersion: v1\nkind: Pod\nmetadata:\n name: kube-apiserver\n namespace: kube-system\n labels:\n k8s-app: kube-apiserver\nspec:\n hostNetwork: true\n containers:\n - name: kube-apiserver\n image: quay.io\/coreos\/hyperkube:${KUBELET_IMAGE_TAG}\n command:\n - \/hyperkube\n - apiserver\n - --admission-control=Initializers,NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota\n - --advertise-address=${INTERNAL_IP}\n - --allow-privileged=true\n - --anonymous-auth=false\n - --insecure-port=0\n - --secure-port=${MASTER_API_HTTPS}\n - --profiling=false\n - --repair-malformed-updates=false\n - --apiserver-count=${CONTROLLER_COUNT}\n - --audit-log-maxage=30\n - --audit-log-maxbackup=10\n - --audit-log-maxsize=100\n - --audit-log-path=\/var\/lib\/audit.log\n - --authorization-mode=Node,RBAC\n - --bind-address=0.0.0.0\n - --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP\n - --event-ttl=1h\n - --service-account-lookup=true\n - --enable-swagger-ui=true\n - --storage-backend=etcd3\n - --etcd-cafile=${ETCD_CERT_PATH}\/etcd-ca.pem\n - --etcd-certfile=${ETCD_CERT_PATH}\/etcd-client.pem\n - --etcd-keyfile=${ETCD_CERT_PATH}\/etcd-client-key.pem\n - --etcd-servers=${ETCD_SERVERS}\n - --experimental-encryption-provider-config=${KUBE_CERT_PATH}\/encryption-config.yaml\n - --tls-ca-file=${KUBE_CERT_PATH}\/ca.pem\n - --tls-cert-file=${KUBE_CERT_PATH}\/kubernetes.pem\n - --tls-private-key-file=${KUBE_CERT_PATH}\/kubernetes-key.pem\n - --kubelet-client-certificate=${KUBE_CERT_PATH}\/kubernetes.pem\n - --kubelet-client-key=${KUBE_CERT_PATH}\/kubernetes-key.pem\n - --kubelet-https=true\n - --runtime-config=api\/all\n - --service-account-key-file=${KUBE_CERT_PATH}\/kubernetes.pem\n - --service-cluster-ip-range=${SERVICE_SUBNET}\n - --service-node-port-range=30000-32767\n - --client-ca-file=${KUBE_CERT_PATH}\/ca.pem\n - --cloud-provider=gce\n - --v=2\n ports:\n - containerPort: ${MASTER_API_HTTPS}\n hostPort: ${MASTER_API_HTTPS}\n name: https\n - containerPort: 8080\n hostPort: 8080\n name: local\n volumeMounts:\n - mountPath: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n readOnly: true\n - mountPath: \/etc\/ssl\/certs\n name: ssl-certs-host\n readOnly: true\n - mountPath: \/etc\/pki\n name: ca-certs-etc-pki\n readOnly: true\n volumes:\n - hostPath:\n path: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n - hostPath:\n path: \/etc\/ssl\/certs\n name: ssl-certs-host\n - hostPath:\n path: \/etc\/pki\n name: ca-certs-etc-pki\" | sudo tee ${KUBE_MANIFEST_PATH}\/kube-apiserver.yaml\n\necho \"apiVersion: v1\nkind: Pod\nmetadata:\n name: kube-proxy\n namespace: kube-system\n labels:\n k8s-app: kube-proxy\nspec:\n hostNetwork: true\n containers:\n - name: kube-proxy\n image: quay.io\/coreos\/hyperkube:${KUBELET_IMAGE_TAG}\n command:\n - \/hyperkube\n - proxy\n - --cluster-cidr=${CLUSTER_SUBNET}\n - --masquerade-all=true\n - --kubeconfig=${KUBE_CERT_PATH}\/kube-proxy.kubeconfig\n - --proxy-mode=iptables\n securityContext:\n privileged: true\n volumeMounts:\n - mountPath: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n readOnly: true\n - mountPath: \/etc\/ssl\/certs\n name: ssl-certs-host\n readOnly: true\n volumes:\n - hostPath:\n path: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n - hostPath:\n path: \/etc\/ssl\/certs\n name: ssl-certs-host\" | sudo tee ${KUBE_MANIFEST_PATH}\/kube-proxy.yaml\n \necho \"apiVersion: v1\nkind: Pod\nmetadata:\n name: kube-controller-manager\n namespace: kube-system\n labels:\n k8s-app: kube-controller-manager\nspec:\n hostNetwork: true\n containers:\n - name: kube-controller-manager\n image: quay.io\/coreos\/hyperkube:${KUBELET_IMAGE_TAG}\n command:\n - \/hyperkube\n - controller-manager\n - --address=0.0.0.0 \n - --allocate-node-cidrs=true \n - --cluster-cidr=${CLUSTER_SUBNET}\n - --cluster-name=kubernetes-prod-cluster \n - --leader-elect=true \n - --kubeconfig=${KUBE_CERT_PATH}\/kube-controller-manager.kubeconfig \n - --service-account-private-key-file=${KUBE_CERT_PATH}\/kubernetes-key.pem \n - --service-cluster-ip-range=${SERVICE_SUBNET}\n - --terminated-pod-gc-threshold=100 \n - --profiling=false \n - --use-service-account-credentials=true\n - --cloud-provider=gce\n - --v=2\n livenessProbe:\n httpGet:\n host: 127.0.0.1\n path: \/healthz\n port: 10252\n initialDelaySeconds: 15\n timeoutSeconds: 1\n volumeMounts:\n - mountPath: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n readOnly: true\n - mountPath: \/etc\/ssl\/certs\n name: ssl-certs-host\n readOnly: true\n - mountPath: \/etc\/pki\n name: ca-certs-etc-pki\n readOnly: true\n volumes:\n - hostPath:\n path: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n - hostPath:\n path: \/etc\/ssl\/certs\n name: ssl-certs-host\n - hostPath:\n path: \/etc\/pki\n name: ca-certs-etc-pki\" | sudo tee ${KUBE_MANIFEST_PATH}\/kube-controller-manager.yaml\n \necho \"apiVersion: v1\nkind: Pod\nmetadata:\n name: kube-scheduler\n namespace: kube-system\n labels:\n k8s-app: kube-scheduler\nspec:\n hostNetwork: true\n containers:\n - name: kube-scheduler\n image: quay.io\/coreos\/hyperkube:${KUBELET_IMAGE_TAG}\n command:\n - \/hyperkube\n - scheduler\n - --kubeconfig=${KUBE_CERT_PATH}\/kube-scheduler.kubeconfig\n - --leader-elect=true\n - --profiling=false\n - --v=2\n livenessProbe:\n httpGet:\n host: 127.0.0.1\n path: \/healthz\n port: 10251\n initialDelaySeconds: 15\n timeoutSeconds: 1\n volumeMounts:\n - mountPath: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n readOnly: true\n volumes:\n - hostPath:\n path: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\" | sudo tee ${KUBE_MANIFEST_PATH}\/kube-scheduler.yaml","type":"","command_line_args":"","exit_status":[],"script_type":"sh"},"timeout_secs":"0","type":"EXEC","variable_list":[]},{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Master"},"retries":"0","description":"","message_list":[],"child_tasks_local_reference_list":[],"name":"Add User Roles","state":"ACTIVE","attrs":{"script":"#!\/bin\/bash\nset -ex\n\nMASTER_API_HTTPS=6443\nINTERNAL_IP=\"@@{private_ip_address}@@\"\nKUBE_CLUSTER_NAME=\"@@{KUBE_CLUSTER_NAME}@@\"\n\nsudo systemctl start etcd docker kubelet\nsudo systemctl enable etcd docker kubelet\n\nexport PATH=$PATH:\/opt\/bin\n\nmkdir CA\nmv admin*.pem ca*.pem etcd-*.pem kubernetes*.pem controller* kube-*.kubeconfig encryption-config.yaml CA\/\nif [ @@{calm_array_index}@@ -ne 0 ];then\n exit\nfi\ncp \/opt\/kube-ssl\/admin*.pem CA\/\n\nCOUNT=0\nwhile [[ $(curl --key CA\/admin-key.pem --cert CA\/admin.pem --cacert CA\/ca.pem https:\/\/${INTERNAL_IP}:${MASTER_API_HTTPS}\/healthz) != \"ok\" ]] ; do\n echo \"sleep for 5 secs\"\n sleep 5\n COUNT=$(($COUNT+1))\n if [[ $COUNT -eq 50 ]]; then\n \techo \"Error: creating cluster\"\n exit 1\n fi\ndone\n\nkubectl config set-cluster ${KUBE_CLUSTER_NAME} --certificate-authority=$HOME\/CA\/ca.pem --embed-certs=true --server=https:\/\/${INTERNAL_IP}:${MASTER_API_HTTPS}\nkubectl config set-credentials admin --client-certificate=$HOME\/CA\/admin.pem --client-key=$HOME\/CA\/admin-key.pem\nkubectl config set-context ${KUBE_CLUSTER_NAME} --cluster=${KUBE_CLUSTER_NAME} --user=admin\nkubectl config use-context ${KUBE_CLUSTER_NAME}\n\ncat <\/dev\/null\") ] ; do echo \"waiting for certs sleeping 5\" && sleep 5; done\n\nscp -o stricthostkeychecking=no ${MASTER_IP}:\/opt\/kube-ssl\/{ca*.pem,kubernetes*.pem,kube-proxy.kubeconfig,ca-config.json} .\n\ninstance=\"minion@@{calm_array_index}@@\"\necho \"{\n \\\"CN\\\": \\\"system:node:${HOSTNAME}\\\",\n \\\"key\\\": {\n \\\"algo\\\": \\\"rsa\\\",\n \\\"size\\\": 2048\n },\n \\\"names\\\": [\n {\n \\\"C\\\": \\\"US\\\",\n \\\"L\\\": \\\"San Jose\\\",\n \\\"O\\\": \\\"system:nodes\\\",\n \\\"OU\\\": \\\"Kubernetes The Hard Way\\\",\n \\\"ST\\\": \\\"California\\\"\n }\n ]\n}\" | tee ${instance}-csr.json\ncfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -hostname=${HOSTNAME},${INSTANCE_IP} -profile=client-server ${instance}-csr.json | cfssljson -bare ${instance}\n\nkubectl config set-cluster ${KUBE_CLUSTER_NAME} --certificate-authority=ca.pem --embed-certs=true --server=https:\/\/${MASTER_IP}:${MASTER_API_HTTPS} --kubeconfig=${instance}.kubeconfig\nkubectl config set-credentials system:node:${HOSTNAME} --client-certificate=${instance}.pem --client-key=${instance}-key.pem --embed-certs=true --kubeconfig=${instance}.kubeconfig\nkubectl config set-context default --cluster=${KUBE_CLUSTER_NAME} --user=system:node:${HOSTNAME} --kubeconfig=${instance}.kubeconfig\nkubectl config use-context default --kubeconfig=${instance}.kubeconfig\n\nsudo cp *.pem *.kubeconfig ${KUBE_CERT_PATH}\/\nsudo chmod +r ${KUBE_CERT_PATH}\/*\n\nrm -rf ${instance}-csr.json","type":"","command_line_args":"","exit_status":[],"script_type":"sh"},"timeout_secs":"0","type":"EXEC","variable_list":[]}],"description":"","name":"927472a7_runbook","state":"ACTIVE","main_task_local_reference":{"kind":"app_task","name":"709f9264_dag"},"message_list":[],"variable_list":[]},"type":"","uninstall_runbook":{"task_definition_list":[{"target_any_local_reference":{"kind":"app_package","name":"GCP_Centos_K8SM_Package"},"retries":"0","description":"","message_list":[],"child_tasks_local_reference_list":[{"kind":"app_task","name":"Remove Node"}],"name":"fef85962_dag","state":"ACTIVE","attrs":{"edges":[],"type":""},"timeout_secs":"0","type":"DAG","variable_list":[]},{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Minion"},"retries":"0","description":"","message_list":[],"child_tasks_local_reference_list":[],"name":"Remove Node","state":"ACTIVE","attrs":{"script":"#!\/bin\/bash\nset -ex\n\nMASTER_IP=\"@@{Kubernetes_Master.address[0]}@@\"\nNODE_NAME=${HOSTNAME}\nssh -o stricthostkeychecking=no ${MASTER_IP} \"kubectl drain '${NODE_NAME}' --ignore-daemonsets --delete-local-data --force\"\nsleep 10\nssh -o stricthostkeychecking=no ${MASTER_IP} \"kubectl delete node '${NODE_NAME}'\"","type":"","command_line_args":"","exit_status":[],"script_type":"sh"},"timeout_secs":"0","type":"EXEC","variable_list":[]}],"description":"","name":"435393f9_runbook","state":"ACTIVE","main_task_local_reference":{"kind":"app_task","name":"fef85962_dag"},"message_list":[],"variable_list":[]}},"variable_list":[]}],"app_profile_list":[{"deployment_create_list":[{"type":"GREENFIELD","description":"","action_list":[],"editables":{"min_replicas":true,"max_replicas":true},"name":"2cbb2f6a_deployment","max_replicas":"3","package_local_reference_list":[{"kind":"app_package","name":"AHV_Centos_K8SC_Package"}],"substrate_local_reference":{"kind":"app_substrate","name":"AHV_Centos_K8SC"},"min_replicas":"3","variable_list":[]},{"type":"GREENFIELD","description":"","action_list":[],"editables":{"min_replicas":true,"max_replicas":true},"name":"39cf7283_deployment","max_replicas":"9","package_local_reference_list":[{"kind":"app_package","name":"AHV_Centos_K8SM_Package"}],"substrate_local_reference":{"kind":"app_substrate","name":"AHV_Centos_K8SM"},"min_replicas":"3","variable_list":[]}],"description":"","action_list":[{"description":"","type":"user","critical":false,"runbook":{"task_definition_list":[{"retries":"0","description":"","child_tasks_local_reference_list":[{"kind":"app_task","name":"Upgrade Controller"},{"kind":"app_task","name":"Upgrade Minion"},{"kind":"app_task","name":"Restart Service"}],"name":"6ee65d8a_dag","attrs":{"edges":[{"from_task_reference":{"kind":"app_task","name":"Upgrade Controller"},"edge_type":"user_defined","type":"","to_task_reference":{"kind":"app_task","name":"Restart Service"}},{"from_task_reference":{"kind":"app_task","name":"Upgrade Minion"},"edge_type":"user_defined","type":"","to_task_reference":{"kind":"app_task","name":"Restart Service"}}],"type":""},"timeout_secs":"0","type":"DAG","variable_list":[]},{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Master"},"retries":"0","description":"","child_tasks_local_reference_list":[],"name":"Upgrade Controller","attrs":{"script":"#!\/bin\/bash\nset -ex\n\nETCD_VERSION=\"v3.2.11\"\nINTERNAL_IP=\"@@{address}@@\"\nCONTROLLER_IPS=\"@@{calm_array_address}@@\"\nNODE_NAME=\"controller@@{calm_array_index}@@\"\nCLUSTER_SUBNET=\"@@{KUBE_CLUSTER_SUBNET}@@\"\nSERVICE_SUBNET=\"@@{KUBE_SERVICE_SUBNET}@@\"\nKUBE_CLUSTER_DNS=\"@@{KUBE_DNS_IP}@@\"\nKUBELET_IMAGE_TAG=\"@@{KUBE_IMAGE_TAG_NEW}@@\"\nDOCKER_VERSION=\"@@{DOCKER_VERSION}@@\"\nETCD_CERT_PATH=\"\/etc\/ssl\/certs\/etcd\"\nKUBE_CERT_PATH=\"\/etc\/kubernetes\/ssl\"\nKUBE_MANIFEST_PATH=\"\/etc\/kubernetes\/manifests\"\nVERSION=$(echo \"${KUBELET_IMAGE_TAG}\" | tr \"_\" \" \" | awk '{print $1}')\nMASTER_API_HTTPS=6443\nETCD_SERVER_PORT=2379\nCONTROLLER_COUNT=$(echo \"@@{calm_array_address}@@\" | tr ',' '\\n' | wc -l)\n\n\ncount=0\nfor ip in $(echo \"${CONTROLLER_IPS}\" | tr \",\" \"\\n\"); do\n ETCD+=\"https:\/\/${ip}:${ETCD_SERVER_PORT}\",\n count=$((count+1))\ndone\nETCD_SERVERS=$(echo $ETCD | sed 's\/,$\/\/')\n\nwget -q https:\/\/storage.googleapis.com\/kubernetes-release\/release\/${VERSION}\/bin\/linux\/amd64\/kubelet\nchmod +x kubelet\nsudo mv kubelet \/usr\/bin\/kubelet\n\necho \"[Unit]\nDescription=Kubernetes Kubelet\nDocumentation=https:\/\/github.com\/GoogleCloudPlatform\/kubernetes\nAfter=docker.service\nRequires=docker.service\n\n[Service]\nExecStart=\/usr\/bin\/kubelet \\\\\n --allow-privileged=true \\\\\n --anonymous-auth=false \\\\\n --authorization-mode=Webhook \\\\\n --cluster-dns=${KUBE_CLUSTER_DNS} \\\\\n --cluster-domain=cluster.local \\\\\n --container-runtime=docker \\\\\n --enable-custom-metrics \\\\\n --kubeconfig=${KUBE_CERT_PATH}\/${NODE_NAME}.kubeconfig \\\\\n --network-plugin=cni \\\\\n --pod-cidr=${CLUSTER_SUBNET} \\\\\n --register-node=true \\\\\n --runtime-request-timeout=10m \\\\\n --client-ca-file=${KUBE_CERT_PATH}\/ca.pem \\\\\n --tls-cert-file=${KUBE_CERT_PATH}\/${NODE_NAME}.pem \\\\\n --tls-private-key-file=${KUBE_CERT_PATH}\/${NODE_NAME}-key.pem \\\\\n --pod-manifest-path=${KUBE_MANIFEST_PATH} \\\\\n --read-only-port=0 \\\\\n --protect-kernel-defaults=false \\\\\n --make-iptables-util-chains=true \\\\\n --keep-terminated-pod-volumes=false \\\\\n --event-qps=0 \\\\\n --cadvisor-port=0 \\\\\n --runtime-cgroups=\/systemd\/system.slice \\\\\n --kubelet-cgroups=\/systemd\/system.slice \\\\\n --node-labels 'node-role.kubernetes.io\/master=true' \\\\\n --node-labels 'node-role.kubernetes.io\/etcd=true' \\\\\n --register-with-taints=node-role.kubernetes.io\/master=true:NoSchedule \\\\\n --v=2\nRestart=on-failure\nRestartSec=5\n\n[Install]\nWantedBy=multi-user.target\" | sudo tee \/etc\/systemd\/system\/kubelet.service\n\necho \"apiVersion: v1\nkind: Pod\nmetadata:\n name: kube-apiserver\n namespace: kube-system\n labels:\n k8s-app: kube-apiserver\nspec:\n hostNetwork: true\n containers:\n - name: kube-apiserver\n image: quay.io\/coreos\/hyperkube:${KUBELET_IMAGE_TAG}\n command:\n - \/hyperkube\n - apiserver\n - --admission-control=Initializers,NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota\n - --advertise-address=${INTERNAL_IP}\n - --allow-privileged=true\n - --anonymous-auth=false\n - --insecure-port=0\n - --secure-port=${MASTER_API_HTTPS}\n - --profiling=false\n - --repair-malformed-updates=false\n - --apiserver-count=${CONTROLLER_COUNT}\n - --audit-log-maxage=30\n - --audit-log-maxbackup=10\n - --audit-log-maxsize=100\n - --audit-log-path=\/var\/lib\/audit.log\n - --authorization-mode=Node,RBAC\n - --bind-address=0.0.0.0\n - --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP\n - --event-ttl=1h\n - --service-account-lookup=true\n - --enable-swagger-ui=true\n - --storage-backend=etcd3\n - --etcd-cafile=${ETCD_CERT_PATH}\/etcd-ca.pem\n - --etcd-certfile=${ETCD_CERT_PATH}\/etcd-client.pem\n - --etcd-keyfile=${ETCD_CERT_PATH}\/etcd-client-key.pem\n - --etcd-servers=${ETCD_SERVERS}\n - --experimental-encryption-provider-config=${KUBE_CERT_PATH}\/encryption-config.yaml\n - --tls-ca-file=${KUBE_CERT_PATH}\/ca.pem\n - --tls-cert-file=${KUBE_CERT_PATH}\/kubernetes.pem\n - --tls-private-key-file=${KUBE_CERT_PATH}\/kubernetes-key.pem\n - --kubelet-client-certificate=${KUBE_CERT_PATH}\/kubernetes.pem\n - --kubelet-client-key=${KUBE_CERT_PATH}\/kubernetes-key.pem\n - --kubelet-https=true\n - --runtime-config=api\/all\n - --service-account-key-file=${KUBE_CERT_PATH}\/kubernetes.pem\n - --service-cluster-ip-range=${SERVICE_SUBNET}\n - --service-node-port-range=30000-32767\n - --client-ca-file=${KUBE_CERT_PATH}\/ca.pem\n - --v=2\n ports:\n - containerPort: ${MASTER_API_HTTPS}\n hostPort: ${MASTER_API_HTTPS}\n name: https\n - containerPort: 8080\n hostPort: 8080\n name: local\n volumeMounts:\n - mountPath: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n readOnly: true\n - mountPath: \/etc\/ssl\/certs\n name: ssl-certs-host\n readOnly: true\n - mountPath: \/etc\/pki\n name: ca-certs-etc-pki\n readOnly: true\n volumes:\n - hostPath:\n path: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n - hostPath:\n path: \/etc\/ssl\/certs\n name: ssl-certs-host\n - hostPath:\n path: \/etc\/pki\n name: ca-certs-etc-pki\" | sudo tee ${KUBE_MANIFEST_PATH}\/kube-apiserver.yaml\n\necho \"apiVersion: v1\nkind: Pod\nmetadata:\n name: kube-proxy\n namespace: kube-system\n labels:\n k8s-app: kube-proxy\nspec:\n hostNetwork: true\n containers:\n - name: kube-proxy\n image: quay.io\/coreos\/hyperkube:${KUBELET_IMAGE_TAG}\n command:\n - \/hyperkube\n - proxy\n - --cluster-cidr=${CLUSTER_SUBNET}\n - --masquerade-all=true\n - --kubeconfig=${KUBE_CERT_PATH}\/kube-proxy.kubeconfig\n - --proxy-mode=iptables\n securityContext:\n privileged: true\n volumeMounts:\n - mountPath: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n readOnly: true\n - mountPath: \/etc\/ssl\/certs\n name: ssl-certs-host\n readOnly: true\n volumes:\n - hostPath:\n path: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n - hostPath:\n path: \/etc\/ssl\/certs\n name: ssl-certs-host\" | sudo tee ${KUBE_MANIFEST_PATH}\/kube-proxy.yaml\n \necho \"apiVersion: v1\nkind: Pod\nmetadata:\n name: kube-controller-manager\n namespace: kube-system\n labels:\n k8s-app: kube-controller-manager\nspec:\n hostNetwork: true\n containers:\n - name: kube-controller-manager\n image: quay.io\/coreos\/hyperkube:${KUBELET_IMAGE_TAG}\n command:\n - \/hyperkube\n - controller-manager\n - --address=0.0.0.0 \n - --allocate-node-cidrs=true \n - --cluster-cidr=${CLUSTER_SUBNET}\n - --cluster-name=kubernetes-prod-cluster \n - --leader-elect=true \n - --kubeconfig=${KUBE_CERT_PATH}\/kube-controller-manager.kubeconfig \n - --service-account-private-key-file=${KUBE_CERT_PATH}\/kubernetes-key.pem \n - --service-cluster-ip-range=${SERVICE_SUBNET}\n - --terminated-pod-gc-threshold=100 \n - --profiling=false \n - --use-service-account-credentials=true \n - --v=2\n livenessProbe:\n httpGet:\n host: 127.0.0.1\n path: \/healthz\n port: 10252\n initialDelaySeconds: 15\n timeoutSeconds: 1\n volumeMounts:\n - mountPath: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n readOnly: true\n - mountPath: \/etc\/ssl\/certs\n name: ssl-certs-host\n readOnly: true\n - mountPath: \/etc\/pki\n name: ca-certs-etc-pki\n readOnly: true\n volumes:\n - hostPath:\n path: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n - hostPath:\n path: \/etc\/ssl\/certs\n name: ssl-certs-host\n - hostPath:\n path: \/etc\/pki\n name: ca-certs-etc-pki\" | sudo tee ${KUBE_MANIFEST_PATH}\/kube-controller-manager.yaml\n \necho \"apiVersion: v1\nkind: Pod\nmetadata:\n name: kube-scheduler\n namespace: kube-system\n labels:\n k8s-app: kube-scheduler\nspec:\n hostNetwork: true\n containers:\n - name: kube-scheduler\n image: quay.io\/coreos\/hyperkube:${KUBELET_IMAGE_TAG}\n command:\n - \/hyperkube\n - scheduler\n - --kubeconfig=${KUBE_CERT_PATH}\/kube-scheduler.kubeconfig\n - --leader-elect=true\n - --profiling=false\n - --v=2\n livenessProbe:\n httpGet:\n host: 127.0.0.1\n path: \/healthz\n port: 10251\n initialDelaySeconds: 15\n timeoutSeconds: 1\n volumeMounts:\n - mountPath: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n readOnly: true\n volumes:\n - hostPath:\n path: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\" | sudo tee ${KUBE_MANIFEST_PATH}\/kube-scheduler.yaml","type":"","command_line_args":"","exit_status":[],"script_type":"sh"},"timeout_secs":"0","type":"EXEC","variable_list":[]},{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Minion"},"retries":"0","description":"","child_tasks_local_reference_list":[],"name":"Upgrade Minion","attrs":{"script":"#!\/bin\/bash\nset -ex\n\nINTERNAL_IP=\"@@{address}@@\"\nNODE_NAME=\"minion@@{calm_array_index}@@\"\nCLUSTER_SUBNET=\"@@{KUBE_CLUSTER_SUBNET}@@\"\nKUBE_CLUSTER_DNS=\"@@{KUBE_DNS_IP}@@\"\nKUBELET_IMAGE_TAG=\"@@{KUBE_IMAGE_TAG_NEW}@@\"\nDOCKER_VERSION=\"@@{DOCKER_VERSION}@@\"\nKUBE_CERT_PATH=\"\/etc\/kubernetes\/ssl\"\nKUBE_MANIFEST_PATH=\"\/etc\/kubernetes\/manifests\"\nVERSION=$(echo \"${KUBELET_IMAGE_TAG}\" | tr \"_\" \" \" | awk '{print $1}')\n\nwget -q https:\/\/storage.googleapis.com\/kubernetes-release\/release\/${VERSION}\/bin\/linux\/amd64\/kubelet\nchmod +x kubelet\nsudo mv kubelet \/usr\/bin\/kubelet\n\necho \"[Unit]\nDescription=Kubernetes Kubelet\nDocumentation=https:\/\/github.com\/GoogleCloudPlatform\/kubernetes\nAfter=docker.service\nRequires=docker.service\n\n[Service]\nExecStart=\/usr\/bin\/kubelet \\\\\n --allow-privileged=true \\\\\n --anonymous-auth=false \\\\\n --authorization-mode=Webhook \\\\\n --cluster-dns=${KUBE_CLUSTER_DNS} \\\\\n --cluster-domain=cluster.local \\\\\n --container-runtime=docker \\\\\n --enable-custom-metrics \\\\\n --kubeconfig=${KUBE_CERT_PATH}\/${NODE_NAME}.kubeconfig \\\\\n --network-plugin=cni \\\\\n --pod-cidr=${CLUSTER_SUBNET} \\\\\n --register-node=true \\\\\n --runtime-request-timeout=10m \\\\\n --client-ca-file=${KUBE_CERT_PATH}\/ca.pem \\\\\n --tls-cert-file=${KUBE_CERT_PATH}\/${NODE_NAME}.pem \\\\\n --tls-private-key-file=${KUBE_CERT_PATH}\/${NODE_NAME}-key.pem \\\\\n --pod-manifest-path=${KUBE_MANIFEST_PATH} \\\\\n --read-only-port=0 \\\\\n --protect-kernel-defaults=false \\\\\n --make-iptables-util-chains=true \\\\\n --keep-terminated-pod-volumes=false \\\\\n --event-qps=0 \\\\\n --cadvisor-port=0 \\\\\n --runtime-cgroups=\/systemd\/system.slice \\\\\n --kubelet-cgroups=\/systemd\/system.slice \\\\\n --eviction-hard=memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5% \\\\\n --node-labels 'node-role.kubernetes.io\/worker=true' \\\\\n --node-labels 'beta.kubernetes.io\/fluentd-ds-ready=true' \\\\\n --v=2\nRestart=on-failure\nRestartSec=5\n\n[Install]\nWantedBy=multi-user.target\" | sudo tee \/etc\/systemd\/system\/kubelet.service\n\necho \"apiVersion: v1\nkind: Pod\nmetadata:\n name: kube-proxy\n namespace: kube-system\n labels:\n k8s-app: kube-proxy\nspec:\n hostNetwork: true\n containers:\n - name: kube-proxy\n image: quay.io\/coreos\/hyperkube:${KUBELET_IMAGE_TAG}\n command:\n - \/hyperkube\n - proxy\n - --cluster-cidr=${CLUSTER_SUBNET}\n - --masquerade-all=true\n - --kubeconfig=${KUBE_CERT_PATH}\/kube-proxy.kubeconfig\n - --proxy-mode=iptables\n securityContext:\n privileged: true\n volumeMounts:\n - mountPath: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n readOnly: true\n - mountPath: \/etc\/ssl\/certs\n name: ssl-certs-host\n readOnly: true\n volumes:\n - hostPath:\n path: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n - hostPath:\n path: \/etc\/ssl\/certs\n name: ssl-certs-host\" | sudo tee ${KUBE_MANIFEST_PATH}\/kube-proxy.yaml","type":"","command_line_args":"","exit_status":[],"script_type":"sh"},"timeout_secs":"0","type":"EXEC","variable_list":[]},{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Master"},"retries":"0","description":"","child_tasks_local_reference_list":[],"name":"Restart Service","attrs":{"script":"#!\/bin\/bash\nset -ex\n\nif [ @@{calm_array_index}@@ -ne 0 ];then\n\texit\nfi\n\nINTERNAL_IP=\"@@{address}@@\"\nCONTROLLER_IPS=\"@@{calm_array_address}@@\"\nMINION_IPS=\"@@{AHV_Centos_K8SM.address}@@\"\nJSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'\nKUBELET_IMAGE_TAG=\"@@{KUBE_IMAGE_TAG_NEW}@@\"\nMASTER_API_HTTPS=6443\nVERSION=$(echo \"${KUBELET_IMAGE_TAG}\" | tr \"_\" \" \" | awk '{print $1}')\nwget -q https:\/\/storage.googleapis.com\/kubernetes-release\/release\/${VERSION}\/bin\/linux\/amd64\/kubectl\nchmod +x kubectl\nsudo mv kubectl \/usr\/local\/bin\/kubectl\n\ncount=0\nwhile [[ $(curl --key CA\/admin-key.pem --cert CA\/admin.pem --cacert CA\/ca.pem https:\/\/${INTERNAL_IP}:${MASTER_API_HTTPS}\/healthz) != \"ok\" ]] ; do\n echo \"Trying to reach master server https:\/\/${INTERNAL_IP}:${MASTER_API_HTTPS} : sleep for 5 secs\"\n sleep 10\n if [[ $count -eq 10 ]]; then\n \techo \"Unable to reach master server: https:\/\/${INTERNAL_IP}:${MASTER_API_HTTPS}\"\n \texit 1\n fi\n count=$((count+1))\ndone\n\ncount=0\nfor ip in $(echo \"${MINION_IPS}\" | tr \",\" \"\\n\"); do\n kubectl drain \"minion${count}\" --ignore-daemonsets --delete-local-data --force\n sleep 20\n ssh -o stricthostkeychecking=no $ip \"sudo yum update -y --quiet && sudo systemctl daemon-reload && sudo systemctl restart kubelet\"\n sleep 20\n while [[ `kubectl get nodes -l kubernetes.io\/hostname=minion${count} -o jsonpath=\"$JSONPATH\" | grep \"Ready=Unknown\" 2>\/dev\/null` ]]; do sleep 10 ; done\n kubectl uncordon minion${count}\n count=$((count+1))\ndone\n\n\nif [[ `kubectl get nodes -o jsonpath=\"$JSONPATH\" | grep \"Ready=Unknown\"` ]]; then \n\techo \"Upgrade failed on minion nodes\"\n exit 1\nfi\n\ncount=0\nfor ip in $(echo \"${CONTROLLER_IPS}\" | tr \",\" \"\\n\"); do\n kubectl drain \"controller${count}\" --ignore-daemonsets --delete-local-data --force\n sleep 20\n ssh -o stricthostkeychecking=no $ip \"sudo yum update -y --quiet && sudo systemctl daemon-reload && sudo systemctl restart kubelet\"\n sleep 20\n while [[ `kubectl get nodes -l kubernetes.io\/hostname=controller${count} -o jsonpath=\"$JSONPATH\" | grep \"Ready=Unknown\" 2>\/dev\/null` ]]; do sleep 10 ; done\n kubectl uncordon controller${count}\n count=$((count+1))\ndone\n\n\nif [[ `kubectl get nodes -o jsonpath=\"$JSONPATH\" | grep \"Ready=Unknown\"` ]]; then \n\techo \"Upgrade failed on nodes: $(kubectl get nodes -o jsonpath='$JSONPATH' | grep 'Ready=Unknown')\"\n exit 1\nfi","type":"","command_line_args":"","exit_status":[],"script_type":"sh"},"timeout_secs":"0","type":"EXEC","variable_list":[]}],"description":"","name":"68a7ffd0_runbook","main_task_local_reference":{"kind":"app_task","name":"6ee65d8a_dag"},"variable_list":[{"val_type":"STRING","description":"","name":"KUBE_IMAGE_TAG_NEW","type":"LOCAL","value":"v1.8.0_coreos.0","label":"","attrs":{"type":""},"editables":{"value":true}}]},"name":"Upgrade"},{"description":"","type":"user","critical":false,"runbook":{"task_definition_list":[{"retries":"0","description":"","child_tasks_local_reference_list":[{"kind":"app_task","name":"Scale Out"},{"kind":"app_task","name":"Set Hosts file"}],"name":"03b33121_dag","attrs":{"edges":[{"from_task_reference":{"kind":"app_task","name":"Scale Out"},"edge_type":"user_defined","type":"","to_task_reference":{"kind":"app_task","name":"Set Hosts file"}}],"type":""},"timeout_secs":"0","type":"DAG","variable_list":[]},{"target_any_local_reference":{"kind":"app_blueprint_deployment","name":"39cf7283_deployment"},"retries":"0","description":"","child_tasks_local_reference_list":[],"name":"Scale Out","attrs":{"scaling_count":"@@{COUNT}@@","type":"","scaling_type":"SCALEOUT"},"timeout_secs":"0","type":"SCALING","variable_list":[]},{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Master"},"retries":"0","description":"","child_tasks_local_reference_list":[],"name":"Set Hosts file","attrs":{"script":"#!\/bin\/bash\nset -ex\n\nMINION_IPS=\"@@{AHV_Centos_K8SM.address}@@\"\nfor ip in $(echo ${MINION_IPS} | tr \",\" \"\\n\"); do\n if ! (( $(grep -c \"${ip} minion${count}\" \/etc\/hosts) )) ; then\n \techo \"${ip} minion${count}\" | sudo tee -a \/etc\/hosts\n fi\n count=$((count+1))\ndone","type":"","command_line_args":"","exit_status":[],"script_type":"sh"},"timeout_secs":"0","type":"EXEC","variable_list":[]}],"description":"","name":"00cf5417_runbook","main_task_local_reference":{"kind":"app_task","name":"03b33121_dag"},"variable_list":[{"val_type":"STRING","description":"","name":"KUBE_IMAGE_TAG_NEW","type":"LOCAL","value":"v1.8.0_coreos.0","label":"","attrs":{"type":""},"editables":{"value":true}},{"val_type":"STRING","description":"","name":"COUNT","type":"LOCAL","value":"1","label":"","attrs":{"type":""},"editables":{"value":true}}]},"name":"ScaleOut"},{"description":"","type":"user","critical":false,"runbook":{"task_definition_list":[{"retries":"0","description":"","child_tasks_local_reference_list":[{"kind":"app_task","name":"Scale In"}],"name":"e72a9133_dag","attrs":{"edges":[],"type":""},"timeout_secs":"0","type":"DAG","variable_list":[]},{"target_any_local_reference":{"kind":"app_blueprint_deployment","name":"39cf7283_deployment"},"retries":"0","description":"","child_tasks_local_reference_list":[],"name":"Scale In","attrs":{"scaling_count":"@@{COUNT}@@","type":"","scaling_type":"SCALEIN"},"timeout_secs":"0","type":"SCALING","variable_list":[]}],"description":"","name":"96546f6c_runbook","main_task_local_reference":{"kind":"app_task","name":"e72a9133_dag"},"variable_list":[{"val_type":"STRING","description":"","name":"COUNT","type":"LOCAL","value":"1","label":"","attrs":{"type":""},"editables":{"value":true}}]},"name":"ScaleIn"}],"name":"Nutanix","variable_list":[{"val_type":"STRING","description":"","name":"KUBE_CLUSTER_NAME","type":"LOCAL","value":"kube-calm","label":"","attrs":{"type":""},"editables":{"value":true}},{"val_type":"STRING","description":"","name":"KUBE_IMAGE_TAG","type":"LOCAL","value":"v1.10.5_coreos.0","label":"","attrs":{"type":""},"editables":{"value":true}},{"val_type":"STRING","description":"","name":"DOCKER_VERSION","type":"LOCAL","value":"17.03.2.ce","label":"","attrs":{"type":""},"editables":{"value":true}},{"val_type":"STRING","description":"","name":"KUBE_CLUSTER_SUBNET","type":"LOCAL","value":"10.200.0.0\/16","label":"","attrs":{"type":""},"editables":{"value":true}},{"val_type":"STRING","description":"","name":"KUBE_SERVICE_SUBNET","type":"LOCAL","value":"10.32.0.0\/24","label":"","attrs":{"type":""},"editables":{"value":true}},{"val_type":"STRING","description":"","name":"KUBE_DNS_IP","type":"LOCAL","value":"10.32.0.10","label":"","attrs":{"type":""},"editables":{"value":true}},{"val_type":"STRING","description":"","name":"PE_CLUSTER_IP","type":"LOCAL","value":"10.132.68.55","label":"","attrs":{"type":""},"editables":{"value":true}},{"val_type":"STRING","description":"","name":"PE_DATA_SERVICE_IP","type":"LOCAL","value":"10.132.68.56","label":"","attrs":{"type":""},"editables":{"value":true}},{"val_type":"STRING","description":"","name":"PE_USERNAME","type":"LOCAL","value":"nutanix","label":"","attrs":{"type":""},"editables":{"value":true}},{"val_type":"STRING","description":"","name":"PE_PASSWORD","type":"SECRET","value":"","label":"","attrs":{"is_secret_modified":false,"secret_reference":{},"type":""},"editables":{"value":true}},{"val_type":"STRING","description":"","name":"PE_CONTAINER_NAME","type":"LOCAL","value":"SelfServiceContainer","label":"","attrs":{"type":""},"editables":{"value":true}},{"val_type":"STRING","description":"","name":"INSTANCE_PUBLIC_KEY","type":"LOCAL","value":"ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEApFiiPDXZ\/in4yknwl33rQkVi3Oo4X\/78ekW93+gGptbV1wCj\/PFay85\/aMA98ZYaAxXoC+MHe3QgUuXER45poH552IWdF0tU7X\/t0jH5KllVFx2yuFzCcjw1aKwdxKagDLX5jP4QvNsvB9pKx4dwECBp4qlKW4cpl\/2cWEpUqRi67PVDYr3TMt\/OCac6FaAMs5Du3N2HhGiWh2wezqUCRameLdni1\/G3ovaKpGL+c68n0peBsK63AJEa+AjVfpFcz6kzEUUMGSvAWD2d5nKCkJzqfBgh6kFJUTXPX0YnorbGsp1SuxmLy1pD7RXh2it5IPFUAFgEQfBwJJd7L4UkWQ== root@centos","label":"","attrs":{"type":""},"editables":{"value":true}}]},{"deployment_create_list":[{"type":"GREENFIELD","description":"","action_list":[],"editables":{"min_replicas":true,"max_replicas":true},"name":"85fa2262_deployment","max_replicas":"3","package_local_reference_list":[{"kind":"app_package","name":"AWS_Centos_K8SC_Package"}],"substrate_local_reference":{"kind":"app_substrate","name":"AWS_Centos_K8SC"},"min_replicas":"3","variable_list":[]},{"type":"GREENFIELD","description":"","action_list":[],"editables":{"min_replicas":true,"max_replicas":true},"name":"d6584f2e_deployment","max_replicas":"6","package_local_reference_list":[{"kind":"app_package","name":"AWS_Centos_K8SM_Package"}],"substrate_local_reference":{"kind":"app_substrate","name":"AWS_Centos_K8SM"},"min_replicas":"3","variable_list":[]}],"description":"","action_list":[{"description":"","type":"user","critical":false,"runbook":{"task_definition_list":[{"retries":"0","description":"","child_tasks_local_reference_list":[{"kind":"app_task","name":"Upgrade Controller"},{"kind":"app_task","name":"Upgrade Minion"},{"kind":"app_task","name":"Restart Services"}],"name":"17adda9e_dag","attrs":{"edges":[{"from_task_reference":{"kind":"app_task","name":"Upgrade Controller"},"edge_type":"user_defined","type":"","to_task_reference":{"kind":"app_task","name":"Restart Services"}},{"from_task_reference":{"kind":"app_task","name":"Upgrade Minion"},"edge_type":"user_defined","type":"","to_task_reference":{"kind":"app_task","name":"Restart Services"}}],"type":""},"timeout_secs":"0","type":"DAG","variable_list":[]},{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Master"},"retries":"0","description":"","child_tasks_local_reference_list":[],"name":"Upgrade Controller","attrs":{"script":"#!\/bin\/bash\nset -ex\n\nETCD_VERSION=\"v3.2.11\"\nINTERNAL_IP=\"@@{private_ip_address}@@\"\nCONTROLLER_IPS=\"@@{calm_array_private_ip_address}@@\"\nNODE_NAME=\"controller@@{calm_array_index}@@\"\nCLUSTER_SUBNET=\"@@{KUBE_CLUSTER_SUBNET}@@\"\nSERVICE_SUBNET=\"@@{KUBE_SERVICE_SUBNET}@@\"\nKUBE_CLUSTER_DNS=\"@@{KUBE_DNS_IP}@@\"\nKUBELET_IMAGE_TAG=\"@@{KUBE_IMAGE_TAG_NEW}@@\"\nDOCKER_VERSION=\"@@{DOCKER_VERSION}@@\"\nETCD_CERT_PATH=\"\/etc\/ssl\/certs\/etcd\"\nKUBE_CERT_PATH=\"\/etc\/kubernetes\/ssl\"\nKUBE_MANIFEST_PATH=\"\/etc\/kubernetes\/manifests\"\nVERSION=$(echo \"${KUBELET_IMAGE_TAG}\" | tr \"_\" \" \" | awk '{print $1}')\nMASTER_API_HTTPS=6443\nETCD_SERVER_PORT=2379\nCONTROLLER_COUNT=$(echo \"@@{calm_array_private_ip_address}@@\" | tr ',' '\\n' | wc -l)\n\ncount=0\nfor ip in $(echo \"${CONTROLLER_IPS}\" | tr \",\" \"\\n\"); do\n ETCD+=\"https:\/\/${ip}:${ETCD_SERVER_PORT}\",\n count=$((count+1))\ndone\nETCD_SERVERS=$(echo $ETCD | sed 's\/,$\/\/')\n\nwget -q https:\/\/storage.googleapis.com\/kubernetes-release\/release\/${VERSION}\/bin\/linux\/amd64\/kubelet\nchmod +x kubelet\nsudo mv kubelet \/usr\/bin\/kubelet\n\necho \"[Unit]\nDescription=Kubernetes Kubelet\nDocumentation=https:\/\/github.com\/GoogleCloudPlatform\/kubernetes\nAfter=docker.service\nRequires=docker.service\n\n[Service]\nExecStart=\/usr\/bin\/kubelet \\\\\n --allow-privileged=true \\\\\n --anonymous-auth=false \\\\\n --authorization-mode=Webhook \\\\\n --cluster-dns=${KUBE_CLUSTER_DNS} \\\\\n --cluster-domain=cluster.local \\\\\n --container-runtime=docker \\\\\n --enable-custom-metrics \\\\\n --kubeconfig=${KUBE_CERT_PATH}\/${NODE_NAME}.kubeconfig \\\\\n --network-plugin=cni \\\\\n --pod-cidr=${CLUSTER_SUBNET} \\\\\n --register-node=true \\\\\n --runtime-request-timeout=10m \\\\\n --client-ca-file=${KUBE_CERT_PATH}\/ca.pem \\\\\n --tls-cert-file=${KUBE_CERT_PATH}\/${NODE_NAME}.pem \\\\\n --tls-private-key-file=${KUBE_CERT_PATH}\/${NODE_NAME}-key.pem \\\\\n --pod-manifest-path=${KUBE_MANIFEST_PATH} \\\\\n --read-only-port=0 \\\\\n --protect-kernel-defaults=false \\\\\n --make-iptables-util-chains=true \\\\\n --keep-terminated-pod-volumes=false \\\\\n --event-qps=0 \\\\\n --cadvisor-port=0 \\\\\n --runtime-cgroups=\/systemd\/system.slice \\\\\n --kubelet-cgroups=\/systemd\/system.slice \\\\\n --node-labels 'node-role.kubernetes.io\/master=true' \\\\\n --node-labels 'node-role.kubernetes.io\/etcd=true' \\\\\n --register-with-taints=node-role.kubernetes.io\/master=true:NoSchedule \\\\\n --cloud-provider=aws \\\\\n --v=2\nRestart=on-failure\nRestartSec=5\n\n[Install]\nWantedBy=multi-user.target\" | sudo tee \/etc\/systemd\/system\/kubelet.service\n\necho \"apiVersion: v1\nkind: Pod\nmetadata:\n name: kube-apiserver\n namespace: kube-system\n labels:\n k8s-app: kube-apiserver\nspec:\n hostNetwork: true\n containers:\n - name: kube-apiserver\n image: quay.io\/coreos\/hyperkube:${KUBELET_IMAGE_TAG}\n command:\n - \/hyperkube\n - apiserver\n - --admission-control=Initializers,NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota\n - --advertise-address=${INTERNAL_IP}\n - --allow-privileged=true\n - --anonymous-auth=false\n - --insecure-port=0\n - --secure-port=${MASTER_API_HTTPS}\n - --profiling=false\n - --repair-malformed-updates=false\n - --apiserver-count=${CONTROLLER_COUNT}\n - --audit-log-maxage=30\n - --audit-log-maxbackup=10\n - --audit-log-maxsize=100\n - --audit-log-path=\/var\/lib\/audit.log\n - --authorization-mode=Node,RBAC\n - --bind-address=0.0.0.0\n - --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP\n - --event-ttl=1h\n - --service-account-lookup=true\n - --enable-swagger-ui=true\n - --storage-backend=etcd3\n - --etcd-cafile=${ETCD_CERT_PATH}\/etcd-ca.pem\n - --etcd-certfile=${ETCD_CERT_PATH}\/etcd-client.pem\n - --etcd-keyfile=${ETCD_CERT_PATH}\/etcd-client-key.pem\n - --etcd-servers=${ETCD_SERVERS}\n - --experimental-encryption-provider-config=${KUBE_CERT_PATH}\/encryption-config.yaml\n - --tls-ca-file=${KUBE_CERT_PATH}\/ca.pem\n - --tls-cert-file=${KUBE_CERT_PATH}\/kubernetes.pem\n - --tls-private-key-file=${KUBE_CERT_PATH}\/kubernetes-key.pem\n - --kubelet-client-certificate=${KUBE_CERT_PATH}\/kubernetes.pem\n - --kubelet-client-key=${KUBE_CERT_PATH}\/kubernetes-key.pem\n - --kubelet-https=true\n - --runtime-config=api\/all\n - --service-account-key-file=${KUBE_CERT_PATH}\/kubernetes.pem\n - --service-cluster-ip-range=${SERVICE_SUBNET}\n - --service-node-port-range=30000-32767\n - --client-ca-file=${KUBE_CERT_PATH}\/ca.pem\n - --cloud-provider=aws\n - --v=2\n ports:\n - containerPort: ${MASTER_API_HTTPS}\n hostPort: ${MASTER_API_HTTPS}\n name: https\n - containerPort: 8080\n hostPort: 8080\n name: local\n volumeMounts:\n - mountPath: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n readOnly: true\n - mountPath: \/etc\/ssl\/certs\n name: ssl-certs-host\n readOnly: true\n - mountPath: \/etc\/pki\n name: ca-certs-etc-pki\n readOnly: true\n volumes:\n - hostPath:\n path: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n - hostPath:\n path: \/etc\/ssl\/certs\n name: ssl-certs-host\n - hostPath:\n path: \/etc\/pki\n name: ca-certs-etc-pki\" | sudo tee ${KUBE_MANIFEST_PATH}\/kube-apiserver.yaml\n\necho \"apiVersion: v1\nkind: Pod\nmetadata:\n name: kube-proxy\n namespace: kube-system\n labels:\n k8s-app: kube-proxy\nspec:\n hostNetwork: true\n containers:\n - name: kube-proxy\n image: quay.io\/coreos\/hyperkube:${KUBELET_IMAGE_TAG}\n command:\n - \/hyperkube\n - proxy\n - --cluster-cidr=${CLUSTER_SUBNET}\n - --masquerade-all=true\n - --kubeconfig=${KUBE_CERT_PATH}\/kube-proxy.kubeconfig\n - --proxy-mode=iptables\n securityContext:\n privileged: true\n volumeMounts:\n - mountPath: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n readOnly: true\n - mountPath: \/etc\/ssl\/certs\n name: ssl-certs-host\n readOnly: true\n volumes:\n - hostPath:\n path: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n - hostPath:\n path: \/etc\/ssl\/certs\n name: ssl-certs-host\" | sudo tee ${KUBE_MANIFEST_PATH}\/kube-proxy.yaml\n \necho \"apiVersion: v1\nkind: Pod\nmetadata:\n name: kube-controller-manager\n namespace: kube-system\n labels:\n k8s-app: kube-controller-manager\nspec:\n hostNetwork: true\n containers:\n - name: kube-controller-manager\n image: quay.io\/coreos\/hyperkube:${KUBELET_IMAGE_TAG}\n command:\n - \/hyperkube\n - controller-manager\n - --address=0.0.0.0 \n - --allocate-node-cidrs=true \n - --cluster-cidr=${CLUSTER_SUBNET}\n - --cluster-name=kubernetes-prod-cluster \n - --leader-elect=true \n - --kubeconfig=${KUBE_CERT_PATH}\/kube-controller-manager.kubeconfig \n - --service-account-private-key-file=${KUBE_CERT_PATH}\/kubernetes-key.pem \n - --service-cluster-ip-range=${SERVICE_SUBNET}\n - --terminated-pod-gc-threshold=100 \n - --profiling=false \n - --use-service-account-credentials=true\n - --cloud-provider=aws\n - --v=2\n livenessProbe:\n httpGet:\n host: 127.0.0.1\n path: \/healthz\n port: 10252\n initialDelaySeconds: 15\n timeoutSeconds: 1\n volumeMounts:\n - mountPath: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n readOnly: true\n - mountPath: \/etc\/ssl\/certs\n name: ssl-certs-host\n readOnly: true\n - mountPath: \/etc\/pki\n name: ca-certs-etc-pki\n readOnly: true\n volumes:\n - hostPath:\n path: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n - hostPath:\n path: \/etc\/ssl\/certs\n name: ssl-certs-host\n - hostPath:\n path: \/etc\/pki\n name: ca-certs-etc-pki\" | sudo tee ${KUBE_MANIFEST_PATH}\/kube-controller-manager.yaml\n \necho \"apiVersion: v1\nkind: Pod\nmetadata:\n name: kube-scheduler\n namespace: kube-system\n labels:\n k8s-app: kube-scheduler\nspec:\n hostNetwork: true\n containers:\n - name: kube-scheduler\n image: quay.io\/coreos\/hyperkube:${KUBELET_IMAGE_TAG}\n command:\n - \/hyperkube\n - scheduler\n - --kubeconfig=${KUBE_CERT_PATH}\/kube-scheduler.kubeconfig\n - --leader-elect=true\n - --profiling=false\n - --v=2\n livenessProbe:\n httpGet:\n host: 127.0.0.1\n path: \/healthz\n port: 10251\n initialDelaySeconds: 15\n timeoutSeconds: 1\n volumeMounts:\n - mountPath: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n readOnly: true\n volumes:\n - hostPath:\n path: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\" | sudo tee ${KUBE_MANIFEST_PATH}\/kube-scheduler.yaml","type":"","command_line_args":"","exit_status":[],"script_type":"sh"},"timeout_secs":"0","type":"EXEC","variable_list":[]},{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Minion"},"retries":"0","description":"","child_tasks_local_reference_list":[],"name":"Upgrade Minion","attrs":{"script":"#!\/bin\/bash\nset -ex\n\nINTERNAL_IP=\"@@{private_ip_address}@@\"\nNODE_NAME=\"minion@@{calm_array_index}@@\"\nCLUSTER_SUBNET=\"@@{KUBE_CLUSTER_SUBNET}@@\"\nKUBE_CLUSTER_DNS=\"@@{KUBE_DNS_IP}@@\"\nKUBELET_IMAGE_TAG=\"@@{KUBE_IMAGE_TAG_NEW}@@\"\nDOCKER_VERSION=\"@@{DOCKER_VERSION}@@\"\nKUBE_CERT_PATH=\"\/etc\/kubernetes\/ssl\"\nKUBE_MANIFEST_PATH=\"\/etc\/kubernetes\/manifests\"\nVERSION=$(echo \"${KUBELET_IMAGE_TAG}\" | tr \"_\" \" \" | awk '{print $1}')\n\nwget -q https:\/\/storage.googleapis.com\/kubernetes-release\/release\/${VERSION}\/bin\/linux\/amd64\/kubelet\nchmod +x kubelet\nsudo mv kubelet \/usr\/bin\/kubelet\n\necho \"[Unit]\nDescription=Kubernetes Kubelet\nDocumentation=https:\/\/github.com\/GoogleCloudPlatform\/kubernetes\nAfter=docker.service\nRequires=docker.service\n\n[Service]\nExecStart=\/usr\/bin\/kubelet \\\\\n --allow-privileged=true \\\\\n --anonymous-auth=false \\\\\n --authorization-mode=Webhook \\\\\n --cluster-dns=${KUBE_CLUSTER_DNS} \\\\\n --cluster-domain=cluster.local \\\\\n --container-runtime=docker \\\\\n --enable-custom-metrics \\\\\n --kubeconfig=${KUBE_CERT_PATH}\/${NODE_NAME}.kubeconfig \\\\\n --network-plugin=cni \\\\\n --pod-cidr=${CLUSTER_SUBNET} \\\\\n --register-node=true \\\\\n --runtime-request-timeout=10m \\\\\n --client-ca-file=${KUBE_CERT_PATH}\/ca.pem \\\\\n --tls-cert-file=${KUBE_CERT_PATH}\/${NODE_NAME}.pem \\\\\n --tls-private-key-file=${KUBE_CERT_PATH}\/${NODE_NAME}-key.pem \\\\\n --pod-manifest-path=${KUBE_MANIFEST_PATH} \\\\\n --read-only-port=0 \\\\\n --protect-kernel-defaults=false \\\\\n --make-iptables-util-chains=true \\\\\n --keep-terminated-pod-volumes=false \\\\\n --event-qps=0 \\\\\n --cadvisor-port=0 \\\\\n --runtime-cgroups=\/systemd\/system.slice \\\\\n --kubelet-cgroups=\/systemd\/system.slice \\\\\n --eviction-hard=memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5% \\\\\n --node-labels 'node-role.kubernetes.io\/worker=true' \\\\\n --node-labels 'beta.kubernetes.io\/fluentd-ds-ready=true' \\\\\n --cloud-provider=aws \\\\\n --v=2\nRestart=on-failure\nRestartSec=5\n\n[Install]\nWantedBy=multi-user.target\" | sudo tee \/etc\/systemd\/system\/kubelet.service\n\necho \"apiVersion: v1\nkind: Pod\nmetadata:\n name: kube-proxy\n namespace: kube-system\n labels:\n k8s-app: kube-proxy\nspec:\n hostNetwork: true\n containers:\n - name: kube-proxy\n image: quay.io\/coreos\/hyperkube:${KUBELET_IMAGE_TAG}\n command:\n - \/hyperkube\n - proxy\n - --cluster-cidr=${CLUSTER_SUBNET}\n - --masquerade-all=true\n - --kubeconfig=${KUBE_CERT_PATH}\/kube-proxy.kubeconfig\n - --proxy-mode=iptables\n securityContext:\n privileged: true\n volumeMounts:\n - mountPath: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n readOnly: true\n - mountPath: \/etc\/ssl\/certs\n name: ssl-certs-host\n readOnly: true\n volumes:\n - hostPath:\n path: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n - hostPath:\n path: \/etc\/ssl\/certs\n name: ssl-certs-host\" | sudo tee ${KUBE_MANIFEST_PATH}\/kube-proxy.yaml","type":"","command_line_args":"","exit_status":[],"script_type":"sh"},"timeout_secs":"0","type":"EXEC","variable_list":[]},{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Master"},"retries":"0","description":"","child_tasks_local_reference_list":[],"name":"Restart Services","attrs":{"script":"#!\/bin\/bash\nset -ex\n\nif [ @@{calm_array_index}@@ -ne 0 ];then\n\texit\nfi\n\nINTERNAL_IP=\"@@{private_ip_address}@@\"\nCONTROLLER_DNS=\"@@{calm_array_private_dns_name}@@\" \nMINION_DNS=\"@@{AWS_Centos_K8SM.private_dns_name}@@\" \nJSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'\nKUBELET_IMAGE_TAG=\"@@{KUBE_IMAGE_TAG_NEW}@@\"\nMASTER_API_HTTPS=6443\nVERSION=$(echo \"${KUBELET_IMAGE_TAG}\" | tr \"_\" \" \" | awk '{print $1}')\nwget -q https:\/\/storage.googleapis.com\/kubernetes-release\/release\/${VERSION}\/bin\/linux\/amd64\/kubectl\nchmod +x kubectl\nsudo mv kubectl \/usr\/local\/bin\/kubectl\n\ncount=0\nwhile [[ $(curl --key CA\/admin-key.pem --cert CA\/admin.pem --cacert CA\/ca.pem https:\/\/${INTERNAL_IP}:${MASTER_API_HTTPS}\/healthz) != \"ok\" ]] ; do\n echo \"Trying to reach master server https:\/\/${INTERNAL_IP}:${MASTER_API_HTTPS} : sleep for 5 secs\"\n sleep 10\n if [[ $count -eq 10 ]]; then\n \techo \"Unable to reach master server: https:\/\/${INTERNAL_IP}:${MASTER_API_HTTPS}\"\n \texit 1\n fi\n count=$((count+1))\ndone\n\nfor name in $(echo \"${MINION_DNS}\" | tr \",\" \"\\n\"); do\n kubectl drain \"$name\" --ignore-daemonsets --delete-local-data --force\n sleep 20\n ssh -o stricthostkeychecking=no $name \"sudo yum update -y --quiet && sudo systemctl daemon-reload && sudo systemctl restart kubelet\"\n sleep 20\n while [[ `kubectl get nodes -l kubernetes.io\/hostname=${name} -o jsonpath=\"$JSONPATH\" | grep \"Ready=Unknown\" 2>\/dev\/null` ]]; do sleep 10 ; done\n kubectl uncordon $name\ndone\n\n\nif [[ `kubectl get nodes -o jsonpath=\"$JSONPATH\" | grep \"Ready=Unknown\"` ]]; then \n\techo \"Upgrade failed on minion nodes\"\n exit 1\nfi\n\nfor name in $(echo \"${CONTROLLER_DNS}\" | tr \",\" \"\\n\"); do\n kubectl drain \"${name}\" --ignore-daemonsets --delete-local-data --force\n sleep 20\n ssh -o stricthostkeychecking=no $name \"sudo yum update -y --quiet && sudo systemctl daemon-reload && sudo systemctl restart kubelet\"\n sleep 20\n while [[ `kubectl get nodes -l kubernetes.io\/hostname=$name -o jsonpath=\"$JSONPATH\" | grep \"Ready=Unknown\" 2>\/dev\/null` ]]; do sleep 10 ; done\n kubectl uncordon $name\ndone\n\n\nif [[ `kubectl get nodes -o jsonpath=\"$JSONPATH\" | grep \"Ready=Unknown\"` ]]; then \n\techo \"Upgrade failed on nodes: $(kubectl get nodes -o jsonpath='$JSONPATH' | grep 'Ready=Unknown')\"\n exit 1\nfi","type":"","command_line_args":"","exit_status":[],"script_type":"sh"},"timeout_secs":"0","type":"EXEC","variable_list":[]}],"description":"","name":"dfce2dff_runbook","main_task_local_reference":{"kind":"app_task","name":"17adda9e_dag"},"variable_list":[{"val_type":"STRING","description":"","name":"KUBE_IMAGE_TAG_NEW","type":"LOCAL","value":"v1.8.0_coreos.0","label":"","attrs":{"type":""},"editables":{"value":true}}]},"name":"Upgrade"},{"description":"","type":"user","critical":false,"runbook":{"task_definition_list":[{"retries":"0","description":"","child_tasks_local_reference_list":[{"kind":"app_task","name":"Scale Out"},{"kind":"app_task","name":"Set Hosts File"}],"name":"a0fb1ecf_dag","attrs":{"edges":[{"from_task_reference":{"kind":"app_task","name":"Scale Out"},"edge_type":"user_defined","type":"","to_task_reference":{"kind":"app_task","name":"Set Hosts File"}}],"type":""},"timeout_secs":"0","type":"DAG","variable_list":[]},{"target_any_local_reference":{"kind":"app_blueprint_deployment","name":"d6584f2e_deployment"},"retries":"0","description":"","child_tasks_local_reference_list":[],"name":"Scale Out","attrs":{"scaling_count":"@@{COUNT}@@","type":"","scaling_type":"SCALEOUT"},"timeout_secs":"0","type":"SCALING","variable_list":[]},{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Master"},"retries":"0","description":"","child_tasks_local_reference_list":[],"name":"Set Hosts File","attrs":{"script":"#!\/bin\/bash\nset -ex\n\nMINION_IPS=\"@@{AWS_Centos_K8SM.private_ip_address}@@\"\nfor ip in $(echo ${MINION_IPS} | tr \",\" \"\\n\"); do\n if ! (( $(grep -c \"${ip} minion${count}\" \/etc\/hosts) )) ; then\n \techo \"${ip} minion${count}\" | sudo tee -a \/etc\/hosts\n fi\n count=$((count+1))\ndone","type":"","command_line_args":"","exit_status":[],"script_type":"sh"},"timeout_secs":"0","type":"EXEC","variable_list":[]}],"description":"","name":"55139b34_runbook","main_task_local_reference":{"kind":"app_task","name":"a0fb1ecf_dag"},"variable_list":[{"val_type":"STRING","description":"","name":"KUBE_IMAGE_TAG_NEW","type":"LOCAL","value":"v1.8.0_coreos.0","label":"","attrs":{"type":""},"editables":{"value":true}},{"val_type":"STRING","description":"","name":"COUNT","type":"LOCAL","value":"1","label":"","attrs":{"type":""},"editables":{"value":true}}]},"name":"ScaleOut"},{"description":"","type":"user","critical":false,"runbook":{"task_definition_list":[{"retries":"0","description":"","child_tasks_local_reference_list":[{"kind":"app_task","name":"Scale In"}],"name":"adfef604_dag","attrs":{"edges":[],"type":""},"timeout_secs":"0","type":"DAG","variable_list":[]},{"target_any_local_reference":{"kind":"app_blueprint_deployment","name":"d6584f2e_deployment"},"retries":"0","description":"","child_tasks_local_reference_list":[],"name":"Scale In","attrs":{"scaling_count":"@@{COUNT}@@","type":"","scaling_type":"SCALEIN"},"timeout_secs":"0","type":"SCALING","variable_list":[]}],"description":"","name":"65df2c70_runbook","main_task_local_reference":{"kind":"app_task","name":"adfef604_dag"},"variable_list":[{"val_type":"STRING","description":"","name":"COUNT","type":"LOCAL","value":"1","label":"","attrs":{"type":""},"editables":{"value":true}}]},"name":"ScaleIn"}],"name":"AWS","variable_list":[{"val_type":"STRING","description":"","name":"KUBE_CLUSTER_NAME","type":"LOCAL","value":"kube-calm-aws1","label":"","attrs":{"type":""},"editables":{"value":true}},{"val_type":"STRING","description":"","name":"KUBE_IMAGE_TAG","type":"LOCAL","value":"v1.8.0_coreos.0","label":"","attrs":{"type":""},"editables":{"value":true}},{"val_type":"STRING","description":"","name":"DOCKER_VERSION","type":"LOCAL","value":"17.03.2.ce","label":"","attrs":{"type":""},"editables":{"value":true}},{"val_type":"STRING","description":"","name":"KUBE_CLUSTER_SUBNET","type":"LOCAL","value":"10.200.0.0\/16","label":"","attrs":{"type":""},"editables":{"value":true}},{"val_type":"STRING","description":"","name":"KUBE_SERVICE_SUBNET","type":"LOCAL","value":"10.32.0.0\/24","label":"","attrs":{"type":""},"editables":{"value":true}},{"val_type":"STRING","description":"","name":"KUBE_DNS_IP","type":"LOCAL","value":"10.32.0.10","label":"","attrs":{"type":""},"editables":{"value":true}}]},{"deployment_create_list":[{"type":"GREENFIELD","description":"","action_list":[],"editables":{"min_replicas":true,"max_replicas":true},"name":"34875daf_deployment","max_replicas":"3","package_local_reference_list":[{"kind":"app_package","name":"GCP_Centos_K8SC_Package"}],"substrate_local_reference":{"kind":"app_substrate","name":"GCP_Centos_K8SC"},"min_replicas":"3","variable_list":[]},{"type":"GREENFIELD","description":"","action_list":[],"editables":{"min_replicas":true,"max_replicas":true},"name":"77b729d5_deployment","max_replicas":"6","package_local_reference_list":[{"kind":"app_package","name":"GCP_Centos_K8SM_Package"}],"substrate_local_reference":{"kind":"app_substrate","name":"GCP_Centos_K8SM"},"min_replicas":"3","variable_list":[]}],"description":"","action_list":[{"description":"","type":"user","critical":false,"runbook":{"task_definition_list":[{"retries":"0","description":"","child_tasks_local_reference_list":[{"kind":"app_task","name":"Upgrade Controller"},{"kind":"app_task","name":"Upgrade Minion"},{"kind":"app_task","name":"Restart Services"}],"name":"cbdb4fa8_dag","attrs":{"edges":[{"from_task_reference":{"kind":"app_task","name":"Upgrade Controller"},"edge_type":"user_defined","type":"","to_task_reference":{"kind":"app_task","name":"Restart Services"}},{"from_task_reference":{"kind":"app_task","name":"Upgrade Minion"},"edge_type":"user_defined","type":"","to_task_reference":{"kind":"app_task","name":"Restart Services"}}],"type":""},"timeout_secs":"0","type":"DAG","variable_list":[]},{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Master"},"retries":"0","description":"","child_tasks_local_reference_list":[],"name":"Upgrade Controller","attrs":{"script":"#!\/bin\/bash\nset -ex\n\nETCD_VERSION=\"v3.2.11\"\nINTERNAL_IP=\"@@{private_ip_address}@@\"\nCONTROLLER_IPS=\"@@{calm_array_private_ip_address}@@\"\nNODE_NAME=\"controller@@{calm_array_index}@@\"\nCLUSTER_SUBNET=\"@@{KUBE_CLUSTER_SUBNET}@@\"\nSERVICE_SUBNET=\"@@{KUBE_SERVICE_SUBNET}@@\"\nKUBE_CLUSTER_DNS=\"@@{KUBE_DNS_IP}@@\"\nKUBELET_IMAGE_TAG=\"@@{KUBE_IMAGE_TAG_NEW}@@\"\nDOCKER_VERSION=\"@@{DOCKER_VERSION}@@\"\nETCD_CERT_PATH=\"\/etc\/ssl\/certs\/etcd\"\nKUBE_CERT_PATH=\"\/etc\/kubernetes\/ssl\"\nKUBE_MANIFEST_PATH=\"\/etc\/kubernetes\/manifests\"\nVERSION=$(echo \"${KUBELET_IMAGE_TAG}\" | tr \"_\" \" \" | awk '{print $1}')\nMASTER_API_HTTPS=6443\nETCD_SERVER_PORT=2379\nCONTROLLER_COUNT=$(echo \"@@{calm_array_private_ip_address}@@\" | tr ',' '\\n' | wc -l)\n\ncount=0\nfor ip in $(echo \"${CONTROLLER_IPS}\" | tr \",\" \"\\n\"); do\n ETCD+=\"https:\/\/${ip}:${ETCD_SERVER_PORT}\",\n count=$((count+1))\ndone\nETCD_SERVERS=$(echo $ETCD | sed 's\/,$\/\/')\n\nwget -q https:\/\/storage.googleapis.com\/kubernetes-release\/release\/${VERSION}\/bin\/linux\/amd64\/kubelet\nchmod +x kubelet\nsudo mv kubelet \/usr\/bin\/kubelet\n\necho \"[Unit]\nDescription=Kubernetes Kubelet\nDocumentation=https:\/\/github.com\/GoogleCloudPlatform\/kubernetes\nAfter=docker.service\nRequires=docker.service\n\n[Service]\nExecStart=\/usr\/bin\/kubelet \\\\\n --allow-privileged=true \\\\\n --anonymous-auth=false \\\\\n --authorization-mode=Webhook \\\\\n --cluster-dns=${KUBE_CLUSTER_DNS} \\\\\n --cluster-domain=cluster.local \\\\\n --container-runtime=docker \\\\\n --enable-custom-metrics \\\\\n --kubeconfig=${KUBE_CERT_PATH}\/${NODE_NAME}.kubeconfig \\\\\n --network-plugin=cni \\\\\n --pod-cidr=${CLUSTER_SUBNET} \\\\\n --register-node=true \\\\\n --runtime-request-timeout=10m \\\\\n --client-ca-file=${KUBE_CERT_PATH}\/ca.pem \\\\\n --tls-cert-file=${KUBE_CERT_PATH}\/${NODE_NAME}.pem \\\\\n --tls-private-key-file=${KUBE_CERT_PATH}\/${NODE_NAME}-key.pem \\\\\n --pod-manifest-path=${KUBE_MANIFEST_PATH} \\\\\n --read-only-port=0 \\\\\n --protect-kernel-defaults=false \\\\\n --make-iptables-util-chains=true \\\\\n --keep-terminated-pod-volumes=false \\\\\n --event-qps=0 \\\\\n --cadvisor-port=0 \\\\\n --runtime-cgroups=\/systemd\/system.slice \\\\\n --kubelet-cgroups=\/systemd\/system.slice \\\\\n --node-labels 'node-role.kubernetes.io\/master=true' \\\\\n --node-labels 'node-role.kubernetes.io\/etcd=true' \\\\\n --register-with-taints=node-role.kubernetes.io\/master=true:NoSchedule \\\\\n --cloud-provider=gce \\\\\n --v=2\nRestart=on-failure\nRestartSec=5\n\n[Install]\nWantedBy=multi-user.target\" | sudo tee \/etc\/systemd\/system\/kubelet.service\n\necho \"apiVersion: v1\nkind: Pod\nmetadata:\n name: kube-apiserver\n namespace: kube-system\n labels:\n k8s-app: kube-apiserver\nspec:\n hostNetwork: true\n containers:\n - name: kube-apiserver\n image: quay.io\/coreos\/hyperkube:${KUBELET_IMAGE_TAG}\n command:\n - \/hyperkube\n - apiserver\n - --admission-control=Initializers,NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota\n - --advertise-address=${INTERNAL_IP}\n - --allow-privileged=true\n - --anonymous-auth=false\n - --insecure-port=0\n - --secure-port=${MASTER_API_HTTPS}\n - --profiling=false\n - --repair-malformed-updates=false\n - --apiserver-count=${CONTROLLER_COUNT}\n - --audit-log-maxage=30\n - --audit-log-maxbackup=10\n - --audit-log-maxsize=100\n - --audit-log-path=\/var\/lib\/audit.log\n - --authorization-mode=Node,RBAC\n - --bind-address=0.0.0.0\n - --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP\n - --event-ttl=1h\n - --service-account-lookup=true\n - --enable-swagger-ui=true\n - --storage-backend=etcd3\n - --etcd-cafile=${ETCD_CERT_PATH}\/etcd-ca.pem\n - --etcd-certfile=${ETCD_CERT_PATH}\/etcd-client.pem\n - --etcd-keyfile=${ETCD_CERT_PATH}\/etcd-client-key.pem\n - --etcd-servers=${ETCD_SERVERS}\n - --experimental-encryption-provider-config=${KUBE_CERT_PATH}\/encryption-config.yaml\n - --tls-ca-file=${KUBE_CERT_PATH}\/ca.pem\n - --tls-cert-file=${KUBE_CERT_PATH}\/kubernetes.pem\n - --tls-private-key-file=${KUBE_CERT_PATH}\/kubernetes-key.pem\n - --kubelet-client-certificate=${KUBE_CERT_PATH}\/kubernetes.pem\n - --kubelet-client-key=${KUBE_CERT_PATH}\/kubernetes-key.pem\n - --kubelet-https=true\n - --runtime-config=api\/all\n - --service-account-key-file=${KUBE_CERT_PATH}\/kubernetes.pem\n - --service-cluster-ip-range=${SERVICE_SUBNET}\n - --service-node-port-range=30000-32767\n - --client-ca-file=${KUBE_CERT_PATH}\/ca.pem\n - --cloud-provider=gce\n - --v=2\n ports:\n - containerPort: ${MASTER_API_HTTPS}\n hostPort: ${MASTER_API_HTTPS}\n name: https\n - containerPort: 8080\n hostPort: 8080\n name: local\n volumeMounts:\n - mountPath: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n readOnly: true\n - mountPath: \/etc\/ssl\/certs\n name: ssl-certs-host\n readOnly: true\n - mountPath: \/etc\/pki\n name: ca-certs-etc-pki\n readOnly: true\n volumes:\n - hostPath:\n path: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n - hostPath:\n path: \/etc\/ssl\/certs\n name: ssl-certs-host\n - hostPath:\n path: \/etc\/pki\n name: ca-certs-etc-pki\" | sudo tee ${KUBE_MANIFEST_PATH}\/kube-apiserver.yaml\n\necho \"apiVersion: v1\nkind: Pod\nmetadata:\n name: kube-proxy\n namespace: kube-system\n labels:\n k8s-app: kube-proxy\nspec:\n hostNetwork: true\n containers:\n - name: kube-proxy\n image: quay.io\/coreos\/hyperkube:${KUBELET_IMAGE_TAG}\n command:\n - \/hyperkube\n - proxy\n - --cluster-cidr=${CLUSTER_SUBNET}\n - --masquerade-all=true\n - --kubeconfig=${KUBE_CERT_PATH}\/kube-proxy.kubeconfig\n - --proxy-mode=iptables\n securityContext:\n privileged: true\n volumeMounts:\n - mountPath: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n readOnly: true\n - mountPath: \/etc\/ssl\/certs\n name: ssl-certs-host\n readOnly: true\n volumes:\n - hostPath:\n path: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n - hostPath:\n path: \/etc\/ssl\/certs\n name: ssl-certs-host\" | sudo tee ${KUBE_MANIFEST_PATH}\/kube-proxy.yaml\n \necho \"apiVersion: v1\nkind: Pod\nmetadata:\n name: kube-controller-manager\n namespace: kube-system\n labels:\n k8s-app: kube-controller-manager\nspec:\n hostNetwork: true\n containers:\n - name: kube-controller-manager\n image: quay.io\/coreos\/hyperkube:${KUBELET_IMAGE_TAG}\n command:\n - \/hyperkube\n - controller-manager\n - --address=0.0.0.0 \n - --allocate-node-cidrs=true \n - --cluster-cidr=${CLUSTER_SUBNET}\n - --cluster-name=kubernetes-prod-cluster \n - --leader-elect=true \n - --kubeconfig=${KUBE_CERT_PATH}\/kube-controller-manager.kubeconfig \n - --service-account-private-key-file=${KUBE_CERT_PATH}\/kubernetes-key.pem \n - --service-cluster-ip-range=${SERVICE_SUBNET}\n - --terminated-pod-gc-threshold=100 \n - --profiling=false \n - --use-service-account-credentials=true\n - --cloud-provider=gce\n - --v=2\n livenessProbe:\n httpGet:\n host: 127.0.0.1\n path: \/healthz\n port: 10252\n initialDelaySeconds: 15\n timeoutSeconds: 1\n volumeMounts:\n - mountPath: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n readOnly: true\n - mountPath: \/etc\/ssl\/certs\n name: ssl-certs-host\n readOnly: true\n - mountPath: \/etc\/pki\n name: ca-certs-etc-pki\n readOnly: true\n volumes:\n - hostPath:\n path: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n - hostPath:\n path: \/etc\/ssl\/certs\n name: ssl-certs-host\n - hostPath:\n path: \/etc\/pki\n name: ca-certs-etc-pki\" | sudo tee ${KUBE_MANIFEST_PATH}\/kube-controller-manager.yaml\n \necho \"apiVersion: v1\nkind: Pod\nmetadata:\n name: kube-scheduler\n namespace: kube-system\n labels:\n k8s-app: kube-scheduler\nspec:\n hostNetwork: true\n containers:\n - name: kube-scheduler\n image: quay.io\/coreos\/hyperkube:${KUBELET_IMAGE_TAG}\n command:\n - \/hyperkube\n - scheduler\n - --kubeconfig=${KUBE_CERT_PATH}\/kube-scheduler.kubeconfig\n - --leader-elect=true\n - --profiling=false\n - --v=2\n livenessProbe:\n httpGet:\n host: 127.0.0.1\n path: \/healthz\n port: 10251\n initialDelaySeconds: 15\n timeoutSeconds: 1\n volumeMounts:\n - mountPath: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n readOnly: true\n volumes:\n - hostPath:\n path: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\" | sudo tee ${KUBE_MANIFEST_PATH}\/kube-scheduler.yaml","type":"","command_line_args":"","exit_status":[],"script_type":"sh"},"timeout_secs":"0","type":"EXEC","variable_list":[]},{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Minion"},"retries":"0","description":"","child_tasks_local_reference_list":[],"name":"Upgrade Minion","attrs":{"script":"#!\/bin\/bash\nset -ex\n\nINTERNAL_IP=\"@@{private_ip_address}@@\"\nNODE_NAME=\"minion@@{calm_array_index}@@\"\nCLUSTER_SUBNET=\"@@{KUBE_CLUSTER_SUBNET}@@\"\nKUBE_CLUSTER_DNS=\"@@{KUBE_DNS_IP}@@\"\nKUBELET_IMAGE_TAG=\"@@{KUBE_IMAGE_TAG_NEW}@@\"\nDOCKER_VERSION=\"@@{DOCKER_VERSION}@@\"\nKUBE_CERT_PATH=\"\/etc\/kubernetes\/ssl\"\nKUBE_MANIFEST_PATH=\"\/etc\/kubernetes\/manifests\"\nVERSION=$(echo \"${KUBELET_IMAGE_TAG}\" | tr \"_\" \" \" | awk '{print $1}')\n\nwget -q https:\/\/storage.googleapis.com\/kubernetes-release\/release\/${VERSION}\/bin\/linux\/amd64\/kubelet\nchmod +x kubelet\nsudo mv kubelet \/usr\/bin\/kubelet\n\necho \"[Unit]\nDescription=Kubernetes Kubelet\nDocumentation=https:\/\/github.com\/GoogleCloudPlatform\/kubernetes\nAfter=docker.service\nRequires=docker.service\n\n[Service]\nExecStart=\/usr\/bin\/kubelet \\\\\n --allow-privileged=true \\\\\n --anonymous-auth=false \\\\\n --authorization-mode=Webhook \\\\\n --cluster-dns=${KUBE_CLUSTER_DNS} \\\\\n --cluster-domain=cluster.local \\\\\n --container-runtime=docker \\\\\n --enable-custom-metrics \\\\\n --kubeconfig=${KUBE_CERT_PATH}\/${NODE_NAME}.kubeconfig \\\\\n --network-plugin=cni \\\\\n --pod-cidr=${CLUSTER_SUBNET} \\\\\n --register-node=true \\\\\n --runtime-request-timeout=10m \\\\\n --client-ca-file=${KUBE_CERT_PATH}\/ca.pem \\\\\n --tls-cert-file=${KUBE_CERT_PATH}\/${NODE_NAME}.pem \\\\\n --tls-private-key-file=${KUBE_CERT_PATH}\/${NODE_NAME}-key.pem \\\\\n --pod-manifest-path=${KUBE_MANIFEST_PATH} \\\\\n --read-only-port=0 \\\\\n --protect-kernel-defaults=false \\\\\n --make-iptables-util-chains=true \\\\\n --keep-terminated-pod-volumes=false \\\\\n --event-qps=0 \\\\\n --cadvisor-port=0 \\\\\n --runtime-cgroups=\/systemd\/system.slice \\\\\n --kubelet-cgroups=\/systemd\/system.slice \\\\\n --eviction-hard=memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5% \\\\\n --node-labels 'node-role.kubernetes.io\/worker=true' \\\\\n --node-labels 'beta.kubernetes.io\/fluentd-ds-ready=true' \\\\\n --cloud-provider=gce \\\\\n --v=2\nRestart=on-failure\nRestartSec=5\n\n[Install]\nWantedBy=multi-user.target\" | sudo tee \/etc\/systemd\/system\/kubelet.service\n\necho \"apiVersion: v1\nkind: Pod\nmetadata:\n name: kube-proxy\n namespace: kube-system\n labels:\n k8s-app: kube-proxy\nspec:\n hostNetwork: true\n containers:\n - name: kube-proxy\n image: quay.io\/coreos\/hyperkube:${KUBELET_IMAGE_TAG}\n command:\n - \/hyperkube\n - proxy\n - --cluster-cidr=${CLUSTER_SUBNET}\n - --masquerade-all=true\n - --kubeconfig=${KUBE_CERT_PATH}\/kube-proxy.kubeconfig\n - --proxy-mode=iptables\n securityContext:\n privileged: true\n volumeMounts:\n - mountPath: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n readOnly: true\n - mountPath: \/etc\/ssl\/certs\n name: ssl-certs-host\n readOnly: true\n volumes:\n - hostPath:\n path: ${KUBE_CERT_PATH}\n name: ssl-certs-kubernetes\n - hostPath:\n path: \/etc\/ssl\/certs\n name: ssl-certs-host\" | sudo tee ${KUBE_MANIFEST_PATH}\/kube-proxy.yaml","type":"","command_line_args":"","exit_status":[],"script_type":"sh"},"timeout_secs":"0","type":"EXEC","variable_list":[]},{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Master"},"retries":"0","description":"","child_tasks_local_reference_list":[],"name":"Restart Services","attrs":{"script":"#!\/bin\/bash\nset -ex\n\nif [ @@{calm_array_index}@@ -ne 0 ];then\n\texit\nfi\n\nINTERNAL_IP=\"@@{private_ip_address}@@\"\nCONTROLLER_NAMES=\"@@{calm_array_name}@@\" \nMINION_NAMES=\"@@{GCP_Centos_K8SM.name}@@\" \nJSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'\nKUBELET_IMAGE_TAG=\"@@{KUBE_IMAGE_TAG_NEW}@@\"\nMASTER_API_HTTPS=6443\nVERSION=$(echo \"${KUBELET_IMAGE_TAG}\" | tr \"_\" \" \" | awk '{print $1}')\nwget -q https:\/\/storage.googleapis.com\/kubernetes-release\/release\/${VERSION}\/bin\/linux\/amd64\/kubectl\nchmod +x kubectl\nsudo mv kubectl \/usr\/local\/bin\/kubectl\n\ncount=0\nwhile [[ $(curl --key CA\/admin-key.pem --cert CA\/admin.pem --cacert CA\/ca.pem https:\/\/${INTERNAL_IP}:${MASTER_API_HTTPS}\/healthz) != \"ok\" ]] ; do\n echo \"Trying to reach master server https:\/\/${INTERNAL_IP}:${MASTER_API_HTTPS} : sleep for 5 secs\"\n sleep 10\n if [[ $count -eq 10 ]]; then\n \techo \"Unable to reach master server: https:\/\/${INTERNAL_IP}:${MASTER_API_HTTPS}\"\n \texit 1\n fi\n count=$((count+1))\ndone\n\ncount=0\nfor name in $(echo \"${MINION_NAMES}\" | tr \",\" \"\\n\"); do\n kubectl drain ${name} --ignore-daemonsets --delete-local-data --force\n sleep 20\n ssh -o stricthostkeychecking=no \"minion${count}\" \"sudo yum update -y --quiet && sudo systemctl daemon-reload && sudo systemctl restart kubelet\"\n sleep 20\n while [[ `kubectl get nodes -l kubernetes.io\/hostname=${name} -o jsonpath=\"$JSONPATH\" | grep \"Ready=Unknown\" 2>\/dev\/null` ]]; do sleep 10 ; done\n kubectl uncordon ${name}\n count=$((count+1))\ndone\n\n\nif [[ `kubectl get nodes -o jsonpath=\"$JSONPATH\" | grep \"Ready=Unknown\"` ]]; then \n\techo \"Upgrade failed on minion nodes\"\n exit 1\nfi\n\ncount=0\nfor name in $(echo \"${CONTROLLER_NAMES}\" | tr \",\" \"\\n\"); do\n kubectl drain $name --ignore-daemonsets --delete-local-data --force\n sleep 20\n ssh -o stricthostkeychecking=no \"controller${count}\" \"sudo yum update -y --quiet && sudo systemctl daemon-reload && sudo systemctl restart kubelet\"\n sleep 20\n while [[ `kubectl get nodes -l kubernetes.io\/hostname=$name -o jsonpath=\"$JSONPATH\" | grep \"Ready=Unknown\" 2>\/dev\/null` ]]; do sleep 10 ; done\n kubectl uncordon $name\n count=$((count+1))\ndone\n\n\nif [[ `kubectl get nodes -o jsonpath=\"$JSONPATH\" | grep \"Ready=Unknown\"` ]]; then \n\techo \"Upgrade failed on nodes: $(kubectl get nodes -o jsonpath='$JSONPATH' | grep 'Ready=Unknown')\"\n exit 1\nfi","type":"","command_line_args":"","exit_status":[],"script_type":"sh"},"timeout_secs":"0","type":"EXEC","variable_list":[]}],"description":"","name":"8d003d32_runbook","main_task_local_reference":{"kind":"app_task","name":"cbdb4fa8_dag"},"variable_list":[{"val_type":"STRING","description":"","name":"KUBE_IMAGE_TAG_NEW","type":"LOCAL","value":"v1.8.0_coreos.0","label":"","attrs":{"type":""},"editables":{"value":true}}]},"name":"Upgrade"},{"description":"","type":"user","critical":false,"runbook":{"task_definition_list":[{"retries":"0","description":"","child_tasks_local_reference_list":[{"kind":"app_task","name":"Scale Out"},{"kind":"app_task","name":"Set Hosts File"}],"name":"f3682593_dag","attrs":{"edges":[{"from_task_reference":{"kind":"app_task","name":"Scale Out"},"edge_type":"user_defined","type":"","to_task_reference":{"kind":"app_task","name":"Set Hosts File"}}],"type":""},"timeout_secs":"0","type":"DAG","variable_list":[]},{"target_any_local_reference":{"kind":"app_blueprint_deployment","name":"77b729d5_deployment"},"retries":"0","description":"","child_tasks_local_reference_list":[],"name":"Scale Out","attrs":{"scaling_count":"@@{COUNT}@@","type":"","scaling_type":"SCALEOUT"},"timeout_secs":"0","type":"SCALING","variable_list":[]},{"target_any_local_reference":{"kind":"app_service","name":"Kubernetes_Master"},"retries":"0","description":"","child_tasks_local_reference_list":[],"name":"Set Hosts File","attrs":{"script":"#!\/bin\/bash\nset -ex\n\nMINION_IPS=\"@@{GCP_Centos_K8SM.private_ip_address}@@\"\nfor ip in $(echo ${MINION_IPS} | tr \",\" \"\\n\"); do\n if ! (( $(grep -c \"${ip} minion${count}\" \/etc\/hosts) )) ; then\n \techo \"${ip} minion${count}\" | sudo tee -a \/etc\/hosts\n fi\n count=$((count+1))\ndone","type":"","command_line_args":"","exit_status":[],"script_type":"sh"},"timeout_secs":"0","type":"EXEC","variable_list":[]}],"description":"","name":"744583cd_runbook","main_task_local_reference":{"kind":"app_task","name":"f3682593_dag"},"variable_list":[{"val_type":"STRING","description":"","name":"KUBE_IMAGE_TAG_NEW","type":"LOCAL","value":"v1.8.0_coreos.0","label":"","attrs":{"type":""},"editables":{"value":true}},{"val_type":"STRING","description":"","name":"COUNT","type":"LOCAL","value":"1","label":"","attrs":{"type":""},"editables":{"value":true}}]},"name":"ScaleOut"},{"description":"","type":"user","critical":false,"runbook":{"task_definition_list":[{"retries":"0","description":"","child_tasks_local_reference_list":[{"kind":"app_task","name":"Scale In"}],"name":"853cb860_dag","attrs":{"edges":[],"type":""},"timeout_secs":"0","type":"DAG","variable_list":[]},{"target_any_local_reference":{"kind":"app_blueprint_deployment","name":"77b729d5_deployment"},"retries":"0","description":"","child_tasks_local_reference_list":[],"name":"Scale In","attrs":{"scaling_count":"@@{COUNT}@@","type":"","scaling_type":"SCALEIN"},"timeout_secs":"0","type":"SCALING","variable_list":[]}],"description":"","name":"9331a25f_runbook","main_task_local_reference":{"kind":"app_task","name":"853cb860_dag"},"variable_list":[{"val_type":"STRING","description":"","name":"COUNT","type":"LOCAL","value":"1","label":"","attrs":{"type":""},"editables":{"value":true}}]},"name":"ScaleIn"}],"name":"GCP","variable_list":[{"val_type":"STRING","description":"","name":"KUBE_CLUSTER_NAME","type":"LOCAL","value":"kube-calm","label":"","attrs":{"type":""},"editables":{"value":true}},{"val_type":"STRING","description":"","name":"KUBE_IMAGE_TAG","type":"LOCAL","value":"v1.8.0_coreos.0","label":"","attrs":{"type":""},"editables":{"value":true}},{"val_type":"STRING","description":"","name":"DOCKER_VERSION","type":"LOCAL","value":"17.03.2.ce","label":"","attrs":{"type":""},"editables":{"value":true}},{"val_type":"STRING","description":"","name":"KUBE_CLUSTER_SUBNET","type":"LOCAL","value":"10.200.0.0\/16","label":"","attrs":{"type":""},"editables":{"value":true}},{"val_type":"STRING","description":"","name":"KUBE_SERVICE_SUBNET","type":"LOCAL","value":"10.32.0.0\/24","label":"","attrs":{"type":""},"editables":{"value":true}},{"val_type":"STRING","description":"","name":"KUBE_DNS_IP","type":"LOCAL","value":"10.32.0.10","label":"","attrs":{"type":""},"editables":{"value":true}}]}],"default_credential_local_reference":{"kind":"app_credential","name":"CENTOS"},"type":"USER"},"name":"k8s-200"},"api_version":"3.0","metadata":{"last_update_time":"1542450914985728","kind":"blueprint","spec_version":8,"creation_time":"1542435524993741","name":"k8s-200"}}