From f4550640669fa90c5f05b48b7afcf322f7f15313 Mon Sep 17 00:00:00 2001 From: Tong Li Date: Thu, 27 Apr 2017 14:03:54 -0400 Subject: [PATCH] Added cockroachdb cluster setup across multiple clouds This patch does the following: 1. Added the apps role so that k8s app deployment can be placed in the role 2. Added cockroachdb-init-pod deployment inthe post role since this has to be deployed first. 3. Added cockroachdb-pod deployment in the apps role 4. Allow this workload to stand up a standalone cockroachdb cluster or join in an existing cockroachdb cluster 5. Added the cockroachdb load generator container so that once the cluster is started, there will be some load generated. 6. Added a way to pre-allocating floating IP addresses so that the workload can use pre-allocated floating IPs for VMs. Change-Id: Ifa9eeb9d761d9801cab580445e6c43c8cf1dfdaa --- .../shade/k8s/roles/apps/tasks/main.yml | 42 +++++++++++++++++ .../roles/apps/templates/cockroachdb-pod.j2 | 45 +++++++++++++++++++ .../shade/k8s/roles/post/tasks/apply.yml | 8 +++- .../post/templates/cockroachdb-init-pod.j2 | 44 ++++++++++++++++++ .../shade/k8s/roles/prepare/tasks/apply.yml | 3 +- .../shade/k8s/roles/provision/tasks/apply.yml | 1 + .../k8s/roles/provision/tasks/destroy.yml | 6 ++- workloads/ansible/shade/k8s/site.yml | 13 ++++++ workloads/ansible/shade/k8s/vars/coreos.yml | 18 ++++++++ workloads/ansible/shade/k8s/vars/ubuntu.yml | 18 ++++++++ 10 files changed, 194 insertions(+), 4 deletions(-) create mode 100755 workloads/ansible/shade/k8s/roles/apps/tasks/main.yml create mode 100755 workloads/ansible/shade/k8s/roles/apps/templates/cockroachdb-pod.j2 create mode 100755 workloads/ansible/shade/k8s/roles/post/templates/cockroachdb-init-pod.j2 diff --git a/workloads/ansible/shade/k8s/roles/apps/tasks/main.yml b/workloads/ansible/shade/k8s/roles/apps/tasks/main.yml new file mode 100755 index 0000000..5350b76 --- /dev/null +++ b/workloads/ansible/shade/k8s/roles/apps/tasks/main.yml @@ -0,0 +1,42 @@ +--- +- name: Setup few variables for coreos target + set_fact: + master_ip: "{{ groups['cmasters'][0] }}" + public_ip: "{{ ansible_host }}" + when: app_env.target_os == "coreos" + +- name: Setup few variables for ubuntu target + set_fact: + master_ip: "{{ groups['umasters'][0] }}" + public_ip: "{{ ansible_host }}" + when: app_env.target_os == "ubuntu" + +- name: Setup private IP variable for master node + set_fact: + private_ip: "{{ hostvars[master_ip].inter_ip }}" + +- name: Setup first node IP when stand alone + set_fact: + first_node_ip: "{{ master_ip }}" + when: app_env.app_setting.own_cluster == True + +- name: Setup first node IP when join others + set_fact: + first_node_ip: "{{ app_env.app_setting.public_node }}" + when: app_env.app_setting.own_cluster == False + +- name: Upload app configuration files + template: + src: "roles/apps/templates/{{ item }}.j2" + dest: "/etc/kubernetes/{{ item }}.yaml" + mode: 0644 + with_items: + - cockroachdb-pod + +- name: Create the app pod + command: >- + /opt/bin/kubectl --server="{{ private_ip }}:8080" create + -f "/etc/kubernetes/{{ item }}.yaml" + with_items: + - cockroachdb-pod + diff --git a/workloads/ansible/shade/k8s/roles/apps/templates/cockroachdb-pod.j2 b/workloads/ansible/shade/k8s/roles/apps/templates/cockroachdb-pod.j2 new file mode 100755 index 0000000..66869b6 --- /dev/null +++ b/workloads/ansible/shade/k8s/roles/apps/templates/cockroachdb-pod.j2 @@ -0,0 +1,45 @@ + +apiVersion: v1 +kind: Pod +metadata: + name: cockroachdb-{{ inter_name }} + labels: + app: cockroachdb +spec: + nodeName: {{ inter_name }} + restartPolicy: Always + containers: + - name: cockroachdb + image: cockroachdb/cockroach + imagePullPolicy: IfNotPresent + ports: + - containerPort: 26257 + hostPort: 26257 + name: grpc + - containerPort: 8080 + hostPort: 8090 + name: http + volumeMounts: + - name: datadir + mountPath: /cockroach/cockroach-data + command: + - "/bin/bash" + - "-ecx" + - | + CRARGS=("start" "--logtostderr" "--insecure" "--http-host" "0.0.0.0" ) + CRARGS+=("--advertise-host" "{{ public_ip }}" ) + CRARGS+=("--locality" "cloud={{ env }}") + CRARGS+=("--join" "{{ first_node_ip }}") + exec /cockroach/cockroach ${CRARGS[*]} + - name: loadgen + image: cockroachdb/interop-demo:1.1 + imagePullPolicy: IfNotPresent + command: + - /kv + - --read-percent=50 + - --max-rate=500 + terminationGracePeriodSeconds: 30 + volumes: + - name: datadir + hostPath: + path: /storage/cockroachdb diff --git a/workloads/ansible/shade/k8s/roles/post/tasks/apply.yml b/workloads/ansible/shade/k8s/roles/post/tasks/apply.yml index ef2457a..b8710b4 100755 --- a/workloads/ansible/shade/k8s/roles/post/tasks/apply.yml +++ b/workloads/ansible/shade/k8s/roles/post/tasks/apply.yml @@ -12,7 +12,7 @@ with_items: - dnscontroller - dashboard - - cockroachdb + - cockroachdb-init-pod - name: Label the master node command: >- @@ -26,5 +26,9 @@ with_items: - dnscontroller - dashboard - - cockroachdb +- name: Setup first cockroachdb node + command: >- + /opt/bin/kubectl --server={{ private_ip }}:8080 create + -f /etc/kubernetes/cockroachdb-init-pod.yaml + when: app_env.app_setting.own_cluster == True diff --git a/workloads/ansible/shade/k8s/roles/post/templates/cockroachdb-init-pod.j2 b/workloads/ansible/shade/k8s/roles/post/templates/cockroachdb-init-pod.j2 new file mode 100755 index 0000000..5e755fe --- /dev/null +++ b/workloads/ansible/shade/k8s/roles/post/templates/cockroachdb-init-pod.j2 @@ -0,0 +1,44 @@ +apiVersion: v1 +kind: Pod +metadata: + name: cockroachdb-{{ inter_name }} + labels: + app: cockroachdb +spec: + nodeName: {{ inter_name }} + restartPolicy: Always + containers: + - name: cockroachdb + image: cockroachdb/cockroach + imagePullPolicy: IfNotPresent + ports: + - containerPort: 26257 + hostPort: 26257 + name: grpc + - containerPort: 8080 + hostPort: 8090 + name: http + volumeMounts: + - name: datadir + mountPath: /cockroach/cockroach-data + command: + - "/bin/bash" + - "-ecx" + - | + CRARGS=("start" "--logtostderr" "--insecure" "--http-host") + CRARGS+=("0.0.0.0" "--advertise-host" "{{ public_ip }}") + CRARGS+=("--locality" "cloud={{ env }}") + exec /cockroach/cockroach ${CRARGS[*]} + - name: loadgen + image: cockroachdb/interop-demo:1.1 + imagePullPolicy: IfNotPresent + command: + - /kv + - --splits=100 + - --read-percent=50 + - --max-rate=500 + terminationGracePeriodSeconds: 30 + volumes: + - name: datadir + hostPath: + path: /storage/cockroachdb diff --git a/workloads/ansible/shade/k8s/roles/prepare/tasks/apply.yml b/workloads/ansible/shade/k8s/roles/prepare/tasks/apply.yml index 7b26e29..35f8142 100755 --- a/workloads/ansible/shade/k8s/roles/prepare/tasks/apply.yml +++ b/workloads/ansible/shade/k8s/roles/prepare/tasks/apply.yml @@ -63,11 +63,12 @@ - { p_min: 53, p_max: 53, dir: ingress, protocol: udp } - { p_min: 53, p_max: 53, dir: egress, protocol: udp } - { p_min: 8080, p_max: 8080, dir: ingress, protocol: tcp } + - { p_min: 8090, p_max: 8090, dir: ingress, protocol: tcp } - { p_min: 8285, p_max: 8285, dir: ingress, protocol: udp } - { p_min: 2379, p_max: 2380, dir: ingress, protocol: tcp } - { p_min: 2379, p_max: 2380, dir: egress, protocol: tcp } - { p_min: 10250, p_max: 10250, dir: ingress, protocol: tcp } - - { p_min: 30000, p_max: 32767, dir: ingress, protocol: tcp } + - { p_min: 20000, p_max: 32767, dir: ingress, protocol: tcp } - { p_min: -1, p_max: -1, dir: ingress, protocol: icmp } - { p_min: -1, p_max: -1, dir: egress, protocol: icmp } diff --git a/workloads/ansible/shade/k8s/roles/provision/tasks/apply.yml b/workloads/ansible/shade/k8s/roles/provision/tasks/apply.yml index e4d539e..c9a03de 100755 --- a/workloads/ansible/shade/k8s/roles/provision/tasks/apply.yml +++ b/workloads/ansible/shade/k8s/roles/provision/tasks/apply.yml @@ -25,6 +25,7 @@ flavor: "{{ app_env.flavor_name }}" network: "{{ app_env.private_net_name }}" floating_ip_pools: "{{ app_env.public_net_name | default(omit) }}" + floating_ips: "{{ app_env.app_setting.ips[inventory_hostname] }}" reuse_ips: False userdata: "{{ lookup('template', tp_path) }}" config_drive: "{{ app_env.config_drive | default('no') }}" diff --git a/workloads/ansible/shade/k8s/roles/provision/tasks/destroy.yml b/workloads/ansible/shade/k8s/roles/provision/tasks/destroy.yml index 079e371..9577835 100755 --- a/workloads/ansible/shade/k8s/roles/provision/tasks/destroy.yml +++ b/workloads/ansible/shade/k8s/roles/provision/tasks/destroy.yml @@ -1,5 +1,9 @@ --- +- name: Setup release floating IP flag + set_fact: + ip_flag: "{{ app_env.app_setting.ips[inventory_hostname] == '' }}" + - name: Destroy the OpenStack VM os_server: state: "absent" @@ -9,7 +13,7 @@ validate_certs: "{{ app_env.validate_certs }}" name: "{{ inventory_hostname }}" image: "{{ app_env.image_name }}" - delete_fip: True + delete_fip: "{{ ip_flag }}" key_name: "k8s" timeout: 200 network: "{{ app_env.private_net_name }}" diff --git a/workloads/ansible/shade/k8s/site.yml b/workloads/ansible/shade/k8s/site.yml index 2436b2f..df30317 100755 --- a/workloads/ansible/shade/k8s/site.yml +++ b/workloads/ansible/shade/k8s/site.yml @@ -110,6 +110,19 @@ environment: "{{ proxy_env }}" tags: "post" +- name: Start up applications + hosts: cworkers, uworkers + gather_facts: true + user: "{{ app_env.ssh_user }}" + become: true + become_user: root + vars_files: + - "vars/{{ env }}.yml" + roles: + - apps + environment: "{{ proxy_env }}" + tags: "apps" + - name: Inform the installer hosts: cloud connection: local diff --git a/workloads/ansible/shade/k8s/vars/coreos.yml b/workloads/ansible/shade/k8s/vars/coreos.yml index 386da5e..f14377d 100755 --- a/workloads/ansible/shade/k8s/vars/coreos.yml +++ b/workloads/ansible/shade/k8s/vars/coreos.yml @@ -40,7 +40,25 @@ app_env: { dns_service_ip: "172.16.0.4", dashboard_service_ip: "172.16.0.5", + app_setting: { + public_node: "", + own_cluster: True, + # The following section shows how to pre allocate floating IPs for each + # server. If you wish not to pre allocate floating IPs or your cloud + # does not support floating IPs, leave them empty. The stack_size + # above should dictate how many worker nodes should be. For example, if + # your stack_size is 10, you will need to add worker-1 to worker-9. + ips: { + master: [], + worker-1: [], + worker-2: [] + } + }, + # The following section shows an example when use a local repo. + # If you have exported some container images such as images that being used + # by this workload, you can place the url point to tar.gz file for + # cimages_repo cimages_repo: "http://10.0.10.12/cimages.tar.gz", flannel_repo: "http://10.0.10.12/flannel-v0.7.0-linux-amd64.tar.gz", k8s_repo: "http://10.0.10.12/v1.5.4/" diff --git a/workloads/ansible/shade/k8s/vars/ubuntu.yml b/workloads/ansible/shade/k8s/vars/ubuntu.yml index 16a3398..4cf2d1c 100755 --- a/workloads/ansible/shade/k8s/vars/ubuntu.yml +++ b/workloads/ansible/shade/k8s/vars/ubuntu.yml @@ -40,7 +40,25 @@ app_env: { dns_service_ip: "172.16.0.4", dashboard_service_ip: "172.16.0.5", + app_setting: { + public_node: "", + own_cluster: True, + # The following section shows how to pre allocate floating IPs for each + # server. If you wish not to pre allocate floating IPs or your cloud + # does not support floating IPs, leave them empty. The stack_size + # above should dictate how many worker nodes should be. For example, if + # your stack_size is 10, you will need to add worker-1 to worker-9. + ips: { + master: [], + worker-1: [], + worker-2: [] + } + }, + # The following section shows an example when use a remote repo. + # If you have exported some container images such as images that being used + # by this workload, you can place the url point to tar.gz file for + # cimages_repo cimages_repo: "", flannel_repo: "https://github.com/coreos/flannel/releases/download/v0.7.0/flannel-v0.7.0-linux-amd64.tar.gz", k8s_repo: "https://storage.googleapis.com/kubernetes-release/release/v1.5.3/bin/linux/amd64/"