Added cockroachdb cluster setup across multiple clouds
This patch does the following: 1. Added the apps role so that k8s app deployment can be placed in the role 2. Added cockroachdb-init-pod deployment inthe post role since this has to be deployed first. 3. Added cockroachdb-pod deployment in the apps role 4. Allow this workload to stand up a standalone cockroachdb cluster or join in an existing cockroachdb cluster 5. Added the cockroachdb load generator container so that once the cluster is started, there will be some load generated. 6. Added a way to pre-allocating floating IP addresses so that the workload can use pre-allocated floating IPs for VMs. Change-Id: Ifa9eeb9d761d9801cab580445e6c43c8cf1dfdaa
This commit is contained in:
Tong Li
litong01
parent
7ff610459e
commit
f455064066
|
|
@ -0,0 +1,42 @@
|
|||
---
|
||||
- name: Setup few variables for coreos target
|
||||
set_fact:
|
||||
master_ip: "{{ groups['cmasters'][0] }}"
|
||||
public_ip: "{{ ansible_host }}"
|
||||
when: app_env.target_os == "coreos"
|
||||
|
||||
- name: Setup few variables for ubuntu target
|
||||
set_fact:
|
||||
master_ip: "{{ groups['umasters'][0] }}"
|
||||
public_ip: "{{ ansible_host }}"
|
||||
when: app_env.target_os == "ubuntu"
|
||||
|
||||
- name: Setup private IP variable for master node
|
||||
set_fact:
|
||||
private_ip: "{{ hostvars[master_ip].inter_ip }}"
|
||||
|
||||
- name: Setup first node IP when stand alone
|
||||
set_fact:
|
||||
first_node_ip: "{{ master_ip }}"
|
||||
when: app_env.app_setting.own_cluster == True
|
||||
|
||||
- name: Setup first node IP when join others
|
||||
set_fact:
|
||||
first_node_ip: "{{ app_env.app_setting.public_node }}"
|
||||
when: app_env.app_setting.own_cluster == False
|
||||
|
||||
- name: Upload app configuration files
|
||||
template:
|
||||
src: "roles/apps/templates/{{ item }}.j2"
|
||||
dest: "/etc/kubernetes/{{ item }}.yaml"
|
||||
mode: 0644
|
||||
with_items:
|
||||
- cockroachdb-pod
|
||||
|
||||
- name: Create the app pod
|
||||
command: >-
|
||||
/opt/bin/kubectl --server="{{ private_ip }}:8080" create
|
||||
-f "/etc/kubernetes/{{ item }}.yaml"
|
||||
with_items:
|
||||
- cockroachdb-pod
|
||||
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: cockroachdb-{{ inter_name }}
|
||||
labels:
|
||||
app: cockroachdb
|
||||
spec:
|
||||
nodeName: {{ inter_name }}
|
||||
restartPolicy: Always
|
||||
containers:
|
||||
- name: cockroachdb
|
||||
image: cockroachdb/cockroach
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- containerPort: 26257
|
||||
hostPort: 26257
|
||||
name: grpc
|
||||
- containerPort: 8080
|
||||
hostPort: 8090
|
||||
name: http
|
||||
volumeMounts:
|
||||
- name: datadir
|
||||
mountPath: /cockroach/cockroach-data
|
||||
command:
|
||||
- "/bin/bash"
|
||||
- "-ecx"
|
||||
- |
|
||||
CRARGS=("start" "--logtostderr" "--insecure" "--http-host" "0.0.0.0" )
|
||||
CRARGS+=("--advertise-host" "{{ public_ip }}" )
|
||||
CRARGS+=("--locality" "cloud={{ env }}")
|
||||
CRARGS+=("--join" "{{ first_node_ip }}")
|
||||
exec /cockroach/cockroach ${CRARGS[*]}
|
||||
- name: loadgen
|
||||
image: cockroachdb/interop-demo:1.1
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /kv
|
||||
- --read-percent=50
|
||||
- --max-rate=500
|
||||
terminationGracePeriodSeconds: 30
|
||||
volumes:
|
||||
- name: datadir
|
||||
hostPath:
|
||||
path: /storage/cockroachdb
|
||||
|
|
@ -12,7 +12,7 @@
|
|||
with_items:
|
||||
- dnscontroller
|
||||
- dashboard
|
||||
- cockroachdb
|
||||
- cockroachdb-init-pod
|
||||
|
||||
- name: Label the master node
|
||||
command: >-
|
||||
|
|
@ -26,5 +26,9 @@
|
|||
with_items:
|
||||
- dnscontroller
|
||||
- dashboard
|
||||
- cockroachdb
|
||||
|
||||
- name: Setup first cockroachdb node
|
||||
command: >-
|
||||
/opt/bin/kubectl --server={{ private_ip }}:8080 create
|
||||
-f /etc/kubernetes/cockroachdb-init-pod.yaml
|
||||
when: app_env.app_setting.own_cluster == True
|
||||
|
|
|
|||
|
|
@ -0,0 +1,44 @@
|
|||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: cockroachdb-{{ inter_name }}
|
||||
labels:
|
||||
app: cockroachdb
|
||||
spec:
|
||||
nodeName: {{ inter_name }}
|
||||
restartPolicy: Always
|
||||
containers:
|
||||
- name: cockroachdb
|
||||
image: cockroachdb/cockroach
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- containerPort: 26257
|
||||
hostPort: 26257
|
||||
name: grpc
|
||||
- containerPort: 8080
|
||||
hostPort: 8090
|
||||
name: http
|
||||
volumeMounts:
|
||||
- name: datadir
|
||||
mountPath: /cockroach/cockroach-data
|
||||
command:
|
||||
- "/bin/bash"
|
||||
- "-ecx"
|
||||
- |
|
||||
CRARGS=("start" "--logtostderr" "--insecure" "--http-host")
|
||||
CRARGS+=("0.0.0.0" "--advertise-host" "{{ public_ip }}")
|
||||
CRARGS+=("--locality" "cloud={{ env }}")
|
||||
exec /cockroach/cockroach ${CRARGS[*]}
|
||||
- name: loadgen
|
||||
image: cockroachdb/interop-demo:1.1
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /kv
|
||||
- --splits=100
|
||||
- --read-percent=50
|
||||
- --max-rate=500
|
||||
terminationGracePeriodSeconds: 30
|
||||
volumes:
|
||||
- name: datadir
|
||||
hostPath:
|
||||
path: /storage/cockroachdb
|
||||
|
|
@ -63,11 +63,12 @@
|
|||
- { p_min: 53, p_max: 53, dir: ingress, protocol: udp }
|
||||
- { p_min: 53, p_max: 53, dir: egress, protocol: udp }
|
||||
- { p_min: 8080, p_max: 8080, dir: ingress, protocol: tcp }
|
||||
- { p_min: 8090, p_max: 8090, dir: ingress, protocol: tcp }
|
||||
- { p_min: 8285, p_max: 8285, dir: ingress, protocol: udp }
|
||||
- { p_min: 2379, p_max: 2380, dir: ingress, protocol: tcp }
|
||||
- { p_min: 2379, p_max: 2380, dir: egress, protocol: tcp }
|
||||
- { p_min: 10250, p_max: 10250, dir: ingress, protocol: tcp }
|
||||
- { p_min: 30000, p_max: 32767, dir: ingress, protocol: tcp }
|
||||
- { p_min: 20000, p_max: 32767, dir: ingress, protocol: tcp }
|
||||
- { p_min: -1, p_max: -1, dir: ingress, protocol: icmp }
|
||||
- { p_min: -1, p_max: -1, dir: egress, protocol: icmp }
|
||||
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@
|
|||
flavor: "{{ app_env.flavor_name }}"
|
||||
network: "{{ app_env.private_net_name }}"
|
||||
floating_ip_pools: "{{ app_env.public_net_name | default(omit) }}"
|
||||
floating_ips: "{{ app_env.app_setting.ips[inventory_hostname] }}"
|
||||
reuse_ips: False
|
||||
userdata: "{{ lookup('template', tp_path) }}"
|
||||
config_drive: "{{ app_env.config_drive | default('no') }}"
|
||||
|
|
|
|||
|
|
@ -1,5 +1,9 @@
|
|||
---
|
||||
|
||||
- name: Setup release floating IP flag
|
||||
set_fact:
|
||||
ip_flag: "{{ app_env.app_setting.ips[inventory_hostname] == '' }}"
|
||||
|
||||
- name: Destroy the OpenStack VM
|
||||
os_server:
|
||||
state: "absent"
|
||||
|
|
@ -9,7 +13,7 @@
|
|||
validate_certs: "{{ app_env.validate_certs }}"
|
||||
name: "{{ inventory_hostname }}"
|
||||
image: "{{ app_env.image_name }}"
|
||||
delete_fip: True
|
||||
delete_fip: "{{ ip_flag }}"
|
||||
key_name: "k8s"
|
||||
timeout: 200
|
||||
network: "{{ app_env.private_net_name }}"
|
||||
|
|
|
|||
|
|
@ -110,6 +110,19 @@
|
|||
environment: "{{ proxy_env }}"
|
||||
tags: "post"
|
||||
|
||||
- name: Start up applications
|
||||
hosts: cworkers, uworkers
|
||||
gather_facts: true
|
||||
user: "{{ app_env.ssh_user }}"
|
||||
become: true
|
||||
become_user: root
|
||||
vars_files:
|
||||
- "vars/{{ env }}.yml"
|
||||
roles:
|
||||
- apps
|
||||
environment: "{{ proxy_env }}"
|
||||
tags: "apps"
|
||||
|
||||
- name: Inform the installer
|
||||
hosts: cloud
|
||||
connection: local
|
||||
|
|
|
|||
|
|
@ -40,7 +40,25 @@ app_env: {
|
|||
dns_service_ip: "172.16.0.4",
|
||||
dashboard_service_ip: "172.16.0.5",
|
||||
|
||||
app_setting: {
|
||||
public_node: "",
|
||||
own_cluster: True,
|
||||
# The following section shows how to pre allocate floating IPs for each
|
||||
# server. If you wish not to pre allocate floating IPs or your cloud
|
||||
# does not support floating IPs, leave them empty. The stack_size
|
||||
# above should dictate how many worker nodes should be. For example, if
|
||||
# your stack_size is 10, you will need to add worker-1 to worker-9.
|
||||
ips: {
|
||||
master: [],
|
||||
worker-1: [],
|
||||
worker-2: []
|
||||
}
|
||||
},
|
||||
|
||||
# The following section shows an example when use a local repo.
|
||||
# If you have exported some container images such as images that being used
|
||||
# by this workload, you can place the url point to tar.gz file for
|
||||
# cimages_repo
|
||||
cimages_repo: "http://10.0.10.12/cimages.tar.gz",
|
||||
flannel_repo: "http://10.0.10.12/flannel-v0.7.0-linux-amd64.tar.gz",
|
||||
k8s_repo: "http://10.0.10.12/v1.5.4/"
|
||||
|
|
|
|||
|
|
@ -40,7 +40,25 @@ app_env: {
|
|||
dns_service_ip: "172.16.0.4",
|
||||
dashboard_service_ip: "172.16.0.5",
|
||||
|
||||
app_setting: {
|
||||
public_node: "",
|
||||
own_cluster: True,
|
||||
# The following section shows how to pre allocate floating IPs for each
|
||||
# server. If you wish not to pre allocate floating IPs or your cloud
|
||||
# does not support floating IPs, leave them empty. The stack_size
|
||||
# above should dictate how many worker nodes should be. For example, if
|
||||
# your stack_size is 10, you will need to add worker-1 to worker-9.
|
||||
ips: {
|
||||
master: [],
|
||||
worker-1: [],
|
||||
worker-2: []
|
||||
}
|
||||
},
|
||||
|
||||
# The following section shows an example when use a remote repo.
|
||||
# If you have exported some container images such as images that being used
|
||||
# by this workload, you can place the url point to tar.gz file for
|
||||
# cimages_repo
|
||||
cimages_repo: "",
|
||||
flannel_repo: "https://github.com/coreos/flannel/releases/download/v0.7.0/flannel-v0.7.0-linux-amd64.tar.gz",
|
||||
k8s_repo: "https://storage.googleapis.com/kubernetes-release/release/v1.5.3/bin/linux/amd64/"
|
||||
|
|
|
|||
Loading…
Reference in New Issue