Switched from docker swarm to Kubernetes with rke

roles/common/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+docker_ce_version_to_install: 19.03.8

roles/common/tasks/main.yml

@@ -7,6 +7,7 @@
 #  apt: upgrade=dist
 - name: Update apt
   apt: update_cache=yes
+  become: true
   
 - name: Install standard programs
   apt: name={{ item }}  state=present force=yes
@@ -31,37 +32,50 @@
     - build-essential
     - tmux
     - sudo
+  become: true
+
+
+- name: Install required packages for Longhorn
+  apt: name={{ item }}  state=present force=yes
+  with_items:
+    - open-iscsi
+    - curl
+  become: true
+  when: longhorn_enabled is defined and longhorn_enabled | bool == True
 
 - name: Add docker key
   apt_key:
     url: https://download.docker.com/linux/{{ ansible_distribution|lower }}/gpg
     state: present
+  become: true
 
 - name: Add docker repo
   apt_repository:
     repo: deb [arch=amd64] https://download.docker.com/linux/{{ ansible_distribution|lower }} {{ ansible_distribution_release }} stable
     state: present
+  become: true
 
 - name: Update apt
   apt: update_cache=yes
+  become: true
 
 - name: Install docker-ce 
   shell: >
     apt-get install -y -qq docker-ce=$(apt-cache madison docker-ce | grep "{{ docker_ce_version_to_install }}" | awk {'print $3'})
+  become: true
 
 - name: Add standard_user to docker group
   user:
     name: "{{ standard_user }}"
     groups: docker
     append: yes
+  become: true
 
 - name: Set timezone to NewYork
   timezone:
       name: "{{ chosen_timezone }}"
   ignore_errors: true
-
-- name: Replace sudoers file
-  template: src=../roles/common/templates/sudoers.j2 dest=/etc/sudoers
+  become: true
 
 - name: Create /etc/docker
   file:
@@ -70,16 +84,16 @@
     owner: root
     mode: 700
     state: directory
+  become: true
 
 - name: Replace docker daemon file
   template: src=../roles/common/templates/docker-daemon.json.j2 dest=/etc/docker/daemon.json
   register: dockerdaemon
+  become: true
 
 - name: Restart docker if daemon changes
   service:
     name: docker
     state: restarted
   when: dockerdaemon.changed
-
-- name: Creates directory
-  file: path=/data state=directory
+  become: true

roles/common/templates/sudoers.j2

@@ -1,30 +0,0 @@
-#
-# This file MUST be edited with the 'visudo' command as root.
-#
-# Please consider adding local content in /etc/sudoers.d/ instead of
-# directly modifying this file.
-#
-# See the man page for details on how to write a sudoers file.
-#
-Defaults	env_reset
-Defaults	mail_badpass
-Defaults	secure_path="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-
-# Host alias specification
-
-# User alias specification
-
-# Cmnd alias specification
-
-# User privilege specification
-root	ALL=(ALL:ALL) ALL
-
-# Members of the admin group may gain root privileges
-%admin ALL=(ALL) ALL
-
-# Allow members of group sudo to execute any command
-%sudo	ALL=(ALL) NOPASSWD:ALL
-# See sudoers(5) for more information on "#include" directives:
-
-#includedir /etc/sudoers.d
-

roles/kubernetes/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+rke_directory: /home/{{ standard_user }}/rke
+rke_node_directory: /opt/rke
+rke_version: 1.0.8
+rke_cluster_name: rke-k8s
+kubernetes_version: 1.15.11

roles/kubernetes/tasks/deploy-rke.yml

@@ -0,0 +1,70 @@
+---
+- name: Create RKE directory
+  file:
+    path: "{{ rke_directory }}"
+    state: directory
+    mode: '0774'
+    owner: "{{ standard_user }}"
+    group: "{{ standard_user }}"
+  delegate_to: localhost
+  run_once: true
+
+- name: Create RKE Configs directory
+  file:
+    path: "{{ rke_directory }}/configs"
+    state: directory
+  delegate_to: localhost
+  run_once: true
+
+- name: Install RKE
+  get_url:
+    dest: "{{ rke_directory }}/rke"
+    url: https://github.com/rancher/rke/releases/download/v{{ rke_version }}/rke_linux-amd64
+  delegate_to: localhost
+  run_once: true
+
+- name: Make RKE executable
+  file:
+    dest: "{{ rke_directory }}/rke"
+    mode: +x
+  delegate_to: localhost
+  run_once: true
+
+- name: Put RKE cluster config in place
+  template:
+    src: ../templates/rke-cluster-deployment.yaml
+    dest: "{{ rke_directory }}/{{ rke_cluster_name }}.yaml"
+  delegate_to: localhost
+  run_once: true
+
+- name: Put RKE configs in place
+  template:
+    src: ../templates/rke-configs/{{ item }}.j2
+    dest: "{{ rke_directory }}/configs/{{ item }}"
+  with_items:
+    - kube-state-metrics-deployment.yaml
+    - kube-state-metrics-service.yaml
+    - kube-state-metrics-rbac.yaml
+  delegate_to: localhost
+  run_once: true
+
+- name: Run RKE
+  shell: >
+    bash -c "{{ rke_directory }}/rke up --config {{ rke_directory }}/{{ rke_cluster_name }}.yaml"
+  delegate_to: localhost
+  run_once: true
+  retries: 5
+  delay: 5
+  register: rke_install
+  until: rke_install.rc == 0
+
+- name: Set permissions on rke directory
+  file:
+    path: "{{ rke_directory }}"
+    state: directory
+    mode: '0774'
+    owner: "{{ standard_user }}"
+    group: "{{ standard_user }}"
+    recurse: yes
+  delegate_to: localhost
+  run_once: true

roles/kubernetes/tasks/main.yml

@@ -1,74 +1,9 @@
 ---
-- name: Create RKE directory
-  file:
-    path: "{{ rke_directory }}"
-    state: directory
-    mode: '0774'
-    owner: "{{ standard_user }}"
-    group: "{{ standard_user }}"
-  delegate_to: localhost
-  run_once: true
-  become: true
+- name: RKE Pre Tasks
+  import_tasks: pre-rke.yml
 
-- name: Create RKE Configs directory
-  file:
-    path: "{{ rke_directory }}/configs"
-    state: directory
-  delegate_to: localhost
-  run_once: true
-  become: true
+- name: RKE Deploy Kubernetes
+  import_tasks: deploy-rke.yml
 
-- name: Install RKE
-  get_url:
-    dest: "{{ rke_directory }}/rke"
-    url: https://github.com/rancher/rke/releases/download/v{{ rke_version }}/rke_linux-amd64
-  delegate_to: localhost
-  run_once: true
-  become: true
-
-- name: Make RKE executable
-  file:
-    dest: "{{ rke_directory }}/rke"
-    mode: +x
-  delegate_to: localhost
-  run_once: true
-  become: true
-
-- name: Put RKE cluster config in place
-  template:
-    src: ../templates/rke-cluster-deployment.yaml.j2
-    dest: "{{ rke_directory }}/{{ rke_cluster_name }}.yaml"
-  delegate_to: localhost
-  run_once: true
-  become: true
-
-- name: Put RKE configs in place
-  template:
-    src: ../templates/rke-configs/{{ item }}.j2
-    dest: "{{ rke_directory }}/configs/{{ item }}"
-  with_items:
-    - kube-state-metrics-deployment.yaml
-    - kube-state-metrics-service.yaml
-    - kube-state-metrics-rbac.yaml
-  delegate_to: localhost
-  run_once: true
-  become: true
-
-- name: Run RKE
-  shell: >
-    bash -c "{{ rke_directory }}/rke up --config {{ rke_directory }}/{{ rke_cluster_name }}.yaml"
-  delegate_to: localhost
-  run_once: true
-  become: true
-
-- name: Set permissions on rke directory
-  file:
-    path: "{{ rke_directory }}"
-    state: directory
-    mode: '0774'
-    owner: "{{ standard_user }}"
-    group: "{{ standard_user }}"
-    recurse: yes
-  delegate_to: localhost
-  run_once: true
-  become: true
+- name: RKE Post Tasks
+  import_tasks: post-rke.yml

roles/kubernetes/tasks/post-rke.yml

@@ -9,4 +9,18 @@
   copy:
     src: "{{ rke_directory }}/{{ rke_cluster_name }}.rkestate"
     dest: "{{ rke_node_directory }}/{{ rke_cluster_name }}.rkestate"
-  become: true 
+  become: true 
+
+- name: Create .kube folder in standard users home directory
+  file: 
+    path: /home/{{ standard_user }}/.kube
+    state: directory
+  become: true
+  delegate_to: localhost
+
+- name: Copy kube config to standard users home directory
+  copy:
+    src:  "{{ rke_directory }}/kube_config_{{ rke_cluster_name }}.yaml"
+    dest: /home/{{ standard_user }}/.kube/config
+  become: true 
+  delegate_to: localhost

roles/kubernetes/tasks/pre-rke.yml

@@ -41,6 +41,36 @@
     group: "{{ standard_user }}"
   become: true 
 
+- name: Create flex volume folder
+  file: 
+    path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
+    state: directory
+  become: true
+
+- name: Create Rook storage Directory
+  file:
+    path: /var/lib/rook
+    state: directory
+    mode: '0774'
+  become: true
+  when: rook_enabled is defined and rook_enabled | bool == True
+
+- name: Create Longhorn Storage Directory
+  file:
+    path: /var/lib/rancher/longhorn
+    state: directory
+    mode: '0774'
+  become: true
+  when: longhorn_enabled is defined and longhorn_enabled | bool == True
+
+- name: Create Longhorn Setting Directory
+  file:
+    path: /var/lib/longhorn-setting
+    state: directory
+    mode: '0774'
+  become: true
+  when: longhorn_enabled is defined and longhorn_enabled | bool == True
+
 - name: Check if RKE cluster state file exists
   stat:
     path: "{{ rke_node_directory }}/{{ rke_cluster_name }}.rkestate"

roles/kubernetes/templates/rke-cluster-deployment.yaml

@@ -0,0 +1,50 @@
+---
+
+ssh_key_path: {{ rke_ssh_key_location }}
+
+cluster_name: {{ rke_cluster_name }}
+ignore_docker_version: true
+system_images:
+    kubernetes: rancher/hyperkube:v{{ kubernetes_version }}-rancher1
+
+{% if (rook_enabled is defined and rook_enabled | bool == True) or (longhorn_enabled is defined and longhorn_enabled | bool == True) %}
+services:
+  kubelet:
+    extra_args:
+      volume-plugin-dir: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
+    extra_binds:
+      - /usr/libexec/kubernetes/kubelet-plugins/volume/exec:/usr/libexec/kubernetes/kubelet-plugins/volume/exec
+{% endif %}
+
+network:
+    plugin: canal
+    {% if (kubernetes_network_interface is defined) %}
+
+    options:
+        canal_iface: {{ kubernetes_network_interface }}
+    {% endif %}
+
+nodes:
+  {% for node in groups['kube-masters'] %}
+
+  - address: {{node}}
+    name: {{node}}
+    user: {{standard_user}}
+    role:
+    - controlplane
+    - etcd
+  {% endfor %}
+  {% for node in groups['kube-workers'] %}
+
+  - address: {{node}}
+    name: {{node}}
+    user: {{standard_user}}
+    role:
+    - worker
+
+  {% endfor %}
+
+authentication:
+    strategy: x509
+    sans:
+      - "kubernetes.{{ root_domain }}"

roles/kubernetes/templates/rke-cluster-deployment.yaml.j2

@@ -1,32 +0,0 @@
----
-
-ssh_key_path: {{ rke_ssh_key_location }}
-
-cluster_name: rke_cluster_name
-ignore_docker_version: true
-system_images:
-    kubernetes: rancher/hyperkube:v1.15.5-rancher1
-
-nodes:
-  {% for node in groups['kube-masters'] %}
-
-  - address: {{node}}
-    name: {{node}}
-    user: {{standard_user}}
-    role:
-    - controlplane
-    - etcd
-  {% endfor %}
-  {% for node in groups['kube-workers'] %}
-
-  - address: {{node}}
-    name: {{node}}
-    user: {{standard_user}}
-    role:
-    - worker
-  {% endfor %}
-
-authentication:
-    strategy: x509
-    sans:
-      - "kubernetes.{{ root_domain }}"

roles/longhorn/defaults/main.yaml

@@ -0,0 +1,2 @@
+---
+longhorn_version: v0.8.1

roles/longhorn/tasks/main.yaml

@@ -0,0 +1,27 @@
+---
+- name: Create Longhorn config Directory
+  file:
+    path: "{{ startup_infrastructure_directory }}/longhorn"
+    state: directory
+
+- name: Get Longhorn Install
+  get_url:
+    url: https://raw.githubusercontent.com/longhorn/longhorn/{{ longhorn_version }}/deploy/longhorn.yaml
+    dest: "{{ startup_infrastructure_directory }}/longhorn/longhorn.yaml"
+
+- name: Put other Longhorn yaml in place
+  template:
+    src: ../templates/{{ item }}.j2
+    dest: "{{ startup_infrastructure_directory }}/longhorn/{{ item }}"
+  with_items:
+    - longhorn-ingress.yaml
+
+- name: Install Longhorn
+  command: kubectl apply -f {{ startup_infrastructure_directory }}/longhorn/longhorn.yaml
+
+- name: Install Longhorn Extras
+  command: kubectl apply -f {{ startup_infrastructure_directory }}/longhorn/
+
+- name: Set default storageclass
+  command: >
+    kubectl patch storageclass longhorn -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'

roles/longhorn/templates/longhorn-ingress.yaml.j2

@@ -0,0 +1,22 @@
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: longhorn-ingress
+  namespace: longhorn-system
+  annotations:
+  #  cert-manager.io/cluster-issuer: "letsencrypt-prod"
+    kubernetes.io/ingress.class: "nginx"
+  #  kubernetes.io/tls-acme: "true"
+spec:
+  rules:
+  - host: longhorn.{{ root_domain }}
+    http:
+      paths:
+      - path: /
+        backend:
+          serviceName: longhorn-frontend
+          servicePort: 80
+  #tls:
+  #  - hosts:
+  #    - longhorn.{{ root_domain }}
+  #    secretName: longhorn-ssl

roles/presslabs-mysql/defaults/main.yml

@@ -0,0 +1,4 @@
+---
+mysql_root_password: password
+mysql_replicas: 3
+mysql_cluster_name: mysql-cluster

roles/presslabs-mysql/tasks/main.yml

@@ -0,0 +1,45 @@
+---
+- name: Create Mysql Directory
+  file:
+    path: "{{ startup_infrastructure_directory }}/presslabs-mysql-operator"
+    state: directory
+
+- name: Add presslabs helm repo
+  command: helm repo add presslabs https://presslabs.github.io/charts
+
+- name: Install the presslabs mysql operator
+  command: helm install presslabs/mysql-operator --name presslabs-mysql-operator
+
+- name: Wait for mysql-operator to be Running
+  command: >
+    bash -c "kubectl get pods --all-namespaces | grep presslabs-mysql-operator | grep Running"
+  retries: 10
+  delay: 30
+  register: mysql_operator_init
+  until: mysql_operator_init.rc == 0
+
+- name: Put mysql cluster yaml in place
+  template:
+    src: presslabs-mysql-cluster.yaml.j2
+    dest: "{{ startup_infrastructure_directory }}/presslabs-mysql-operator/presslabs-mysql-cluster.yaml"
+
+- name: Deploy mysql cluster
+  command: >
+    kubectl apply -f {{ startup_infrastructure_directory }}/presslabs-mysql-operator/presslabs-mysql-cluster.yaml
+
+- name: Wait for cluster to be Running
+  command: >
+    bash -c "kubectl get mysql | grep {{ mysql_cluster_name }} | grep True"
+  retries: 10
+  delay: 60
+  register: mysql_cluster_init
+  until: mysql_cluster_init.rc == 0
+
+- name: Test a query on the cluster
+  command: >
+    kubectl run mysql-client --image=mysql:5.7 -it --rm --restart=Never \
+      -- mysql -h {{ mysql_cluster_name }}-mysql -u root -p{{ mysql_root_password }} -e "show databases; show slave status;"
+  retries: 10
+  delay: 20
+  register: mysql_cluster_query
+  until: mysql_cluster_query.rc == 0

roles/presslabs-mysql/templates/presslabs-mysql-cluster.yaml.j2

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ mysql_cluster_name }}-secret
+type: Opaque
+# use data if password is base64 encoded
+#data:
+#  ROOT_PASSWORD: {{ mysql_root_password }}
+stringData:
+  ROOT_PASSWORD: {{ mysql_root_password }}
+---
+apiVersion: mysql.presslabs.org/v1alpha1
+kind: MysqlCluster
+metadata:
+  name: {{ mysql_cluster_name }}
+spec:
+  replicas: {{ mysql_replicas }}
+  secretName: {{ mysql_cluster_name }}-secret

roles/startup-infrastructure/tasks/main.yml

@@ -1,13 +1,12 @@
 ---
 - name: Create Startup Infrastructure Directory
   file:
-    path: /opt/startup-infrastructure
+    path: "{{ startup_infrastructure_directory }}"
     state: directory
-  become: true
 
 - name: Download Helm
   get_url:
-    url: https://get.helm.sh/helm-v2.14.3-linux-amd64.tar.gz
+    url: https://get.helm.sh/helm-v2.16.6-linux-amd64.tar.gz
     dest: /tmp/helm.tar.gz
   become: true
 
@@ -42,21 +41,29 @@
 
 - name: Create Tiller serviceaccount
   command: kubectl -n kube-system create serviceaccount tiller
-  become: true
-  environment:
-    KUBECONFIG: "{{ rke_directory }}/kube_config_{{ rke_cluster_name }}.yaml"
+  ignore_errors: true
 
 - name: Create Tiller rbac
-  become: true
   command: >
     kubectl create clusterrolebinding tiller \
       --clusterrole=cluster-admin \
       --serviceaccount=kube-system:tiller
-  environment:
-    KUBECONFIG: "{{ rke_directory }}/kube_config_{{ rke_cluster_name }}.yaml"
 
 - name: Initialize Tiller
   command: helm init --service-account tiller
-  become: true
-  environment:
-    KUBECONFIG: "{{ rke_directory }}/kube_config_{{ rke_cluster_name }}.yaml"
+
+- name: Wait for Tiller to be running
+  command: >
+    bash -c "kubectl -n kube-system get pods | grep tiller | grep Running"
+  retries: 10
+  delay: 30
+  register: tiller_running
+  until: tiller_running.rc == 0
+
+- name: Wait for Tiller to be initialized
+  command: >
+    bash -c "kubectl rollout status -w deployment/tiller-deploy --namespace=kube-system | grep "successfully rolled out""
+  retries: 10
+  delay: 30
+  register: tiller_init
+  until: tiller_init.rc == 0