Switch to flannel, ignore k8s interfaces for zerotier, hard code dns for both k8s and docker due to systemd resolvd, and disable ipv6

ansible/roles/kubernetes/defaults/main.yml

@@ -1,6 +1,6 @@
 ---
 rke_directory: /home/{{ standard_user }}/rke
 rke_node_directory: /opt/rke
-rke_version: 1.1.3
+rke_version: 1.2.1
 rke_cluster_name: rke-k8s
-kubernetes_version: 1.15.12
+kubernetes_version: 1.17.13

ansible/roles/kubernetes/tasks/deploy-rke.yml

@@ -46,16 +46,17 @@
   delegate_to: localhost
   run_once: true
 
-# - name: Put RKE configs in place
+- name: Put RKE configs in place
-#   template:
+  template:
-#     src: ../templates/rke-configs/{{ item }}.j2
+    src: ../templates/{{ item }}
-#     dest: "{{ rke_directory }}/configs/{{ item }}"
+    dest: "{{ rke_directory }}/configs/{{ item }}"
-#   with_items:
+  with_items:
-#     - kube-state-metrics-deployment.yaml
+    - cloudflare-updater.yaml
-#     - kube-state-metrics-service.yaml
+    - test-app.yaml
-#     - kube-state-metrics-rbac.yaml
+    - cert-manager-namespace.yaml
-#   delegate_to: localhost
+    - cert-manager-prod-issuer.yaml
-#   run_once: true
+  delegate_to: localhost
+  run_once: true
 
 - name: Run RKE
   shell: >

ansible/roles/kubernetes/templates/cert-manager-namespace.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: cert-manager
+  labels:
+    certmanager.k8s.io/disable-validation: "true"

ansible/roles/kubernetes/templates/cert-manager-prod-issuer.yaml

@@ -0,0 +1,19 @@
+apiVersion: cert-manager.io/v1alpha2
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt-prod
+spec:
+  acme:
+    # The ACME server URL
+    server: https://acme-v02.api.letsencrypt.org/directory
+    # Email address used for ACME registration
+    email: jacob.wimer@gmail.com
+    # Name of a secret used to store the ACME account private key
+    privateKeySecretRef:
+      name: letsencrypt-prod
+    # Enable the HTTP-01 challenge provider
+    solvers:
+    - http01:
+        ingress:
+          class: nginx
+      selector: {}

ansible/roles/kubernetes/templates/cloudflare-updater.yaml

@@ -0,0 +1,37 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cloudflare-updater
+  labels:
+    app: cloudflare-updater
+spec:
+  strategy:
+    type: Recreate
+  selector:
+     matchLabels:
+       app: cloudflare-updater
+  template:
+    metadata:
+      labels:
+        app: cloudflare-updater
+    spec:
+      containers:
+      - image: jcwimer/cloudflare-updater
+        name: cloudflare-updater
+        env:
+        - name: CLOUDFLARE_API
+          value: "{{cloudflare_api}}"
+        - name: PROXIED
+          value: "false"
+        - name: ZONE_NAME
+          value: "codywimer.com"
+        - name: RECORD_TO_MODIFY
+          value: "test"
+        resources:
+          limits:
+            cpu: "0.1"
+            memory: "50Mi"
+          requests:
+            memory: "50Mi"
+            cpu: "0.1"

ansible/roles/kubernetes/templates/rke-cluster-deployment.yaml

@@ -15,13 +15,15 @@ services:
     extra_binds:
       - /usr/libexec/kubernetes/kubelet-plugins/volume/exec:/usr/libexec/kubernetes/kubelet-plugins/volume/exec
 {% endif %}
+  kube_api:
+    service_cluster_ip_range: 192.168.0.0/16
 
 network:
-    plugin: canal
+    plugin: flannel
     {% if (kubernetes_network_interface is defined) %}
 
     options:
-        canal_iface: {{ kubernetes_network_interface }}
+        flannel_iface: {{ kubernetes_network_interface }}
     {% endif %}
 
 nodes:
@@ -35,6 +37,7 @@ nodes:
     role:
     - controlplane
     - etcd
+
   {% endfor %}
   {% for node in groups['workers'] %}
 
@@ -47,3 +50,16 @@ nodes:
     - worker
 
   {% endfor %}
+
+dns:
+    provider: coredns
+    upstreamnameservers:
+    - 1.1.1.1
+    - 8.8.4.4
+
+addons_include:
+  - {{ rke_directory }}/cert-manager-namespace.yaml
+  - {{ rke_directory }}/configs/cloudflare-updater.yaml
+  - {{ rke_directory }}/configs/test-app.yaml
+  - https://github.com/jetstack/cert-manager/releases/download/v0.13.1/cert-manager.yaml
+  - {{ rke_directory }}/cert-manager-prod-issuer.yaml

ansible/roles/kubernetes/templates/test-app.yaml

@@ -0,0 +1,73 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: hello
+  labels:
+    app: hello
+spec:
+  ports:
+    - port: 80
+      targetPort: 80
+  selector:
+    app: hello
+  clusterIP: None
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hello
+  labels:
+    app: hello
+spec:
+  replicas: 2
+  strategy:
+    type: Recreate
+  selector:
+     matchLabels:
+       app: hello
+  template:
+    metadata:
+      labels:
+        app: hello
+    spec:
+      containers:
+      - image: tutum/hello-world
+        name: hello
+        ports:
+        - containerPort: 80
+          name: http
+        resources:
+          limits:
+            cpu: "0.5"
+            memory: "512Mi"
+          requests:
+            memory: "256Mi"
+            cpu: "0.2"
+        livenessProbe:
+          httpGet:
+            path: /
+            port: 80
+          initialDelaySeconds: 180
+          periodSeconds: 20
+          timeoutSeconds: 10
+---
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: hello-ingress-external
+  annotations:
+    cert-manager.io/cluster-issuer: "letsencrypt-prod"
+    kubernetes.io/tls-acme: "true"
+spec:
+  rules:
+  - host: test.codywimer.com
+    http:
+      paths:
+      - path: /
+        backend:
+          serviceName: hello
+          servicePort: 80
+  tls:
+    - hosts:
+      - test.codywimer.com
+      secretName: letsencrypt-prod-test

ansible/run-ansible.sh

@@ -5,5 +5,5 @@ cd ${project_dir}/ansible
 ansible-playbook --inventory-file=${project_dir}/terraform-code/inventory --private-key ~/.ssh/id_home \
   -e rke_ssh_key_location=~/.ssh/id_home \
   -e rke_directory=${project_dir}/rke \
-  -e cloudflare_api=${CLOUDFLARE_API}
+  -e cloudflare_api=${CLOUDFLARE_API} \
   playbooks/site.yml

terraform-code/user-data.sh

@@ -1,13 +1,43 @@
 #!/bin/bash
 apt-get update                                                  
-apt-get install python-dev python-pip curl sudo -y
+apt-get install python-dev python-pip curl sudo open-iscsi -y
+
+# Disable ipv6
+sysctl -w net.ipv6.conf.all.disable_ipv6=1
+sysctl -w net.ipv6.conf.default.disable_ipv6=1
+
+cat <<EOF > /etc/sysctl.d/ipv6.conf
+net.ipv6.conf.all.disable_ipv6 = 1
+net.ipv6.conf.default.disable_ipv6 = 1
+net.ipv6.conf.lo.disable_ipv6 = 1
+EOF
+
+# enable tun module
+modprobe tun
+cat <<EOF > /etc/modules-load.d/tun.conf
+tun
+EOF
+
+service networking restart
+
+# force zerotier to ignore kubernetes interfaces when it's looking for a gateway
+mkdir -p /var/lib/zerotier-one
+cat <<EOF > /var/lib/zerotier-one/local.conf
+{
+  "settings": {
+    "interfacePrefixBlacklist": [ "flannel", "veth", "cni", "docker" ],
+    "allowTcpFallbackRelay": false
+  }
+}
+EOF
+
+curl -s https://install.zerotier.com | bash
+zerotier-cli join ${zerotier_network}
 
 if ! which docker > /dev/null; then                                                                 
   curl -s -L https://raw.githubusercontent.com/rancher/install-docker/master/19.03.9.sh | bash      
 fi 
 
-curl -s https://install.zerotier.com | bash
-zerotier-cli join ${zerotier_network}
 user=debian
 if ! cat /etc/passwd | grep debian; then
   # Add the user (--gecos "" ensures that this runs non-interactively)
@@ -32,5 +62,11 @@ if ! cat /etc/passwd | grep debian; then
 fi
 usermod -a -G docker $user
 
-# for RKE
+mkdir -p /etc/docker
-# iptables -I INPUT -j ACCEPT
+cat <<EOF > /etc/docker/daemon.json
+{
+  "dns": ["1.1.1.1", "8.8.4.4"]
+}
+EOF
+
+service docker restart