Fixed yaml error in k8s manifest, added a passenger pool size secret, and added multiple workers

2026-03-24 17:04:43 +00:00 · 2023-03-01 21:58:24 +00:00
parent 87353da05e
commit 92bd1ec3c9
6 changed files with 78 additions and 69 deletions
--- a/deploy/kubernetes/README.md
+++ b/deploy/kubernetes/README.md
@@ -11,10 +11,10 @@
 4. Run `kubectl apply -f deploy/kubernetes/manifests/`

 ## What do I get?
-1. Wrestlingdev deployed with 2 replicas. Autoscaling is turned on up to 4 replcias. 
-2. A standalone mariadb that can back up to S3 compatable storage if you set the values in `deploy/kubernetes/secrets/secrets.yaml` and prometheus ready metrics
-3. A standalone memcahced.
-4. A single job runner to run wrestlingdev background jobs.
+1. Wrestlingdev deployed with 2 replicas.
+2. Two workers are deployed to run background jobs
+3. A standalone mariadb that can back up to S3 compatable storage if you set the values in `deploy/kubernetes/secrets/secrets.yaml` and prometheus ready metrics
+4. A standalone memcahced.

 ## How do I update the app?
 First, be sure your secrets.yaml has all envs up to date. Then, make sure you get all manifest changes
@@ -23,12 +23,17 @@ First, be sure your secrets.yaml has all envs up to date. Then, make sure you ge
 Each push to master updates the docker `prod` tag and also pushes a tag with the git hash. You will want to update to those tags.
 1. Set the git hash as a variable `TAG=$(git rev-parse --verify HEAD)`
 2. Update the wrestlingdev deployment tag `kubectl --record deployment.apps/wrestlingdev-app-deployment set image deployment.v1.apps/wrestlingdev-app-deployment wrestlingdev-app=jcwimer/wrestlingdev:${TAG}`
-3. Update the wrestlingdev job runner tag `kubectl --record deployment.apps/wrestlingdev-worker-deployment set image deployment.v1.apps/wrestlingdev-worker-deployment wrestlingdev-worker=jcwimer/wrestlingdev:${TAG}`
+3. Update the wrestlingdev job runner tag `kubectl --record statefulset.apps/wrestlingdev-worker set image statefulset.v1.apps/wrestlingdev-worker wrestlingdev-worker=jcwimer/wrestlingdev:${TAG}`

 Finally, run db-migrations
 1. Delete the db migrations job so you can re-run it `kubectl delete job wrestlingdev-db-create-migrate`
 2. Re-run the db migrations job `kubectl apply -f https://raw.githubusercontent.com/jcwimer/wrestlingApp/master/deploy/kubernetes/manifests/db-migration.yaml`

+## How do I see logs?
+
+For workers: `kubectl logs -f --tail=100 -l app=wrestlingdev -l tier=worker`
+For app logs: `kubectl logs -f --tail=100 -l app=wrestlingdev -l tier=app`
+
 ## I'm a pro. What's bad about this?
 Right now, mariadb's root password comes from the secrets.yaml and wrestlingdev uses the root password to run. Ideally, you'd create another secret for mariadb's root password and you'd create a user specifically for wrestlingdev.
 From a mysql shell> `CREATE USER ${username} IDENTIFIED BY '${password}'; GRANT ALL PRIVILEGES ON  ${database}.* TO ${username}; FLUSH PRIVILEGES;` $database would be wrestlingdev. I'll do this automatically later.
--- a/deploy/kubernetes/manifests/influxdb.yaml
+++ b/deploy/kubernetes/manifests/influxdb.yaml
@@ -46,12 +46,12 @@ spec:
        volumeMounts:
        - name: wrestlingdev-influxdb-persistent-storage
          mountPath: /var/lib/influxdb
-        resources:
-          limits:
-            memory: "512Mi"
-          requests:
-            memory: "128Mi"
-            cpu: "0.5"
+        # resources:
+        #   limits:
+        #     memory: "512Mi"
+        #   requests:
+        #     memory: "128Mi"
+        #     cpu: "0.5"
      volumes:
      - name: wrestlingdev-influxdb-persistent-storage
        persistentVolumeClaim:
@@ -65,9 +65,9 @@ metadata:
    app: wrestlingdev
    tier: metrics
 spec:
-  storageClassName: standard
+  # storageClassName: standard
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
-      storage: 100Gi         
+      storage: 20Gi
--- a/deploy/kubernetes/manifests/mariadb-standalone.yaml
+++ b/deploy/kubernetes/manifests/mariadb-standalone.yaml
@@ -19,7 +19,7 @@ metadata:
  labels:
    app: wrestlingdev
 spec:
-  storageClassName: standard
+  # storageClassName: standard
  accessModes:
  - ReadWriteOnce
  resources:
@@ -62,12 +62,12 @@ spec:
        volumeMounts:
        - name: wrestlingdev-mariadb-persistent-storage
          mountPath: /var/lib/mysql
-        resources:
-          limits:
-            memory: "512Mi"
-          requests:
-            memory: "256Mi"
-            cpu: "0.2"
+        # resources:
+        #   limits:
+        #     memory: "512Mi"
+        #   requests:
+        #     memory: "256Mi"
+        #     cpu: "0.2"
      - image: jcwimer/mariadb-rclone-backup-docker:10.3
        name: mariadb-backup
        env:
@@ -77,7 +77,7 @@ spec:
             name: wrestlingdev-secrets
             key: dbpassword
        - name: CRON_SCHEDULE
-          value: "@hourly"  # hourly
+          value: "*/5 * * * *"  # every 5 minutes
        - name: DB_USERNAME
          valueFrom:
           secretKeyRef:
@@ -125,17 +125,17 @@ spec:
        volumeMounts:
        - name: wrestlingdev-mariadb-persistent-storage
          mountPath: /var/lib/mysql
-        resources:
-          limits:
-            memory: "100Mi"
-          requests:
-            memory: "50Mi"
-            cpu: "0.1"
+        # resources:
+        #   limits:
+        #     memory: "100Mi"
+        #   requests:
+        #     memory: "50Mi"
+        #     cpu: "0.1"
      - image: prom/mysqld-exporter:v0.11.0
        name: mariadb-exporter
        ports:
        - containerPort: 9125
-          name: http"
+          name: "http"
        args:
        - --web.listen-address=0.0.0.0:9125
        - --web.telemetry-path=/metrics
@@ -158,12 +158,12 @@ spec:
          value: $(DB_USERNAME):$(DB_PASSWORD)@($(DB_HOST))/
        - name: DAYS_TO_KEEP
          value: "7"
-        resources:
-          limits:
-            memory: "128Mi"
-          requests:
-            memory: "32Mi"
-            cpu: "10m"
+        # resources:
+        #   limits:
+        #     memory: "128Mi"
+        #   requests:
+        #     memory: "32Mi"
+        #     cpu: "10m"
        livenessProbe:
          httpGet:
            path: /metrics
--- a/deploy/kubernetes/manifests/memcached-standalone.yaml
+++ b/deploy/kubernetes/manifests/memcached-standalone.yaml
@@ -37,6 +37,6 @@ spec:
        resources:
          limits:
            memory: "64Mi"
-          requests:
-            memory: "64Mi"
-            cpu: "0.1"
+          # requests:
+          #   memory: "64Mi"
+          #   cpu: "0.1"
--- a/deploy/kubernetes/manifests/wrestlingdev.yaml
+++ b/deploy/kubernetes/manifests/wrestlingdev.yaml
@@ -41,7 +41,10 @@ spec:
        - name: RAILS_ENV
          value: production
        - name: PASSENGER_POOL_SIZE
-          value: "2"
+          valueFrom:
+           secretKeyRef:
+             name: wrestlingdev-secrets
+             key: passenger_pool_size
        - name: WRESTLINGDEV_DB_NAME
          value: wrestlingdev
        - name: WRESTLINGDEV_DB_USR
@@ -98,12 +101,12 @@ spec:
           secretKeyRef:
             name: wrestlingdev-secrets
             key: influxdb_port
-        resources:
-          limits:
-            memory: "768Mi"
-          requests:
-            memory: "300Mi"
-            cpu: "0.3"
+        # resources:
+        #   limits:
+        #     memory: "768Mi"
+        #   requests:
+        #     memory: "300Mi"
+        #     cpu: "0.3"
        livenessProbe:
          httpGet:
            path: /
@@ -111,23 +114,23 @@ spec:
          initialDelaySeconds: 180
          periodSeconds: 20
          timeoutSeconds: 10
---
-apiVersion: autoscaling/v2beta1
-kind: HorizontalPodAutoscaler
-metadata:
-  name: wrestlingdev-app-deployment-autoscale
-spec:
-  scaleTargetRef:
-    apiVersion: apps/v1
-    kind: Deployment
-    name: wrestlingdev-app-deployment
-  minReplicas: 2
-  maxReplicas: 5
-  metrics:
-  - type: Resource
-    resource:
-      name: cpu
-      targetAverageUtilization: 75
+# ---
+# apiVersion: autoscaling/v2beta1
+# kind: HorizontalPodAutoscaler
+# metadata:
+#   name: wrestlingdev-app-deployment-autoscale
+# spec:
+#   scaleTargetRef:
+#     apiVersion: apps/v1
+#     kind: Deployment
+#     name: wrestlingdev-app-deployment
+#   minReplicas: 2
+#   maxReplicas: 5
+#   metrics:
+#   - type: Resource
+#     resource:
+#       name: cpu
+#       targetAverageUtilization: 75
  # - type: Resource
  #   resource:
  #     name: memory
@@ -140,7 +143,7 @@ metadata:
  labels:
    app: wrestlingdev
 spec:
-  replicas: 1
+  replicas: 2
  selector:
    matchLabels:
      app: wrestlingdev
@@ -216,10 +219,10 @@ spec:
             key: influxdb_port
        command: ["bundle"]
        args: ["exec", "bin/delayed_job", "-n", "1", "run"]
-        resources:
-          limits:
-            memory: "512Mi"
-          requests:
-            memory: "200Mi"
-            cpu: "0.2"
+        # resources:
+        #   limits:
+        #     memory: "512Mi"
+        #   requests:
+        #     memory: "200Mi"
+        #     cpu: "0.2"

--- a/deploy/kubernetes/secrets/secrets.yaml
+++ b/deploy/kubernetes/secrets/secrets.yaml
@@ -11,8 +11,9 @@ stringData:
  dbhost: wrestlingdev-mariadb # if using the mariadb-standalone from this repo, otherwise use your hostname for your database
  devisesecretkey: PUT_SECRET_HERE # get from running rails secret
  railssecretkey: PUT_SECRET_HERE # get from running rails secret
-  emailpassword: PUT_EMAIL_PASSWORD_HERE # gmail password
+  gmailpassword: PUT_EMAIL_PASSWORD_HERE # gmail password
  gmailemail: PUT EMAIL ADDRESS HERE
+  passenger_pool_size: "2"
  # OPTIONAL
  # DELETE THESE LINES IF YOU'RE NOT USING THEM
  influxdb_database: PUT INFLUXDB DATABASE NAME HERE