From 92bd1ec3c909de63403397e9699880f73ac31d9c Mon Sep 17 00:00:00 2001 From: Jacob Cody Wimer Date: Wed, 1 Mar 2023 21:58:24 +0000 Subject: [PATCH] Fixed yaml error in k8s manifest, added a passenger pool size secret, and added multiple workers --- deploy/kubernetes/README.md | 15 +++-- deploy/kubernetes/manifests/influxdb.yaml | 16 ++--- .../manifests/mariadb-standalone.yaml | 42 ++++++------ .../manifests/memcached-standalone.yaml | 6 +- deploy/kubernetes/manifests/wrestlingdev.yaml | 65 ++++++++++--------- deploy/kubernetes/secrets/secrets.yaml | 3 +- 6 files changed, 78 insertions(+), 69 deletions(-) diff --git a/deploy/kubernetes/README.md b/deploy/kubernetes/README.md index 08e405d..ababaa0 100644 --- a/deploy/kubernetes/README.md +++ b/deploy/kubernetes/README.md @@ -11,10 +11,10 @@ 4. Run `kubectl apply -f deploy/kubernetes/manifests/` ## What do I get? -1. Wrestlingdev deployed with 2 replicas. Autoscaling is turned on up to 4 replcias. -2. A standalone mariadb that can back up to S3 compatable storage if you set the values in `deploy/kubernetes/secrets/secrets.yaml` and prometheus ready metrics -3. A standalone memcahced. -4. A single job runner to run wrestlingdev background jobs. +1. Wrestlingdev deployed with 2 replicas. +2. Two workers are deployed to run background jobs +3. A standalone mariadb that can back up to S3 compatable storage if you set the values in `deploy/kubernetes/secrets/secrets.yaml` and prometheus ready metrics +4. A standalone memcahced. ## How do I update the app? First, be sure your secrets.yaml has all envs up to date. Then, make sure you get all manifest changes @@ -23,12 +23,17 @@ First, be sure your secrets.yaml has all envs up to date. Then, make sure you ge Each push to master updates the docker `prod` tag and also pushes a tag with the git hash. You will want to update to those tags. 1. Set the git hash as a variable `TAG=$(git rev-parse --verify HEAD)` 2. Update the wrestlingdev deployment tag `kubectl --record deployment.apps/wrestlingdev-app-deployment set image deployment.v1.apps/wrestlingdev-app-deployment wrestlingdev-app=jcwimer/wrestlingdev:${TAG}` -3. Update the wrestlingdev job runner tag `kubectl --record deployment.apps/wrestlingdev-worker-deployment set image deployment.v1.apps/wrestlingdev-worker-deployment wrestlingdev-worker=jcwimer/wrestlingdev:${TAG}` +3. Update the wrestlingdev job runner tag `kubectl --record statefulset.apps/wrestlingdev-worker set image statefulset.v1.apps/wrestlingdev-worker wrestlingdev-worker=jcwimer/wrestlingdev:${TAG}` Finally, run db-migrations 1. Delete the db migrations job so you can re-run it `kubectl delete job wrestlingdev-db-create-migrate` 2. Re-run the db migrations job `kubectl apply -f https://raw.githubusercontent.com/jcwimer/wrestlingApp/master/deploy/kubernetes/manifests/db-migration.yaml` +## How do I see logs? + +For workers: `kubectl logs -f --tail=100 -l app=wrestlingdev -l tier=worker` +For app logs: `kubectl logs -f --tail=100 -l app=wrestlingdev -l tier=app` + ## I'm a pro. What's bad about this? Right now, mariadb's root password comes from the secrets.yaml and wrestlingdev uses the root password to run. Ideally, you'd create another secret for mariadb's root password and you'd create a user specifically for wrestlingdev. From a mysql shell> `CREATE USER ${username} IDENTIFIED BY '${password}'; GRANT ALL PRIVILEGES ON ${database}.* TO ${username}; FLUSH PRIVILEGES;` $database would be wrestlingdev. I'll do this automatically later. diff --git a/deploy/kubernetes/manifests/influxdb.yaml b/deploy/kubernetes/manifests/influxdb.yaml index f4a8377..53730d1 100644 --- a/deploy/kubernetes/manifests/influxdb.yaml +++ b/deploy/kubernetes/manifests/influxdb.yaml @@ -46,12 +46,12 @@ spec: volumeMounts: - name: wrestlingdev-influxdb-persistent-storage mountPath: /var/lib/influxdb - resources: - limits: - memory: "512Mi" - requests: - memory: "128Mi" - cpu: "0.5" + # resources: + # limits: + # memory: "512Mi" + # requests: + # memory: "128Mi" + # cpu: "0.5" volumes: - name: wrestlingdev-influxdb-persistent-storage persistentVolumeClaim: @@ -65,9 +65,9 @@ metadata: app: wrestlingdev tier: metrics spec: - storageClassName: standard + # storageClassName: standard accessModes: - ReadWriteOnce resources: requests: - storage: 100Gi \ No newline at end of file + storage: 20Gi \ No newline at end of file diff --git a/deploy/kubernetes/manifests/mariadb-standalone.yaml b/deploy/kubernetes/manifests/mariadb-standalone.yaml index 358afce..0e1d8cb 100644 --- a/deploy/kubernetes/manifests/mariadb-standalone.yaml +++ b/deploy/kubernetes/manifests/mariadb-standalone.yaml @@ -19,7 +19,7 @@ metadata: labels: app: wrestlingdev spec: - storageClassName: standard + # storageClassName: standard accessModes: - ReadWriteOnce resources: @@ -62,12 +62,12 @@ spec: volumeMounts: - name: wrestlingdev-mariadb-persistent-storage mountPath: /var/lib/mysql - resources: - limits: - memory: "512Mi" - requests: - memory: "256Mi" - cpu: "0.2" + # resources: + # limits: + # memory: "512Mi" + # requests: + # memory: "256Mi" + # cpu: "0.2" - image: jcwimer/mariadb-rclone-backup-docker:10.3 name: mariadb-backup env: @@ -77,7 +77,7 @@ spec: name: wrestlingdev-secrets key: dbpassword - name: CRON_SCHEDULE - value: "@hourly" # hourly + value: "*/5 * * * *" # every 5 minutes - name: DB_USERNAME valueFrom: secretKeyRef: @@ -125,17 +125,17 @@ spec: volumeMounts: - name: wrestlingdev-mariadb-persistent-storage mountPath: /var/lib/mysql - resources: - limits: - memory: "100Mi" - requests: - memory: "50Mi" - cpu: "0.1" + # resources: + # limits: + # memory: "100Mi" + # requests: + # memory: "50Mi" + # cpu: "0.1" - image: prom/mysqld-exporter:v0.11.0 name: mariadb-exporter ports: - containerPort: 9125 - name: http" + name: "http" args: - --web.listen-address=0.0.0.0:9125 - --web.telemetry-path=/metrics @@ -158,12 +158,12 @@ spec: value: $(DB_USERNAME):$(DB_PASSWORD)@($(DB_HOST))/ - name: DAYS_TO_KEEP value: "7" - resources: - limits: - memory: "128Mi" - requests: - memory: "32Mi" - cpu: "10m" + # resources: + # limits: + # memory: "128Mi" + # requests: + # memory: "32Mi" + # cpu: "10m" livenessProbe: httpGet: path: /metrics diff --git a/deploy/kubernetes/manifests/memcached-standalone.yaml b/deploy/kubernetes/manifests/memcached-standalone.yaml index 91803e9..bcb0ffa 100644 --- a/deploy/kubernetes/manifests/memcached-standalone.yaml +++ b/deploy/kubernetes/manifests/memcached-standalone.yaml @@ -37,6 +37,6 @@ spec: resources: limits: memory: "64Mi" - requests: - memory: "64Mi" - cpu: "0.1" + # requests: + # memory: "64Mi" + # cpu: "0.1" diff --git a/deploy/kubernetes/manifests/wrestlingdev.yaml b/deploy/kubernetes/manifests/wrestlingdev.yaml index 815c462..7327d69 100644 --- a/deploy/kubernetes/manifests/wrestlingdev.yaml +++ b/deploy/kubernetes/manifests/wrestlingdev.yaml @@ -41,7 +41,10 @@ spec: - name: RAILS_ENV value: production - name: PASSENGER_POOL_SIZE - value: "2" + valueFrom: + secretKeyRef: + name: wrestlingdev-secrets + key: passenger_pool_size - name: WRESTLINGDEV_DB_NAME value: wrestlingdev - name: WRESTLINGDEV_DB_USR @@ -98,12 +101,12 @@ spec: secretKeyRef: name: wrestlingdev-secrets key: influxdb_port - resources: - limits: - memory: "768Mi" - requests: - memory: "300Mi" - cpu: "0.3" + # resources: + # limits: + # memory: "768Mi" + # requests: + # memory: "300Mi" + # cpu: "0.3" livenessProbe: httpGet: path: / @@ -111,23 +114,23 @@ spec: initialDelaySeconds: 180 periodSeconds: 20 timeoutSeconds: 10 ---- -apiVersion: autoscaling/v2beta1 -kind: HorizontalPodAutoscaler -metadata: - name: wrestlingdev-app-deployment-autoscale -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: wrestlingdev-app-deployment - minReplicas: 2 - maxReplicas: 5 - metrics: - - type: Resource - resource: - name: cpu - targetAverageUtilization: 75 +# --- +# apiVersion: autoscaling/v2beta1 +# kind: HorizontalPodAutoscaler +# metadata: +# name: wrestlingdev-app-deployment-autoscale +# spec: +# scaleTargetRef: +# apiVersion: apps/v1 +# kind: Deployment +# name: wrestlingdev-app-deployment +# minReplicas: 2 +# maxReplicas: 5 +# metrics: +# - type: Resource +# resource: +# name: cpu +# targetAverageUtilization: 75 # - type: Resource # resource: # name: memory @@ -140,7 +143,7 @@ metadata: labels: app: wrestlingdev spec: - replicas: 1 + replicas: 2 selector: matchLabels: app: wrestlingdev @@ -216,10 +219,10 @@ spec: key: influxdb_port command: ["bundle"] args: ["exec", "bin/delayed_job", "-n", "1", "run"] - resources: - limits: - memory: "512Mi" - requests: - memory: "200Mi" - cpu: "0.2" + # resources: + # limits: + # memory: "512Mi" + # requests: + # memory: "200Mi" + # cpu: "0.2" diff --git a/deploy/kubernetes/secrets/secrets.yaml b/deploy/kubernetes/secrets/secrets.yaml index bf9f796..37c3b31 100644 --- a/deploy/kubernetes/secrets/secrets.yaml +++ b/deploy/kubernetes/secrets/secrets.yaml @@ -11,8 +11,9 @@ stringData: dbhost: wrestlingdev-mariadb # if using the mariadb-standalone from this repo, otherwise use your hostname for your database devisesecretkey: PUT_SECRET_HERE # get from running rails secret railssecretkey: PUT_SECRET_HERE # get from running rails secret - emailpassword: PUT_EMAIL_PASSWORD_HERE # gmail password + gmailpassword: PUT_EMAIL_PASSWORD_HERE # gmail password gmailemail: PUT EMAIL ADDRESS HERE + passenger_pool_size: "2" # OPTIONAL # DELETE THESE LINES IF YOU'RE NOT USING THEM influxdb_database: PUT INFLUXDB DATABASE NAME HERE