From 865b475bfce5c490f7e311f1663c29a58c8d7050 Mon Sep 17 00:00:00 2001 From: erik Date: Wed, 27 Jul 2022 16:48:04 +0200 Subject: [PATCH] Configurable storage sizes Storage sizes can now be configured. Due to limitations of yaml/load-as-edn volumes.yaml was split into datavolume.yaml and rootvolume.yaml. This way, we are able to follow volume size recommendations described in jira. Valid config was updated. Tests were updated. --- src/main/cljc/dda/c4k_gitea/core.cljc | 6 +-- src/main/cljc/dda/c4k_gitea/gitea.cljc | 56 ++++++++++++++++++--- src/main/resources/gitea/datavolume.yaml | 15 ++++++ src/main/resources/gitea/rootvolume.yaml | 14 ++++++ src/main/resources/gitea/volumes.yaml | 30 ----------- src/test/cljc/dda/c4k_gitea/gitea_test.cljc | 11 ++++ valid-config.edn | 1 + 7 files changed, 93 insertions(+), 40 deletions(-) create mode 100644 src/main/resources/gitea/datavolume.yaml create mode 100644 src/main/resources/gitea/rootvolume.yaml delete mode 100644 src/main/resources/gitea/volumes.yaml diff --git a/src/main/cljc/dda/c4k_gitea/core.cljc b/src/main/cljc/dda/c4k_gitea/core.cljc index 492539c..6584d14 100644 --- a/src/main/cljc/dda/c4k_gitea/core.cljc +++ b/src/main/cljc/dda/c4k_gitea/core.cljc @@ -7,11 +7,10 @@ (defn k8s-objects [config] (let [storage-class (if (contains? config :postgres-data-volume-path) :manual :local-path)] - (cm/concat-vec - [(yaml/load-resource "gitea/volumes.yaml") + (cm/concat-vec (yaml/load-resource "gitea/deployment.yaml") (yaml/load-resource "gitea/services.yaml") - (yaml/load-resource "gitea/traefik-middleware.yaml")] + (yaml/load-resource "gitea/traefik-middleware.yaml") (map yaml/to-string [(postgres/generate-config {:postgres-size :2gb :db-name "gitea"}) @@ -23,6 +22,7 @@ (postgres/generate-deployment {:postgres-image "postgres:14" :postgres-size :2gb}) (postgres/generate-service) + (gitea/generate-volumes config) (gitea/generate-appini-env config) (gitea/generate-secrets config) (gitea/generate-ingress config) diff --git a/src/main/cljc/dda/c4k_gitea/gitea.cljc b/src/main/cljc/dda/c4k_gitea/gitea.cljc index daa9345..6626e85 100644 --- a/src/main/cljc/dda/c4k_gitea/gitea.cljc +++ b/src/main/cljc/dda/c4k_gitea/gitea.cljc @@ -1,8 +1,6 @@ (ns dda.c4k-gitea.gitea (:require - [clojure.spec.alpha :as s] - [clojure.string :as str] - [clojure.core :as c] + [clojure.spec.alpha :as s] #?(:cljs [shadow.resource :as rc]) #?(:clj [orchestra.core :refer [defn-spec]] :cljs [orchestra.core :refer-macros [defn-spec]]) @@ -23,14 +21,38 @@ (s/def ::mailer-user pred/bash-env-string?) (s/def ::mailer-pw pred/bash-env-string?) (s/def ::issuer pred/letsencrypt-issuer?) +(s/def ::volume-total-storage-size int?) ;TODO extend this for checking lower size limits (def config-defaults {:issuer "staging"}) -(def config? (s/keys :req-un [::fqdn ::mailer-from ::mailer-host-port ::service-noreply-address] - :opt-un [::issuer ::default-app-name ::service-domain-whitelist])) +(def config? (s/keys :req-un [::fqdn + ::mailer-from + ::mailer-host-port + ::service-noreply-address] + :opt-un [::issuer + ::default-app-name + ::service-domain-whitelist])) (def auth? (s/keys :req-un [::postgres/postgres-db-user ::postgres/postgres-db-password ::mailer-user ::mailer-pw])) +(def vol? (s/keys :req-un [::volume-total-storage-size])) + +(defn root-storage-by-volume-size + [in] + (cond + (<= in 5) (throw (Exception. "Volume smaller or equal 5Gi!\nIncrease volume-total-storage-size to value > 5")) + (and (> in 5) (<= in 20)) 5 + (and (> in 20) (<= in 100)) 10 + (> in 100) 20)) + +(defn data-storage-by-volume-size + [total root] + (cond + (and (<= total 20) (> total 5)) (- total root) + (and (<= total 100) (> total 20)) (- total root) + (> total 100) (- total root))) + + #?(:cljs (defmethod yaml/load-resource :gitea [resource-name] (case resource-name @@ -41,7 +63,8 @@ "gitea/secrets.yaml" (rc/inline "gitea/secrets.yaml") "gitea/services.yaml" (rc/inline "gitea/services.yaml") "gitea/traefik-middleware.yaml" (rc/inline "gitea/traefik-middleware.yaml") - "gitea/volumes.yaml" (rc/inline "gitea/volumes.yaml") + "gitea/rootvolume.yaml" (rc/inline "gitea/rootvolume.yaml") + "gitea/datavolume.yaml" (rc/inline "gitea/datavolume.yaml") (throw (js/Error. "Undefined Resource!"))))) #?(:cljs @@ -71,7 +94,10 @@ (defn-spec generate-secrets pred/map-or-seq? [auth auth?] - (let [{:keys [postgres-db-user postgres-db-password mailer-user mailer-pw]} auth] + (let [{:keys [postgres-db-user + postgres-db-password + mailer-user + mailer-pw]} auth] (-> (yaml/load-as-edn "gitea/secrets.yaml") (cm/replace-all-matching-values-by-new-value "DBUSER" (b64/encode postgres-db-user)) @@ -96,3 +122,19 @@ (assoc-in [:spec :issuerRef :name] letsencrypt-issuer) (cm/replace-all-matching-values-by-new-value "FQDN" fqdn)))) +(defn-spec generate-root-volume pred/map-or-seq? + [config vol?] + (let [{:keys [volume-total-storage-size]} config + root-storage-size (root-storage-by-volume-size volume-total-storage-size)] + (-> + (yaml/load-as-edn "gitea/rootvolume.yaml") + (cm/replace-all-matching-values-by-new-value "ROOTSTORAGESIZE" (str (str root-storage-size) "Gi"))))) + +(defn-spec generate-data-volume pred/map-or-seq? + [config vol?] + (let [{:keys [volume-total-storage-size]} config + root-storage-size (root-storage-by-volume-size volume-total-storage-size) + data-storage-size (data-storage-by-volume-size volume-total-storage-size root-storage-size)] + (-> + (yaml/load-as-edn "gitea/datavolume.yaml") + (cm/replace-all-matching-values-by-new-value "DATASTORAGESIZE" (str (str data-storage-size) "Gi"))))) \ No newline at end of file diff --git a/src/main/resources/gitea/datavolume.yaml b/src/main/resources/gitea/datavolume.yaml new file mode 100644 index 0000000..75f825b --- /dev/null +++ b/src/main/resources/gitea/datavolume.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: gitea-data-pvc + namespace: default + labels: + app: gitea +spec: + storageClassName: local-path + accessModes: + - ReadWriteOnce + resources: + requests: + storage: DATASTORAGESIZE + \ No newline at end of file diff --git a/src/main/resources/gitea/rootvolume.yaml b/src/main/resources/gitea/rootvolume.yaml new file mode 100644 index 0000000..19c5fe7 --- /dev/null +++ b/src/main/resources/gitea/rootvolume.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: gitea-root-pvc + namespace: default + labels: + app: gitea +spec: + storageClassName: local-path + accessModes: + - ReadWriteOnce + resources: + requests: + storage: ROOTSTORAGESIZE diff --git a/src/main/resources/gitea/volumes.yaml b/src/main/resources/gitea/volumes.yaml deleted file mode 100644 index c491cea..0000000 --- a/src/main/resources/gitea/volumes.yaml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: gitea-root-pvc - namespace: default - labels: - app: gitea -spec: - storageClassName: local-path - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: gitea-data-pvc - namespace: default - labels: - app: gitea -spec: - storageClassName: local-path - accessModes: - - ReadWriteOnce - resources: - requests: - # TODO: Review jem - 2022/07/26 - should be configurable & as large as posible ... - storage: 50Gi diff --git a/src/test/cljc/dda/c4k_gitea/gitea_test.cljc b/src/test/cljc/dda/c4k_gitea/gitea_test.cljc index fffdc22..b9f0dcd 100644 --- a/src/test/cljc/dda/c4k_gitea/gitea_test.cljc +++ b/src/test/cljc/dda/c4k_gitea/gitea_test.cljc @@ -66,3 +66,14 @@ :mailer-user "maileruser" :mailer-pw "mailerpw"}))))) +(deftest should-generate-root-volume + (is (= {:storage-c1 "5Gi", + :storage-c2 "20Gi"} + (ct/map-diff (cut/generate-root-volume {:volume-total-storage-size 6}) + (cut/generate-root-volume {:volume-total-storage-size 101}))))) + +(deftest should-generate-data-volume + (is (= {:storage-c1 "1Gi", + :storage-c2 "15Gi"} + (ct/map-diff (cut/generate-data-volume {:volume-total-storage-size 6}) + (cut/generate-data-volume {:volume-total-storage-size 20}))))) \ No newline at end of file diff --git a/valid-config.edn b/valid-config.edn index 4874c80..cf37f4d 100644 --- a/valid-config.edn +++ b/valid-config.edn @@ -6,4 +6,5 @@ :mailer-host-port "test.de:123" :service-whitelist-domains "test.de" :service-noreply-address "noreply@test.de" +:volume-total-storage-size 6 }