From 1ed850aea243de21198d89bc8528db075f7a1294 Mon Sep 17 00:00:00 2001 From: erik Date: Tue, 16 Jan 2024 15:18:18 +0100 Subject: [PATCH 01/22] Initial rate limit middleware --- src/main/resources/forgejo/middleware-ratelimit.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 src/main/resources/forgejo/middleware-ratelimit.yaml diff --git a/src/main/resources/forgejo/middleware-ratelimit.yaml b/src/main/resources/forgejo/middleware-ratelimit.yaml new file mode 100644 index 0000000..4c614e4 --- /dev/null +++ b/src/main/resources/forgejo/middleware-ratelimit.yaml @@ -0,0 +1,10 @@ +# Here, an average of 100 requests per second is allowed. +# In addition, a burst of 50 requests is allowed. +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: ratelimit +spec: + rateLimit: # ToDo: Config options for rate limiting: https://doc.traefik.io/traefik/middlewares/http/ratelimit/ + average: 100 + burst: 50 \ No newline at end of file From c5e777c9c5ea4f863be08edd148ff7d76f932b2a Mon Sep 17 00:00:00 2001 From: erik Date: Tue, 16 Jan 2024 15:44:10 +0100 Subject: [PATCH 02/22] WIP: Add defn-spec for rate-limiting ingress --- src/main/cljc/dda/c4k_forgejo/forgejo.cljc | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/main/cljc/dda/c4k_forgejo/forgejo.cljc b/src/main/cljc/dda/c4k_forgejo/forgejo.cljc index 23b370e..0953f98 100644 --- a/src/main/cljc/dda/c4k_forgejo/forgejo.cljc +++ b/src/main/cljc/dda/c4k_forgejo/forgejo.cljc @@ -119,6 +119,18 @@ :fqdns [fqdn]} config)))) +(defn-spec generate-rate-limit-ingress-and-cert pred/map-or-seq? + [config config?] + (let [{:keys [fqdn average burst period]} config] + (-> + (generate-ingress-and-cert config) + (#(cm/replace-key-value % + :traefik.ingress.kubernetes.io/router.middlewares + (str + (:traefik.ingress.kubernetes.io/router.middlewares + (:annotations (:metadata %))) + ", default-ratelimit@kubernetescrd")))))) + (defn-spec generate-data-volume pred/map-or-seq? [config vol?] (let [{:keys [volume-total-storage-size]} config From 8a3194e7151d4627035752bec9fe9d9a2e9baebb Mon Sep 17 00:00:00 2001 From: erik Date: Tue, 16 Jan 2024 15:50:08 +0100 Subject: [PATCH 03/22] Add ToDo --- src/main/cljc/dda/c4k_forgejo/forgejo.cljc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/cljc/dda/c4k_forgejo/forgejo.cljc b/src/main/cljc/dda/c4k_forgejo/forgejo.cljc index 0953f98..fc55e81 100644 --- a/src/main/cljc/dda/c4k_forgejo/forgejo.cljc +++ b/src/main/cljc/dda/c4k_forgejo/forgejo.cljc @@ -129,7 +129,7 @@ (str (:traefik.ingress.kubernetes.io/router.middlewares (:annotations (:metadata %))) - ", default-ratelimit@kubernetescrd")))))) + ", default-ratelimit@kubernetescrd")))))) ; ToDo: Rate Limit Konfig Optionen (defn-spec generate-data-volume pred/map-or-seq? [config vol?] From 220eb337f903f5aa0f410df5b39726d0556ef168 Mon Sep 17 00:00:00 2001 From: erik Date: Wed, 17 Jan 2024 11:35:20 +0100 Subject: [PATCH 04/22] No default values for optional rate limiting --- src/main/resources/forgejo/middleware-ratelimit.yaml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/main/resources/forgejo/middleware-ratelimit.yaml b/src/main/resources/forgejo/middleware-ratelimit.yaml index 4c614e4..25184cd 100644 --- a/src/main/resources/forgejo/middleware-ratelimit.yaml +++ b/src/main/resources/forgejo/middleware-ratelimit.yaml @@ -1,10 +1,8 @@ -# Here, an average of 100 requests per second is allowed. -# In addition, a burst of 50 requests is allowed. apiVersion: traefik.io/v1alpha1 kind: Middleware metadata: name: ratelimit spec: - rateLimit: # ToDo: Config options for rate limiting: https://doc.traefik.io/traefik/middlewares/http/ratelimit/ - average: 100 - burst: 50 \ No newline at end of file + rateLimit: # Config options for rate limiting: https://doc.traefik.io/traefik/middlewares/http/ratelimit/ + average: AVG + burst: BRS \ No newline at end of file From a63f170ace42dc16ec8d7b7b970056729c1501ba Mon Sep 17 00:00:00 2001 From: erik Date: Wed, 17 Jan 2024 11:36:43 +0100 Subject: [PATCH 05/22] Generate ingress with rate limit conditionally --- src/main/cljc/dda/c4k_forgejo/core.cljc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/cljc/dda/c4k_forgejo/core.cljc b/src/main/cljc/dda/c4k_forgejo/core.cljc index 356751f..ba9e12a 100644 --- a/src/main/cljc/dda/c4k_forgejo/core.cljc +++ b/src/main/cljc/dda/c4k_forgejo/core.cljc @@ -49,8 +49,11 @@ (forgejo/generate-service-ssh) (forgejo/generate-data-volume config) (forgejo/generate-appini-env config) - (forgejo/generate-secrets auth)] - (forgejo/generate-ingress-and-cert config) + (forgejo/generate-secrets auth)] + (if (contains? config :average) + (do (forgejo/generate-rate-limit-ingress-and-cert config) + (forgejo/generate-rate-limit-middleware config)) + (forgejo/generate-ingress-and-cert config)) (when (contains? config :restic-repository) [(backup/generate-config config) (backup/generate-secret auth) From 52e43fe23c9dd5c5ddfe12592d01787b82600b46 Mon Sep 17 00:00:00 2001 From: erik Date: Wed, 17 Jan 2024 11:37:31 +0100 Subject: [PATCH 06/22] Add specs for rate limit options --- src/main/cljc/dda/c4k_forgejo/forgejo.cljc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/cljc/dda/c4k_forgejo/forgejo.cljc b/src/main/cljc/dda/c4k_forgejo/forgejo.cljc index fc55e81..515d370 100644 --- a/src/main/cljc/dda/c4k_forgejo/forgejo.cljc +++ b/src/main/cljc/dda/c4k_forgejo/forgejo.cljc @@ -42,6 +42,8 @@ (s/def ::mailer-pw pred/bash-env-string?) (s/def ::issuer pred/letsencrypt-issuer?) (s/def ::volume-total-storage-size (partial pred/int-gt-n? 5)) +(s/def ::average int?) +(s/def ::burst int?) (def config? (s/keys :req-un [::fqdn ::mailer-from @@ -51,7 +53,9 @@ :opt-un [::issuer ::deploy-federated ::default-app-name - ::service-domain-whitelist])) + ::service-domain-whitelist + ::average + ::burst])) (def auth? (s/keys :req-un [::postgres/postgres-db-user ::postgres/postgres-db-password ::mailer-user ::mailer-pw])) From 13e718ca37b02a83a1d8e2731f1ca98e7af1aa1e Mon Sep 17 00:00:00 2001 From: erik Date: Wed, 17 Jan 2024 11:40:58 +0100 Subject: [PATCH 07/22] Implement rate limit ingress --- src/main/cljc/dda/c4k_forgejo/forgejo.cljc | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/main/cljc/dda/c4k_forgejo/forgejo.cljc b/src/main/cljc/dda/c4k_forgejo/forgejo.cljc index 515d370..1368396 100644 --- a/src/main/cljc/dda/c4k_forgejo/forgejo.cljc +++ b/src/main/cljc/dda/c4k_forgejo/forgejo.cljc @@ -125,15 +125,13 @@ (defn-spec generate-rate-limit-ingress-and-cert pred/map-or-seq? [config config?] - (let [{:keys [fqdn average burst period]} config] - (-> - (generate-ingress-and-cert config) - (#(cm/replace-key-value % - :traefik.ingress.kubernetes.io/router.middlewares - (str - (:traefik.ingress.kubernetes.io/router.middlewares - (:annotations (:metadata %))) - ", default-ratelimit@kubernetescrd")))))) ; ToDo: Rate Limit Konfig Optionen + (-> + (generate-ingress-and-cert config) ; returns a vector + (#(assoc-in % ; Attention: heavily relying on the output order of ing/generate-ingress-and-cert + [1 :metadata :annotations :traefik.ingress.kubernetes.io/router.middlewares] + (str + (-> (second %) :metadata :annotations :traefik.ingress.kubernetes.io/router.middlewares) + ", default-ratelimit@kubernetescrd"))))) (defn-spec generate-data-volume pred/map-or-seq? [config vol?] From 2a6b6ccf3f58e1c70002fc9144ccda0103caf71b Mon Sep 17 00:00:00 2001 From: erik Date: Wed, 17 Jan 2024 11:43:15 +0100 Subject: [PATCH 08/22] Implement rate limit middleware --- src/main/cljc/dda/c4k_forgejo/forgejo.cljc | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/main/cljc/dda/c4k_forgejo/forgejo.cljc b/src/main/cljc/dda/c4k_forgejo/forgejo.cljc index 1368396..8c9249e 100644 --- a/src/main/cljc/dda/c4k_forgejo/forgejo.cljc +++ b/src/main/cljc/dda/c4k_forgejo/forgejo.cljc @@ -133,6 +133,17 @@ (-> (second %) :metadata :annotations :traefik.ingress.kubernetes.io/router.middlewares) ", default-ratelimit@kubernetescrd"))))) + +; using :average and :burst seems sensible, :period may be interesting for fine tuning later on +(defn-spec generate-rate-limit-middleware pred/map-or-seq? + [config config?] + (let [{:keys [average burst]} config] + (-> + (yaml/load-as-edn "forgejo/middleware-ratelimit.yaml") + (cm/replace-key-value :average average) + (cm/replace-key-value :burst burst) + ))) + (defn-spec generate-data-volume pred/map-or-seq? [config vol?] (let [{:keys [volume-total-storage-size]} config From 054e6954af9ff90e159227f2bdb21adfa7c6bfab Mon Sep 17 00:00:00 2001 From: erik Date: Wed, 17 Jan 2024 11:43:32 +0100 Subject: [PATCH 09/22] Implement tests --- .../cljc/dda/c4k_forgejo/forgejo_test.cljc | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/test/cljc/dda/c4k_forgejo/forgejo_test.cljc b/src/test/cljc/dda/c4k_forgejo/forgejo_test.cljc index 71805a3..7e5b3b2 100644 --- a/src/test/cljc/dda/c4k_forgejo/forgejo_test.cljc +++ b/src/test/cljc/dda/c4k_forgejo/forgejo_test.cljc @@ -130,3 +130,32 @@ :storage-c2 "15Gi"} (th/map-diff (cut/generate-data-volume {:volume-total-storage-size 1}) (cut/generate-data-volume {:volume-total-storage-size 15}))))) + +(deftest should-generate-middleware-ratelimit + (is (= {:apiVersion "traefik.io/v1alpha1", + :kind "Middleware", + :metadata {:name "ratelimit"}, + :spec {:rateLimit {:average 10, :burst 5}}} + (cut/generate-rate-limit-middleware {:fqdn "test.de" + :mailer-from "" + :mailer-host "m.t.de" + :mailer-port "123" + :service-noreply-address "" + :average 10 + :burst 5})))) + +(deftest should-generate-middleware-ratelimit-ingress-and-cert + (is (= {:traefik.ingress.kubernetes.io/router.entrypoints "web, websecure", + :traefik.ingress.kubernetes.io/router.middlewares + "default-redirect-https@kubernetescrd, default-ratelimit@kubernetescrd", + :metallb.universe.tf/address-pool "public"} + (-> (second + (cut/generate-rate-limit-ingress-and-cert + {:fqdn "test.de" + :mailer-from "" + :mailer-host "m.t.de" + :mailer-port "123" + :service-noreply-address "" + :average 10 + :burst 5})) + :metadata :annotations)))) From 010ab3d8fd93b9e3202d5c16481a34d360a0b2de Mon Sep 17 00:00:00 2001 From: erik Date: Wed, 17 Jan 2024 11:57:19 +0100 Subject: [PATCH 10/22] Split if into multiple whens Otherwise weird behavior. --- src/main/cljc/dda/c4k_forgejo/core.cljc | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/cljc/dda/c4k_forgejo/core.cljc b/src/main/cljc/dda/c4k_forgejo/core.cljc index ba9e12a..0236572 100644 --- a/src/main/cljc/dda/c4k_forgejo/core.cljc +++ b/src/main/cljc/dda/c4k_forgejo/core.cljc @@ -46,14 +46,15 @@ (postgres/generate-service) (forgejo/generate-deployment config) (forgejo/generate-service) - (forgejo/generate-service-ssh) + (forgejo/generate-service-ssh) (forgejo/generate-data-volume config) (forgejo/generate-appini-env config) (forgejo/generate-secrets auth)] - (if (contains? config :average) - (do (forgejo/generate-rate-limit-ingress-and-cert config) - (forgejo/generate-rate-limit-middleware config)) - (forgejo/generate-ingress-and-cert config)) + (when (contains? config :average) + (forgejo/generate-rate-limit-ingress-and-cert config) ; this function has a vector as output + [(forgejo/generate-rate-limit-middleware config)]) ; this does not + (when (not (contains? config :average)) + (forgejo/generate-ingress-and-cert config)) (when (contains? config :restic-repository) [(backup/generate-config config) (backup/generate-secret auth) From d9cb19242b8becc13aaadb25df62ff6f0713b91a Mon Sep 17 00:00:00 2001 From: erik Date: Wed, 17 Jan 2024 11:57:40 +0100 Subject: [PATCH 11/22] Format --- src/main/cljc/dda/c4k_forgejo/forgejo.cljc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/cljc/dda/c4k_forgejo/forgejo.cljc b/src/main/cljc/dda/c4k_forgejo/forgejo.cljc index 8c9249e..ebeb9e7 100644 --- a/src/main/cljc/dda/c4k_forgejo/forgejo.cljc +++ b/src/main/cljc/dda/c4k_forgejo/forgejo.cljc @@ -141,8 +141,7 @@ (-> (yaml/load-as-edn "forgejo/middleware-ratelimit.yaml") (cm/replace-key-value :average average) - (cm/replace-key-value :burst burst) - ))) + (cm/replace-key-value :burst burst)))) (defn-spec generate-data-volume pred/map-or-seq? [config vol?] From 777b94a3406edc183c67e358e4cbfd0050a557dc Mon Sep 17 00:00:00 2001 From: erik Date: Wed, 17 Jan 2024 11:57:55 +0100 Subject: [PATCH 12/22] Add average and burst keys --- src/test/resources/forgejo-test/valid-config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/resources/forgejo-test/valid-config.yaml b/src/test/resources/forgejo-test/valid-config.yaml index d847ff9..0b207cb 100644 --- a/src/test/resources/forgejo-test/valid-config.yaml +++ b/src/test/resources/forgejo-test/valid-config.yaml @@ -9,6 +9,8 @@ service-noreply-address: "noreply@test.de" volume-total-storage-size: 6 restic-repository: "repo-path" deploy-federated: "false" +average: 10 +burst: 4 mon-cfg: grafana-cloud-url: "url-for-your-prom-remote-write-endpoint" cluster-name: "forgejo" From 56b843981f7550203bc6b215445ab45ee56c7843 Mon Sep 17 00:00:00 2001 From: erik Date: Wed, 17 Jan 2024 12:35:48 +0100 Subject: [PATCH 13/22] Correct api version --- src/main/resources/forgejo/middleware-ratelimit.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/forgejo/middleware-ratelimit.yaml b/src/main/resources/forgejo/middleware-ratelimit.yaml index 25184cd..0f6c49d 100644 --- a/src/main/resources/forgejo/middleware-ratelimit.yaml +++ b/src/main/resources/forgejo/middleware-ratelimit.yaml @@ -1,4 +1,4 @@ -apiVersion: traefik.io/v1alpha1 +apiVersion: traefik.containo.us/v1alpha1 kind: Middleware metadata: name: ratelimit From 260d08623261807a6a29b2741f4f9892b835d357 Mon Sep 17 00:00:00 2001 From: erik Date: Wed, 17 Jan 2024 12:36:24 +0100 Subject: [PATCH 14/22] Further split flow control --- src/main/cljc/dda/c4k_forgejo/core.cljc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/cljc/dda/c4k_forgejo/core.cljc b/src/main/cljc/dda/c4k_forgejo/core.cljc index 0236572..d5c172d 100644 --- a/src/main/cljc/dda/c4k_forgejo/core.cljc +++ b/src/main/cljc/dda/c4k_forgejo/core.cljc @@ -51,8 +51,9 @@ (forgejo/generate-appini-env config) (forgejo/generate-secrets auth)] (when (contains? config :average) - (forgejo/generate-rate-limit-ingress-and-cert config) ; this function has a vector as output - [(forgejo/generate-rate-limit-middleware config)]) ; this does not + (forgejo/generate-rate-limit-ingress-and-cert config)) ; this function has a vector as output + (when (contains? config :average) + [(forgejo/generate-rate-limit-middleware config)]) ; this does not (when (not (contains? config :average)) (forgejo/generate-ingress-and-cert config)) (when (contains? config :restic-repository) From 7d21f5aff15308bcd0d0ca3e6e8a182db4fbc819 Mon Sep 17 00:00:00 2001 From: erik Date: Wed, 17 Jan 2024 12:44:22 +0100 Subject: [PATCH 15/22] Fix test --- src/test/cljc/dda/c4k_forgejo/forgejo_test.cljc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/cljc/dda/c4k_forgejo/forgejo_test.cljc b/src/test/cljc/dda/c4k_forgejo/forgejo_test.cljc index 7e5b3b2..b6a8661 100644 --- a/src/test/cljc/dda/c4k_forgejo/forgejo_test.cljc +++ b/src/test/cljc/dda/c4k_forgejo/forgejo_test.cljc @@ -132,7 +132,7 @@ (cut/generate-data-volume {:volume-total-storage-size 15}))))) (deftest should-generate-middleware-ratelimit - (is (= {:apiVersion "traefik.io/v1alpha1", + (is (= {:apiVersion "traefik.containo.us/v1alpha1", :kind "Middleware", :metadata {:name "ratelimit"}, :spec {:rateLimit {:average 10, :burst 5}}} From 3f0de27055fa59d54c87839239f0098f83925dae Mon Sep 17 00:00:00 2001 From: erik Date: Wed, 17 Jan 2024 15:40:47 +0100 Subject: [PATCH 16/22] Add Middleware to be skipped --- build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.py b/build.py index fb2b9dd..3df17fa 100644 --- a/build.py +++ b/build.py @@ -56,7 +56,7 @@ def test_schema(project): "java -jar target/uberjar/c4k-forgejo-standalone.jar " + "src/test/resources/forgejo-test/valid-config.yaml " + "src/test/resources/forgejo-test/valid-auth.yaml | " - + "kubeconform --kubernetes-version 1.23.0 --strict --skip Certificate -", + + """kubeconform --kubernetes-version 1.23.0 --strict --skip "Certificate, Middleware" -""", shell=True, check=True, ) From aec67352d52ea35b67b7e651209f4829cc1e395f Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 19 Jan 2024 09:44:55 +0100 Subject: [PATCH 17/22] [Skip-CI] Add ToDos --- src/main/cljc/dda/c4k_forgejo/core.cljc | 2 +- src/main/cljc/dda/c4k_forgejo/forgejo.cljc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/cljc/dda/c4k_forgejo/core.cljc b/src/main/cljc/dda/c4k_forgejo/core.cljc index d5c172d..504fb01 100644 --- a/src/main/cljc/dda/c4k_forgejo/core.cljc +++ b/src/main/cljc/dda/c4k_forgejo/core.cljc @@ -50,7 +50,7 @@ (forgejo/generate-data-volume config) (forgejo/generate-appini-env config) (forgejo/generate-secrets auth)] - (when (contains? config :average) + (when (contains? config :average) ; ToDo: just leave this out and make sensible defaults (forgejo/generate-rate-limit-ingress-and-cert config)) ; this function has a vector as output (when (contains? config :average) [(forgejo/generate-rate-limit-middleware config)]) ; this does not diff --git a/src/main/cljc/dda/c4k_forgejo/forgejo.cljc b/src/main/cljc/dda/c4k_forgejo/forgejo.cljc index ebeb9e7..da9632e 100644 --- a/src/main/cljc/dda/c4k_forgejo/forgejo.cljc +++ b/src/main/cljc/dda/c4k_forgejo/forgejo.cljc @@ -42,7 +42,7 @@ (s/def ::mailer-pw pred/bash-env-string?) (s/def ::issuer pred/letsencrypt-issuer?) (s/def ::volume-total-storage-size (partial pred/int-gt-n? 5)) -(s/def ::average int?) +(s/def ::average int?) (s/def ::burst int?) (def config? (s/keys :req-un [::fqdn @@ -137,7 +137,7 @@ ; using :average and :burst seems sensible, :period may be interesting for fine tuning later on (defn-spec generate-rate-limit-middleware pred/map-or-seq? [config config?] - (let [{:keys [average burst]} config] + (let [{:keys [average burst]} config] ; ToDo: Set defaults, don't read config ; refactor ":average" KW to smth more speaking (-> (yaml/load-as-edn "forgejo/middleware-ratelimit.yaml") (cm/replace-key-value :average average) From 62fb2a37a042ea558600e9bd3c56648806ffa422 Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 19 Jan 2024 10:14:44 +0100 Subject: [PATCH 18/22] [WIP] Use defaults for rate limit --- src/main/cljc/dda/c4k_forgejo/core.cljc | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/main/cljc/dda/c4k_forgejo/core.cljc b/src/main/cljc/dda/c4k_forgejo/core.cljc index 504fb01..71338b8 100644 --- a/src/main/cljc/dda/c4k_forgejo/core.cljc +++ b/src/main/cljc/dda/c4k_forgejo/core.cljc @@ -9,6 +9,7 @@ [dda.c4k-common.postgres :as postgres])) (def config-defaults {:issuer "staging", :deploy-federated "false"}) +(def rate-limit-defaults {:max-rate 10, :max-concurrent-requests 5}) (def config? (s/keys :req-un [::forgejo/fqdn ::forgejo/mailer-from @@ -30,7 +31,7 @@ (def vol? (s/keys :req-un [::forgejo/volume-total-storage-size])) -(defn k8s-objects [config auth] +(defn k8s-objects [config auth] ; ToDo: ADR for generate functions - vector or no vector? (let [storage-class (if (contains? config :postgres-data-volume-path) :manual :local-path)] (map yaml/to-string (filter #(not (nil? %)) @@ -49,13 +50,9 @@ (forgejo/generate-service-ssh) (forgejo/generate-data-volume config) (forgejo/generate-appini-env config) - (forgejo/generate-secrets auth)] - (when (contains? config :average) ; ToDo: just leave this out and make sensible defaults - (forgejo/generate-rate-limit-ingress-and-cert config)) ; this function has a vector as output - (when (contains? config :average) - [(forgejo/generate-rate-limit-middleware config)]) ; this does not - (when (not (contains? config :average)) - (forgejo/generate-ingress-and-cert config)) + (forgejo/generate-secrets auth) + (forgejo/generate-rate-limit-middleware rate-limit-defaults)] ; this does not have a vector as output + (forgejo/generate-rate-limit-ingress-and-cert config) ; this function has a vector as output (when (contains? config :restic-repository) [(backup/generate-config config) (backup/generate-secret auth) From 38183f7bf112895a4ffcfda6728fa92f145ef44b Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 19 Jan 2024 10:18:02 +0100 Subject: [PATCH 19/22] [Skip-CI, WIP] Refactor middleware generation --- src/main/cljc/dda/c4k_forgejo/forgejo.cljc | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/main/cljc/dda/c4k_forgejo/forgejo.cljc b/src/main/cljc/dda/c4k_forgejo/forgejo.cljc index da9632e..22befbe 100644 --- a/src/main/cljc/dda/c4k_forgejo/forgejo.cljc +++ b/src/main/cljc/dda/c4k_forgejo/forgejo.cljc @@ -42,8 +42,8 @@ (s/def ::mailer-pw pred/bash-env-string?) (s/def ::issuer pred/letsencrypt-issuer?) (s/def ::volume-total-storage-size (partial pred/int-gt-n? 5)) -(s/def ::average int?) -(s/def ::burst int?) +(s/def ::max-rate int?) +(s/def ::max-concurrent-requests int?) (def config? (s/keys :req-un [::fqdn ::mailer-from @@ -53,9 +53,10 @@ :opt-un [::issuer ::deploy-federated ::default-app-name - ::service-domain-whitelist - ::average - ::burst])) + ::service-domain-whitelist])) + +(def rate-limit-config? (s/keys :req-un [::max-rate + ::max-concurrent-requests])) (def auth? (s/keys :req-un [::postgres/postgres-db-user ::postgres/postgres-db-password ::mailer-user ::mailer-pw])) @@ -136,12 +137,12 @@ ; using :average and :burst seems sensible, :period may be interesting for fine tuning later on (defn-spec generate-rate-limit-middleware pred/map-or-seq? - [config config?] - (let [{:keys [average burst]} config] ; ToDo: Set defaults, don't read config ; refactor ":average" KW to smth more speaking + [config rate-limit-config?] + (let [{:keys [max-rate max-concurrent-requests]} config] ; ToDo: Set defaults, don't read config ; refactor ":average" KW to smth more speaking (-> (yaml/load-as-edn "forgejo/middleware-ratelimit.yaml") - (cm/replace-key-value :average average) - (cm/replace-key-value :burst burst)))) + (cm/replace-key-value :average max-rate) + (cm/replace-key-value :burst max-concurrent-requests)))) (defn-spec generate-data-volume pred/map-or-seq? [config vol?] From 4881ea3c0d5fbd3962518513a3ec79f38b5fa4f5 Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 19 Jan 2024 11:38:33 +0100 Subject: [PATCH 20/22] Refactor Keywords --- src/main/cljc/dda/c4k_forgejo/forgejo.cljc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/cljc/dda/c4k_forgejo/forgejo.cljc b/src/main/cljc/dda/c4k_forgejo/forgejo.cljc index 22befbe..7d2a5fb 100644 --- a/src/main/cljc/dda/c4k_forgejo/forgejo.cljc +++ b/src/main/cljc/dda/c4k_forgejo/forgejo.cljc @@ -138,7 +138,7 @@ ; using :average and :burst seems sensible, :period may be interesting for fine tuning later on (defn-spec generate-rate-limit-middleware pred/map-or-seq? [config rate-limit-config?] - (let [{:keys [max-rate max-concurrent-requests]} config] ; ToDo: Set defaults, don't read config ; refactor ":average" KW to smth more speaking + (let [{:keys [max-rate max-concurrent-requests]} config] (-> (yaml/load-as-edn "forgejo/middleware-ratelimit.yaml") (cm/replace-key-value :average max-rate) From 12034502ac2fb2e26419a48dc9a10016d5ce99ff Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 19 Jan 2024 11:39:42 +0100 Subject: [PATCH 21/22] Use default values in tests --- src/test/cljc/dda/c4k_forgejo/forgejo_test.cljc | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/test/cljc/dda/c4k_forgejo/forgejo_test.cljc b/src/test/cljc/dda/c4k_forgejo/forgejo_test.cljc index b6a8661..54a6070 100644 --- a/src/test/cljc/dda/c4k_forgejo/forgejo_test.cljc +++ b/src/test/cljc/dda/c4k_forgejo/forgejo_test.cljc @@ -136,13 +136,7 @@ :kind "Middleware", :metadata {:name "ratelimit"}, :spec {:rateLimit {:average 10, :burst 5}}} - (cut/generate-rate-limit-middleware {:fqdn "test.de" - :mailer-from "" - :mailer-host "m.t.de" - :mailer-port "123" - :service-noreply-address "" - :average 10 - :burst 5})))) + (cut/generate-rate-limit-middleware {:max-rate 10, :max-concurrent-requests 5})))) (deftest should-generate-middleware-ratelimit-ingress-and-cert (is (= {:traefik.ingress.kubernetes.io/router.entrypoints "web, websecure", From bba058afa01a1e38e3fb85c74f7d58b7cb1fb324 Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 19 Jan 2024 11:40:01 +0100 Subject: [PATCH 22/22] [Skip-CI] Remove keywords from valid config --- src/test/resources/forgejo-test/valid-config.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/resources/forgejo-test/valid-config.yaml b/src/test/resources/forgejo-test/valid-config.yaml index 0b207cb..d847ff9 100644 --- a/src/test/resources/forgejo-test/valid-config.yaml +++ b/src/test/resources/forgejo-test/valid-config.yaml @@ -9,8 +9,6 @@ service-noreply-address: "noreply@test.de" volume-total-storage-size: 6 restic-repository: "repo-path" deploy-federated: "false" -average: 10 -burst: 4 mon-cfg: grafana-cloud-url: "url-for-your-prom-remote-write-endpoint" cluster-name: "forgejo"