Merge branch 'backup' into 'main'

Backup

See merge request domaindrivenarchitecture/c4k-gitea!2

doc/BackupAndRestore.md

@@ -0,0 +1,41 @@
+# Backup Architecture details
+
+![](backup.svg)
+
+* we use restic to produce small & encrypted backups
+* backup is scheduled at `schedule: "10 23 * * *"`
+* Gitea stores files in `/data/gitea` and `/data/git/repositories`, these files are backed up. 
+* The postgres db is also backed up
+
+## Manual init the restic repository for the first time
+
+1. apply backup-and-restore pod:   
+   `kubectl scale deployment backup-restore --replicas=1`
+2. exec into pod and execute restore pod (press tab to get your exact pod name)   
+   `kubectl exec -it backup-restore-... -- /usr/local/bin/init.sh`
+3. remove backup-and-restore pod:   
+   `kubectl scale deployment backup-restore --replicas=0`
+
+
+## Manual backup the restic repository for the first time
+
+1. apply backup-and-restore pod:   
+  `kubectl scale deployment backup-restore --replicas=1`
+2. exec into pod and execute restore pod (press tab to get your exact pod name)   
+   `kubectl exec -it backup-restore-... -- /usr/local/bin/backup.sh`
+3. remove backup-and-restore pod:   
+   `kubectl scale deployment backup-restore --replicas=0`
+
+
+## Manual restore
+
+1. apply backup-and-restore pod:   
+  `kubectl scale deployment backup-restore --replicas=1`
+2. Scale down gitea deployment:
+   `kubectl scale deployment gitea --replicas=0`
+3. exec into pod and execute restore pod (press tab to get your exact pod name)   
+   `kubectl exec -it backup-restore-... -- /usr/local/bin/restore.sh`
+4. Start gitea again:
+   `kubectl scale deployment gitea --replicas=1`
+5. remove backup-and-restore pod:   
+   `kubectl scale deployment backup-restore --replicas=0`

infrastructure/docker-backup/build.py

@@ -0,0 +1,51 @@
+from os import environ
+from pybuilder.core import task, init
+from ddadevops import *
+import logging
+
+name = 'c4k-gitea-backup'
+MODULE = 'docker'
+PROJECT_ROOT_PATH = '../..'
+
+class MyBuild(DevopsDockerBuild):
+    pass
+
+@init
+def initialize(project):
+    project.build_depends_on('ddadevops>=0.12.4')
+    stage = 'prod'
+    dockerhub_user = environ.get('DOCKERHUB_USER')
+    if not dockerhub_user:
+        dockerhub_user = gopass_field_from_path('meissa/web/docker.com', 'login')
+    dockerhub_password = environ.get('DOCKERHUB_PASSWORD')
+    if not dockerhub_password:
+        dockerhub_password = gopass_password_from_path('meissa/web/docker.com')
+    tag = environ.get('CI_COMMIT_TAG')
+    if not tag:
+        tag = get_tag_from_latest_commit()
+    config = create_devops_docker_build_config(
+        stage, PROJECT_ROOT_PATH, MODULE, dockerhub_user, dockerhub_password, docker_publish_tag=tag)
+    build = MyBuild(project, config)
+    build.initialize_build_dir()
+
+
+@task
+def image(project):
+    build = get_devops_build(project)
+    build.image()
+
+@task
+def drun(project):
+    build = get_devops_build(project)
+    build.drun()
+
+@task
+def publish(project):
+    build = get_devops_build(project)
+    build.dockerhub_login()
+    build.dockerhub_publish()
+
+@task
+def test(project):
+    build = get_devops_build(project)
+    build.test()

infrastructure/docker-backup/image/Dockerfile

@@ -0,0 +1,5 @@
+FROM domaindrivenarchitecture/dda-backup:1.0.6
+
+# Prepare Entrypoint Script
+ADD resources /tmp
+RUN /tmp/install.sh

infrastructure/docker-backup/image/resources/backup.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -o pipefail
+
+function main() {
+    file_env AWS_ACCESS_KEY_ID
+    file_env AWS_SECRET_ACCESS_KEY
+    file_env RESTIC_DAYS_TO_KEEP 30
+    file_env RESTIC_MONTHS_TO_KEEP 12
+
+    backup-db-dump
+    backup-fs-from-directory '/var/backups/' 'gitea/' 'git/repositories/'
+}
+
+source /usr/local/lib/functions.sh
+source /usr/local/lib/pg-functions.sh
+source /usr/local/lib/file-functions.sh
+
+main

infrastructure/docker-backup/image/resources/entrypoint-start-and-wait.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+function main() {
+    create-pg-pass
+
+    while true; do
+        sleep 1m
+    done
+}
+
+source /usr/local/lib/functions.sh
+source /usr/local/lib/pg-functions.sh
+main

infrastructure/docker-backup/image/resources/entrypoint.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+function main() {
+    create-pg-pass
+
+	/usr/local/bin/backup.sh
+}
+
+source /usr/local/lib/functions.sh
+source /usr/local/lib/pg-functions.sh
+main

infrastructure/docker-backup/image/resources/init.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+function main() {
+    file_env AWS_ACCESS_KEY_ID
+    file_env AWS_SECRET_ACCESS_KEY
+
+    init-database-repo
+    init-file-repo
+}
+
+source /usr/local/lib/functions.sh
+source /usr/local/lib/pg-functions.sh
+source /usr/local/lib/file-functions.sh
+main

infrastructure/docker-backup/image/resources/install.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+apt-get update > /dev/null;
+
+install -m 0700 /tmp/entrypoint.sh /
+install -m 0700 /tmp/entrypoint-start-and-wait.sh /
+
+install -m 0700 /tmp/init.sh /usr/local/bin/
+install -m 0700 /tmp/backup.sh /usr/local/bin/
+install -m 0700 /tmp/restore.sh /usr/local/bin/
+install -m 0700 /tmp/restic-snapshots.sh /usr/local/bin/

infrastructure/docker-backup/image/resources/restic-snapshots.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -o pipefail
+
+function main() {
+    file_env AWS_ACCESS_KEY_ID
+    file_env AWS_SECRET_ACCESS_KEY
+
+    restic -r ${RESTIC_REPOSITORY}/files snapshots
+    restic -r ${RESTIC_REPOSITORY}/pg-database snapshots
+}
+
+source /usr/local/lib/functions.sh
+source /usr/local/lib/file-functions.sh
+
+main

infrastructure/docker-backup/image/resources/restore.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -Eeo pipefail
+
+function main() {
+
+    file_env AWS_ACCESS_KEY_ID
+    file_env AWS_SECRET_ACCESS_KEY
+
+    file_env POSTGRES_DB
+    file_env POSTGRES_PASSWORD
+    file_env POSTGRES_USER
+
+    # Restore latest snapshot into /var/backups/restore
+    rm -rf /var/backups/restore
+    restore-directory '/var/backups/restore'
+
+    rm -rf /var/backups/gitea/*
+    rm -rf /var/backups/git/repositories/*
+    cp -r /var/backups/restore/gitea/* /var/backups/gitea/
+    cp -r /var/backups/restore/git/repositories/* /var/backups/git/repositories/
+    
+    # adjust file permissions for the git user
+    chown -R 1000:1000 /var/backups
+
+    # TODO: Regenerate Git Hooks? Do we need this?
+    #/usr/local/bin/gitea -c '/data/gitea/conf/app.ini' admin regenerate hooks
+
+    # Restore db
+    drop-create-db
+    restore-db
+}
+
+source /usr/local/lib/functions.sh
+source /usr/local/lib/pg-functions.sh
+source /usr/local/lib/file-functions.sh
+
+main

infrastructure/docker-backup/test/Dockerfile

@@ -0,0 +1,11 @@
+FROM c4k-jira-backup
+
+RUN apt update
+RUN apt -yqq --no-install-recommends --yes install curl default-jre-headless
+
+RUN curl -L -o /tmp/serverspec.jar \
+    https://github.com/DomainDrivenArchitecture/dda-serverspec-crate/releases/download/2.0.0/dda-serverspec-standalone.jar
+
+COPY serverspec.edn /tmp/serverspec.edn
+
+RUN java -jar /tmp/serverspec.jar /tmp/serverspec.edn -v

infrastructure/docker-backup/test/serverspec.edn

@@ -0,0 +1,6 @@
+{:file [{:path "/usr/local/bin/init.sh" :mod "700"}
+        {:path "/usr/local/bin/backup.sh" :mod "700"}
+        {:path "/usr/local/bin/restore.sh" :mod "700"}
+        {:path "/usr/local/bin/restic-snapshots.sh" :mod "700"}
+        {:path "/entrypoint.sh" :mod "700"}
+        {:path "/entrypoint-start-and-wait.sh" :mod "700"}]}

src/main/clj/dda/c4k_gitea/uberjar.clj

@@ -6,4 +6,4 @@
    [dda.c4k-common.uberjar :as uberjar]))
 
 (defn -main [& cmd-args]
-  (uberjar/main-common "c4k-gitea" gitea/config? gitea/auth? gitea/config-defaults core/k8s-objects cmd-args))
+  (uberjar/main-common "c4k-gitea" core/config? core/auth? core/config-defaults core/k8s-objects cmd-args))

src/main/cljc/dda/c4k_gitea/backup.cljc

@@ -0,0 +1,44 @@
+(ns dda.c4k-gitea.backup
+ (:require
+  [clojure.spec.alpha :as s]
+  #?(:cljs [shadow.resource :as rc])
+  [dda.c4k-common.yaml :as yaml]
+  [dda.c4k-common.base64 :as b64]
+  [dda.c4k-common.common :as cm]))
+
+(s/def ::aws-access-key-id cm/bash-env-string?)
+(s/def ::aws-secret-access-key cm/bash-env-string?)
+(s/def ::restic-password cm/bash-env-string?)
+(s/def ::restic-repository cm/bash-env-string?)
+
+#?(:cljs
+   (defmethod yaml/load-resource :backup [resource-name]
+     (case resource-name
+       "backup/config.yaml" (rc/inline "backup/config.yaml")
+       "backup/cron.yaml" (rc/inline "backup/cron.yaml")
+       "backup/secret.yaml" (rc/inline "backup/secret.yaml")
+       "backup/backup-restore-deployment.yaml" (rc/inline "backup/backup-restore-deployment.yaml")
+       (throw (js/Error. "Undefined Resource!")))))
+
+(defn generate-config [my-conf]
+  (let [{:keys [restic-repository]} my-conf]
+    (->
+     (yaml/from-string (yaml/load-resource "backup/config.yaml"))
+     (cm/replace-key-value :restic-repository restic-repository))))
+
+(defn generate-cron []
+   (yaml/from-string (yaml/load-resource "backup/cron.yaml")))
+
+(defn generate-backup-restore-deployment [my-conf]
+  (let [backup-restore-yaml (yaml/from-string (yaml/load-resource "backup/backup-restore-deployment.yaml"))]
+    (if (and (contains? my-conf :local-integration-test) (= true (:local-integration-test my-conf)))
+      (cm/replace-named-value backup-restore-yaml "CERTIFICATE_FILE" "/var/run/secrets/localstack-secrets/ca.crt")
+      backup-restore-yaml)))
+
+(defn generate-secret [my-auth]
+  (let [{:keys [aws-access-key-id aws-secret-access-key restic-password]} my-auth]
+    (->
+     (yaml/from-string (yaml/load-resource "backup/secret.yaml"))
+     (cm/replace-key-value :aws-access-key-id (b64/encode aws-access-key-id))
+     (cm/replace-key-value :aws-secret-access-key (b64/encode aws-secret-access-key))
+     (cm/replace-key-value :restic-password (b64/encode restic-password)))))

src/main/cljc/dda/c4k_gitea/core.cljc

@@ -1,15 +1,35 @@
 (ns dda.c4k-gitea.core
  (:require
+  [clojure.spec.alpha :as s]
   [dda.c4k-common.yaml :as yaml]
   [dda.c4k-common.common :as cm]
   [dda.c4k-gitea.gitea :as gitea]
+  [dda.c4k-gitea.backup :as backup]
   [dda.c4k-common.postgres :as postgres]))
 
+(def config-defaults {:issuer "staging"})
+
+(def config? (s/keys :req-un [::gitea/fqdn 
+                              ::gitea/mailer-from 
+                              ::gitea/mailer-host-port 
+                              ::gitea/service-noreply-address]
+                     :opt-un [::gitea/issuer 
+                              ::gitea/default-app-name 
+                              ::gitea/service-domain-whitelist
+                              ::backup/restic-repository]))
+
+(def auth? (s/keys :req-un [::postgres/postgres-db-user ::postgres/postgres-db-password
+                            ::gitea/mailer-user ::gitea/mailer-pw
+                            ::backup/aws-access-key-id ::backup/aws-secret-access-key]
+                   :opt-un [::backup/restic-password])) ; TODO gec: Is restic password opt or req?
+
+(def vol? (s/keys :req-un [::gitea/volume-total-storage-size]))
+
 (defn k8s-objects [config]
   (let [storage-class (if (contains? config :postgres-data-volume-path) :manual :local-path)]
-    (cm/concat-vec
+    (map yaml/to-string
-     (map yaml/to-string
+         (filter #(not (nil? %))
-          (filter #(not (nil? %))
+                 (cm/concat-vec
                   [(postgres/generate-config {:postgres-size :2gb :db-name "gitea"})
                    (postgres/generate-secret config)
                    (when (contains? config :postgres-data-volume-path)
@@ -26,4 +46,9 @@
                    (gitea/generate-appini-env config)
                    (gitea/generate-secrets config)
                    (gitea/generate-ingress config)
-                   (gitea/generate-certificate config)])))))
+                   (gitea/generate-certificate config)]
+                  (when (contains? config :restic-repository)
+                    [(backup/generate-config config)
+                     (backup/generate-secret config)
+                     (backup/generate-cron)
+                     (backup/generate-backup-restore-deployment config)]))))))

src/main/cljc/dda/c4k_gitea/gitea.cljc

@@ -66,8 +66,8 @@
    (defmethod yaml/load-as-edn :gitea [resource-name]
      (yaml/from-string (yaml/load-resource resource-name))))
 
-(defn-spec generate-appini-env pred/map-or-seq?  
+(defn generate-appini-env
-  [config config?]
+  [config]
   (let [{:keys [default-app-name
                 fqdn
                 mailer-from
@@ -87,8 +87,8 @@
      (cm/replace-all-matching-values-by-new-value "WHITELISTDOMAINS" service-domain-whitelist)
      (cm/replace-all-matching-values-by-new-value "NOREPLY" service-noreply-address))))
 
-(defn-spec generate-secrets pred/map-or-seq?
+(defn generate-secrets
-  [auth auth?]
+  [auth]
   (let [{:keys [postgres-db-user 
                 postgres-db-password 
                 mailer-user 
@@ -100,15 +100,15 @@
      (cm/replace-all-matching-values-by-new-value "MAILERUSER" (b64/encode mailer-user))
      (cm/replace-all-matching-values-by-new-value "MAILERPW" (b64/encode mailer-pw)))))
 
-(defn-spec generate-ingress pred/map-or-seq?
+(defn generate-ingress
-  [config config?]
+  [config]
   (let [{:keys [fqdn]} config]
     (->
      (yaml/load-as-edn "gitea/ingress.yaml")
      (cm/replace-all-matching-values-by-new-value "FQDN" fqdn))))
 
-(defn-spec generate-certificate pred/map-or-seq?
+(defn generate-certificate
-  [config config?]
+  [config]
   (let [{:keys [fqdn issuer]
          :or {issuer "staging"}} config
         letsencrypt-issuer (name issuer)]
@@ -125,14 +125,14 @@
      (yaml/load-as-edn "gitea/datavolume.yaml")
      (cm/replace-all-matching-values-by-new-value "DATASTORAGESIZE" (str (str data-storage-size) "Gi")))))
 
-(defn-spec generate-deployment pred/map-or-seq?
+(defn generate-deployment
   []
   (yaml/load-as-edn "gitea/deployment.yaml"))
 
-(defn-spec generate-service pred/map-or-seq?
+(defn generate-service
   []
   (yaml/load-as-edn "gitea/service.yaml"))
 
-(defn-spec generate-service-ssh pred/map-or-seq?
+(defn generate-service-ssh
   []
   (yaml/load-as-edn "gitea/service-ssh.yaml"))

src/main/cljs/dda/c4k_gitea/browser.cljs

@@ -87,7 +87,7 @@
   (br/validate! "app-name" ::gitea/default-app-name :optional true)
   (br/validate! "domain-whitelist" ::gitea/service-domain-whitelist :optional true)  
   (br/validate! "volume-total-storage-size" ::gitea/volume-total-storage-size :deserializer js/parseInt)
-  (br/validate! "auth" gitea/auth? :deserializer edn/read-string)
+  (br/validate! "auth" core/auth? :deserializer edn/read-string)
   (br/set-form-validated!))
 
 (defn add-validate-listener [name]

src/main/resources/backup/backup-restore-deployment.yaml

@@ -0,0 +1,73 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: backup-restore
+spec:
+  replicas: 0
+  selector:
+    matchLabels:
+      app: backup-restore
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: backup-restore
+        app.kubernetes.io/name: backup-restore
+        app.kubernetes.io/part-of: gitea
+    spec:
+      containers:
+      - image: domaindrivenarchitecture/c4k-gitea-backup
+        name: backup-app
+        imagePullPolicy: IfNotPresent
+        command: ["/entrypoint-start-and-wait.sh"]
+        env:
+        - name: POSTGRES_USER
+          valueFrom:
+            secretKeyRef:
+              name: postgres-secret
+              key: postgres-user
+        - name: POSTGRES_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: postgres-secret
+              key: postgres-password
+        - name: POSTGRES_DB
+          valueFrom:
+            configMapKeyRef:
+              name: postgres-config
+              key: postgres-db
+        - name: POSTGRES_HOST
+          value: "postgresql-service:5432"
+        - name: POSTGRES_SERVICE
+          value: "postgresql-service"
+        - name: POSTGRES_PORT
+          value: "5432"
+        - name: AWS_DEFAULT_REGION
+          value: eu-central-1
+        - name: AWS_ACCESS_KEY_ID_FILE
+          value: /var/run/secrets/backup-secrets/aws-access-key-id
+        - name: AWS_SECRET_ACCESS_KEY_FILE
+          value: /var/run/secrets/backup-secrets/aws-secret-access-key
+        - name: RESTIC_REPOSITORY
+          valueFrom:
+            configMapKeyRef:
+              name: backup-config
+              key: restic-repository
+        - name: RESTIC_PASSWORD_FILE
+          value: /var/run/secrets/backup-secrets/restic-password
+        - name: CERTIFICATE_FILE
+          value: ""
+        volumeMounts:
+        - name: gitea-data-volume
+          mountPath: /var/backups
+        - name: backup-secret-volume
+          mountPath: /var/run/secrets/backup-secrets
+          readOnly: true
+      volumes:
+      - name: gitea-data-volume
+        persistentVolumeClaim:
+          claimName: gitea-data-pvc
+      - name: backup-secret-volume
+        secret:
+          secretName: backup-secret

src/main/resources/backup/config.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: backup-config
+  labels:
+    app.kubernetes.io/name: backup
+    app.kubernetes.io/part-of: gitea
+data:
+  restic-repository: restic-repository

src/main/resources/backup/cron.yaml

@@ -0,0 +1,47 @@
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: gitea-backup
+  labels:
+    app.kubernetes.part-of: gitea
+spec:
+  schedule: "10 23 * * *"
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 1
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: backup-app
+            image: domaindrivenarchitecture/c4k-gitea-backup
+            imagePullPolicy: IfNotPresent
+            command: ["/entrypoint.sh"]
+            env:
+            - name: AWS_DEFAULT_REGION
+              value: eu-central-1
+            - name: AWS_ACCESS_KEY_ID_FILE
+              value: /var/run/secrets/backup-secrets/aws-access-key-id
+            - name: AWS_SECRET_ACCESS_KEY_FILE
+              value: /var/run/secrets/backup-secrets/aws-secret-access-key
+            - name: RESTIC_REPOSITORY
+              valueFrom:
+                configMapKeyRef:
+                  name: backup-config
+                  key: restic-repository
+            - name: RESTIC_PASSWORD_FILE
+              value: /var/run/secrets/backup-secrets/restic-password
+            volumeMounts:
+            - name: gitea-data-volume
+              mountPath: /var/backups
+            - name: backup-secret-volume
+              mountPath: /var/run/secrets/backup-secrets
+              readOnly: true
+          volumes:
+          - name: gitea-data-volume
+            persistentVolumeClaim:
+              claimName: gitea-data-pvc
+          - name: backup-secret-volume
+            secret:
+              secretName: backup-secret
+          restartPolicy: OnFailure

src/main/resources/backup/secret.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: backup-secret
+type: Opaque
+data:
+  aws-access-key-id: aws-access-key-id
+  aws-secret-access-key: aws-secret-access-key
+  restic-password: restic-password

valid-auth.edn

@@ -1,4 +1,7 @@
 {:postgres-db-user "gitea"
  :postgres-db-password "gitea-db-password"
  :mailer-user ""
- :mailer-pw ""}
+ :mailer-pw ""
+ :aws-access-key-id "AWS_KEY_ID"
+ :aws-secret-access-key "AWS_KEY_SECRET"
+ :restic-password ""}

valid-config.edn

@@ -1,10 +1,9 @@
-{
+{:default-app-name "Meissas awesome gitea"
-:default-app-name "Meissas awesome gitea"
+ :fqdn "test.de"
-:fqdn "test.de"
+ :issuer "staging"
-:issuer "staging"
+ :mailer-from "test@test.de"
-:mailer-from "test@test.de"
+ :mailer-host-port "test.de:123"
-:mailer-host-port "test.de:123"
+ :service-whitelist-domains "test.de"
-:service-whitelist-domains "test.de"
+ :service-noreply-address "noreply@test.de"
-:service-noreply-address "noreply@test.de"
+ :volume-total-storage-size 6
-:volume-total-storage-size 6
+ :restic-repository "repo-path"}
- }