diff --git a/flux/.gitignore b/flux/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..af0a1bc837429c57cd5e2e453c61c0a43f8a8669 --- /dev/null +++ b/flux/.gitignore @@ -0,0 +1,3 @@ +temp/ +dummy_secrets.json +videoag_config_path \ No newline at end of file diff --git a/flux/Dockerfile b/flux/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..c7cd276a212986a423b373ee29c8bf2ab7fb3222 --- /dev/null +++ b/flux/Dockerfile @@ -0,0 +1,26 @@ +FROM alpine:3.20 + +RUN apk update && apk add openssh-server openssh-keygen git +RUN mkdir /var/run/sshd +RUN echo $'PermitRootLogin yes\nPasswordAuthentication yes\nPubKeyAuthentication yes\nPermitEmptyPasswords yes\nHostKey /etc/ssh/ssh_host_ed25519_key' > /etc/ssh/sshd_config + +# Ensure that we always use the same key +RUN echo "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAsETHXp+4fFZiNM6MyZDd/OhL2U+alOPETSjQMsgALg" > /etc/ssh/ssh_host_ed25519_key.pub +RUN echo "-----BEGIN OPENSSH PRIVATE KEY-----" > /etc/ssh/ssh_host_ed25519_key +RUN echo "b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW" >> /etc/ssh/ssh_host_ed25519_key +RUN echo "QyNTUxOQAAACALBEx16fuHxWYjTOjMmQ3fzoS9lPmpTjxE0o0DLIAC4AAAAJBaI64BWiOu" >> /etc/ssh/ssh_host_ed25519_key +RUN echo "AQAAAAtzc2gtZWQyNTUxOQAAACALBEx16fuHxWYjTOjMmQ3fzoS9lPmpTjxE0o0DLIAC4A" >> /etc/ssh/ssh_host_ed25519_key +RUN echo "AAAEAhmXAof7EXMDU9K6L4S3spOY9JZii9N1r5G8KV9lozvwsETHXp+4fFZiNM6MyZDd/O" >> /etc/ssh/ssh_host_ed25519_key +RUN echo "hL2U+alOPETSjQMsgALgAAAADXVidW51dEBVYnVudXQ=" >> /etc/ssh/ssh_host_ed25519_key +RUN echo "-----END OPENSSH PRIVATE KEY-----" >> /etc/ssh/ssh_host_ed25519_key +RUN chmod og-rwx /etc/ssh/ssh_host_ed25519_key.pub +RUN chmod og-rwx /etc/ssh/ssh_host_ed25519_key + +RUN mkdir ~/.ssh +RUN echo "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIlgnUL0ojZChGGSXPQqs4n759brNL6zLZtOhEVgrv/D" > ~/.ssh/authorized_keys + +RUN echo "root:$(mkpasswd -s </dev/null)" | chpasswd -e + +RUN git config --global --add safe.directory /flux_repo + +ENTRYPOINT ["/usr/sbin/sshd", "-D"] \ No newline at end of file diff --git a/flux/README.md b/flux/README.md new file mode 100644 index 0000000000000000000000000000000000000000..a01d9b2def6289338ad73c3d26003297540922cb --- /dev/null +++ b/flux/README.md @@ -0,0 +1,70 @@ + +This folder contains a few rudimentary scripts to setup a local kubernetes cluster (minikube) to test configurations. + +These have **not** been tested extensively. USE AT YOUR OWN RISK! + +## Requirements + +* minikube +* kubectl +* kubeseal +* tmux + +Minikube and kubectl can be installed with `install_dependencies.sh` (Note that this script is for x86_64) + +## Setup + +First you need to put the **absolute** path to the videoag config into the `videoag_config_path` file. Example path: +`/home/simon/fsmpi/video_ag_website/flux/apps/production/videoag` (Note that `/home/simon/fsmpi/video_ag_website/flux` is the flux repo) +(When updating the config, files from here are copied to a temporary local repo from which the cluster syncs) + +You need to execute `bootstrap_cluster.sh` once to create a new minikube cluster and bootstrap flux. This script will +ask you `Please give the key access to your repository?` and `Have you added the deploy key to your repository`. Say yes +in both cases. + +Note that sometimes, bootstrapping flux fails because it can't find the repo 🤷. Just run `bootstrap_cluster.sh` again + +Also: Be patient. Some steps take reeeeaaaally long + +After that you can execute `run_cluster.sh` to run the cluster. See [below](#run-cluster) + +To delete the cluster use `delete_cluster.sh` + +## Run cluster + +Use `run_cluster.sh` to run your cluster. This will open a tmux session with 3 consoles + +* Top left: The dashboard. This takes a bit to startup and then automatically opens the dashboard in your browser +* Bottom left: The minikube tunnel. This will ask for your sudo password and starts a tunnel to access the database externally. + Not required unless you want to access the database externally and can be ignored. +* Right: The console to update the config. + * Pressing `u` will update the config (by copying the videoag config files, see [setup](#setup)) and synchronize flux + * This will also fetch the error messages. If there are no errors the following will be displayed: + `No resources found in flux-system namespace.`. + Note the 'last seen' time at the beginning of the message. Not all message displayed are still relevant. + * Pressing `q` will stop the cluster + +DO NOT modify the files in `temp/`. They will be overwritten + +To access the site, use `minikube ip` to get minikube's ip. The ip and port of the database is visible in the dashboard. + +Note that, the database is not initialized automatically and the site won't work mostly. + +## Secrets + +Your local cluster obviously can't decode the sealed secrets. As a way around this, when the config files are copied, +all `sealedSecret.json` files are replaced: A new secret with the same keys but random values is created and sealed by +your local cluster. If you want to specify some of these random values yourself (for example to specify the registry key), +you can put them in `dummy_secrets.json`. This is a key-value mapping where the key is `SECRET_NAME.SECRETKEY` (the +namespace is ignored for this). Example file: + +<pre> +{ + "registry-secret..dockerconfigjson": "{\"auths\":{\"registry.git.fsmpi.rwth-aachen.de/videoag_infra/production\":{\"auth\":\"...\"}}}" +} +</pre> + +## Some notes: + +* Increase minikube cpus: `minikube config set cpus 4` +* Increase minikube memory: `minikube config set memory 8000` \ No newline at end of file diff --git a/flux/_update_flux.sh b/flux/_update_flux.sh new file mode 100755 index 0000000000000000000000000000000000000000..09647f781efe9bccf22b08585b8570e4e8a748c5 --- /dev/null +++ b/flux/_update_flux.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +echo $'\nUpdating flux:' + +ACTUAL_VIDEOAG_CONFIG_PATH=$(cat videoag_config_path) +if [[ "$ACTUAL_VIDEOAG_CONFIG_PATH" == "" ]] +then + echo "Missing path for actual videoag config. Put into 'videoag_config_path' file. Example: '/home/simon/fsmpi/video_ag_website/flux/apps/production/videoag'" + exit \b +fi + +echo "Copying data..." +cd temp/local_flux \ + || { echo "cd failed"; exit 1; } +rm -r -d videoag +cp -r -p $ACTUAL_VIDEOAG_CONFIG_PATH videoag +echo "Replacing secrets..." +python3 ../../create_dummy_secrets.py "../../dummy_secrets.json" +python3 ../../replace_image_paths.py "../../replace_image_paths.json" +echo " +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: video + namespace: flux-system +spec: + interval: 10m0s + path: ./videoag + prune: true + sourceRef: + kind: GitRepository + name: flux-system +" > clusters/infratest/app.yaml +echo "Pushing to git repo..." +git add . +git commit -m "Update" +git push +echo "Updating flux..." +flux reconcile source git flux-system + +echo $'Errors from flux:' +kubectl get events -n flux-system --field-selector type=Warning + +echo $'Updated flux\n' \ No newline at end of file diff --git a/flux/_update_flux_loop.sh b/flux/_update_flux_loop.sh new file mode 100644 index 0000000000000000000000000000000000000000..0390f0392bdebda7722e158ae67f4a118f72d537 --- /dev/null +++ b/flux/_update_flux_loop.sh @@ -0,0 +1,14 @@ +#!/bin/bash +read -rsp $'Press u to update config, q to exit...\n' -n 1 key +while [[ "$key" != "q" ]] +do + if [[ "$key" == "u" ]] + then + ./_update_flux.sh + fi + read -rsp $'Press u to update config, q to exit...\n' -n 1 key +done +echo "Shutting down" +minikube stop +docker compose stop +tmux kill-session -t video_local_cluster diff --git a/flux/bootstrap_cluster.sh b/flux/bootstrap_cluster.sh new file mode 100755 index 0000000000000000000000000000000000000000..430e5d9394dc249df06563f995ea10f4c4a3b048 --- /dev/null +++ b/flux/bootstrap_cluster.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +HOST_IP=$(hostname -i | awk '{print $1}') +HOST_IP="10.0.2.15" + +rm -r temp/local_flux_remote +mkdir -p temp/local_flux_remote \ + || { echo "Create dir failed"; exit 1; } + +docker compose stop +docker compose up --build --detach \ + || { echo "Docker start failed"; exit 1; } + +cd temp/local_flux_remote \ + || { echo "cd failed"; exit 1; } +git init --bare --initial-branch=main \ + || { echo "git init failed"; exit 1; } +cd ../../ \ + || { echo "cd failed"; exit 1; } + +minikube delete +# Disable verification, because that takes soooo long +minikube start --extra-config kubeadm.ignore-preflight-errors=SystemVerification \ + || { echo "Minicube start failed"; exit 1; } +minikube addons enable ingress \ + || { echo "Ingress install failed"; exit 1; } +kubectl apply -k github.com/zalando/postgres-operator/manifests \ + || { echo "Postgres operator install failed"; exit 1; } +flux bootstrap git \ + --url=ssh://root@$HOST_IP:2020/flux_repo \ + --private-key-file ./local_flux_key \ + --branch=main --path=clusters/infratest \ + --components-extra=image-reflector-controller,image-automation-controller \ + || { + echo "Bootstrap failed. Trying again"; + flux bootstrap git \ + --url=ssh://root@$HOST_IP:2020/flux_repo \ + --private-key-file ./local_flux_key \ + --branch=main --path=clusters/infratest \ + --components-extra=image-reflector-controller,image-automation-controller \ + || { echo "Bootstrap failed again"; exit 1;} + } +#flux create source git video --url=ssh://root@$HOST_IP:2020/flux_repo --branch=main --ignore-paths=clusters \ +# || { echo "Add git source failed"; exit 1;} +flux create source helm sealed-secrets \ + --interval=1h \ + --url=https://bitnami-labs.github.io/sealed-secrets +flux create helmrelease sealed-secrets \ + --interval=1h \ + --release-name=sealed-secrets-controller \ + --target-namespace=flux-system \ + --source=HelmRepository/sealed-secrets \ + --chart=sealed-secrets \ + --chart-version=">=1.15.0-0" \ + --crds=CreateReplace + + +rm -r temp/local_flux +git clone ssh://root@localhost:2020/flux_repo temp/local_flux \ + || { echo "Cloning failed"; exit 1; } + +echo "Installed" + +minikube stop +docker compose stop \ No newline at end of file diff --git a/flux/create_dummy_secrets.py b/flux/create_dummy_secrets.py new file mode 100644 index 0000000000000000000000000000000000000000..29cbe7d1ee4be7ffd746a2ad27af92e7c83da7dc --- /dev/null +++ b/flux/create_dummy_secrets.py @@ -0,0 +1,52 @@ +import base64 +import json +import os +import sys +from pathlib import Path + + +def get_value_overrides(value_overrides_file: str): + path = Path(value_overrides_file).resolve() + print(f"Getting dummy values from: {path}") + if not path.exists(): + return {} + return json.loads(path.read_text(encoding="UTF8")) + + +def create_dummy_secret(value_overrides: dict, sealed_file: Path): + sealed_secret = json.loads(sealed_file.read_text(encoding="UTF8")) + assert sealed_secret["kind"] == "SealedSecret" + + name: str = sealed_secret["spec"]["template"]["metadata"]["name"] + namespace: str or None = sealed_secret["spec"]["template"]["metadata"].get("namespace", None) + type: str or None = sealed_secret["spec"]["template"].get("type", None) + + encrypted_data = sealed_secret["spec"]["encryptedData"] + + data_string = "\n ".join(map(lambda item: item[0] + ": " + base64.b64encode( + value_overrides.get(f"{name}.{item[0]}", "some random value").encode() + ).decode(), encrypted_data.items())) + + dummy_secret = f"""\ +apiVersion: v1 +kind: Secret +metadata: + name: {name} + {"" if namespace is None else f"namespace: {namespace}"} +data: + {data_string} +{"" if type is None else f"type: {type}"} +""" + temp_file = Path(sealed_file.parent.joinpath(".temp_secret.yaml")) + temp_file.write_text(dummy_secret, encoding="UTF8") + os.system("kubectl config set-context minikube") + cmd = f"kubeseal --format=json --controller-namespace=flux-system < {temp_file.resolve()} > {sealed_file.resolve()}" + print(cmd) + os.system(cmd) + os.remove(temp_file) + + +if __name__ == "__main__": + value_overrides = get_value_overrides(sys.argv[1]) + for file in Path().resolve().glob("**/sealedSecret.json"): + create_dummy_secret(value_overrides, file) diff --git a/flux/delete_cluster.sh b/flux/delete_cluster.sh new file mode 100644 index 0000000000000000000000000000000000000000..15bc67b716cc967d6c5d40352430a17f423adfb0 --- /dev/null +++ b/flux/delete_cluster.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +minikube delete +rm -r temp \ No newline at end of file diff --git a/flux/docker-compose.yml b/flux/docker-compose.yml new file mode 100644 index 0000000000000000000000000000000000000000..f1b0815d03a080a209fbd55dc1d6f3b6e60d9c95 --- /dev/null +++ b/flux/docker-compose.yml @@ -0,0 +1,9 @@ +services: + flux_test: + build: . + ports: + - 2020:22 + volumes: + - ./temp/local_flux_remote/:/flux_repo/ + extra_hosts: + host.docker.internal: host-gateway \ No newline at end of file diff --git a/flux/install_dependencies.sh b/flux/install_dependencies.sh new file mode 100644 index 0000000000000000000000000000000000000000..e29ea35e2b7b4017f62b9ce506aa76017e667ac7 --- /dev/null +++ b/flux/install_dependencies.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 +sudo install minikube-linux-amd64 /usr/local/bin/minikube +rm minikube-linux-amd64 +curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" +sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl +rm kubectl diff --git a/flux/local_flux_key b/flux/local_flux_key new file mode 100644 index 0000000000000000000000000000000000000000..390b8afe98b056fd4f8673bbae2797ceaac9bae4 --- /dev/null +++ b/flux/local_flux_key @@ -0,0 +1,7 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW +QyNTUxOQAAACCJYJ1C9KI2QoRhklz0KrOJ++fW6zS+sy2bToRFYK7/wwAAAJgmH/DgJh/w +4AAAAAtzc2gtZWQyNTUxOQAAACCJYJ1C9KI2QoRhklz0KrOJ++fW6zS+sy2bToRFYK7/ww +AAAEBY5ErsbSuk4xcO8W4bgkbT9iSfroV4HtyIL4XdYaNwf4lgnUL0ojZChGGSXPQqs4n7 +59brNL6zLZtOhEVgrv/DAAAAD3NpbW9uQGFyY2hsaW51eAECAwQFBg== +-----END OPENSSH PRIVATE KEY----- diff --git a/flux/local_flux_key.pub b/flux/local_flux_key.pub new file mode 100644 index 0000000000000000000000000000000000000000..0eb62dcf48f9695a1fe3d3686d9377dd688e957b --- /dev/null +++ b/flux/local_flux_key.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIlgnUL0ojZChGGSXPQqs4n759brNL6zLZtOhEVgrv/D simon@archlinux diff --git a/flux/replace_image_paths.py b/flux/replace_image_paths.py new file mode 100644 index 0000000000000000000000000000000000000000..115fa388ec81a5687d913bd0bd007362403dddf9 --- /dev/null +++ b/flux/replace_image_paths.py @@ -0,0 +1,21 @@ +import base64 +import json +import os +import sys +import re +from pathlib import Path + + +def replace_image_paths(file: Path): + file_content = file.read_text(encoding="UTF8") + file_content = re.sub( + "image: registry\\.git\\.fsmpi\\.rwth-aachen\\.de\\/videoag_infra\\/production\\/([a-zA-Z0-9_]+):[a-zA-Z0-9_.]+", + "image: registry.git.fsmpi.rwth-aachen.de/videoag/development/\\1:latest", + file_content + ) + file.write_text(file_content) + + +if __name__ == "__main__": + for file in Path().resolve().glob("**/*.yaml"): + replace_image_paths(file) diff --git a/flux/run_cluster.sh b/flux/run_cluster.sh new file mode 100644 index 0000000000000000000000000000000000000000..e772bfdf9b85017d8c5c90f0a946ecbb4bc901d0 --- /dev/null +++ b/flux/run_cluster.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +HOST_IP=$(hostname -I | awk '{print $1}') + +docker compose up --build --detach \ + || { echo "Docker start failed"; exit 1; } + +minikube start \ + || { echo "Minikube start failed"; exit 1; } +tmux \ + new-session -s video_local_cluster "minikube dashboard; read -rsp $'Dashboard terminated. Press enter to close...\n'" \; \ + split-window -v "echo 'Minikube tunnel. You will need to enter your sudo password here (Default: Press Ctrl+B shortly and then o to switch console)';minikube tunnel; read -rsp $'Tunnel terminated. Press enter to close...\n'" \; \ + split-window -fh -l '70%' "./_update_flux_loop.sh" \; + +minikube stop +docker compose stop + +echo "Cluster stopped"