From 5cda5a0147cdad1ba8b99dd9b65875a6db5dc634 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20K=C3=BCnzel?= <simonk@fsmpi.rwth-aachen.de> Date: Sun, 13 Oct 2024 22:03:27 +0200 Subject: [PATCH] Add video storage --- flux/.gitignore | 1 + flux/README.md | 33 +++++++++---- flux/_update_flux.sh | 3 +- flux/bootstrap_cluster.sh | 2 + ...mmy_secrets.py => do_file_replacements.py} | 48 +++++++++++++++++-- flux/replace_image_paths.py | 21 -------- flux/run_cluster.sh | 1 + 7 files changed, 75 insertions(+), 34 deletions(-) rename flux/{create_dummy_secrets.py => do_file_replacements.py} (59%) delete mode 100644 flux/replace_image_paths.py diff --git a/flux/.gitignore b/flux/.gitignore index af0a1bc..c247936 100644 --- a/flux/.gitignore +++ b/flux/.gitignore @@ -1,3 +1,4 @@ temp/ +video_data/ dummy_secrets.json videoag_config_path \ No newline at end of file diff --git a/flux/README.md b/flux/README.md index 0899eed..2b1f22e 100644 --- a/flux/README.md +++ b/flux/README.md @@ -10,8 +10,9 @@ These have **not** been tested extensively. USE AT YOUR OWN RISK! * kubeseal * tmux * docker compose +* flux -Minikube and kubectl can be installed with `install_dependencies.sh` (Note that this script is for x86_64) +Minikube and kubectl can be installed with your package manager or with `install_dependencies.sh` if your lucky (Note that this script is for x86_64) ## Setup @@ -38,8 +39,9 @@ To delete the cluster use `delete_cluster.sh` Use `run_cluster.sh` to run your cluster. This will open a tmux session with 3 consoles * Top left: The dashboard. This takes a bit to startup and then automatically opens the dashboard in your browser -* Bottom left: The minikube tunnel. This will ask for your sudo password and starts a tunnel to access the database externally. +* Middle left: The minikube tunnel. This will ask for your sudo password and starts a tunnel to access the database externally. Not required unless you want to access the database externally and can be ignored. +* Bottom left: Minikube mount for video_data * Right: The console to update the config. * Pressing `u` will update the config (by copying the videoag config files, see [setup](#setup)) and synchronize flux * This will also fetch the error messages. If there are no errors the following will be displayed: @@ -60,13 +62,24 @@ Then open `https://api.video.fsmpi.rwth-aachen.de` in your browser and allow ins The ip and port of the database is visible in the dashboard. -## Secrets +## Flux on-the-fly modifications -Your local cluster obviously can't decode the sealed secrets. As a way around this, when the config files are copied, -all `sealedSecret.json` files are replaced: A new secret with the same keys but random values is created and sealed by -your local cluster. If you want to specify some of these random values yourself (for example to specify the registry key), -you can put them in `dummy_secrets.json`. This is a key-value mapping where the key is `SECRET_NAME.SECRETKEY` (the -namespace is ignored for this). Example file: +When synchronizing the flux repo, some things are automatically modified by the script copying the flux config. These +changes are required so that the flux config works on your local cluster for development + +### Image paths + +All image paths are replaced with the development version. E.g. +`registry.git.fsmpi.rwth-aachen.de/videoag_infra/production/videoag_api:v0.42` is replaced with +`registry.git.fsmpi.rwth-aachen.de/videoag/development/videoag_api:latest` + +### Secrets + +Your local cluster obviously can't decode the sealed secrets. As a way around this, when all `sealedSecret.json` files +are replaced: A new secret with the same keys but random values is created and sealed by your local cluster. If you want +to specify some of these random values yourself (for example to specify the container registry key for the production +images), you can put them in `dummy_secrets.json`. This is a key-value mapping where the key is `SECRET_NAME.SECRETKEY` +(the namespace is ignored for this). Example file: <pre> { @@ -74,6 +87,10 @@ namespace is ignored for this). Example file: } </pre> +### Data Storage + +The persistent volume `video-data` is modified so that it uses a `hostPath` pointing to `./video_data/` instead of `nfs` + ## Some notes: * Increase minikube cpus: `minikube config set cpus 4` diff --git a/flux/_update_flux.sh b/flux/_update_flux.sh index 09647f7..3b0b7ed 100755 --- a/flux/_update_flux.sh +++ b/flux/_update_flux.sh @@ -15,8 +15,7 @@ cd temp/local_flux \ rm -r -d videoag cp -r -p $ACTUAL_VIDEOAG_CONFIG_PATH videoag echo "Replacing secrets..." -python3 ../../create_dummy_secrets.py "../../dummy_secrets.json" -python3 ../../replace_image_paths.py "../../replace_image_paths.json" +python3 ../../do_file_replacements.py echo " --- apiVersion: kustomize.toolkit.fluxcd.io/v1 diff --git a/flux/bootstrap_cluster.sh b/flux/bootstrap_cluster.sh index d18ff9e..f194ab9 100755 --- a/flux/bootstrap_cluster.sh +++ b/flux/bootstrap_cluster.sh @@ -6,6 +6,8 @@ rm -r temp/local_flux_remote mkdir -p temp/local_flux_remote \ || { echo "Create dir failed"; exit 1; } +mkdir -p video_data/ + docker compose stop docker compose up --build --detach \ || { echo "Docker start failed"; exit 1; } diff --git a/flux/create_dummy_secrets.py b/flux/do_file_replacements.py similarity index 59% rename from flux/create_dummy_secrets.py rename to flux/do_file_replacements.py index 29cbe7d..5b9fa06 100644 --- a/flux/create_dummy_secrets.py +++ b/flux/do_file_replacements.py @@ -1,11 +1,14 @@ import base64 import json import os +import re import sys from pathlib import Path -def get_value_overrides(value_overrides_file: str): +# Secret Replacement + +def get_secret_value_overrides(value_overrides_file: str): path = Path(value_overrides_file).resolve() print(f"Getting dummy values from: {path}") if not path.exists(): @@ -46,7 +49,46 @@ data: os.remove(temp_file) -if __name__ == "__main__": - value_overrides = get_value_overrides(sys.argv[1]) +def do_secret_replacement(): + value_overrides = get_secret_value_overrides("../../dummy_secrets.json") for file in Path().resolve().glob("**/sealedSecret.json"): create_dummy_secret(value_overrides, file) + + +# Image replacement + +def replace_image_paths(file: Path): + file_content = file.read_text(encoding="UTF8") + file_content = re.sub( + "image: registry\\.git\\.fsmpi\\.rwth-aachen\\.de\\/videoag_infra\\/production\\/([a-zA-Z0-9_]+):[a-zA-Z0-9_.]+", + "image: registry.git.fsmpi.rwth-aachen.de/videoag/development/\\1:latest", + file_content + ) + file.write_text(file_content) + + +def do_image_path_replacement(): + for file in Path().resolve().glob("**/*.yaml"): + replace_image_paths(file) + + +def do_video_data_mount_replacement(): + file = Path("videoag/storage.yaml") + file_content = file.read_text(encoding="UTF8") + file_content = re.sub( + """\ + nfs: + path: /srv/videoag + server: video-main.fsmpi.rwth-aachen.de\ +""", """\ + hostPath: + path: /mnt/video_data\ +""", file_content + ) + file.write_text(file_content) + + +if __name__ == "__main__": + do_secret_replacement() + do_image_path_replacement() + do_video_data_mount_replacement() diff --git a/flux/replace_image_paths.py b/flux/replace_image_paths.py deleted file mode 100644 index 115fa38..0000000 --- a/flux/replace_image_paths.py +++ /dev/null @@ -1,21 +0,0 @@ -import base64 -import json -import os -import sys -import re -from pathlib import Path - - -def replace_image_paths(file: Path): - file_content = file.read_text(encoding="UTF8") - file_content = re.sub( - "image: registry\\.git\\.fsmpi\\.rwth-aachen\\.de\\/videoag_infra\\/production\\/([a-zA-Z0-9_]+):[a-zA-Z0-9_.]+", - "image: registry.git.fsmpi.rwth-aachen.de/videoag/development/\\1:latest", - file_content - ) - file.write_text(file_content) - - -if __name__ == "__main__": - for file in Path().resolve().glob("**/*.yaml"): - replace_image_paths(file) diff --git a/flux/run_cluster.sh b/flux/run_cluster.sh index d1571ce..b052763 100644 --- a/flux/run_cluster.sh +++ b/flux/run_cluster.sh @@ -10,6 +10,7 @@ minikube start \ tmux \ new-session -s video_local_cluster "minikube dashboard; read -rsp $'Dashboard terminated. Press enter to close...\n'" \; \ split-window -v "echo 'Minikube tunnel. You will need to enter your sudo password here (Default: Press Ctrl+B shortly and then o to switch console)';minikube tunnel; read -rsp $'Tunnel terminated. Press enter to close...\n'" \; \ + split-window -v "minikube mount ./video_data:/mnt/video_data; read -rsp $'Tunnel terminated. Press enter to close...\n'" \; \ split-window -fh -l '70%' "./_update_flux_loop.sh" \; minikube stop -- GitLab