Add video storage

flux/.gitignore

 temp/
+video_data/
 dummy_secrets.json
 videoag_config_path
\ No newline at end of file

flux/README.md

@@ -10,8 +10,9 @@ These have **not** been tested extensively. USE AT YOUR OWN RISK!
 * kubeseal
 * tmux
 * docker compose
+* flux
 
-Minikube and kubectl can be installed with `install_dependencies.sh` (Note that this script is for x86_64)
+Minikube and kubectl can be installed with your package manager or with `install_dependencies.sh` if your lucky (Note that this script is for x86_64)
 
 ## Setup
 
@@ -38,8 +39,9 @@ To delete the cluster use `delete_cluster.sh`
 Use `run_cluster.sh` to run your cluster. This will open a tmux session with 3 consoles
 
 * Top left: The dashboard. This takes a bit to startup and then automatically opens the dashboard in your browser
-* Bottom left: The minikube tunnel. This will ask for your sudo password and starts a tunnel to access the database externally.
+* Middle left: The minikube tunnel. This will ask for your sudo password and starts a tunnel to access the database externally.
   Not required unless you want to access the database externally and can be ignored.
+* Bottom left: Minikube mount for video_data
 * Right: The console to update the config.
   * Pressing `u` will update the config (by copying the videoag config files, see [setup](#setup)) and synchronize flux
   * This will also fetch the error messages. If there are no errors the following will be displayed:
@@ -60,13 +62,24 @@ Then open `https://api.video.fsmpi.rwth-aachen.de` in your browser and allow ins
 
 The ip and port of the database is visible in the dashboard.
 
-## Secrets
+## Flux on-the-fly modifications
 
-Your local cluster obviously can't decode the sealed secrets. As a way around this, when the config files are copied,
-all `sealedSecret.json` files are replaced: A new secret with the same keys but random values is created and sealed by
-your local cluster. If you want to specify some of these random values yourself (for example to specify the registry key),
-you can put them in `dummy_secrets.json`. This is a key-value mapping where the key is `SECRET_NAME.SECRETKEY` (the
-namespace is ignored for this). Example file:
+When synchronizing the flux repo, some things are automatically modified by the script copying the flux config. These
+changes are required so that the flux config works on your local cluster for development
+
+### Image paths
+
+All image paths are replaced with the development version. E.g.
+`registry.git.fsmpi.rwth-aachen.de/videoag_infra/production/videoag_api:v0.42` is replaced with
+`registry.git.fsmpi.rwth-aachen.de/videoag/development/videoag_api:latest`
+
+### Secrets
+
+Your local cluster obviously can't decode the sealed secrets. As a way around this, when all `sealedSecret.json` files 
+are replaced: A new secret with the same keys but random values is created and sealed by your local cluster. If you want
+to specify some of these random values yourself (for example to specify the container registry key for the production 
+images), you can put them in `dummy_secrets.json`. This is a key-value mapping where the key is `SECRET_NAME.SECRETKEY` 
+(the namespace is ignored for this). Example file:
 
 <pre>
 {
@@ -74,6 +87,10 @@ namespace is ignored for this). Example file:
 }
 </pre>
 
+### Data Storage
+
+The persistent volume `video-data` is modified so that it uses a `hostPath` pointing to `./video_data/` instead of `nfs`
+
 ## Some notes:
 
 * Increase minikube cpus: `minikube config set cpus 4`

flux/_update_flux.sh

@@ -15,8 +15,7 @@ cd temp/local_flux \
 rm -r -d videoag
 cp -r -p $ACTUAL_VIDEOAG_CONFIG_PATH videoag
 echo "Replacing secrets..."
-python3 ../../create_dummy_secrets.py "../../dummy_secrets.json"
-python3 ../../replace_image_paths.py "../../replace_image_paths.json"
+python3 ../../do_file_replacements.py
 echo "
 ---
 apiVersion: kustomize.toolkit.fluxcd.io/v1

flux/bootstrap_cluster.sh

@@ -6,6 +6,8 @@ rm -r temp/local_flux_remote
 mkdir -p temp/local_flux_remote \
   || { echo "Create dir failed"; exit 1; }
 
+mkdir -p video_data/
+
 docker compose stop
 docker compose up --build --detach \
   || { echo "Docker start failed"; exit 1; }

flux/create_dummy_secrets.py → flux/do_file_replacements.py

 import base64
 import json
 import os
+import re
 import sys
 from pathlib import Path
 
 
-def get_value_overrides(value_overrides_file: str):
+# Secret Replacement
+
+def get_secret_value_overrides(value_overrides_file: str):
     path = Path(value_overrides_file).resolve()
     print(f"Getting dummy values from: {path}")
     if not path.exists():
@@ -46,7 +49,46 @@ data:
     os.remove(temp_file)
 
 
-if __name__ == "__main__":
-    value_overrides = get_value_overrides(sys.argv[1])
+def do_secret_replacement():
+    value_overrides = get_secret_value_overrides("../../dummy_secrets.json")
     for file in Path().resolve().glob("**/sealedSecret.json"):
         create_dummy_secret(value_overrides, file)
+
+
+# Image replacement
+
+def replace_image_paths(file: Path):
+    file_content = file.read_text(encoding="UTF8")
+    file_content = re.sub(
+        "image: registry\\.git\\.fsmpi\\.rwth-aachen\\.de\\/videoag_infra\\/production\\/([a-zA-Z0-9_]+):[a-zA-Z0-9_.]+",
+        "image: registry.git.fsmpi.rwth-aachen.de/videoag/development/\\1:latest",
+        file_content
+    )
+    file.write_text(file_content)
+
+
+def do_image_path_replacement():
+    for file in Path().resolve().glob("**/*.yaml"):
+        replace_image_paths(file)
+
+
+def do_video_data_mount_replacement():
+    file = Path("videoag/storage.yaml")
+    file_content = file.read_text(encoding="UTF8")
+    file_content = re.sub(
+        """\
+  nfs:
+    path: /srv/videoag
+    server: video-main.fsmpi.rwth-aachen.de\
+""", """\
+  hostPath:
+    path: /mnt/video_data\
+""", file_content
+    )
+    file.write_text(file_content)
+
+
+if __name__ == "__main__":
+    do_secret_replacement()
+    do_image_path_replacement()
+    do_video_data_mount_replacement()

flux/replace_image_paths.py

-import base64
-import json
-import os
-import sys
-import re
-from pathlib import Path
-
-
-def replace_image_paths(file: Path):
-    file_content = file.read_text(encoding="UTF8")
-    file_content = re.sub(
-        "image: registry\\.git\\.fsmpi\\.rwth-aachen\\.de\\/videoag_infra\\/production\\/([a-zA-Z0-9_]+):[a-zA-Z0-9_.]+",
-        "image: registry.git.fsmpi.rwth-aachen.de/videoag/development/\\1:latest",
-        file_content
-    )
-    file.write_text(file_content)
-
-
-if __name__ == "__main__":
-    for file in Path().resolve().glob("**/*.yaml"):
-        replace_image_paths(file)

flux/run_cluster.sh

@@ -10,6 +10,7 @@ minikube start \
 tmux \
   new-session -s video_local_cluster "minikube dashboard; read -rsp $'Dashboard terminated. Press enter to close...\n'" \; \
   split-window -v "echo 'Minikube tunnel. You will need to enter your sudo password here (Default: Press Ctrl+B shortly and then o to switch console)';minikube tunnel; read -rsp $'Tunnel terminated. Press enter to close...\n'" \; \
+  split-window -v "minikube mount ./video_data:/mnt/video_data; read -rsp $'Tunnel terminated. Press enter to close...\n'" \; \
   split-window -fh -l '70%' "./_update_flux_loop.sh" \;
 
 minikube stop