From e6a1175a69087c21cfc262f84223cfe3e758bfca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Simon=20K=C3=BCnzel?= <simonk@fsmpi.rwth-aachen.de>
Date: Mon, 3 Mar 2025 18:10:58 +0100
Subject: [PATCH] Update README and move license

---
 api/LICENSE => LICENSE                        |   2 +-
 README.md                                     | 128 ++++++++++++++++++
 api/README.md                                 |  39 ------
 common_py/LICENSE.txt                         |   7 -
 .../generate_dockerfiles.sh                   |   1 +
 dev/install_requirements.sh                   |   8 +-
 dev/run.sh                                    |   4 +
 7 files changed, 141 insertions(+), 48 deletions(-)
 rename api/LICENSE => LICENSE (94%)
 create mode 100644 README.md
 delete mode 100644 api/README.md
 delete mode 100644 common_py/LICENSE.txt
 rename generate_dockerfiles.sh => dev/generate_dockerfiles.sh (80%)
 create mode 100755 dev/run.sh

diff --git a/api/LICENSE b/LICENSE
similarity index 94%
rename from api/LICENSE
rename to LICENSE
index ac54e30..4c0dd10 100644
--- a/api/LICENSE
+++ b/LICENSE
@@ -1,4 +1,4 @@
-Copyright (c) 2024 Fachschaft I/1 der RWTH Aachen
+Copyright (c) 2024-2025 Fachschaft I/1 der RWTH Aachen
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..9eff5d4
--- /dev/null
+++ b/README.md
@@ -0,0 +1,128 @@
+
+This is the repository for the backend of the Video AG Website. It is mainly written in Python and uses Flask,
+Sqlalchemy, ffmpeg, Docker and Kubernetes. 
+
+# Running the API (e.g. if you are working on the frontend)
+
+You need to have Docker installed.
+
+Then just execute `dev/run.sh` inside the **project root** directory. The API should be available at
+`localhost:5000`.
+
+If you have problems with 403 errors, the permissions probably need to be adjusted. For the Docker container to work,
+the 'other' group needs to have the e**x**ecute bit set for every directory (including the `api/` directory) and the
+**r**ead bit set for every file.
+
+In the debug mode (`DEBUG=True` in the config `api/config/api_example_config.py`) the login data is:
+* Username: `videoag`
+* Password: `videoag`
+
+
+Note: In the database, sometimes warnings appear that no transaction is running. This is because we always perform a
+ROLLBACK to ensure the next transactions can run flawlessly.
+
+# Logical Overview
+
+Everything is executed inside a Kubernetes Cluster
+
+* Postgres Database
+  * Used to store all non-file data
+  * Provided by the K8s Cluster (See flux repo)
+* API
+  * Provides controlled access to the database for the outside world
+  * Does NOT have access to any media files
+  * Inside the `api/` subdirectory of this repo
+* Job Controller
+  * Takes jobs from the queue in the database and creates a K8s Job
+  * Stores the result in the database when the job finished/failed
+  * Adds jobs to the queue if they are to be executed periodically
+  * Inside the `job_controller/` subdirectory of this repo
+* The different Jobs
+  * Executed by the K8s cluster to do some work
+  * e.g. any file operations are performed by jobs
+  * Have access to the media files
+  * Inside the `job_controller/jobs/` subdirectory of this repo
+* File Provider
+  * Serves the media files to the outside world
+  * Simple nginx image (See flux repo)
+
+# Implementation Overview
+
+In addition to the subdirectories mentioned above, there is also `common_py/` which contains some code used by the api,
+job_controller and the jobs.
+
+## Build Pipeline
+
+The Dockerfiles and the GitLab CI pipeline is dynamically generated by `build_pipeline_generator.py` based on the 
+`build_config.py` inside the different modules.
+
+Currently, the `build_config.py` can contain the following variables/instructions (Also see the API which currently 
+utilizes all of them):
+
+| Name                       | Type        | Optional | Description                                                                                                                                                                                                                                                                         |
+|----------------------------|-------------|----------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `TARGET_IMAGE_NAME`        | `str`       | No       | The name of the final docker image. Note that this name will be prepended by `development_` or `production_` before the image is put into the registry (As anyone can modify `development_` images in the registry)                                                                 |
+| `BUILD_DEPENDENCIES`       | `list[str]` | Yes      | Specify the location of other modules to load. The string must be a path to a directory with a `build_config.py`, relative to the directory of the current file.                                                                                                                    |
+| `PIP_REQUIREMENTS_FILE`    | `str`       | Yes      | The path to the pip requirements file, relative to the directory of the current file                                                                                                                                                                                                |
+| `APT_RUNTIME_DEPENDENCIES` | `list[str]` | Yes      | List of apt packages to install for the final image                                                                                                                                                                                                                                 |
+| `APT_BUILD_DEPENDENCIES`   | `list[str]` | Yes      | List of apt packages to install only for the pip install                                                                                                                                                                                                                            |
+| `DOCKERFILE_EXTRA`         | `str`       | Yes      | Dockerfile statements which are appended to the generated pip and apt install statements. The build context for docker is always the project root and the current workdir is `/code`. The term `$MODULE_DIR` is replaced by the path from the project root to the module directory. |
+| `CI_TEST_JOB_TEMPLATE`     | `str`       | Yes      | A GitLab CI job description in yaml using 4 space indentation. The template itself must not have any top-level indentation. The keys `stage`, `needs` and `image` are added automatically to the job.                                                                               |
+
+# Working on the Backend
+
+## Run tests & Coverage
+
+* Execute `dev/run.sh api-test` (execute from the project root!) to run the tests with docker
+* The coverage report is put into `api/coverage/report.txt` and `api/coverage/html/index.html`
+
+## Run locally
+
+This shows you, how to run the backend components locally and debug them:
+
+* Install all the dependencies with `dev/install_requirements.sh` (execute from the project root!)
+  * You may need to install some system packages for psycopg and uwsgi. See the apt requirements in the 
+    `build_config.py` files.
+* (PyCharm specific, but probably similar for other IDEs):
+  * Set `.venv/bin/python` as your python interpreter for this project
+  * Mark `api/src`, `common_py/src`, `job_controller/src` as your Sources Root.
+  * Add run profile for `api/src/run_tests.py`
+    * Working directory: `backend/api/src`
+    * Environment Variables: `VIDEOAG_CONFIG=../config/api_example_config.py;VIDEOAG_TEST_CONFIG_OVERRIDE=../config/test_config_override.py`
+    * Ensure option `Add sources roots to PYTHONPATH` is enabled
+* Execute `dev/run.sh db` (execute from the project root!) to start the database
+* Execute your `run_tests.py` profile with the IDE and enjoy debugging!
+
+# Authors
+
+Initially written by Simon Künzel and Dorian Koch
+
+# A few technical notes
+
+## Dockerfiles
+
+### Caching with apt and pip (Currently NOT implemented!, just keeping it here as a note)
+
+There are two types of caching used: One for local building and one in the CI with kaniko.
+
+#### Local Building
+
+The `PIP_CACHE_DIR` and `APT_CACHE_DIR` variables are empty, pip and apt use their default cache location. However, the
+`--mount=type=CACHE` option before the RUN command mounts these cache locations and so the cache can be reused between
+different builds. Since the locations are mounted, the files put into there (during the installation command), are not
+put into the final image.
+
+Additionally, Docker provides a script for apt to automatically clean the cache (so it does not stay in the final image).
+However, we don't want that, and so we need to remove the `/etc/apt/apt.conf.d/docker-clean` file.
+
+#### CI Building
+
+The `PIP_CACHE_DIR` and `APT_CACHE_DIR` variables are set by the CI to a location (`.cache/pip`, `.cache/apt`) inside
+the project dir. The cache option of the GitLab CI means these locations are persistent between different job runs. They
+need to be inside the project dir since GitLab cannot cache locations outside. When kaniko now executes the Dockerfile 
+it will provide the environment vars with the caching locations. Pip directly reads the `PIP_CACHE_DIR` env var (Note 
+that the ARG command in Docker provides the build argument as an environment variable to the container). For apt we 
+need to put the location into `/etc/apt/apt.conf.d/`. The `--ignore-path` option of kaniko ensures that the cache is not
+included in the final image.
+
+Also see https://github.com/GoogleContainerTools/kaniko/issues/969#issuecomment-2160910028.
\ No newline at end of file
diff --git a/api/README.md b/api/README.md
deleted file mode 100644
index 0fbbcca..0000000
--- a/api/README.md
+++ /dev/null
@@ -1,39 +0,0 @@
-# VideoAG Website Backend API
-
-Dies ist die API der VideoAG Website. Sie ist in Python basierend auf Flask und uWSGI geschrieben.
-
-## Entwicklung
-
-Zum lokalen Testen wird am besten Docker verwendet. Wenn Docker installiert ist, einfach `docker compose up --build` ausführen und
-die API sollte laufen. (Wenn die nötigen Dependencies installiert sind, sollte man auch direkt, ohne Docker,
-`docker_start.sh` ausführen können)
-
-Wenn du Probleme mit 403 Errors hast, sind wahrscheinlich die Berechtigungen deiner Ordner nicht richtig angepasst.
-Damit der Docker Container funktionieren kann, muss für die "Other" Gruppe jeder Ordner (Auch der Projekt ordner selbst!)
-das E**x**ecute Bit und jede Datei das **R**ead bit gesetzt haben.
-
-Es wird eine uWSGI Instanz gestartet, welcher unter `localhost:5000` verfügbar ist. Zur Entwicklung ist es auch sinnvoll das
-[Frontend](https://git.fsmpi.rwth-aachen.de/videoag/frontend) zu starten. Dieses sollte unter `localhost:3000` verfügbar sein.
-
-Im Debug Modus (`DEBUG=True` in der Config) sind die Logindaten: Nutzer: `videoag` Passwort: `videoag`
-
-Hinweis: In der Datenbank tauchen manchmal Warnungen auf, dass keine Transaktion am Laufen ist. Dies liegt daran, dass
-wir immer ein ROLLBACK machen, wenn es irgendwelche Probleme gegeben hat damit die nächste Transaktion einwandfrei läuft.
-
-### Unittests
-
-Test laufen auch über docker und können mit `run_tests.sh` gestartet werden.
-
-### Zum Mitmachen:
-
-1. Repo für den eigenen User forken, dafür den "Fork-Button" auf der Website verwenden
-2. Sicherstellen, dass der Upstream richtig konfiguriert ist:  
-   [Link](https://help.github.com/articles/configuring-a-remote-for-a-fork/)
-   Origin stellt hier euren User da, Upstream das Original der Gruppe videoag
-3. Erstellt euch eine eigene Branch, diese könnt ihr nennen wie ihr wollt, entweder nach der Änderung oder eurem Namen (git branch username), danach switched ihr in diese Branch (git checkout username)
-4. Die Initialisierung ist unter "Zum Testen" bereits erklärt worden
-5. Änderungen machen, committen, upstream mergen (git fetch upstream; git merge upstream/master)
-6. Ins eigene Repo pushen (git push)
-7. Pull-Request an uns, dazu unter "Merge-Requests" einmal auf "New Merge Request" und das Private Repo auswählen; oder ihr geht auf euer privates repo, da taucht dann eine Benachrichtigung über einen möglichen Merge-Request auf
-8. Warten
-9. Wir mergen die Änderungen
diff --git a/common_py/LICENSE.txt b/common_py/LICENSE.txt
deleted file mode 100644
index ac54e30..0000000
--- a/common_py/LICENSE.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-Copyright (c) 2024 Fachschaft I/1 der RWTH Aachen
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/generate_dockerfiles.sh b/dev/generate_dockerfiles.sh
similarity index 80%
rename from generate_dockerfiles.sh
rename to dev/generate_dockerfiles.sh
index 49dd60c..a16d0e7 100755
--- a/generate_dockerfiles.sh
+++ b/dev/generate_dockerfiles.sh
@@ -1,2 +1,3 @@
 #!/bin/bash
+# Execute from project ROOT!
 python build_pipeline_generator.py api job_controller $(find job_controller/jobs/ -mindepth 1 -maxdepth 1)
diff --git a/dev/install_requirements.sh b/dev/install_requirements.sh
index 9119325..e69af24 100755
--- a/dev/install_requirements.sh
+++ b/dev/install_requirements.sh
@@ -1,2 +1,8 @@
 #!/bin/bash
-pip install -r ../common_py/requirements.txt -r ../api/extra_requirements.txt -r ../job_controller/extra_requirements.txt
+if [ -f "install_requirements.sh" ]; then
+  echo "Inside dev/ directory. Execute only from the project ROOT!"
+  exit -1
+fi
+
+python -m venv .venv/
+.venv/bin/pip install -r common_py/requirements.txt -r api/requirements.txt -r job_controller/requirements.txt
diff --git a/dev/run.sh b/dev/run.sh
new file mode 100755
index 0000000..8de0f57
--- /dev/null
+++ b/dev/run.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+# Execute from project ROOT!
+dev/generate_dockerfiles.sh
+docker compose up "$@" --build
\ No newline at end of file
-- 
GitLab