fixup! Add package caching for CI

.gitignore

@@ -7,3 +7,4 @@ __pycache__
 
 *.sqlite
 child-pipeline.yml
+.dockerfiles

.gitlab-ci.yml

@@ -6,11 +6,18 @@ generate-pipeline:
     stage: build
     image: python:3.13-slim
     script:
-        - python generate_ci_pipeline.py
+        - >-
+            python build_pipeline_generator.py \
+            --ci-pipeline-dest child-pipeline.yml \
+            --commit-sha $CI_COMMIT_SHA \
+            --commit-tag $CI_COMMIT_TAG \
+            api job_controller \
+            $(find job_controller/jobs/ -mindepth 1 -maxdepth 1)
     artifacts:
         expire_in: 1 week
         paths:
             - child-pipeline.yml
+            - .dockerfiles
 
 run-pipeline:
     stage: run

api/Dockerfile

-# Can be "development" or "production"
-ARG ENV_TYPE
-ARG GIT_COMMIT_SHA
-FROM registry.git.fsmpi.rwth-aachen.de/videoag/backend/${ENV_TYPE}_common_py:${GIT_COMMIT_SHA} AS base
-
-# READ THE NOTE on caching in the README before changing this/for more info!
-ARG PIP_CACHE_DIR=
-ARG APT_CACHE_DIR=
-RUN rm -f /etc/apt/apt.conf.d/docker-clean
-RUN if ! [ -z "$APT_CACHE_DIR" ]; then echo "Dir::Cache::Archives '$APT_CACHE_DIR';" > /etc/apt/apt.conf.d/ci_caching; fi
-
-
-ENV VIDEOAG_API_GIT_COMMIT_HASH $GIT_COMMIT_SHA
-
-
-# Install and build requirements in different image too reduce final image size
-FROM base AS builder
-
-# READ THE NOTE on caching in the README before changing this/for more info!
-COPY extra_requirements.txt ./
-RUN --mount=type=cache,target=/root/.cache/pip \
-    --mount=type=cache,target=/var/cache/apt,sharing=locked \
-    --mount=type=cache,target=/var/lib/apt,sharing=locked \
-    if ! pip --python .venv/bin/python install -r extra_requirements.txt; \
-    then echo "Pip failed (packages weren't cached) but you can ignore the error above. We will install the build dependencies and try again" \
-         # Packages needed to build lxml with pip
-         && apt-get update && apt-get --no-install-recommends install -y gcc python3-dev libxml2-dev libxslt-dev \
-         && pip --python .venv/bin/python install -r extra_requirements.txt; \
-    fi
-
-FROM base AS final
-
-COPY --from=builder /code/.venv /code/.venv
-
-COPY docker_start.sh ./
-COPY .pylintrc ./
-COPY tests/ ./tests/
-COPY config/api_example_config.py ./config/
-COPY config/test_config_override.py ./config/
-COPY config/uwsgi_example.ini ./config/
-COPY config/db_test_data.sql ./config/
-
-# The source has a symlink file at src/videoag_common
-# The actual files are already in the image at src/videoag_common
-# So we move the actual files temporarily, copy the src directory, remove the symlink and move the actual files back
-# In the future, COPY --exclude src/videoag_common might work (but right now it doesn't, some "failed to compute cache key")
-RUN mv src/videoag_common src/.temp
-COPY src/ ./src/
-RUN rm src/videoag_common
-RUN mv src/.temp src/videoag_common
-
-ENTRYPOINT ["/code/docker_start.sh"]
\ No newline at end of file

api/build_config.py

+TARGET_IMAGE_NAME = "api"
+BUILD_DEPENDENCIES = ["../common_py/"]
+
+PIP_REQUIREMENTS_FILE = "requirements.txt"
+APT_RUNTIME_DEPENDENCIES = [
+    "libxml2",  # For uwsgi
+]
+APT_BUILD_DEPENDENCIES = [
+    "gcc", "python3-dev", "libxslt-dev", "libxml2-dev"
+]
+DOCKERFILE_EXTRA = """
+COPY $MODULE_DIR/docker_start.sh ./
+COPY $MODULE_DIR/.pylintrc ./
+COPY $MODULE_DIR/src/ ./src/
+COPY $MODULE_DIR/tests/ ./tests/
+COPY $MODULE_DIR/config/api_example_config.py ./config/
+COPY $MODULE_DIR/config/test_config_override.py ./config/
+COPY $MODULE_DIR/config/uwsgi_example.ini ./config/
+COPY $MODULE_DIR/config/db_test_data.sql ./config/
+
+ENTRYPOINT ["/code/docker_start.sh"]
+"""
+
+# stage, needs, image is added automatically
+CI_TEST_JOB_TEMPLATE = """\
+timeout: 30m
+variables:
+    VIDEOAG_CONFIG: /code/config/api_example_config.py
+command: -test
+artifacts:
+    paths:
+        - /code/coverage/report.txt
+        - /code/coverage/html/*
+services:
+    - name: postgres:17
+      # So we can have the same host for local and CI testing
+      alias: host.docker.internal
+      variables:
+          POSTGRES_DB: videoag
+          POSTGRES_USER: videoag
+          POSTGRES_PASSWORD: videoag
+"""
\ No newline at end of file

api/config/api_example_config.py

@@ -65,7 +65,7 @@ DATABASE = {
     "engine": "postgres",
     "postgres": {
         "host": "host.docker.internal",
-        "port": 9343,
+        "port": 5432,
         "user": "videoag",
         "password": "videoag",
         "database": "videoag",

api/config/test_config_override.py

@@ -8,11 +8,11 @@ DB_DATA = "../config/db_test_data.sql"
 
 DATABASE = {
     "postgres": {
-        "host": "ci-database",
+        "host": "host.docker.internal",
         "port": 5432,
-        "user": "videoagtest",
-        "password": "LetMeTest...",
-        "database": "videoagtest",
+        "user": "videoag",
+        "password": "videoag",
+        "database": "videoag",
         "auto_migration": True
     },
     "log_all_statements": False

api/docker_start.sh

@@ -10,14 +10,14 @@ if [ $# = 1 ] && [ $1 = "-test" ]; then
   cd src
   uname -a
   export VIDEOAG_TEST_CONFIG_OVERRIDE="../config/test_config_override.py"
-  ../.venv/bin/python -V
-  ../.venv/bin/python -m coverage run --data-file "../coverage/.data" run_tests.py ||
+  python -V
+  python -m coverage run --data-file "../coverage/.data" run_tests.py ||
     { echo "Test failed!"; exit 1; }
-  ../.venv/bin/python -m coverage report --data-file "../coverage/.data" --include "./*" ||
+  python -m coverage report --data-file "../coverage/.data" --include "./*" ||
     { echo "Coverage report stdout failed"; exit 1; }
-  ../.venv/bin/python -m coverage report --data-file "../coverage/.data" -m --include "./*" > ../coverage/report.txt ||
+  python -m coverage report --data-file "../coverage/.data" -m --include "./*" > ../coverage/report.txt ||
     { echo "Coverage report report.txt failed"; exit 1; }
-  ../.venv/bin/python -m coverage html -d "../coverage/html/" --data-file "../coverage/.data" --include "./*" ||
+  python -m coverage html -d "../coverage/html/" --data-file "../coverage/.data" --include "./*" ||
     { echo "Coverage report html failed"; exit 1; }
 else
   echo "Running uWSGI"

api/extra_requirements.txt → api/requirements.txt

 # The Api Dependencies (Versions picked in January 2025)
+# Note that this does NOT include the common_py dependencies!
 
 # required
 flask==3.1.0

api/src/app.py

@@ -30,3 +30,6 @@ if "SECRET_KEY" not in app.config:
 # Import routes AFTER initialization
 # noinspection PyUnresolvedReferences
 import api.routes
+
+# Flush all statements after startup
+print(flush=True)

api/src/videoag_common

-../../common_py/src/videoag_common
\ No newline at end of file

api/tests/api_test.py

@@ -29,6 +29,7 @@ class ApiTest(DatabaseTest):
                   f"{pretty_json_or_string(self._last_request[1])}\n"
                   f"Response:\n"
                   f"{pretty_json_or_string(self._last_request[2])}")
+        print(flush=True, end="")
         return res
     
     def moderator_login(self) -> str:

build_pipeline_generator.py

+import os
+import re
+from pathlib import Path
+from argparse import ArgumentParser
+
+
+class ModuleBuildConfig:
+    
+    def __init__(self, context: "BuildContext", module_dir: Path):
+        super().__init__()
+        
+        self.context = context
+        self.module_dir = module_dir.resolve()
+        self.config_file = self.module_dir.joinpath("build_config.py")
+        self.name = str(self.module_dir.relative_to(context.build_dir))
+        
+        globals = {}
+        exec(self.config_file.read_text(), globals)
+        
+        self.target_image_name = globals.pop("TARGET_IMAGE_NAME", None)
+        if self.target_image_name is not None and  not isinstance(self.target_image_name, str):
+            raise TypeError("TARGET_IMAGE_NAME must be a str (or absent)")
+        
+        context.add_module(self)
+        
+        self.dependencies: list[ModuleBuildConfig] = []
+        for dependency in globals.pop("BUILD_DEPENDENCIES", []):
+            if not isinstance(dependency, str):
+                raise ValueError("BUILD_DEPENDENCIES must be list of str")
+            dependency_name = str(module_dir.joinpath(Path(dependency)).resolve().relative_to(context.build_dir))
+            self.dependencies.append(context.get_or_load_module(dependency_name))
+        
+        self.pip_req_file = None
+        pip_req_file_name = globals.pop("PIP_REQUIREMENTS_FILE", None)
+        if pip_req_file_name is not None:
+            assert isinstance(pip_req_file_name, str)
+            self.pip_req_file = self.module_dir.joinpath(Path(pip_req_file_name))
+            if not self.pip_req_file.is_file():
+                raise ValueError(f"Cannot find pip requirements file {self.pip_req_file}")
+        
+        self.apt_runtime_dependencies = globals.pop("APT_RUNTIME_DEPENDENCIES", [])
+        if any(not isinstance(r, str) for r in self.apt_runtime_dependencies):
+            raise TypeError(f"APT_RUNTIME_DEPENDENCIES must be list of str")
+        
+        self.apt_build_dependencies = globals.pop("APT_BUILD_DEPENDENCIES", [])
+        if any(not isinstance(r, str) for r in self.apt_build_dependencies):
+            raise TypeError(f"APT_BUILD_DEPENDENCIES must be list of str")
+        
+        self.dockerfile_extra = globals.pop("DOCKERFILE_EXTRA", "")
+        if not isinstance(self.dockerfile_extra, str):
+            raise TypeError("DOCKERFILE_EXTRA must be str (or absent)")
+        self.dockerfile_extra = self.dockerfile_extra.replace(
+            "$MODULE_DIR",
+            str(self.module_dir.relative_to(context.build_dir))
+        )
+        
+        self.ci_test_job_template: str or None = globals.pop("CI_TEST_JOB_TEMPLATE", None)
+        if self.ci_test_job_template is not None and not isinstance(self.dockerfile_extra, str):
+            raise TypeError("CI_TEST_JOB_TEMPLATE must be str (or absent)")
+        
+        for g in globals.keys():
+            assert isinstance(g, str)
+            if g.isupper():
+                raise ValueError(f"Unknown key {g} in config file")
+    
+    def check_cyclic_dependency(self, dependents_stack: list[str]):
+        if self.name in dependents_stack:
+            raise ValueError(f"Dependency cycle involving {self.name} detected")
+        dependents_stack = dependents_stack + [self.name]
+        for dependency in self.dependencies:
+            dependency.check_cyclic_dependency(dependents_stack)
+    
+    def collect_docker_dependencies(
+            self,
+            pip_requirement_files: list[Path],
+            apt_runtime_dependencies: list[str],
+            apt_build_dependencies: list[str],
+            dockerfile_extras: list[str]
+    ):
+        for dep in self.dependencies:
+            dep.collect_docker_dependencies(
+                pip_requirement_files,
+                apt_runtime_dependencies,
+                apt_build_dependencies,
+                dockerfile_extras
+            )
+        
+        if self.pip_req_file is not None:
+            pip_requirement_files.append(self.pip_req_file)
+        
+        apt_runtime_dependencies.extend(self.apt_runtime_dependencies)
+        apt_build_dependencies.extend(self.apt_build_dependencies)
+        dockerfile_extras.append(self.dockerfile_extra)
+    
+    def generate_dockerfile(self):
+        pip_requirement_files: list[Path] = []
+        apt_runtime_dependencies: list[str] = []
+        apt_build_dependencies: list[str] = []
+        dockerfile_extras: list[str] = []
+        self.collect_docker_dependencies(
+            pip_requirement_files,
+            apt_runtime_dependencies,
+            apt_build_dependencies,
+            dockerfile_extras
+        )
+        # Remove duplicates and ensure same order every time
+        apt_runtime_dependencies = list(set(apt_runtime_dependencies))
+        apt_runtime_dependencies.sort()
+        apt_build_dependencies = list(set(apt_build_dependencies))
+        apt_build_dependencies.sort()
+        
+        res = f"""\
+#####################################################################
+### WARNING: THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT EDIT ! ###
+#####################################################################
+FROM python:3.13-slim AS base
+
+RUN mkdir -p /code
+WORKDIR /code
+
+ENV PIP_CACHE_DIR=/tmp/pip-cache
+RUN apt-get update && apt-get --no-install-recommends install -y {' '.join(apt_runtime_dependencies)}
+"""
+        for path, i in zip(pip_requirement_files, range(len(pip_requirement_files))):
+            res += f"COPY {str(path.relative_to(self.context.build_dir))} /tmp/req-{i}.txt\n"
+        
+        pip_requirement_list_arg = " ".join(f"-r /tmp/req-{i}.txt" for i in range(len(pip_requirement_files)))
+        
+        res += f"""
+FROM base AS builder
+
+# This step builds (and installs) the requirements and also puts the build result into the pip cache
+RUN apt-get update && apt-get --no-install-recommends install -y {' '.join(apt_build_dependencies)}
+RUN pip install {pip_requirement_list_arg}
+
+FROM base AS final
+
+# Here we copy the pip cache with the built packages and install them again. Pip will use the cache and won't need the
+# apt build dependencies. This saves a lot of space, compared to leaving the build dependencies in the final image
+# (reduces the final image size by about half)
+COPY --from=builder /tmp/pip-cache /tmp/pip-cache
+RUN pip install {pip_requirement_list_arg}
+"""
+        for docker in dockerfile_extras:
+            res += "\n" + docker + "\n"
+        
+        return res
+    
+    def generate_ci_jobs(self):
+        self.context.ensure_in_ci()
+        if self.target_image_name is None:
+            raise ValueError("This module has no target image name and therefore cannot be built")
+        return (self._generate_ci_build_job()
+                + "\n" + self._generate_ci_test_job()
+                + "\n" + self._generate_ci_deploy_job())
+    
+    def output_dockerfile_path(self) -> Path:
+        return self.context.build_dir.joinpath(".dockerfiles").joinpath(self.target_image_name)
+    
+    def image_full_name(self):
+        return f"{self.context.env_type()}/{self.target_image_name}"
+    
+    def ci_build_job_name(self):
+        return f"build-{self.target_image_name}"
+    
+    @staticmethod
+    def _get_auth_echo():
+        return """\
+echo "{\\"auths\\":{\\"$CI_REGISTRY\\":{\\"username\\":\\"$CI_REGISTRY_USER\\",\\"password\\":\\"$CI_REGISTRY_PASSWORD\\"}}}" > /kaniko/.docker/config.json\
+"""
+
+    def _generate_ci_build_job(self):
+        kaniko_args = [
+            f"--context=git://git.fsmpi.rwth-aachen.de/videoag/backend.git#{self.context.commit_sha}",
+            f"--dockerfile={str(self.output_dockerfile_path().relative_to(self.context.build_dir))}"
+            f"--git recurse-submodules=true",
+            f"--destination=$CI_REGISTRY_IMAGE/{self.image_full_name()}#{self.context.commit_sha}",
+            f"--build-arg=GIT_COMMIT_SHA={self.context.commit_sha}",
+            f"--cache=true",
+        ]
+        
+        if self.context.commit_tag is not None:
+            kaniko_args.append(f"--build-arg=GIT_COMMIT_TAG={self.context.commit_tag}")
+        
+        return f"""
+{self.ci_build_job_name()}:
+    stage: build-and-test
+    timeout: 1h
+    needs:
+        - pipeline: $PARENT_PIPELINE_ID
+          job: generate-pipeline
+    image:
+        name: gcr.io/kaniko-project/executor:v1.23.2-debug
+        entrypoint: [""]
+    script:
+        - {self._get_auth_echo()}
+        - echo {self.context.commit_sha}
+        - >-
+            /kaniko/executor
+            {"\n            ".join(kaniko_args)}
+"""
+    
+    def _generate_ci_test_job(self):
+        if self.ci_test_job_template is None:
+            return ""
+        assert isinstance(self.ci_test_job_template, str)
+        res = f"""
+test-{self.target_image_name}:
+    stage: build-and-test
+    needs: [{self.ci_build_job_name()}]
+    image:
+        name: $CI_REGISTRY_IMAGE/{self.image_full_name()}#{self.context.commit_sha}
+"""
+        res += "    " + "\n    ".join(self.ci_test_job_template.splitlines()) + "\n"
+        return res
+    
+    def _generate_ci_deploy_job(self):
+        destination_args = [
+            f"--destination=$CI_REGISTRY_IMAGE/{self.image_full_name()}:latest"
+        ]
+        if self.context.is_production:
+            destination_args.append(f"--destination=$CI_REGISTRY_IMAGE/{self.image_full_name()}:{self.context.commit_tag}")
+        
+        return f"""
+deploy-{self.target_image_name}:
+    stage: deploy
+    timeout: 1h
+    image:
+        name: gcr.io/kaniko-project/executor:v1.23.2-debug
+        entrypoint: [""]
+    script:
+        - {self._get_auth_echo()}
+        - mkdir /workdir
+        - echo "FROM $CI_REGISTRY_IMAGE/{self.image_full_name()} > /workdir/Dockerfile"
+        - echo {self.context.commit_sha}
+        - >-
+            /kaniko/executor
+            --context=dir:///workdir
+            {"\n            ".join(destination_args)}
+"""
+
+
+class BuildContext:
+    
+    def __init__(self, build_dir: Path, commit_sha: str or None, commit_tag: str or None):
+        super().__init__()
+        self.build_dir = build_dir
+        self.modules: dict[str, ModuleBuildConfig] = {}
+        self._target_image_names: set[str] = set()
+        self.commit_sha = commit_sha or None  # Make empty string to None
+        self.commit_tag = commit_tag or None
+        self.is_production = commit_tag is not None and re.fullmatch("v.*", commit_tag) is not None
+    
+    def env_type(self) -> str:
+        return "production" if self.is_production else "development"
+    
+    def ensure_in_ci(self):
+        if self.commit_sha is None:
+            raise Exception("Not in GitLab CI. No commit sha given")
+    
+    def add_module(self, module: ModuleBuildConfig):
+        if module.target_image_name in self._target_image_names:
+            raise ValueError(f"Duplicate target image name {module.target_image_name}")
+        self.modules[module.name] = module
+    
+    def get_or_load_module(self, name: str):
+        if name in self.modules:
+            return self.modules[name]
+        module_dir = self.build_dir.joinpath(name)
+        try:
+            return ModuleBuildConfig(self, module_dir)
+        except Exception as e:
+            raise Exception(f"Exception while loading module {module_dir}", e)
+    
+    def generate_ci_pipeline(self):
+        self.ensure_in_ci()
+        
+        pipeline = """
+####################################################################
+##### AUTOMATICALLY GENERATED PIPELINE. DO NOT CHANGE MANUALLY! ####
+####################################################################
+
+stages:
+    - build-and-test
+    - deploy
+"""
+        for module in self.modules.values():
+            if module.target_image_name is not None:
+                pipeline += module.generate_ci_jobs()
+        
+        return pipeline
+
+
+def main():
+    parser = ArgumentParser()
+    parser.add_argument("--build-dir", type=Path, default=Path("."))
+    parser.add_argument("--commit-sha", type=str, required=False)
+    parser.add_argument("--commit-tag", type=str, required=False)
+    parser.add_argument("--ci-pipeline-dest", type=Path, required=False)
+    parser.add_argument("modules", nargs="+", type=Path)
+    
+    args = parser.parse_args()
+    
+    context = BuildContext(args.build_dir.resolve(), args.commit_sha, args.commit_tag)
+    for module in args.modules:
+        context.get_or_load_module(str(module.resolve().relative_to(context.build_dir)))
+    
+    for module in context.modules.values():
+        if module.target_image_name is None:
+            continue
+        module.output_dockerfile_path().parent.mkdir(parents=True, exist_ok=True)
+        module.output_dockerfile_path().write_text(module.generate_dockerfile())
+    
+    if args.ci_pipeline_dest is not None:
+        args.ci_pipeline_dest.parent.mkdir(parents=True, exist_ok=True)
+        args.ci_pipeline_dest.write_text(context.generate_ci_pipeline())
+
+
+if __name__ == "__main__":
+    main()

common_py/Dockerfile

-FROM python:3.13-slim AS base
-
-# READ THE NOTE on caching in the README before changing this/for more info!
-ARG PIP_CACHE_DIR=
-ARG APT_CACHE_DIR=
-RUN rm -f /etc/apt/apt.conf.d/docker-clean
-RUN if ! [ -z "$APT_CACHE_DIR" ]; then echo "Dir::Cache::Archives '$APT_CACHE_DIR';" > /etc/apt/apt.conf.d/ci_caching; fi
-
-
-WORKDIR /code
-
-RUN mkdir -p /code
-WORKDIR /code
-
-
-# Install and build requirements in different image too reduce final image size
-FROM base AS builder
-
-RUN python -m venv --without-pip .venv/
-
-# READ THE NOTE on caching in the README before changing this/for more info!
-COPY requirements.txt /code
-RUN --mount=type=cache,target=/root/.cache/pip \
-    --mount=type=cache,target=/var/cache/apt,sharing=locked \
-    --mount=type=cache,target=/var/lib/apt,sharing=locked \
-    if ! pip --python .venv/bin/python install -r requirements.txt; \
-    then echo "Pip failed (packages weren't cached) but you can ignore the error above. We will install the build dependencies and try again" \
-         # Packages needed to build psycopg with pip
-         && apt-get update && apt-get --no-install-recommends install -y gcc python3-dev libpq-dev \
-         && pip --python .venv/bin/python install -r requirements.txt; \
-    fi
-
-FROM base AS final
-
-# Copy venv created in builder image with the dependencies
-COPY --from=builder /code/.venv /code/.venv
-
-COPY src/videoag_common /code/src/videoag_common

common_py/build_config.py

+PIP_REQUIREMENTS_FILE = "requirements.txt"
+APT_RUNTIME_DEPENDENCIES = [
+    "libpq-dev",  # For psycopg
+]
+APT_BUILD_DEPENDENCIES = [
+    "gcc", "python3-dev",
+]
+DOCKERFILE_EXTRA = """
+COPY $MODULE_DIR/src ./src/
+"""
\ No newline at end of file

common_py/src/videoag_common/miscellaneous/util.py

@@ -102,7 +102,7 @@ def hash_json_sha256(data: JsonTypes) -> str:
 
 def load_config_file(path: Path):
     config_module = ModuleType("config")
-    config_module.__file__ = path.name
+    config_module.__file__ = str(path.absolute())
     try:
         exec(compile(path.read_text(), path.name, "exec"), config_module.__dict__)
     except Exception as e:

api/docker-compose.yaml → docker-compose.yaml

 services:
-  videoag_api:
-    build: .
+  api:
+    build:
+      context: .
+      dockerfile: .dockerfiles/api
     ports:
       - 5000:5000
-    volumes:
-      - ./:/code
     extra_hosts:
       host.docker.internal: host-gateway
     environment:
@@ -12,22 +12,39 @@ services:
       - VIDEOAG_API_LIVE_CONFIG=../config/live_config.json
       - VIDEOAG_UWSGI_CONFIG=/code/config/uwsgi_example.ini
     depends_on:
-      database:
+      db:
+        condition: service_healthy
+
+  api-test:
+    profiles: ["test"]
+    build:
+      context: .
+      dockerfile: .dockerfiles/api
+    extra_hosts:
+      host.docker.internal: host-gateway
+    environment:
+      - VIDEOAG_CONFIG=../config/api_example_config.py
+    volumes:
+      - ./api/coverage/:/code/coverage
+    command: -test
+    depends_on:
+      db:
         condition: service_healthy
-  database:
+
+  db:
     image: postgres:16-alpine
     environment:
       - POSTGRES_USER=videoag
       - POSTGRES_PASSWORD=videoag
     ports:
-      - "9343:5432"
+      - "5432:5432"
     volumes:
-      - .data/:/var/lib/postgresql/data
+      - db-data:/var/lib/postgresql/data
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U videoag"]
       interval: 2s
       timeout: 5s
       retries: 5
+
 volumes:
-  database:
-    driver: local
+  db-data:

generate_ci_pipeline.py

-import os
-import re
-import traceback
-from dataclasses import dataclass
-from pathlib import Path
-
-
-# This file dynamically generates a GitLab CI pipeline. This is needed since the GitLab CI is not quite flexible enough
-# to handle the different "development" and "production" builds and to automatically build all the jobs.
-# For development (everything that is not a tag pipeline whose tag starts with v) the images are put into the registry
-# with the "development" prefix. For production the images are put into the registry with the "production" prefix and
-# that prefix is write-protected, so only maintainers (or pipelines triggered by them) can push new images to production.
-# Kubernetes directly uses those images from the registry.
-
-
-class BuildContext:
-    
-    def __init__(self, commit_sha: str, commit_tag: str or None):
-        super().__init__()
-        self.commit_sha = commit_sha
-        self.commit_tag = commit_tag
-        self.is_production = commit_tag is not None and re.fullmatch("v.*", commit_tag) is not None
-        self.targets: dict[str, "ImageTarget"] = {}
-    
-    def env_type(self) -> str:
-        return "production" if self.is_production else "development"
-    
-    def add_image_target(self,
-                         image_name: str,
-                         dependency_targets: list["ImageTarget"],
-                         context_sub_path: str,
-                         only_intermediate: bool
-                         ) -> "ImageTarget":
-        target = ImageTarget(
-            self,
-            image_name,
-            dependency_targets,
-            context_sub_path,
-            only_intermediate
-        )
-        self.targets[target.image_name] = target
-        return target
-
-
-@dataclass
-class ImageTarget:
-    context: BuildContext
-    image_name: str
-    dependency_targets: list["ImageTarget" or str]
-    context_sub_path: str
-    only_intermediate: bool
-    
-    def full_name(self):
-        return f"{self.context.env_type()}_{self.image_name}"
-    
-    def versioned_full_name(self):
-        return f"{self.full_name()}:{self.context.commit_sha}"
-    
-    def build_job_name(self):
-        return f"build-{self.image_name}"
-    
-    def deploy_job_name(self):
-        return f"deploy-{self.image_name}"
-    
-    def _validate(self):
-        for i in range(len(self.dependency_targets)):
-            target = self.dependency_targets[i]
-            if isinstance(target, ImageTarget):
-                if target not in self.context.targets.values():
-                    raise ValueError(f"Unknown target {target.image_name} (object not in context)")
-            else:
-                assert isinstance(target, str)
-                if target not in self.context.targets:
-                    raise ValueError(f"Unknown target {target}")
-                self.dependency_targets[i] = self.context.targets[target]
-    
-    def gen_jobs(self):
-        self._validate()
-        return self._gen_build_job() + self._gen_deploy_job()
-    
-    @staticmethod
-    def _get_auth_echo():
-        return """\
-echo "{\\"auths\\":{\\"$CI_REGISTRY\\":{\\"username\\":\\"$CI_REGISTRY_USER\\",\\"password\\":\\"$CI_REGISTRY_PASSWORD\\"}}}" > /kaniko/.docker/config.json\
-"""
-    
-    def _gen_build_job(self):
-        kaniko_args = [
-            f"--context=git://git.fsmpi.rwth-aachen.de/videoag/backend.git#{self.context.commit_sha}",
-            f"--context-sub-path={self.context_sub_path}",
-            f"--git recurse-submodules=true",
-            f"--destination=$CI_REGISTRY_IMAGE/{self.versioned_full_name()}",
-            f"--build-arg=GIT_COMMIT_SHA={self.context.commit_sha}",
-            f"--build-arg=ENV_TYPE={self.context.env_type()}",
-            
-            # READ THE NOTE on caching in the README before changing this/for more info!
-            f"--ignore-path=$CI_PROJECT_DIR/.cache",
-            f"--build-arg=PIP_CACHE_DIR=$CI_PROJECT_DIR/.cache/pip",
-            f"--build-arg=APT_CACHE_DIR=$CI_PROJECT_DIR/.cache/apt",
-            f"--cache=true",
-        ]
-        
-        if self.context.commit_tag is not None:
-            kaniko_args.append(f"--build-arg=GIT_COMMIT_TAG={self.context.commit_tag}")
-        
-        return f"""
-{self.build_job_name()}:
-    stage: build-and-test
-    needs: [{",".join(t.build_job_name() for t in self.dependency_targets)}]
-    timeout: 1h
-    image:
-        name: gcr.io/kaniko-project/executor:v1.23.2-debug
-        entrypoint: [""]
-    cache:
-        # READ THE NOTE on caching in the README before changing this/for more info!
-        key: videoag-cache-{self.context.env_type()}
-        paths:
-            - .cache/pip
-            - .cache/apt
-    script:
-        - {self._get_auth_echo()}
-        - echo {self.context.commit_sha}
-        - >-
-            /kaniko/executor
-            {"\n            ".join(kaniko_args)}
-"""
-    
-    def _gen_deploy_job(self):
-        if self.only_intermediate:
-            return ""
-        
-        destination_args = [
-            f"--destination=$CI_REGISTRY_IMAGE/{self.full_name()}:latest"
-        ]
-        if self.context.is_production:
-            destination_args.append(f"--destination=$CI_REGISTRY_IMAGE/{self.full_name()}:{self.context.commit_tag}")
-        
-        return f"""
-{self.deploy_job_name()}:
-    stage: deploy
-    timeout: 1h
-    image:
-        name: gcr.io/kaniko-project/executor:v1.23.2-debug
-        entrypoint: [""]
-    script:
-        - {self._get_auth_echo()}
-        - mkdir /workdir
-        - echo "FROM registry.git.fsmpi.rwth-aachen.de/videoag/backend/{self.versioned_full_name()} > /workdir/Dockerfile"
-        - echo {self.context.commit_sha}
-        - >-
-            /kaniko/executor
-            --context=dir:///workdir
-            {"\n            ".join(destination_args)}
-"""
-
-
-def gen_test_api(context: BuildContext) -> str:
-    return f"""
-run-api-tests:
-    stage: build-and-test
-    needs: [build-api]
-    timeout: 30m
-    variables:
-        VIDEOAG_CONFIG: /code/config/api_example_config.py
-    image:
-        name: registry.git.fsmpi.rwth-aachen.de/videoag/backend/{context.env_type()}_api:{context.commit_sha}
-        entrypoint: [""]
-    script:
-        - cd /code
-        - /code/docker_start.sh -test
-    artifacts:
-        paths:
-            - /code/coverage/report.txt
-            - /code/coverage/html/*
-    services:
-        - name: postgres:17
-          alias: ci-database
-          variables:
-              POSTGRES_DB: videoagtest
-              POSTGRES_USER: videoagtest
-              POSTGRES_PASSWORD: LetMeTest...
-
-"""
-
-
-def gen_pipeline(context: BuildContext) -> str:
-    pipeline = """
-####################################################################
-##### AUTOMATICALLY GENERATED PIPELINE. DO NOT CHANGE MANUALLY! ####
-####################################################################
-
-stages:
-    - build-and-test
-    - deploy
-"""
-    
-    for target in context.targets.values():
-        pipeline += target.gen_jobs()
-    
-    pipeline += gen_test_api(context)
-    
-    return pipeline
-
-
-def main():
-    commit_sha = os.environ["CI_COMMIT_SHA"]
-    if not isinstance(commit_sha, str) or commit_sha == "":
-        raise ValueError("Empty or invalid commit sha")
-    
-    commit_tag = os.environ.get("CI_COMMIT_TAG", None)
-    if commit_tag == "":
-        commit_tag = None
-    if commit_tag is not None and not isinstance(commit_tag, str):
-        raise ValueError("Invalid commit tag")
-    
-    context = BuildContext(commit_sha, commit_tag)
-    
-    common_py = context.add_image_target("common_py", [], "common_py/", True)
-    context.add_image_target("api", [common_py], "api/", False)
-    context.add_image_target("job_controller", [common_py], "job_controller/", False)
-    
-    for job_dir in Path("job_controller/jobs").iterdir():
-        assert isinstance(job_dir, Path)
-        job_name = job_dir.name
-        
-        job_dir = Path("job_controller/jobs").joinpath(job_name)
-        dockerfile = job_dir.joinpath("Dockerfile")
-        pattern = "FROM registry\\.git\\.fsmpi\\.rwth-aachen\\.de\\/videoag\\/backend/\\$\\{ENV_TYPE}_([a-zA-Z0-9-_]+):\\$\\{GIT_COMMIT_SHA}"
-        matches = re.findall(pattern, dockerfile.read_text())
-        if len(matches) != 1:
-            raise Exception(f"{dockerfile}: Unable to determine base image for pipeline dependencies. Cannot find"
-                            f"special FROM instruction (Or found multiple) (See other job's images)")
-        base_image_name = matches[0]
-        
-        context.add_image_target(
-            f"job_{job_name}",
-            [base_image_name],
-            f"job_controller/jobs/{job_dir.name}",
-            not job_dir.joinpath("metadata.json").exists()
-        )
-    
-    Path("child-pipeline.yml").write_text(gen_pipeline(context))
-
-
-if __name__ == "__main__":
-    try:
-        main()
-    except Exception as e:
-        traceback.print_exception(e)
-        exit(-1)

generate_dockerfiles.sh

+#!/bin/bash
+python build_pipeline_generator.py api job_controller $(find job_controller/jobs/ -mindepth 1 -maxdepth 1)

job_controller/Dockerfile

-# Can be "development" or "production"
-ARG ENV_TYPE
-ARG GIT_COMMIT_SHA
-FROM registry.git.fsmpi.rwth-aachen.de/videoag/backend/${ENV_TYPE}_common_py:${GIT_COMMIT_SHA}
-
-# READ THE NOTE on caching in the README before changing this/for more info!
-ARG PIP_CACHE_DIR=
-
-
-# Empty by default
-ARG GIT_COMMIT_TAG=
-
-ENV VIDEOAG_JOB_CONTROLLER_GIT_COMMIT_HASH $GIT_COMMIT_SHA
-ENV VIDEOAG_JOB_CONTROLLER_GIT_COMMIT_TAG $GIT_COMMIT_TAG
-
-COPY extra_requirements.txt ./
-# READ THE NOTE on caching in the README before changing this/for more info!
-RUN --mount=type=cache,target=/root/.cache/pip \
-    pip --python .venv/bin/python install -r extra_requirements.txt
-
-COPY jobs ./jobs/
-
-# The source has a symlink file at src/videoag_common
-# The actual files are already in the image at src/videoag_common
-# So we move the actual files temporarily, copy the src directory, remove the symlink and move the actual files back
-# In the future, COPY --exclude src/videoag_common might work (but right now it doesn't, some "failed to compute cache key")
-RUN mv src/videoag_common src/.temp
-COPY src/ ./src/
-RUN rm src/videoag_common
-RUN mv src/.temp src/videoag_common
-
-
-WORKDIR src
-
-CMD ["python", "run.py"]

job_controller/build_config.py

+TARGET_IMAGE_NAME = "job_controller"
+BUILD_DEPENDENCIES = ["../common_py/"]
+
+PIP_REQUIREMENTS_FILE = "requirements.txt"
+DOCKERFILE_EXTRA = """
+COPY $MODULE_DIR/src ./src/
+COPY $MODULE_DIR/jobs/ ./jobs/
+
+WORKDIR src
+
+CMD ["python", "run.py"]
+"""
\ No newline at end of file