Select Git revision
Code owners
Assign users and groups as approvers for specific file changes. Learn more.
generate_ci_pipeline.py 8.24 KiB
import os
import re
import traceback
from dataclasses import dataclass
from pathlib import Path
# This file dynamically generates a GitLab CI pipeline. This is needed since the GitLab CI is not quite flexible enough
# to handle the different "development" and "production" builds and to automatically build all the jobs.
# For development (everything that is not a tag pipeline whose tag starts with v) the images are put into the registry
# with the "development" prefix. For production the images are put into the registry with the "production" prefix and
# that prefix is write-protected, so only maintainers (or pipelines triggered by them) can push new images to production.
# Kubernetes directly uses those images from the registry.
class BuildContext:
def __init__(self, commit_sha: str, commit_tag: str or None):
super().__init__()
self.commit_sha = commit_sha
self.commit_tag = commit_tag
self.is_production = commit_tag is not None and re.fullmatch("v.*", commit_tag) is not None
self.targets: dict[str, "ImageTarget"] = {}
def env_type(self) -> str:
return "production" if self.is_production else "development"
def add_image_target(self,
image_name: str,
dependency_targets: list["ImageTarget"],
context_sub_path: str,
only_intermediate: bool
) -> "ImageTarget":
target = ImageTarget(
self,
image_name,
dependency_targets,
context_sub_path,
only_intermediate
)
self.targets[target.image_name] = target
return target
@dataclass
class ImageTarget:
context: BuildContext
image_name: str
dependency_targets: list["ImageTarget" or str]
context_sub_path: str
only_intermediate: bool
def full_name(self):
return f"{self.context.env_type()}_{self.image_name}"
def versioned_full_name(self):
return f"{self.full_name()}:{self.context.commit_sha}"
def build_job_name(self):
return f"build-{self.image_name}"
def deploy_job_name(self):
return f"deploy-{self.image_name}"
def _validate(self):
for i in range(len(self.dependency_targets)):
target = self.dependency_targets[i]
if isinstance(target, ImageTarget):
if target not in self.context.targets.values():
raise ValueError(f"Unknown target {target.image_name} (object not in context)")
else:
assert isinstance(target, str)
if target not in self.context.targets:
raise ValueError(f"Unknown target {target}")
self.dependency_targets[i] = self.context.targets[target]
def gen_jobs(self):
self._validate()
return self._gen_build_job() + self._gen_deploy_job()
@staticmethod
def _get_auth_echo():
return """\
echo "{\\"auths\\":{\\"$CI_REGISTRY\\":{\\"username\\":\\"$CI_REGISTRY_USER\\",\\"password\\":\\"$CI_REGISTRY_PASSWORD\\"}}}" > /kaniko/.docker/config.json\
"""
def _gen_build_job(self):
kaniko_args = [
f"--context=git://git.fsmpi.rwth-aachen.de/videoag/backend.git#{self.context.commit_sha}",
f"--context-sub-path={self.context_sub_path}",
f"--git recurse-submodules=true",
f"--destination=$CI_REGISTRY_IMAGE/{self.versioned_full_name()}",
f"--build-arg=GIT_COMMIT_SHA={self.context.commit_sha}",
f"--build-arg=ENV_TYPE={self.context.env_type()}",
# See https://github.com/GoogleContainerTools/kaniko/issues/969#issuecomment-2160910028
f"--build-arg=PIP_CACHE_DIR=$CI_PROJECT_DIR/.cache/pip",
f"--cache=true",
]
if self.context.commit_tag is not None:
kaniko_args.append(f"--build-arg=GIT_COMMIT_TAG={self.context.commit_tag}")
return f"""
{self.build_job_name()}:
stage: build-and-test
needs: [{",".join(t.build_job_name() for t in self.dependency_targets)}]
timeout: 1h
image:
name: gcr.io/kaniko-project/executor:v1.23.2-debug
entrypoint: [""]
cache:
key: videoag-cache-{self.context.env_type()}
paths:
- .cache/pip
script:
- {self._get_auth_echo()}
- echo {self.context.commit_sha}
- >-
/kaniko/executor
{"\n ".join(kaniko_args)}
"""
def _gen_deploy_job(self):
if self.only_intermediate:
return ""
destination_args = [
f"--destination=$CI_REGISTRY_IMAGE/{self.full_name()}:latest"
]
if self.context.is_production:
destination_args.append(f"--destination=$CI_REGISTRY_IMAGE/{self.full_name()}:{self.context.commit_tag}")
return f"""
{self.deploy_job_name()}:
stage: deploy
timeout: 1h
image:
name: gcr.io/kaniko-project/executor:v1.23.2-debug
entrypoint: [""]
script:
- {self._get_auth_echo()}
- mkdir /workdir
- echo "FROM registry.git.fsmpi.rwth-aachen.de/videoag/backend/{self.versioned_full_name()} > /workdir/Dockerfile"
- echo {self.context.commit_sha}
- >-
/kaniko/executor
--context=dir:///workdir
{"\n ".join(destination_args)}
"""
def gen_test_api(context: BuildContext) -> str:
return f"""
run-api-tests:
stage: build-and-test
needs: [build-api]
timeout: 30m
variables:
VIDEOAG_CONFIG: /code/config/api_example_config.py
image:
name: registry.git.fsmpi.rwth-aachen.de/videoag/backend/{context.env_type()}_api:{context.commit_sha}
entrypoint: [""]
script:
- cd /code
- /code/docker_start.sh -test
artifacts:
paths:
- /code/coverage/report.txt
- /code/coverage/html/*
services:
- name: postgres:17
alias: ci-database
variables:
POSTGRES_DB: videoagtest
POSTGRES_USER: videoagtest
POSTGRES_PASSWORD: LetMeTest...
"""
def gen_pipeline(context: BuildContext) -> str:
pipeline = """
stages:
- build-and-test
- deploy
"""
for target in context.targets.values():
pipeline += target.gen_jobs()
pipeline += gen_test_api(context)
return pipeline
def main():
commit_sha = os.environ["CI_COMMIT_SHA"]
if not isinstance(commit_sha, str) or commit_sha == "":
raise ValueError("Empty or invalid commit sha")
commit_tag = os.environ.get("CI_COMMIT_TAG", None)
if commit_tag == "":
commit_tag = None
if commit_tag is not None and not isinstance(commit_tag, str):
raise ValueError("Invalid commit tag")
context = BuildContext(commit_sha, commit_tag)
common_py = context.add_image_target("common_py", [], "common_py/", True)
context.add_image_target("api", [common_py], "api/", False)
context.add_image_target("job_controller", [common_py], "job_controller/", False)
for job_dir in Path("job_controller/jobs").iterdir():
assert isinstance(job_dir, Path)
job_name = job_dir.name
job_dir = Path("job_controller/jobs").joinpath(job_name)
dockerfile = job_dir.joinpath("Dockerfile")
pattern = "FROM registry\\.git\\.fsmpi\\.rwth-aachen\\.de\\/videoag\\/backend/\\$\\{ENV_TYPE}_([a-zA-Z0-9-_]+):\\$\\{GIT_COMMIT_SHA}"
matches = re.findall(pattern, dockerfile.read_text())
if len(matches) != 1:
raise Exception(f"{dockerfile}: Unable to determine base image for pipeline dependencies. Cannot find"
f"special FROM instruction (Or found multiple) (See other job's images)")
base_image_name = matches[0]
context.add_image_target(
f"job_{job_name}",
[base_image_name],
f"job_controller/jobs/{job_dir.name}",
not job_dir.joinpath("metadata.json").exists()
)
Path("child-pipeline.yml").write_text(gen_pipeline(context))
if __name__ == "__main__":
try:
main()
except Exception as e:
traceback.print_exception(e)
exit(-1)