Skip to content
Snippets Groups Projects
Select Git revision
  • 9ceabf71e75688e28521f99572e9ecfe27c4a775
  • master default protected
2 results

tasks.py

Blame
  • Forked from protokollsystem / proto3
    Source project has a limited visibility.
    Code owners
    Assign users and groups as approvers for specific file changes. Learn more.
    generate_ci_pipeline.py 7.92 KiB
    import os
    import re
    import traceback
    from dataclasses import dataclass
    from pathlib import Path
    
    
    # This file dynamically generates a GitLab CI pipeline. This is needed since the GitLab CI is not quite flexible enough
    # to handle the different "development" and "production" builds and to automatically build all the jobs.
    # For development (everything that is not a tag pipeline whose tag starts with v) the images are put into the registry
    # with the "development" prefix. For production the images are put into the registry with the "production" prefix and
    # that prefix is write-protected, so only maintainers (or pipelines triggered by them) can push new images to production.
    # Kubernetes directly uses those images from the registry.
    
    
    class BuildContext:
        
        def __init__(self, commit_sha: str, commit_tag: str or None):
            super().__init__()
            self.commit_sha = commit_sha
            self.commit_tag = commit_tag
            self.is_production = commit_tag is not None and re.fullmatch("v.*", commit_tag) is not None
            self.targets: dict[str, "ImageTarget"] = {}
        
        def env_type(self) -> str:
            return "production" if self.is_production else "development"
        
        def add_image_target(self,
                             image_name: str,
                             dependency_targets: list["ImageTarget"],
                             context_sub_path: str,
                             only_intermediate: bool
                             ) -> "ImageTarget":
            target = ImageTarget(
                self,
                image_name,
                dependency_targets,
                context_sub_path,
                only_intermediate
            )
            self.targets[target.image_name] = target
            return target
    
    
    @dataclass
    class ImageTarget:
        context: BuildContext
        image_name: str
        dependency_targets: list["ImageTarget" or str]
        context_sub_path: str
        only_intermediate: bool
        
        def full_name(self):
            return f"{self.context.env_type()}_{self.image_name}"
        
        def versioned_full_name(self):
            return f"{self.full_name()}:{self.context.commit_sha}"
        
        def build_job_name(self):
            return f"build-{self.image_name}"
        
        def deploy_job_name(self):
            return f"deploy-{self.image_name}"
        
        def _validate(self):
            for i in range(len(self.dependency_targets)):
                target = self.dependency_targets[i]
                if isinstance(target, ImageTarget):
                    if target not in self.context.targets.values():
                        raise ValueError(f"Unknown target {target.image_name} (object not in context)")
                else:
                    assert isinstance(target, str)
                    if target not in self.context.targets:
                        raise ValueError(f"Unknown target {target}")
                    self.dependency_targets[i] = self.context.targets[target]
        
        def gen_jobs(self):
            self._validate()
            return self._gen_build_job() + self._gen_deploy_job()
        
        @staticmethod
        def _get_auth_echo():
            return """\
    echo "{\\"auths\\":{\\"$CI_REGISTRY\\":{\\"username\\":\\"$CI_REGISTRY_USER\\",\\"password\\":\\"$CI_REGISTRY_PASSWORD\\"}}}" > /kaniko/.docker/config.json\
    """
        
        def _gen_build_job(self):
            kaniko_args = [
                f"--context=git://git.fsmpi.rwth-aachen.de/videoag/backend.git#{self.context.commit_sha}",
                f"--context-sub-path={self.context_sub_path}",
                f"--git recurse-submodules=true",
                f"--destination=$CI_REGISTRY_IMAGE/{self.versioned_full_name()}",
                f"--build-arg=GIT_COMMIT_SHA={self.context.commit_sha}",
                f"--build-arg=ENV_TYPE={self.context.env_type()}",
                f"--cache=true",
            ]
            
            if self.context.commit_tag is not None:
                kaniko_args.append(f"--build-arg=GIT_COMMIT_TAG={self.context.commit_tag}")
            
            return f"""
    {self.build_job_name()}:
        stage: build-and-test
        needs: [{",".join(t.build_job_name() for t in self.dependency_targets)}]
        timeout: 1h
        image:
            name: gcr.io/kaniko-project/executor:v1.23.2-debug
            entrypoint: [""]
        script:
            - {self._get_auth_echo()}
            - echo {self.context.commit_sha}
            - >-
                /kaniko/executor
                {"\n            ".join(kaniko_args)}
    """
        
        def _gen_deploy_job(self):
            if self.only_intermediate:
                return ""
            
            destination_args = [
                f"--destination=$CI_REGISTRY_IMAGE/{self.full_name()}:latest"
            ]
            if self.context.is_production:
                destination_args.append(f"--destination=$CI_REGISTRY_IMAGE/{self.full_name()}:{self.context.commit_tag}")
            
            return f"""
    {self.deploy_job_name()}:
        stage: deploy
        timeout: 1h
        image:
            name: gcr.io/kaniko-project/executor:v1.23.2-debug
            entrypoint: [""]
        script:
            - {self._get_auth_echo()}
            - mkdir /workdir
            - echo "FROM registry.git.fsmpi.rwth-aachen.de/videoag/backend/{self.versioned_full_name()} > /workdir/Dockerfile"
            - echo {self.context.commit_sha}
            - >-
                /kaniko/executor
                --context=dir:///workdir
                {"\n            ".join(destination_args)}
    """
    
    
    def gen_test_api(context: BuildContext) -> str:
        return f"""
    run-api-tests:
        stage: build-and-test
        needs: [build-api]
        timeout: 30m
        variables:
            VIDEOAG_CONFIG: /code/config/api_example_config.py
        image:
            name: registry.git.fsmpi.rwth-aachen.de/videoag/backend/{context.env_type()}_api:{context.commit_sha}
            entrypoint: [""]
        script:
            - cd /code
            - /code/docker_start.sh -test
        artifacts:
            paths:
                - /code/coverage/report.txt
                - /code/coverage/html/*
        services:
            - name: postgres:17
              alias: ci-database
              variables:
                  POSTGRES_USER: videoagtest
                  POSTGRES_PASSWORD: LetMeTest...
    
    """
    
    
    def gen_pipeline(context: BuildContext) -> str:
        pipeline = """
    stages:
        - build-and-test
        - deploy
    """
        
        for target in context.targets.values():
            pipeline += target.gen_jobs()
        
        pipeline += gen_test_api(context)
        
        return pipeline
    
    
    def main():
        commit_sha = os.environ["CI_COMMIT_SHA"]
        if not isinstance(commit_sha, str) or commit_sha == "":
            raise ValueError("Empty or invalid commit sha")
        
        commit_tag = os.environ.get("CI_COMMIT_TAG", None)
        if commit_tag == "":
            commit_tag = None
        if commit_tag is not None and not isinstance(commit_tag, str):
            raise ValueError("Invalid commit tag")
        
        context = BuildContext(commit_sha, commit_tag)
        
        common_py = context.add_image_target("common_py", [], "common_py/", True)
        context.add_image_target("api", [common_py], "api/", False)
        context.add_image_target("job_controller", [common_py], "job_controller/", False)
        
        for job_dir in Path("job_controller/jobs").iterdir():
            assert isinstance(job_dir, Path)
            job_name = job_dir.name
            
            job_dir = Path("job_controller/jobs").joinpath(job_name)
            dockerfile = job_dir.joinpath("Dockerfile")
            pattern = "FROM registry\\.git\\.fsmpi\\.rwth-aachen\\.de\\/videoag\\/backend/\\$\\{ENV_TYPE}_([a-zA-Z0-9-_]+):\\$\\{GIT_COMMIT_SHA}"
            matches = re.findall(pattern, dockerfile.read_text())
            if len(matches) != 1:
                raise Exception(f"{dockerfile}: Unable to determine base image for pipeline dependencies. Cannot find"
                                f"special FROM instruction (Or found multiple) (See other job's images)")
            base_image_name = matches[0]
            
            context.add_image_target(
                f"job_{job_name}",
                [base_image_name],
                f"job_controller/jobs/{job_dir.name}",
                not job_dir.joinpath("metadata.json").exists()
            )
        
        Path("child-pipeline.yml").write_text(gen_pipeline(context))
    
    
    if __name__ == "__main__":
        try:
            main()
        except Exception as e:
            traceback.print_exception(e)
            exit(-1)