From aeefd324e10b783ef96905b509016f3628019469 Mon Sep 17 00:00:00 2001
From: Dorian Koch <doriank@fsmpi.rwth-aachen.de>
Date: Thu, 26 Sep 2024 16:14:22 +0200
Subject: [PATCH] Use more resources for CutAndTranscode
---
src/jobs/cut_and_transcode.py | 1 +
src/jobs/wrapped_job.py | 9 +++++++++
src/kubernetes_api.py | 19 +++++++++----------
3 files changed, 19 insertions(+), 10 deletions(-)
diff --git a/src/jobs/cut_and_transcode.py b/src/jobs/cut_and_transcode.py
index b41cca2..4498337 100644
--- a/src/jobs/cut_and_transcode.py
+++ b/src/jobs/cut_and_transcode.py
@@ -43,6 +43,7 @@ class CutAndTranscode(WrappedJob):
def config(self) -> ContainerConfig:
cfg = ContainerConfig(image="registry.git.fsmpi.rwth-aachen.de/videoag_infra/production/dominic_render_video:latest", args=["-C", self.in_dir])
cfg.mount_video_fs("/mnt/video_data")
+ cfg.resources("8", "12Gi")
return cfg
def prepare(self, cstate: "ControllerState"):
diff --git a/src/jobs/wrapped_job.py b/src/jobs/wrapped_job.py
index 4d7534a..8f28541 100644
--- a/src/jobs/wrapped_job.py
+++ b/src/jobs/wrapped_job.py
@@ -10,10 +10,19 @@ class ContainerConfig:
self.command = command
self.args = args
self.mount_video_fs_path: str | None = None
+ self.resources_dict: dict | None = None
def mount_video_fs(self, mount_path: str):
self.mount_video_fs_path = mount_path
+ def resources(self, cpu: str, memory: str):
+ self.resources_dict = {
+ "limits": {
+ "cpu": cpu,
+ "memory": memory
+ }
+ }
+
class WrappedJob(Event):
diff --git a/src/kubernetes_api.py b/src/kubernetes_api.py
index 47a0f79..2b1e50c 100644
--- a/src/kubernetes_api.py
+++ b/src/kubernetes_api.py
@@ -34,15 +34,7 @@ class K8sApi():
def list_worker_jobs(self) -> V1JobList:
return self.batch_v1.list_namespaced_job(NAMESPACE, label_selector=f"app={WORKER_LABEL}")
- def create_job(self, job: WrappedJob, resources={}):
- default_res = {
- "limits": {
- "cpu": "1",
- "memory": "1Gi"
- }
- }
- resources = {**default_res, **resources}
-
+ def create_job(self, job: WrappedJob):
cfg = job.config()
job_manifest = {
@@ -76,7 +68,12 @@ class K8sApi():
"image": cfg.image,
"args": cfg.args,
"command": cfg.command,
- "resources": resources
+ "resources": {
+ "limits": {
+ "cpu": "1",
+ "memory": "1Gi"
+ }
+ }
}
],
"priorityClassName": "videoag-worker-priority",
@@ -118,6 +115,8 @@ class K8sApi():
}
}
]
+ if cfg.resources_dict is not None:
+ job_manifest["spec"]["template"]["spec"]["containers"][0]["resources"] = cfg.resources_dict
# print(f"Creating job with manifest: {json.dumps(job_manifest)}")
return self.batch_v1.create_namespaced_job(NAMESPACE, job_manifest)
--
GitLab