From f54dd511095584ad7b193b7b6640ab0fceb0cef4 Mon Sep 17 00:00:00 2001
From: Dorian Koch <doriank@fsmpi.rwth-aachen.de>
Date: Thu, 26 Sep 2024 15:52:41 +0200
Subject: [PATCH] Don't immediately delete everything after processing
---
config/job_controller_config.py | 5 +++++
src/jobs/wrapped_job.py | 7 +++++--
2 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/config/job_controller_config.py b/config/job_controller_config.py
index c33c2d8..707b6dc 100644
--- a/config/job_controller_config.py
+++ b/config/job_controller_config.py
@@ -11,3 +11,8 @@ POSTGRES = {
"database": "videoag"
}
'''
+
+# Whether to immediately delete jobs after they have been processed
+# Ideally everything that is needed will be extracted from the job and stored in the database, so this should be set to True
+DELETE_JOBS_AFTER_SUCCESS = True
+DELETE_JOBS_AFTER_FAILURE = False
diff --git a/src/jobs/wrapped_job.py b/src/jobs/wrapped_job.py
index 45e636c..4d7534a 100644
--- a/src/jobs/wrapped_job.py
+++ b/src/jobs/wrapped_job.py
@@ -40,13 +40,16 @@ class WrappedJob(Event):
if self.job_state == JobState.FINISHED:
self.success(cstate)
self.jobData.update_state(cstate, JobState.FINISHED_AND_PROCESSED)
+ if cstate.config.get("DELETE_JOBS_AFTER_SUCCESS", True):
+ cstate.k8s.delete_job_by_id(self.job_id)
elif self.job_state == JobState.FAILED:
self.failure(cstate)
self.jobData.update_state(cstate, JobState.FAILED_AND_PROCESSED)
+ if cstate.config.get("DELETE_JOBS_AFTER_FAILURE", True):
+ cstate.k8s.delete_job_by_id(self.job_id)
else:
raise Exception(f"Job is in unexpected state: {self.job_state}")
- # delete from k8s
- cstate.k8s.delete_job_by_id(self.job_id)
+
# update state to processed
if self.job_state == JobState.FINISHED_AND_PROCESSED or self.job_state == JobState.FAILED_AND_PROCESSED:
return EventResult.DONE
--
GitLab