diff --git a/config/job_controller_config.py b/config/job_controller_config.py index c33c2d8f3e45da6d1ab0829162ec14fa570644ca..707b6dce254ee38b55261fee5050a05ad8b30b10 100644 --- a/config/job_controller_config.py +++ b/config/job_controller_config.py @@ -11,3 +11,8 @@ POSTGRES = { "database": "videoag" } ''' + +# Whether to immediately delete jobs after they have been processed +# Ideally everything that is needed will be extracted from the job and stored in the database, so this should be set to True +DELETE_JOBS_AFTER_SUCCESS = True +DELETE_JOBS_AFTER_FAILURE = False diff --git a/src/jobs/wrapped_job.py b/src/jobs/wrapped_job.py index 45e636c1092c6093635d07ef24e28c35ba005a7b..4d7534a71f7c2ea209f02e31db65f40735e8b4e7 100644 --- a/src/jobs/wrapped_job.py +++ b/src/jobs/wrapped_job.py @@ -40,13 +40,16 @@ class WrappedJob(Event): if self.job_state == JobState.FINISHED: self.success(cstate) self.jobData.update_state(cstate, JobState.FINISHED_AND_PROCESSED) + if cstate.config.get("DELETE_JOBS_AFTER_SUCCESS", True): + cstate.k8s.delete_job_by_id(self.job_id) elif self.job_state == JobState.FAILED: self.failure(cstate) self.jobData.update_state(cstate, JobState.FAILED_AND_PROCESSED) + if cstate.config.get("DELETE_JOBS_AFTER_FAILURE", True): + cstate.k8s.delete_job_by_id(self.job_id) else: raise Exception(f"Job is in unexpected state: {self.job_state}") - # delete from k8s - cstate.k8s.delete_job_by_id(self.job_id) + # update state to processed if self.job_state == JobState.FINISHED_AND_PROCESSED or self.job_state == JobState.FAILED_AND_PROCESSED: return EventResult.DONE