From 05e0fb676223fabe34fb920fd1140eb078c5aa81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20K=C3=BCnzel?= <simonk@fsmpi.rwth-aachen.de> Date: Fri, 2 May 2025 02:26:40 +0200 Subject: [PATCH] Handle sorter transaction failure better --- job_controller/jobs/source_file_sorter/job.py | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/job_controller/jobs/source_file_sorter/job.py b/job_controller/jobs/source_file_sorter/job.py index c089b95..8d897e2 100644 --- a/job_controller/jobs/source_file_sorter/job.py +++ b/job_controller/jobs/source_file_sorter/job.py @@ -140,7 +140,14 @@ def _check_files(session: SessionDb) -> list[str]: return to_sort_file_db_paths -def _sort_file(session: SessionDb, own_job_id: int, db_path: str): +class FunctionContainer: + + def __init__(self): + super().__init__() + self.func = None + + +def _sort_file(session: SessionDb, own_job_id: int, db_path: str, on_transaction_failure: FunctionContainer): sorter_file = session.scalar( SorterFile.sudo_select().where(SorterFile.file_path == db_path) ) @@ -248,6 +255,7 @@ def _sort_file(session: SessionDb, own_job_id: int, db_path: str): else: dest_file.parent.mkdir(parents=True, exist_ok=True) shutil.move(file, dest_file) + on_transaction_failure.func = lambda: shutil.move(dest_file, file) # Set AFTER moving the file, as moving is more likely to throw exception than setting the variable # Note that with an exception the file's variables will still be updated in the database sorter_file.file_path = _get_db_path(dest_file) @@ -279,6 +287,17 @@ def execute(database, own_job_id, input_data: CJsonObject): to_sort_file_db_paths = database.execute_write_transaction_and_commit(_check_files) for db_path in to_sort_file_db_paths: + on_transaction_failure = FunctionContainer() # Sort every file in its own transaction to reduce blocking the database - database.execute_write_transaction_and_commit(_sort_file, own_job_id, db_path) + try: + database.execute_write_transaction_and_commit( + _sort_file, own_job_id, db_path, + ) + except Exception: + if on_transaction_failure.func is not None: + try: + on_transaction_failure.func() + except Exception as e: + logger.warning("Error while calling failed transaction handler: ", e) + raise -- GitLab