From c0dcdd10d7915c97db7b649c49d4f8182bdf38be Mon Sep 17 00:00:00 2001
From: Dorian Koch <doriank@fsmpi.rwth-aachen.de>
Date: Thu, 5 Dec 2024 10:45:48 +0100
Subject: [PATCH] Add export
---
export.py | 74 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 74 insertions(+)
create mode 100644 export.py
diff --git a/export.py b/export.py
new file mode 100644
index 0000000..948ae2e
--- /dev/null
+++ b/export.py
@@ -0,0 +1,74 @@
+import os
+import sys
+import csv
+import time
+import psycopg2
+
+
+def export_response_times_to_csv():
+ # Get environment variables
+ db_host = os.getenv('DB_HOST')
+ db_name = os.getenv('DB_NAME')
+ db_user = os.getenv('DB_USER')
+ db_password = os.getenv('DB_PASSWORD')
+
+ try:
+ # Log start of export
+ print("Starting database export...", file=sys.stderr)
+ start_time = time.time()
+
+ # Connect to PostgreSQL database
+ print(f"Connecting to database {db_name} on host {db_host}...", file=sys.stderr)
+ conn = psycopg2.connect(
+ host=db_host,
+ database=db_name,
+ user=db_user,
+ password=db_password
+ )
+ cur = conn.cursor()
+
+ # Count total rows
+ cur.execute("SELECT COUNT(*) FROM response_times")
+ total_rows = cur.fetchone()[0]
+ print(f"Total rows to export: {total_rows}", file=sys.stderr)
+
+ # Execute query to fetch all rows from response_times table
+ print("Fetching data...", file=sys.stderr)
+ cur.execute("SELECT * FROM response_times")
+
+ # Get column names
+ column_names = [desc[0] for desc in cur.description]
+
+ # Create a CSV writer that writes to stdout
+ writer = csv.writer(sys.stdout)
+
+ # Write headers
+ writer.writerow(column_names)
+
+ # Write data rows with progress tracking
+ rows_exported = 0
+ for row in cur:
+ writer.writerow(row)
+ rows_exported += 1
+
+ # Periodic progress update
+ if rows_exported % 1000 == 0:
+ print(f"Exported {rows_exported}/{total_rows} rows...", file=sys.stderr)
+
+ # Log completion
+ end_time = time.time()
+ export_duration = end_time - start_time
+ print(f"\nExport completed in {export_duration:.2f} seconds.", file=sys.stderr)
+ print(f"Exported {rows_exported} rows in total.", file=sys.stderr)
+
+ # Close cursor and connection
+ cur.close()
+ conn.close()
+
+ except Exception as e:
+ print(f"An error occurred: {e}", file=sys.stderr)
+ sys.exit(1)
+
+
+if __name__ == "__main__":
+ export_response_times_to_csv()
--
GitLab