More commands for importing data

server.py

@@ -23,7 +23,7 @@ import config
 from shared import db, date_filter, datetime_filter, date_filter_long, date_filter_short, time_filter, ldap_manager, security_manager, current_user, check_login, login_required, group_required, class_filter, needs_date_test, todostate_name_filter, code_filter, indent_tab_filter
 from utils import is_past, mail_manager, url_manager, get_first_unused_int, set_etherpad_text, get_etherpad_text, split_terms, optional_int_arg
 from models.database import ProtocolType, Protocol, DefaultTOP, TOP, Document, Todo, Decision, MeetingReminder, Error, TodoMail, DecisionDocument, TodoState, Meta, DefaultMeta
-from views.forms import LoginForm, ProtocolTypeForm, DefaultTopForm, MeetingReminderForm, NewProtocolForm, DocumentUploadForm, KnownProtocolSourceUploadForm, NewProtocolSourceUploadForm, ProtocolForm, TopForm, SearchForm, NewProtocolFileUploadForm, NewTodoForm, TodoForm, TodoMailForm, DefaultMetaForm, MetaForm
+from views.forms import LoginForm, ProtocolTypeForm, DefaultTopForm, MeetingReminderForm, NewProtocolForm, DocumentUploadForm, KnownProtocolSourceUploadForm, NewProtocolSourceUploadForm, ProtocolForm, TopForm, SearchForm, NewProtocolFileUploadForm, NewTodoForm, TodoForm, TodoMailForm, DefaultMetaForm, MetaForm, MergeTodosForm
 from views.tables import ProtocolsTable, ProtocolTypesTable, ProtocolTypeTable, DefaultTOPsTable, MeetingRemindersTable, ErrorsTable, TodosTable, DocumentsTable, DecisionsTable, TodoTable, ErrorTable, TodoMailsTable, DefaultMetasTable
 from legacy import import_old_todos, import_old_protocols, import_old_todomails
 
@@ -82,6 +82,40 @@ def import_legacy():
         import_old_protocols(content)
         import_old_todomails(content)
 
+@manager.command
+def recompile_all():
+    for protocol in sorted(Protocol.query.all(), key=lambda p: p.date):
+        if protocol.is_done():
+            print(protocol.get_identifier())
+            tasks.parse_protocol(protocol)
+
+@manager.command
+def merge_todos():
+    todo_by_id = {}
+    todos = Todo.query.all()
+    for todo in todos:
+        todo_id = todo.get_id()
+        if todo_id in todo_by_id:
+            todo1, todo2 = todo, todo_by_id[todo_id]
+            print(todo1)
+            print(todo2)
+            if todo2.id > todo1.id:
+                todo2, todo1 = todo1, todo2
+            for protocol in todo2.protocols:
+                if protocol not in todo1.protocols:
+                    todo1.protocols.append(protocol)
+                todo2.protocols.remove(protocol)
+            db.session.delete(todo2)
+            db.session.commit()
+            todo_by_id[todo_id] = todo1
+        else:
+            todo_by_id[todo_id] = todo
+
+@manager.command
+def runserver():
+    app.run()
+    make_scheduler()
+
 # cause uwsgi currently has a bug
 def send_file(file_like, cache_timeout, as_attachment, attachment_filename):
     mimetype, _ = mimetypes.guess_type(attachment_filename)
@@ -935,6 +969,29 @@ def delete_todo(todo_id):
     flash("Todo gelöscht.", "alert-success")
     return redirect(request.args.get("next") or url_for("list_todos", protocoltype=type_id))
 
+@app.route("/todo/merge", methods=["GET", "POST"])
+@login_required
+@group_required(config.ADMIN_GROUP)
+def merge_todos():
+    form = MergeTodosForm(request.args.get("todo_id"))
+    if form.validate_on_submit():
+        todo1 = Todo.query.filter_by(id=form.todo1.data).first()
+        todo2 = Todo.query.filter_by(id=todo.todo2.data).first()
+        if todo1 is None or todo2 is None:
+            flash("Missing todos.", "alert-error")
+        else:
+            id1 = todo1.id
+            id2 = todo2.id
+            for protocol in todo2.protocols:
+                if protocol not in todo1.protocols:
+                    todo1.protocols.append(protocol)
+                todo2.protocols.remove(protocol)
+            db.session.delete(todo2)
+            db.session.commit()
+            flash("Merged todos {} and {}.".format(id1, id2), "alert-success")
+            return redirect(request.args.get("next") or url_for("show_todos"))
+    return render_template("todos-merge.html", form=form, next_url=request.args.get("next"))
+
 @app.route("/decisions/list")
 def list_decisions():
     is_logged_In = check_login()
@@ -1240,7 +1297,6 @@ try:
             check_and_send_reminders()
 except ImportError as exc:
     def make_scheduler():
-        print(exc)
         print("uwsgi not found, falling back to apscheduler for cron-like tasks")
         scheduler = BackgroundScheduler()
         scheduler.start()
@@ -1273,5 +1329,4 @@ def check_and_send_reminders():
                     tasks.send_reminder(reminder, protocol)
 
 if __name__ == "__main__":
-    make_scheduler()
     manager.run()

tasks.py

@@ -6,6 +6,7 @@ import shutil
 import tempfile
 from datetime import datetime
 import traceback
+from copy import copy
 
 from models.database import Document, Protocol, Error, Todo, Decision, TOP, DefaultTOP, MeetingReminder, TodoMail, DecisionDocument, TodoState, OldTodo
 from models.errors import DateNotMatchingException
@@ -89,8 +90,8 @@ def parse_protocol_async_inner(protocol, encoded_kwargs):
     for error in old_errors:
         protocol.errors.remove(error)
     db.session.commit()
-    if protocol.source is None:
-        error = protocol.create_error("Parsing", "Protocol source is None", "")
+    if protocol.source is None or len(protocol.source.strip()) == 0:
+        error = protocol.create_error("Parsing", "Protocol source is empty", "")
         db.session.add(error)
         db.session.commit()
         return
@@ -111,7 +112,7 @@ def parse_protocol_async_inner(protocol, encoded_kwargs):
         db.session.commit()
         return
     remarks = {element.name: element for element in tree.children if isinstance(element, Remark)}
-    required_fields = KNOWN_KEYS
+    required_fields = copy(KNOWN_KEYS)
     for default_meta in protocol.protocoltype.metas:
         required_fields.append(default_meta.key)
     if not config.PARSER_LAZY:
@@ -151,7 +152,6 @@ def parse_protocol_async_inner(protocol, encoded_kwargs):
     old_todos = list(protocol.todos)
     for todo in old_todos:
         protocol.todos.remove(todo)
-    print(old_todo_number_map)
     db.session.commit()
     tags = tree.get_tags()
     todo_tags = [tag for tag in tags if tag.name == "todo"]

templates/todos-merge.html

+{% extends "layout.html" %}
+{% from "macros.html" import render_form %}
+{% block title %}Todo mergen{% endblock %}
+
+{% block content %}
+<div class="container">
+    {{render_form(form, action_url=url_for("merge_todos"), action_text="Mergen")}}
+</div>
+{% endblock %}

views/forms.py

@@ -220,3 +220,11 @@ class MetaForm(FlaskForm):
 class DefaultMetaForm(FlaskForm):
     key = StringField("Key", validators=[InputRequired("Bitte gib den Protokoll-Syntax-Schlüssel der Metadaten an.")])
     name = StringField("Name", validators=[InputRequired("Bitte gib den Namen der Metadaten an.")])
+
+class MergeTodosForm(FlaskForm):
+    todo1 = IntegerField("todo 1", validators=[InputRequired()])
+    todo2 = IntegerField("todo 2", validators=[InputRequired()])
+
+    def __init__(self, todo=None):
+        if todo is not None:
+            self.todo1.data = todo.id