server.py

#!/usr/bin/env python3
import locale
locale.setlocale(locale.LC_TIME, "de_DE.utf8")

from flask import Flask, g, current_app, request, session, flash, redirect, url_for, abort, render_template, Response, send_file as flask_send_file, Markup
from werkzeug.utils import secure_filename
from flask_script import Manager, prompt
from flask_migrate import Migrate, MigrateCommand
#from flask_socketio import SocketIO
from celery import Celery
from sqlalchemy import or_, and_
from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.triggers.cron import CronTrigger
from apscheduler.triggers.interval import IntervalTrigger
import atexit
from io import StringIO, BytesIO
import os
from datetime import datetime
import math
import mimetypes
import subprocess

import config
from shared import db, date_filter, datetime_filter, date_filter_long, date_filter_short, time_filter, time_filter_short, user_manager, security_manager, current_user, check_login, login_required, group_required, class_filter, needs_date_test, todostate_name_filter, code_filter, indent_tab_filter
from utils import is_past, mail_manager, get_first_unused_int, set_etherpad_text, get_etherpad_text, split_terms, optional_int_arg, fancy_join, footnote_hash
from decorators import db_lookup, require_public_view_right, require_private_view_right, require_modify_right, require_publish_right, require_admin_right
from models.database import ProtocolType, Protocol, DefaultTOP, TOP, LocalTOP, Document, Todo, Decision, MeetingReminder, Error, TodoMail, DecisionDocument, TodoState, Meta, DefaultMeta, DecisionCategory, Like
from views.forms import LoginForm, ProtocolTypeForm, DefaultTopForm, MeetingReminderForm, NewProtocolForm, DocumentUploadForm, KnownProtocolSourceUploadForm, NewProtocolSourceUploadForm, generate_protocol_form, TopForm, LocalTopForm, SearchForm, DecisionSearchForm, ProtocolSearchForm, TodoSearchForm, NewProtocolFileUploadForm, NewTodoForm, TodoForm, TodoMailForm, DefaultMetaForm, MetaForm, MergeTodosForm, DecisionCategoryForm, DocumentEditForm
from views.tables import ProtocolsTable, ProtocolTypesTable, ProtocolTypeTable, DefaultTOPsTable, MeetingRemindersTable, ErrorsTable, TodosTable, DocumentsTable, DecisionsTable, TodoTable, ErrorTable, TodoMailsTable, DefaultMetasTable, DecisionCategoriesTable
from legacy import import_old_todos, import_old_protocols, import_old_todomails

app = Flask(__name__)
app.config.from_object(config)
db.init_app(app)
migrate = Migrate(app, db)
manager = Manager(app)
manager.add_command("db", MigrateCommand)

def make_celery(app, config):
    celery = Celery(app.import_name, broker=config.CELERY_BROKER_URL)
    celery.conf.update(app.config)
    return celery
celery = make_celery(app, config)

#def make_socketio(app, config):
#    socketio = SocketIO(app)
#    return socketio
#socketio = make_socketio(app, config)

app.jinja_env.trim_blocks = True
app.jinja_env.lstrip_blocks = True
app.jinja_env.filters["datify"] = date_filter
app.jinja_env.filters["datetimify"] = datetime_filter
app.jinja_env.filters["timify"] = time_filter
app.jinja_env.filters["timify_short"] = time_filter_short
app.jinja_env.filters["datify_short"] = date_filter_short
app.jinja_env.filters["datify_long"] = date_filter_long
#app.jinja_env.filters["url_complete"] = url_manager.complete
app.jinja_env.filters["class"] = class_filter
app.jinja_env.filters["todo_get_name"] = todostate_name_filter
app.jinja_env.filters["code"] = code_filter
app.jinja_env.filters["indent_tab"] = indent_tab_filter
app.jinja_env.filters["fancy_join"] = fancy_join
app.jinja_env.filters["footnote_hash"] = footnote_hash
app.jinja_env.tests["auth_valid"] = security_manager.check_user
app.jinja_env.tests["needs_date"] = needs_date_test

additional_templates = getattr(config, "LATEX_LOCAL_TEMPLATES", None)
if additional_templates is not None and os.path.isdir(additional_templates):
    if additional_templates not in app.jinja_loader.searchpath:
        app.jinja_loader.searchpath.append(additional_templates)
    

import tasks

app.jinja_env.globals.update(check_login=check_login)
app.jinja_env.globals.update(current_user=current_user)
app.jinja_env.globals.update(zip=zip)
app.jinja_env.globals.update(min=min)
app.jinja_env.globals.update(max=max)
app.jinja_env.globals.update(dir=dir)
app.jinja_env.globals.update(now=datetime.now)

def get_git_revision():
    gitlab_url = "https://git.fsmpi.rwth-aachen.de/protokollsystem/proto3"
    commit_hash = subprocess.check_output(["git", "log", "-g", "-1", "--pretty=%H"]).decode("UTF-8").strip()
    timestamp = int(subprocess.check_output(["git", "log", "-g", "-1", "--pretty=%at"]).strip())
    commit_date = datetime.fromtimestamp(timestamp)
    return {"url": gitlab_url, "hash": commit_hash, "date": commit_date}

try:
    app.jinja_env.globals["git_revision"] = get_git_revision()
except:
    pass

# blueprints here

@manager.command
def import_legacy():
    """Import the old todos and protocols from an sql dump"""
    filename = prompt("SQL-file")
    #filename = "legacy.sql"
    with open(filename, "rb") as sqlfile:
        content = sqlfile.read().decode("utf-8")
        import_old_todos(content)
        import_old_protocols(content)
        import_old_todomails(content)

@manager.command
def recompile_all():
    for protocol in sorted(Protocol.query.all(), key=lambda p: p.date):
        if protocol.is_done():
            print(protocol.get_short_identifier())
            tasks.parse_protocol(protocol)

@manager.command
def merge_todos():
    todo_by_id = {}
    todos = Todo.query.all()
    for todo in todos:
        todo_id = todo.get_id()
        if todo_id in todo_by_id:
            todo1, todo2 = todo, todo_by_id[todo_id]
            print(todo1)
            print(todo2)
            if todo2.state.value > todo1.state.value:
                todo2, todo1 = todo1, todo2
            for protocol in todo2.protocols:
                if protocol not in todo1.protocols:
                    todo1.protocols.append(protocol)
                todo2.protocols.remove(protocol)
            db.session.delete(todo2)
            db.session.commit()
            todo_by_id[todo_id] = todo1
        else:
            todo_by_id[todo_id] = todo

@manager.command
def runserver():
    app.run()
    make_scheduler()

# cause uwsgi currently has a bug
def send_file(file_like, cache_timeout, as_attachment, attachment_filename):
    mimetype, _ = mimetypes.guess_type(attachment_filename)
    response = Response(file_like.read(), mimetype)
    if as_attachment:
        response.headers["Content-Disposition"] = 'attachment; filename="{}"'.format(attachment_filename)
    content_type = mimetype
    if mimetype.startswith("text/"):
        content_type = "{}; charset=utf-8".format(content_type)
    response.headers["Content-Type"] = content_type
    response.headers["Cache-Control"] = "public, max-age={}".format(cache_timeout)
    response.headers["Connection"] = "close"
    return response

@app.route("/")
def index():
    user = current_user()
    protocols = [
        protocol for protocol in Protocol.query.all()
        if protocol.protocoltype.has_public_view_right(user,
            check_networks=False)
    ]
    def _protocol_sort_key(protocol):
        if protocol.date is not None:
            return protocol.date
        return datetime.now().date()
    current_day = datetime.now().date()
    open_protocols = sorted(
        [
            protocol for protocol in protocols
            if not protocol.done
            and (protocol.date - current_day).days < config.MAX_INDEX_DAYS
            and (current_day - protocol.date).days < config.MAX_PAST_INDEX_DAYS
        ],
        key=_protocol_sort_key
    )
    finished_protocols = sorted(
        [
            protocol for protocol in protocols
            if protocol.done and protocol.public
            and (protocol.has_private_view_right(user)
                or protocol.protocoltype.has_public_view_right(user, check_networks=False))
        ],
        key=_protocol_sort_key,
        reverse=True
    )
    protocol = None
    show_private = False
    has_public_view_right = False
    if len(finished_protocols) > 0:
        protocol = finished_protocols[0]
        show_private = protocol.has_private_view_right(user)
        has_public_view_right = protocol.protocoltype.has_public_view_right(user)
    todos = None
    if check_login():
        todos = [
            todo for todo in Todo.query.all()
            if todo.protocoltype.has_private_view_right(user)
            and not todo.is_done()
        ]
        user_todos = [
            todo for todo in todos
            if user.username.lower() in list(map(str.strip, todo.who.lower().split(",")))
        ]
        if len(user_todos) > 0:
            todos = user_todos
        def _todo_sort_key(todo):
            protocol = todo.get_first_protocol()
            return protocol.date if protocol is not None and protocol.date is not None else datetime.now().date()
        todos = sorted(todos, key=_todo_sort_key, reverse=True)
    todos_table = TodosTable(todos) if todos is not None else None
    return render_template("index.html", open_protocols=open_protocols, protocol=protocol, todos=todos, show_private=show_private, has_public_view_right=has_public_view_right)

@app.route("/documentation")
@login_required
def documentation():
    todostates = list(TodoState)
    name_to_state = TodoState.get_name_to_state()
    return render_template("documentation.html", todostates=todostates, name_to_state=name_to_state)

@app.route("/types/list")
@login_required
def list_types():
    is_logged_in = check_login()
    user = current_user()
    types = [
        protocoltype for protocoltype in ProtocolType.query.all()
        if (protocoltype.has_private_view_right(user)
        or protocoltype.has_public_view_right(user)
        or protocoltype.is_public)]
    types = sorted(types, key=lambda t: t.short_name)
    types_table = ProtocolTypesTable(types)
    return render_template("types-list.html", types=types, types_table=types_table)

@app.route("/type/new", methods=["GET", "POST"])
@login_required
def new_type():
    form = ProtocolTypeForm()
    if form.validate_on_submit():
        user = current_user()
        if form.private_group.data not in user.groups:
            flash("Du kannst keinen internen Protokolltypen anlegen, zu dem du selbst keinen Zugang hast.", "alert-error")
        else:
            protocoltype = ProtocolType()
            form.populate_obj(protocoltype)
            db.session.add(protocoltype)
            db.session.commit()
            flash("Der Protokolltyp {} wurde angelegt.".format(protocoltype.name), "alert-success")
        return redirect(request.args.get("next") or url_for("list_types"))
    return render_template("type-new.html", form=form)

@app.route("/type/edit/<int:protocoltype_id>", methods=["GET", "POST"])
@login_required
@db_lookup(ProtocolType)
@require_private_view_right()
def edit_type(protocoltype):
    user = current_user()
    form = ProtocolTypeForm(obj=protocoltype)
    if form.validate_on_submit():
        if form.private_group.data not in user.groups:
            flash("Du kannst keinen internen Protokolltypen anlegen, zu dem du selbst keinen Zugang hast.", "alert-error")
        else:
            form.populate_obj(protocoltype)
            db.session.commit()
            return redirect(request.args.get("next") or url_for("show_type", protocoltype_id=protocoltype.id))
    return render_template("type-edit.html", form=form, protocoltype=protocoltype)

@app.route("/type/show/<int:protocoltype_id>")
@login_required
@db_lookup(ProtocolType)
@require_private_view_right()
def show_type(protocoltype):
    protocoltype_table = ProtocolTypeTable(protocoltype)
    default_tops_table = DefaultTOPsTable(protocoltype.default_tops, protocoltype)
    reminders_table = MeetingRemindersTable(protocoltype.reminders, protocoltype)
    metas_table = DefaultMetasTable(protocoltype.metas, protocoltype)
    categories_table = DecisionCategoriesTable(protocoltype.decisioncategories, protocoltype)
    return render_template("type-show.html", protocoltype=protocoltype, protocoltype_table=protocoltype_table, default_tops_table=default_tops_table, metas_table=metas_table, reminders_table=reminders_table, mail_active=config.MAIL_ACTIVE, categories_table=categories_table)

@app.route("/type/delete/<int:protocoltype_id>")
@login_required
@group_required(config.ADMIN_GROUP)
@db_lookup(ProtocolType)
@require_modify_right()
def delete_type(protocoltype):
    name = protocoltype.name
    db.session.delete(protocoltype) 
    db.session.commit()
    flash("Der Protokolltype {} wurde gelöscht.".format(name), "alert-success")
    return redirect(request.args.get("next") or url_for("list_types"))

@app.route("/type/reminders/new/<int:protocoltype_id>", methods=["GET", "POST"])
@login_required
@db_lookup(ProtocolType)
@require_modify_right()
def new_reminder(protocoltype):
    form = MeetingReminderForm()
    if form.validate_on_submit():
        meetingreminder = MeetingReminder(protocoltype_id=protocoltype.id)
        form.populate_obj(meetingreminder)
        db.session.add(meetingreminder)
        db.session.commit()
        return redirect(request.args.get("next") or url_for("show_type", protocoltype_id=protocoltype.id))
    return render_template("reminder-new.html", form=form, protocoltype=protocoltype)

@app.route("/type/reminder/edit/<int:meetingreminder_id>", methods=["GET", "POST"])
@login_required
@db_lookup(MeetingReminder)
@require_modify_right()
def edit_reminder(meetingreminder):
    form = MeetingReminderForm(obj=meetingreminder)
    if form.validate_on_submit():
        form.populate_obj(meetingreminder)
        db.session.commit()
        return redirect(request.args.get("next") or url_for("show_type", protocoltype_id=protocoltype.id))
    return render_template("reminder-edit.html", form=form, meetingreminder=meetingreminder)

@app.route("/type/reminder/delete/<int:meetingreminder_id>")
@login_required
@db_lookup(MeetingReminder)
@require_modify_right()
def delete_reminder(meetingreminder):
    protocoltype = meetingreminder.protocoltype
    db.session.delete(meetingreminder)
    db.session.commit()
    return redirect(request.args.get("next") or url_for("show_type", protocoltype_id=protocoltype.id))

@app.route("/type/tops/new/<int:protocoltype_id>", methods=["GET", "POST"])
@login_required
@db_lookup(ProtocolType)
@require_modify_right()
def new_default_top(protocoltype):
    form = DefaultTopForm()
    if form.validate_on_submit():
        defaulttop = DefaultTOP(protocoltype_id=protocoltype.id)
        form.populate_obj(defaulttop)
        db.session.add(defaulttop)
        db.session.commit()
        for protocol in protocoltype.protocols:
            if not protocol.done:
                localtop = LocalTOP(protocol_id=protocol.id,
                    defaulttop_id=defaulttop.id, description="")
                db.session.add(localtop)
        db.session.commit()
        flash("Der Standard-TOP {} wurde für dem Protokolltyp {} hinzugefügt.".format(defaulttop.name, protocoltype.name), "alert-success")
        return redirect(request.args.get("next") or url_for("index"))
    return render_template("default-top-new.html", form=form, protocoltype=protocoltype)

@app.route("/type/tops/edit/<int:protocoltype_id>/<int:defaulttop_id>", methods=["GET", "POST"])
@login_required
@db_lookup(ProtocolType, DefaultTOP)
@require_modify_right()
def edit_default_top(protocoltype, defaulttop):
    form = DefaultTopForm(obj=defaulttop)
    if form.validate_on_submit():
        form.populate_obj(defaulttop)
        db.session.commit()
        return redirect(request.args.get("next") or url_for("show_type", protocoltype_id=protocoltype.id))
    return render_template("default-top-edit.html", form=form, protocoltype=protocoltype, defaulttop=defaulttop)

@app.route("/type/tops/delete/<int:defaulttop_id>")
@login_required
@db_lookup(DefaultTOP)
@require_modify_right()
def delete_default_top(defaulttop):
    db.session.delete(defaulttop)
    db.session.commit()
    return redirect(request.args.get("next") or url_for("show_type", protocoltype_id=defaulttop.protocoltype.id))

@app.route("/type/tops/move/<int:defaulttop_id>/<diff>/")
@login_required
@db_lookup(DefaultTOP)
@require_modify_right()
def move_default_top(defaulttop, diff):
    try:
        defaulttop.number += int(diff)
        db.session.commit()
    except ValueError:
        flash("Die angegebene Differenz ist keine Zahl.", "alert-error")
    return redirect(request.args.get("next") or url_for("show_type", protocoltype_id=defaulttop.protocoltype.id))

@app.route("/protocols/list")
def list_protocols():
    is_logged_in = check_login()
    user = current_user()
    protocoltype = None
    protocoltype_id = None
    try:
        protocoltype_id = int(request.args.get("protocoltype_id"))
    except (ValueError, TypeError):
        pass
    state_open = -1
    try:
        state_open = int(request.args.get("state_open"))
    except (ValueError, TypeError):
        pass
    search_term = request.args.get("search")
    protocoltypes = ProtocolType.get_public_protocoltypes(user, check_networks=False)
    search_form = ProtocolSearchForm(protocoltypes)
    if protocoltype_id is not None:
        search_form.protocoltype_id.data = protocoltype_id
        protocoltype = ProtocolType.query.filter_by(id=protocoltype_id).first()
    if state_open is not None:
        search_form.state_open.data = state_open
    if search_term is not None:
        search_form.search.data = search_term
    protocol_query = Protocol.query
    shall_search = search_term is not None and len(search_term.strip()) > 0
    search_terms = []
    if shall_search:
        search_terms = list(map(str.lower, split_terms(search_term)))
        for term in search_terms:
            protocol_query = protocol_query.filter(or_(
                Protocol.content_public.ilike("%{}%".format(term)),
                Protocol.content_private.ilike("%{}%".format(term))
            ))
    protocols = [
        protocol for protocol in protocol_query.all()
        if protocol.protocoltype.has_public_view_right(user, check_networks=False)
    ]
    def _matches_search(content):
        content = content.lower()
        for search_term in search_terms:
            if search_term.lower() not in content:
                return False
        return True
    def _matches_search_lazy(content):
        content = content.lower()
        for search_term in search_terms:
            if search_term.lower() in content:
                return True
        return False
    search_results = {} if shall_search else None
    if protocoltype_id is not None and protocoltype_id != -1:
        protocols = [
            protocol for protocol in protocols
            if protocol.protocoltype.id == protocoltype_id
        ]
    if state_open is not None and state_open != -1:
        protocol_done = bool(state_open)
        protocols = [
            protocol for protocol in protocols
            if (protocol.is_done() or False) == protocol_done
        ]
    if shall_search:
        protocols = [
            protocol for protocol in protocols
            if (protocol.protocoltype.has_private_view_right(user)
                and _matches_search(protocol.content_private))
            or (protocol.has_public_view_right(user)
                and _matches_search(protocol.content_public))
        ]
        for protocol in protocols:
            content = protocol.content_private if protocol.protocoltype.has_private_view_right(user) else protocol.content_public
            lines = content.splitlines()
            matches = [line for line in lines if _matches_search_lazy(line)]
            formatted_lines = []
            for line in matches:
                parts = []
                lower_line = line.lower()
                last_index = 0
                while last_index < len(line):
                    index_candidates = list(filter(lambda t: t[0] != -1, 
                        [(lower_line.find(term, last_index), term) for term in search_terms]))
                    if len(index_candidates) == 0:
                        parts.append((line[last_index:], False))
                        break
                    else:
                        new_index, term = min(index_candidates, key=lambda t: t[0])
                        new_end_index = new_index + len(term)
                        parts.append((line[last_index:new_index], False))
                        parts.append((line[new_index:new_end_index], True))
                        last_index = new_end_index
                formatted_lines.append("".join([
                    "<b>{}</b>".format(text) if matched else text
                    for text, matched in parts
                ]))
            search_results[protocol] = " …<br />\n".join(formatted_lines)
    protocols = sorted(protocols, key=lambda protocol: protocol.date, reverse=True)
    page = _get_page()
    page_length = _get_page_length()
    page_count = int(math.ceil(len(protocols) / page_length))
    if page >= page_count:
        page = 0
    begin_index = page * page_length
    end_index = (page + 1) * page_length
    max_page_length_exp = math.ceil(math.log10(len(protocols))) if len(protocols) > 0 else 1
    protocols = protocols[begin_index:end_index]
    protocols_table = ProtocolsTable(protocols, search_results=search_results)
    return render_template("protocols-list.html", protocols=protocols, protocols_table=protocols_table, search_form=search_form, page=page, page_count=page_count, page_diff=config.PAGE_DIFF, protocoltype_id=protocoltype_id, search_term=search_term, state_open=state_open, page_length=page_length, max_page_length_exp=max_page_length_exp)

@app.route("/protocol/new", methods=["GET", "POST"])
@login_required
def new_protocol():
    user = current_user()
    protocoltypes = ProtocolType.get_modifiable_protocoltypes(user)
    form = NewProtocolForm(protocoltypes)
    upload_form = NewProtocolSourceUploadForm(protocoltypes)
    file_upload_form = NewProtocolFileUploadForm(protocoltypes)
    if form.validate_on_submit():
        protocoltype = ProtocolType.query.filter_by(id=form.protocoltype_id.data).first()
        if protocoltype is None or not protocoltype.has_modify_right(user):
            flash("Dir fehlen die nötigen Zugriffsrechte.", "alert-error")
            return redirect(request.args.get("next") or url_for("index"))
        protocol = Protocol.create_new_protocol(protocoltype,
            form.date.data, form.start_time.data)
        return redirect(request.args.get("next") or url_for("show_protocol", protocol_id=protocol.id))
    type_id = request.args.get("protocoltype_id")
    if type_id is not None:
        form.protocoltype.data = type_id
        upload_form.protocoltype.data = type_id
    return render_template("protocol-new.html", form=form, upload_form=upload_form, file_upload_form=file_upload_form, protocoltypes=protocoltypes)

@app.route("/protocol/show/<int:protocol_id>")
@db_lookup(Protocol)
def show_protocol(protocol):
    user = current_user()
    errors_table = ErrorsTable(protocol.errors)
    if not protocol.protocoltype.has_public_view_right(user, check_networks=False):
        flash("Dir fehlen die nötigen Zugriffsrechte.", "alert-error")
        if check_login():
            return redirect(url_for("index"))
        return redirect(request.args.get("next") or url_for("login", next=request.url))
    visible_documents = [
        document for document in protocol.documents
        if (not document.is_private and document.protocol.has_public_view_right(user))
        or (document.is_private and document.protocol.protocoltype.has_private_view_right(user))
    ]
    documents_table = DocumentsTable(visible_documents, protocol)
    document_upload_form = DocumentUploadForm()
    source_upload_form = KnownProtocolSourceUploadForm()
    time_diff = protocol.date - datetime.now().date()
    large_time_diff = not protocol.is_done() and time_diff.days > 0
    content_html = (protocol.content_html_private
        if protocol.has_private_view_right(user)
        else protocol.content_html_public)
    if content_html is not None:
        content_html = Markup(content_html)
    return render_template("protocol-show.html", protocol=protocol, errors_table=errors_table, documents_table=documents_table, document_upload_form=document_upload_form, source_upload_form=source_upload_form, time_diff=time_diff, large_time_diff=large_time_diff, content_html=content_html)

@app.route("/protocol/delete/<int:protocol_id>")
@login_required
@group_required(config.ADMIN_GROUP)
@db_lookup(Protocol)
@require_modify_right()
def delete_protocol(protocol):
    name = protocol.get_short_identifier()
    protocol.delete_orphan_todos()
    db.session.delete(protocol)
    db.session.commit()
    flash("Protokoll {} ist gelöscht.".format(name), "alert-success")
    return redirect(request.args.get("next") or url_for("list_protocols"))

@app.route("/protocol/etherpull/<int:protocol_id>")
@login_required
@db_lookup(Protocol)
@require_modify_right()
def etherpull_protocol(protocol):
    if not config.ETHERPAD_ACTIVE:
        flash("Die Etherpadfunktion ist nicht aktiviert.", "alert-error")
        return redirect(request.args.get("next") or url_for("show_protocol", protocol_id=protocol_id))
    protocol.source = get_etherpad_text(protocol.get_identifier())
    db.session.commit()
    tasks.parse_protocol(protocol)
    flash("Das Protokoll wird kompiliert.", "alert-success")
    return redirect(request.args.get("next") or url_for("show_protocol", protocol_id=protocol.id))

@app.route("/protocol/upload/known/<int:protocol_id>", methods=["POST"])
@login_required
@db_lookup(Protocol)
@require_modify_right()
def upload_source_to_known_protocol(protocol):
    form = KnownProtocolSourceUploadForm()
    if form.validate_on_submit():
        if form.source.data is None:
            flash("Es wurde keine Datei ausgewählt.", "alert-error")
        else:
            file = form.source.data
            if file.filename == "":
                flash("Es wurde keine Datei ausgewählt.", "alert-error")
            else:
                # todo: Prüfen, ob es Text ist?
                source = file.stream.read().decode("utf-8")
                protocol.source = source
                db.session.commit()
                tasks.parse_protocol(protocol)
                flash("Das Protokoll wird kompiliert.", "alert-success")
    return redirect(request.args.get("next") or url_for("show_protocol", protocol_id=protocol.id))

@app.route("/protocol/upload/new/", methods=["POST"])
@login_required
def upload_new_protocol():
    user = current_user()
    available_types = ProtocolType.get_modifiable_protocoltypes(user)
    form = NewProtocolSourceUploadForm(protocoltypes=available_types)
    if form.validate_on_submit():
        if form.source.data is None:
            flash("Es wurde keine Datei ausgewählt.", "alert-error")
            return redirect(request.args.get("fail") or url_for("new_protocol"))
        file = form.source.data
        if file.filename == "":
            flash("Es wurde keine Datei ausgewählt.", "alert-error")
            return redirect(request.args.get("fail") or url_for("new_protocol"))
        source = file.stream.read().decode("utf-8")
        protocoltype = ProtocolType.query.filter_by(id=form.protocoltype_id.data).first()
        if protocoltype is None or not protocoltype.has_modify_right(user):
            flash("Invalider Protokolltyp oder keine Rechte.", "alert-error")
            return redirect(request.args.get("fail") or url_for("new_protocol"))
        protocol = Protocol(protocoltype_id=protocoltype.id, source=source)
        db.session.add(protocol)
        db.session.commit()
        for local_top in protocol.create_localtops:
            db.session.add(local_top)
        db.session.commit()
        tasks.parse_protocol(protocol)
        return redirect(request.args.get("next") or url_for("show_protocol", protocol_id=protocol.id))
    return redirect(request.args.get("fail") or url_for("new_protocol"))

@app.route("/protocol/upload/new/file/", methods=["POST"])
@login_required
def upload_new_protocol_by_file():
    user = current_user()
    available_types = ProtocolType.get_modifiable_protocoltypes(user)
    form = NewProtocolFileUploadForm(protocoltypes=available_types)
    if form.validate_on_submit():
        if form.file.data is None:
            flash("Es wurde keine Datei ausgewählt.", "alert-error")
            return redirect(request.args.get("fail") or url_for("new_protocol"))
        file = form.file.data
        if file.filename == "":
            flash("Es wurde keine Datei ausgewählt.", "alert-error")
            return redirect(request.args.get("fail") or url_for("new_protocol"))
        filename = secure_filename(file.filename)
        protocoltype = ProtocolType.query.filter_by(id=form.protocoltype_id.data).first()
        if protocoltype is None or not protocoltype.has_modify_right(user):
            flash("Invalider Protokolltyp oder keine Rechte.", "alert-error")
            return redirect(request.args.get("fail") or url_for("new_protocol"))
        protocol = Protocol(protocoltype_id=protocoltype.id, date=datetime.now().date(), done=True)
        db.session.add(protocol)
        db.session.commit()
        for local_top in protocol.create_localtops:
            db.session.add(local_top)
        db.session.commit()
        document = Document(protocol_id=protocol.id, name=filename,
            filename="", is_compiled=False)
        form.populate_obj(document)
        db.session.add(document)
        db.session.commit()
        internal_filename = "{}-{}-{}".format(protocol.id, document.id, filename)
        document.filename = internal_filename
        file.save(os.path.join(config.DOCUMENTS_PATH, internal_filename))
        db.session.commit()
        return redirect(request.args.get("next") or url_for("show_protocol", protocol_id=protocol.id))
    return redirect(request.args.get("fail") or url_for("new_protocol"))

@app.route("/protocol/recompile/<int:protocol_id>")
@login_required
@group_required(config.ADMIN_GROUP)
@db_lookup(Protocol)
@require_modify_right()
def recompile_protocol(protocol):
    tasks.parse_protocol(protocol)
    return redirect(request.args.get("next") or url_for("show_protocol", protocol_id=protocol.id))

@app.route("/protocol/source/<int:protocol_id>")
@login_required
@db_lookup(Protocol)
@require_modify_right()
def get_protocol_source(protocol):
    file_like = BytesIO(protocol.source.encode("utf-8"))
    return send_file(file_like, cache_timeout=1, as_attachment=True, attachment_filename="{}.txt".format(protocol.get_short_identifier()))

@app.route("/protocol/template/<int:protocol_id>")
@login_required
@db_lookup(Protocol)
@require_modify_right()
def get_protocol_template(protocol):
    file_like = BytesIO(protocol.get_template().encode("utf-8"))
    return send_file(file_like, cache_timeout=1, as_attachment=True, attachment_filename="{}-template.txt".format(protocol.get_short_identifier()))

@app.route("/protocol/etherpush/<int:protocol_id>")
@login_required
@db_lookup(Protocol)
@require_modify_right()
def etherpush_protocol(protocol):
    if not config.ETHERPAD_ACTIVE:
        flash("Die Etherpadfunktion ist nicht aktiviert.", "alert-error")
        return redirect(request.args.get("next") or url_for("show_protocol", protocol_id=protocol_id))
    if not protocol.is_done():
        tasks.set_etherpad_content(protocol)
    return redirect(request.args.get("next") or protocol.get_etherpad_link())

@app.route("/protocol/update/<int:protocol_id>", methods=["GET", "POST"])
@login_required
@db_lookup(Protocol)
@require_publish_right()
def update_protocol(protocol):
    upload_form = KnownProtocolSourceUploadForm()
    edit_form = generate_protocol_form(protocol)(obj=protocol)
    if edit_form.validate_on_submit():
        edit_form.populate_obj(protocol)
        for meta in protocol.metas:
            meta.value = getattr(edit_form.metas, meta.name).data
        db.session.commit()
        tasks.push_tops_to_calendar(protocol)
        return redirect(request.args.get("next") or url_for("show_protocol", protocol_id=protocol.id))
    for meta in protocol.metas:
        getattr(edit_form.metas, meta.name).data = meta.value
    return render_template("protocol-update.html", upload_form=upload_form, edit_form=edit_form, protocol=protocol)

@app.route("/protocol/publish/<int:protocol_id>")
@login_required
@db_lookup(Protocol)
@require_publish_right()
def publish_protocol(protocol):
    protocol.public = True
    db.session.commit()
    return redirect(request.args.get("next") or url_for("show_protocol", protocol_id=protocol.id))

@app.route("/prococol/send/private/<int:protocol_id>")
@login_required
@db_lookup(Protocol)
@require_modify_right()
def send_protocol_private(protocol):
    if not config.MAIL_ACTIVE:
        flash("Die Mailfunktion ist nicht aktiviert.", "alert-error")
        return redirect(request.args.get("next") or url_for("show_protocol", protocol_id=protocol_id))
    tasks.send_protocol_private(protocol)
    flash("Das Protokoll wurde versandt.", "alert-success")
    return redirect(request.args.get("next") or url_for("show_protocol", protocol_id=protocol.id))

@app.route("/prococol/send/public/<int:protocol_id>")
@login_required
@db_lookup(Protocol)
@require_publish_right()
def send_protocol_public(protocol):
    if not config.MAIL_ACTIVE:
        flash("Die Mailfunktion ist nicht aktiviert.", "alert-error")
        return redirect(request.args.get("next") or url_for("show_protocol", protocol_id=protocol_id))
    tasks.send_protocol_public(protocol)
    flash("Das Protokoll wurde versandt.", "alert-success")
    return redirect(request.args.get("next") or url_for("show_protocol", protocol_id=protocol.id))

@app.route("/protocol/reminder/<int:protocol_id>")
@login_required
@db_lookup(Protocol)
@require_modify_right()
def send_protocol_reminder(protocol):
    if not config.MAIL_ACTIVE:
        flash("Die Mailfunktion ist nicht aktiviert.", "alert-error")
        return redirect(request.args.get("next") or url_for("show_protocol", protocol_id=protocol_id))
    meetingreminders = MeetingReminder.query.filter_by(protocoltype_id=protocol.protocoltype.id).all()
    if len(meetingreminders) == 0:
        flash("Für diesen Protokolltyp sind keine Einladungsmails konfiguriert.", "alert-error")
        return redirect(request.args.get("next") or url_for("show_protocol", protocol_id=protocol_id))
    day_difference = (protocol.date - datetime.now().date()).days
    past_reminders = [
        meetingreminder for meetingreminder in meetingreminders
        if meetingreminder.days_before > day_difference
    ]
    if len(past_reminders) == 0:
        flash("Bisher hätte keine Einladungsmail verschickt werden sollen, schicke letzte.", "alert-info")
        past_reminders = meetingreminders
    past_reminders = sorted(past_reminders, key=lambda r: r.days_before)
    choosen_reminder = past_reminders[0]
    tasks.send_reminder(choosen_reminder, protocol)
    flash("Einladungsmail ist versandt.", "alert-success")
    return redirect(request.args.get("next") or url_for("show_protocol", protocol_id=protocol.id))
        

@app.route("/protocol/tops/new/<int:protocol_id>", methods=["GET", "POST"])
@login_required
@db_lookup(Protocol)
@require_modify_right()
def new_top(protocol):
    form = TopForm()
    if form.validate_on_submit():
        top = TOP(protocol_id=protocol.id, planned=True)
        form.populate_obj(top)
        db.session.add(top)
        db.session.commit()
        tasks.push_tops_to_calendar(top.protocol)
        return redirect(request.args.get("next") or url_for("show_protocol", protocol_id=protocol.id))
    else:
        current_numbers = list(map(lambda t: t.number, protocol.tops))
        suggested_number = get_first_unused_int(current_numbers)
        form.number.data = suggested_number
    return render_template("top-new.html", form=form, protocol=protocol)

@app.route("/protocol/top/edit/<int:top_id>", methods=["GET", "POST"])
@login_required
@db_lookup(TOP)
@require_modify_right()
def edit_top(top):
    form = TopForm(obj=top)
    if form.validate_on_submit():
        form.populate_obj(top)
        db.session.commit()
        tasks.push_tops_to_calendar(top.protocol)
        return redirect(request.args.get("next") or url_for("show_protocol", protocol_id=top.protocol.id))
    return render_template("top-edit.html", form=form, top=top)

@app.route("/protocol/top/delete/<int:top_id>")
@login_required
@db_lookup(TOP)
@require_modify_right()
def delete_top(top):
    name = top.name
    protocol = top.protocol
    db.session.delete(top)
    db.session.commit()
    tasks.push_tops_to_calendar(protocol)
    flash("Der TOP {} wurde gelöscht.".format(name), "alert-success")
    return redirect(request.args.get("next") or url_for("show_protocol", protocol_id=protocol.id))

@app.route("/protocol/top/move/<int:top_id>/<diff>")
@login_required
@db_lookup(TOP)
@require_modify_right()
def move_top(top, diff):
    try:
        top.number += int(diff)
        db.session.commit()
        tasks.push_tops_to_calendar(top.protocol)
    except ValueError:
        flash("Die angegebene Differenz ist keine Zahl.", "alert-error")
    return redirect(request.args.get("next") or url_for("show_protocol", protocol_id=top.protocol.id))

@app.route("/protocol/localtop/edit/<int:localtop_id>", methods=["GET", "POST"])
@login_required
@db_lookup(LocalTOP)
@require_modify_right()
def edit_localtop(localtop):
    form = LocalTopForm(obj=localtop)
    if form.validate_on_submit():
        form.populate_obj(localtop)
        db.session.commit()
        return redirect(request.args.get("next") or url_for("show_protocol", protocol_id=localtop.protocol.id))
    return render_template("localtop-edit.html", form=form, localtop=localtop)

def _get_page():
    try:
        page = request.args.get("page")
        if page is None:
            return 0
        return int(page)
    except ValueError:
        return 0

def _get_page_length():
    try:
        page_length = request.args.get("page_length")
        if page_length is None:
            return config.PAGE_LENGTH
        return int(page_length)
    except ValueError:
        return config.PAGE_LENGTH

@app.route("/todos/list")
@login_required
def list_todos():
    user = current_user()
    protocoltype = None
    protocoltype_id = None
    try:
        protocoltype_id = int(request.args.get("protocoltype_id"))
    except (ValueError, TypeError):
        pass
    state_open = -1
    try:
        state_open = int(request.args.get("state_open"))
    except (ValueError, TypeError):
        pass
    search_term = request.args.get("search")
    protocoltypes = ProtocolType.get_public_protocoltypes(user)
    search_form = TodoSearchForm(protocoltypes)
    if protocoltype_id is not None:
        search_form.protocoltype_id.data = protocoltype_id
        protocoltype = ProtocolType.query.filter_by(id=protocoltype_id).first()
    if state_open is not None:
        search_form.state_open.data = state_open