tasks.py 19.5 KB
Newer Older
1
2
3
4
5
from flask import render_template

import os
import subprocess
import shutil
6
import tempfile
7

Robin Sonnabend's avatar
Robin Sonnabend committed
8
from models.database import Document, Protocol, Error, Todo, Decision, TOP, DefaultTOP, MeetingReminder, TodoMail, DecisionDocument
Robin Sonnabend's avatar
Robin Sonnabend committed
9
from models.errors import DateNotMatchingException
10
from server import celery, app
11
from shared import db, escape_tex, unhyphen, date_filter, datetime_filter, date_filter_long, date_filter_short, time_filter, class_filter
12
from utils import mail_manager, url_manager, encode_kwargs, decode_kwargs
Robin Sonnabend's avatar
Robin Sonnabend committed
13
from parser import parse, ParserException, Element, Content, Text, Tag, Remark, Fork, RenderType
Robin Sonnabend's avatar
Robin Sonnabend committed
14
from wiki import WikiClient, WikiException
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

import config

texenv = app.create_jinja_environment()
texenv.block_start_string = r"\ENV{"
texenv.block_end_string = r"}"
texenv.variable_start_string = r"\VAR{"
texenv.variable_end_string = r"}"
texenv.comment_start_string = r"\COMMENT{"
texenv.comment_end_string = r"}"
texenv.filters["escape_tex"] = escape_tex
texenv.filters["unhyphen"] = unhyphen
texenv.trim_blocks = True
texenv.lstrip_blocks = True
texenv.filters["url_complete"] = url_manager.complete
texenv.filters["datify"] = date_filter
texenv.filters["datify_long"] = date_filter_long
32
texenv.filters["datify_short"] = date_filter_short
33
34
texenv.filters["datetimify"] = datetime_filter
texenv.filters["timify"] = time_filter
35
texenv.filters["class"] = class_filter
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

mailenv = app.create_jinja_environment()
mailenv.trim_blocks = True
mailenv.lstrip_blocks = True
mailenv.filters["url_complete"] = url_manager.complete
mailenv.filters["datify"] = date_filter
mailenv.filters["datetimify"] = datetime_filter

ID_FIELD_BEGINNING = "id "

def parse_protocol(protocol, **kwargs):
    parse_protocol_async.delay(protocol.id, encode_kwargs(kwargs))

@celery.task
def parse_protocol_async(protocol_id, encoded_kwargs):
    with app.app_context():
        with app.test_request_context("/"):
            kwargs = decode_kwargs(encoded_kwargs)
            protocol = Protocol.query.filter_by(id=protocol_id).first()
            if protocol is None:
                raise Exception("No protocol given. Aborting parsing.")
57
58
            old_errors = list(protocol.errors)
            for error in old_errors:
Robin Sonnabend's avatar
Robin Sonnabend committed
59
60
                protocol.errors.remove(error)
            db.session.commit()
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
            if protocol.source is None:
                error = protocol.create_error("Parsing", "Protocol source is None", "")
                db.session.add(error)
                db.session.commit()
                return
            tree = None
            try:
                tree = parse(protocol.source)
            except ParserException as exc:
                context = ""
                if exc.linenumber is not None:
                    source_lines = source.splitlines()
                    start_index = max(0, exc.linenumber - config.ERROR_CONTEXT_LINES)
                    end_index = min(len(source_lines) - 1, exc.linenumber + config.ERROR_CONTEXT_LINES)
                    context = "\n".join(source_lines[start_index:end_index])
                error = protocol.create_error("Parsing", str(exc), context)
                db.session.add(error)
                db.session.commit()
                return
Robin Sonnabend's avatar
Robin Sonnabend committed
80
            remarks = {element.name: element for element in tree.children if isinstance(element, Remark)}
81
            required_fields = ["Datum", "Anwesende", "Beginn", "Ende", "Autor", "Ort"]
Robin Sonnabend's avatar
Robin Sonnabend committed
82
            missing_fields = [field for field in required_fields if field not in remarks]
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
            if len(missing_fields) > 0:
                error = protocol.create_error("Parsing", "Missing fields", ", ".join(missing_fields))
                db.session.add(error)
                db.session.commit()
                return
            try:
                protocol.fill_from_remarks(remarks)
            except ValueError:
                error = protocol.create_error(
                    "Parsing", "Invalid fields",
                    "Date or time fields are not '%d.%m.%Y' respectively '%H:%M', "
                    "but rather {}".format(
                    ", ".join([remarks["Datum"], remarks["Beginn"], remarks["Ende"]])))
                db.session.add(error)
                db.session.commit()
                return
Robin Sonnabend's avatar
Robin Sonnabend committed
99
100
101
102
103
104
105
106
            except DateNotMatchingException as exc:
                error = protocol.create_error("Parsing", "Date not matching",
                    "This protocol's date should be {}, but the protocol source says {}.".format(date_filter(exc.original_date), date_filter(exc.protocol_date)))
                db.session.add(error)
                db.session.commit()
                return
            protocol.delete_orphan_todos()
            db.session.commit()
107
            old_todos = list(protocol.todos)
Robin Sonnabend's avatar
Robin Sonnabend committed
108
            for todo in old_todos:
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
                protocol.todos.remove(todo)
            db.session.commit()
            tags = tree.get_tags()
            todo_tags = [tag for tag in tags if tag.name == "todo"]
            for todo_tag in todo_tags:
                if len(todo_tag.values) < 2:
                    error = protocol.create_error("Parsing", "Invalid todo-tag",
                        "The todo tag in line {} needs at least "
                        "information on who and what, "
                        "but has less than that.".format(todo_tag.linenumber))
                    db.session.add(error)
                    db.session.commit()
                    return
                who = todo_tag.values[0]
                what = todo_tag.values[1]
                todo = None
Robin Sonnabend's avatar
Robin Sonnabend committed
125
                field_id = None
126
127
128
129
130
131
132
133
134
135
136
                for other_field in todo_tag.values[2:]:
                    if other_field.startswith(ID_FIELD_BEGINNING):
                        try:
                            field_id = int(other_field[len(ID_FIELD_BEGINNING):])
                        except ValueError:
                            error = protocol.create_error("Parsing", "Non-numerical todo ID",
                            "The todo in line {} has a nonnumerical ID, but needs "
                            "something like \"id 1234\"".format(todo_tag.linenumber))
                            db.session.add(error)
                            db.session.commit()
                            return
Robin Sonnabend's avatar
Robin Sonnabend committed
137
138
139
                        todo = Todo.query.filter_by(number=field_id).first()
                who = who.strip()
                what = what.strip()
140
                if todo is None:
Robin Sonnabend's avatar
Robin Sonnabend committed
141
142
143
144
145
146
147
148
                    if field_id is not None:
                        candidate = Todo.query.filter_by(who=who, description=what, number=None).first()
                        if candidate is None:
                            candidate = Todo.query.filter_by(description=what, number=None).first()
                        if candidate is not None:
                            candidate.number = field_id
                            todo = candidate
                        else:
Robin Sonnabend's avatar
Robin Sonnabend committed
149
                            todo = Todo(type_id=protocol.protocoltype.id, who=who, description=what, tags="", done=False)
Robin Sonnabend's avatar
Robin Sonnabend committed
150
151
152
153
154
155
                            todo.number = field_id
                    else:
                        candidate = Todo.query.filter_by(who=who, description=what).first()
                        if candidate is not None:
                            todo = candidate
                        else:
Robin Sonnabend's avatar
Robin Sonnabend committed
156
                            todo = Todo(type_id=protocol.protocoltype.id, who=who, description=what, tags="", done=False)
Robin Sonnabend's avatar
Robin Sonnabend committed
157
                            db.session.add(todo)
158
159
160
161
162
163
164
165
166
167
                todo.protocols.append(protocol)
                todo_tags_internal = todo.tags.split(";")
                for other_field in todo_tag.values[2:]:
                    if other_field.startswith(ID_FIELD_BEGINNING):
                        continue
                    elif other_field == "done":
                        todo.done = True
                    elif other_field not in todo_tags_internal:
                        todo_tags_internal.append(other_field)
                todo.tags = ";".join(todo_tags_internal)
Robin Sonnabend's avatar
Robin Sonnabend committed
168
                todo_tag.todo = todo
169
170
                db.session.commit()
            old_decisions = list(protocol.decisions)
Robin Sonnabend's avatar
Robin Sonnabend committed
171
            for decision in old_decisions:
172
173
174
175
176
177
178
179
180
181
182
183
184
                protocol.decisions.remove(decision)
            db.session.commit()
            decision_tags = [tag for tag in tags if tag.name == "beschluss"]
            for decision_tag in decision_tags:
                if len(decision_tag.values) == 0:
                    error = protocol.create_error("Parsing", "Empty decision found.",
                        "The decision in line {} is empty.".format(decision_tag.linenumber))
                    db.session.add(error)
                    db.session.commit()
                    return
                decision = Decision(protocol_id=protocol.id, content=decision_tag.values[0])
                db.session.add(decision)
                db.session.commit()
Robin Sonnabend's avatar
Robin Sonnabend committed
185
186
187
                decision_content = texenv.get_template("decision.tex").render(render_type=RenderType.latex, decision=decision, protocol=protocol, top=decision_tag.fork.get_top(), show_private=False)
                print(decision_content)
                compile_decision(decision_content, decision)
Robin Sonnabend's avatar
Robin Sonnabend committed
188
189
190
191
192
193
194
195
            old_tops = list(protocol.tops)
            for top in old_tops:
                protocol.tops.remove(top)
            tops = []
            for index, fork in enumerate((child for child in tree.children if isinstance(child, Fork))):
                top = TOP(protocol.id, fork.name, index, False)
                db.session.add(top)
            db.session.commit()
196

197
198
199
200
201
202
203
204
205
206
207
            render_kwargs = {
                "protocol": protocol,
                "tree": tree
            }
            privacy_states = [False]
            content_private = render_template("protocol.txt", render_type=RenderType.plaintext, show_private=True, **render_kwargs)
            content_public = render_template("protocol.txt", render_type=RenderType.plaintext, show_private=False, **render_kwargs)
            if content_private != content_public:
                privacy_states.append(True)
            protocol.content_private = content_private
            protocol.content_public = content_public
208

209
210
211
            for show_private in privacy_states:
                latex_source = texenv.get_template("protocol.tex").render(render_type=RenderType.latex, show_private=show_private, **render_kwargs)
                compile(latex_source, protocol, show_private=show_private)
Robin Sonnabend's avatar
Robin Sonnabend committed
212
213
214
215

            if protocol.protocoltype.use_wiki:
                wiki_source = render_template("protocol.wiki", render_type=RenderType.wikitext, show_private=not protocol.protocoltype.wiki_only_public, **render_kwargs).replace("\n\n\n", "\n\n")
                push_to_wiki(protocol, wiki_source, "Automatisch generiert vom Protokollsystem 3.0")
216
217
            protocol.done = True
            db.session.commit()
218

Robin Sonnabend's avatar
Robin Sonnabend committed
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
def push_to_wiki(protocol, content, summary):
    push_to_wiki_async.delay(protocol.id, content, summary)

@celery.task
def push_to_wiki_async(protocol_id, content, summary):
    with WikiClient() as wiki_client, app.app_context():
        protocol = Protocol.query.filter_by(id=protocol_id).first()
        try:
            wiki_client.edit_page(
                title=protocol.get_wiki_title(),
                content=content,
                summary="Automatisch generiert vom Protokollsystem 3.")
        except WikiException as exc:
            error = protocol.create_error("Pushing to Wiki", "Pushing to Wiki failed.", str(exc))

Robin Sonnabend's avatar
Robin Sonnabend committed
234
def compile(content, protocol, show_private):
Robin Sonnabend's avatar
Robin Sonnabend committed
235
236
237
238
   compile_async.delay(content, protocol.id, show_private=show_private)

def compile_decision(content, decision):
    compile_async.delay(content, decision.id, use_decision=True)
239
240

@celery.task
Robin Sonnabend's avatar
Robin Sonnabend committed
241
def compile_async(content, protocol_id, show_private=False, use_decision=False):
242
    with tempfile.TemporaryDirectory() as compile_dir, app.app_context():
Robin Sonnabend's avatar
Robin Sonnabend committed
243
244
245
246
247
248
249
        decision = None
        protocol = None
        if use_decision:
            decision = Decision.query.filter_by(id=protocol_id).first()
            protocol = decision.protocol
        else:
            protocol = Protocol.query.filter_by(id=protocol_id).first()
250
251
252
        try:
            current = os.getcwd()
            protocol_source_filename = "protocol.tex"
253
            protocol_target_filename = "protocol.pdf"
254
255
256
            log_filename = "protocol.log"
            with open(os.path.join(compile_dir, protocol_source_filename), "w") as source_file:
                source_file.write(content)
257
258
259
            protocol2_class_source = texenv.get_template("protokoll2.cls").render(fonts=config.FONTS)
            with open(os.path.join(compile_dir, "protokoll2.cls"), "w") as protocol2_class_file:
                protocol2_class_file.write(protocol2_class_source)
260
261
262
263
264
265
266
267
268
269
            os.chdir(compile_dir)
            command = [
                "/usr/bin/xelatex",
                "-halt-on-error",
                "-file-line-error",
                protocol_source_filename
            ]
            subprocess.check_call(command, universal_newlines=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
            subprocess.check_call(command, universal_newlines=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
            os.chdir(current)
Robin Sonnabend's avatar
Robin Sonnabend committed
270
271
272
273
274
275
276
277
            document = None
            if not use_decision:
                for old_document in [document for document in protocol.documents if document.is_compiled and document.is_private == show_private]:
                    protocol.documents.remove(old_document)
                db.session.commit()
                document = Document(protocol.id, name="protokoll{}_{}_{}.pdf".format("_intern" if show_private else "", protocol.protocoltype.short_name, date_filter_short(protocol.date)), filename="", is_compiled=True, is_private=show_private)
            else:
                document = DecisionDocument(decision.id, name="beschluss_{}_{}_{}.pdf".format(protocol.protocoltype.short_name, date_filter_short(protocol.date), decision.id), filename="")
278
279
            db.session.add(document)
            db.session.commit()
Robin Sonnabend's avatar
Robin Sonnabend committed
280
            target_filename = "compiled-{}-{}.pdf".format(document.id, "internal" if show_private else "public")
Robin Sonnabend's avatar
Robin Sonnabend committed
281
282
            if use_decision:
                target_filename = "decision-{}-{}-{}.pdf".format(protocol.id, decision.id, document.id)
283
            document.filename = target_filename
284
            shutil.copy(os.path.join(compile_dir, protocol_target_filename), os.path.join(config.DOCUMENTS_PATH, target_filename))
285
286
287
288
289
290
291
292
293
294
295
296
297
298
            db.session.commit()
        except subprocess.SubprocessError:
            log = ""
            total_log_filename = os.path.join(compile_dir, log_filename)
            if os.path.isfile(total_log_filename):
                with open(total_log_filename, "r") as log_file:
                    log = log_file.read()
            else:
                log = "Logfile not found."
            error = protocol.create_error("Compiling", "Compiling LaTeX failed", log)
            db.session.add(error)
            db.session.commit()
        finally:
            os.chdir(current)
299

Robin Sonnabend's avatar
Robin Sonnabend committed
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
def print_file(filename, protocol):
    if config.PRINTING_ACTIVE:
        print_file_async.delay(filename, protocol.id)

@celery.task
def print_file_async(filename, protocol_id):
    with app.app_context():
        protocol = Protocol.query.filter_by(id=protocol_id).first()
        if protocol.protocoltype.printer is None:
            error = protocol.create_error("Printing", "No printer configured.", "You don't have any printer configured for the protocoltype {}. Please do so before printing a protocol.".format(protocol.protocoltype.name))
        try:
            command = [
                "/usr/bin/lpr",
                "-H", config.PRINTING_SERVER,
                "-P", protocol.protocoltype.printer,
                "-U", config.PRINTING_USER,
                "-T", protocol.get_identifier(),
            ]
            for option in config.PRINTING_PRINTERS[protocol.protocoltype.printer]:
                command.extend(["-o", '"{}"'.format(option) if " " in option else option])
            command.append(filename)
            subprocess.check_call(command, universal_newlines=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
        except subprocess.SubprocessError:
            error = protocol.create_error("Printing", "Printing {} failed.".format(protocol.get_identifier()), "")
            db.session.add(error)
            db.session.commit()

Robin Sonnabend's avatar
Robin Sonnabend committed
327
328
def send_reminder(reminder, protocol):
    send_reminder_async.delay(reminder.id, protocol.id)
329
330

@celery.task
Robin Sonnabend's avatar
Robin Sonnabend committed
331
def send_reminder_async(reminder_id, protocol_id):
332
    with app.app_context():
Robin Sonnabend's avatar
Robin Sonnabend committed
333
334
        reminder = MeetingReminder.query.filter_by(id=reminder_id).first()
        protocol = Protocol.query.filter_by(id=protocol_id).first()
335
        reminder_text = render_template("reminder-mail.txt", reminder=reminder, protocol=protocol)
Robin Sonnabend's avatar
Robin Sonnabend committed
336
337
338
339
340
        if reminder.send_public:
            send_mail(protocol, protocol.protocoltype.public_mail, "Tagesordnung der {}".format(protocol.protocoltype.name), reminder_text)
        if reminder.send_private:
            send_mail(protocol, protocol.protocoltype.private_mail, "Tagesordnung der {}".format(protocol.protocoltype.name), reminder_text)

341
342
343
def send_protocol(protocol):
    send_protocol_async.delay(protocol.id, show_private=True)
    send_protocol_async.delay(protocol.id, show_private=False)
Robin Sonnabend's avatar
Robin Sonnabend committed
344
    send_todomails_async.delay(protocol.id)
345
346
347
348
349
350
351
352
353
354
355
356
357
358

@celery.task
def send_protocol_async(protocol_id, show_private):
    with app.app_context():
        protocol = Protocol.query.filter_by(id=protocol_id).first()
        to_addr = protocol.protocoltype.private_mail if show_private else protocol.protocoltype.public_mail
        subject = "{}{}-Protokoll vom {}".format("Internes " if show_private else "", protocol.protocoltype.short_name, date_filter(protocol.date))
        mail_content = render_template("protocol-mail.txt", protocol=protocol)
        appendix = [(document.name, document.as_file_like())
            for document in protocol.documents
            if show_private or not document.is_private
        ]
        send_mail(protocol, to_addr, subject, mail_content, appendix)

Robin Sonnabend's avatar
Robin Sonnabend committed
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
@celery.task
def send_todomails_async(protocol_id):
    with app.app_context():
        protocol = Protocol.query.filter_by(id=protocol_id).first()
        all_todos = Todo.query.filter(Todo.done == False).all()
        users = {user for todo in all_todos for user in todo.get_users()}
        grouped_todos = {
            user: [todo for todo in all_todos if user in todo.get_users()]
            for user in users
        }
        subject = "Du hast noch was zu tun!"
        for user in users:
            todomail = TodoMail.query.filter(TodoMail.name.ilike(user)).first()
            if todomail is None:
                error = protocol.create_error("Sending Todomail", "Sending Todomail failed.", "User {} has no Todo-Mail-Assignment.".format(user))
                db.session.add(error)
                db.session.commit()
                continue
            to_addr = todomail.get_formatted_mail()
            mail_content = render_template("todo-mail.txt", protocol=protocol, todomail=todomail, todos=grouped_todos[user])
            send_mail(protocol, to_addr, subject, mail_content)
380
381

def send_mail(protocol, to_addr, subject, content, appendix=None):
Robin Sonnabend's avatar
Robin Sonnabend committed
382
    if to_addr is not None and len(to_addr.strip()) > 0:
383
        send_mail_async.delay(protocol.id, to_addr, subject, content, appendix)
384

Robin Sonnabend's avatar
Robin Sonnabend committed
385
@celery.task
386
def send_mail_async(protocol_id, to_addr, subject, content, appendix):
Robin Sonnabend's avatar
Robin Sonnabend committed
387
388
389
    with app.app_context():
        protocol = Protocol.query.filter_by(id=protocol_id).first()
        try:
390
            mail_manager.send(to_addr, subject, content, appendix)
Robin Sonnabend's avatar
Robin Sonnabend committed
391
392
393
394
        except Exception as exc:
            error = protocol.create_error("Sending Mail", "Sending mail failed", str(exc))
            db.session.add(error)
            db.session.commit()
395