tasks.py 39.2 KB
Newer Older
1
2
3
4
5
from flask import render_template

import os
import subprocess
import shutil
6
import tempfile
7
from datetime import datetime
Robin Sonnabend's avatar
Robin Sonnabend committed
8
import traceback
9
from copy import copy
10
import xmlrpc.client
11

12
13
14
from models.database import (
    Document, Protocol, Todo, Decision, TOP, MeetingReminder,
    TodoMail, DecisionDocument, TodoState, OldTodo, DecisionCategory)
Robin Sonnabend's avatar
Robin Sonnabend committed
15
from models.errors import DateNotMatchingException
16
from server import celery, app
17
18
from shared import (
    db, escape_tex, unhyphen, date_filter, datetime_filter, date_filter_long,
19
    date_filter_short, time_filter, class_filter, KNOWN_KEYS, WikiType, config)
20
21
22
23
from utils import (
    mail_manager, add_line_numbers,
    set_etherpad_text, parse_datetime_from_string)
from protoparser import parse, ParserException, Tag, Remark, Fork, RenderType
Robin Sonnabend's avatar
Robin Sonnabend committed
24
from wiki import WikiClient, WikiException
Robin Sonnabend's avatar
Robin Sonnabend committed
25
from calendarpush import Client as CalendarClient, CalendarException
26
from legacy import lookup_todo_id
27
28
29
30
31
32
33
34
35
36
37
38
39
40

texenv = app.create_jinja_environment()
texenv.block_start_string = r"\ENV{"
texenv.block_end_string = r"}"
texenv.variable_start_string = r"\VAR{"
texenv.variable_end_string = r"}"
texenv.comment_start_string = r"\COMMENT{"
texenv.comment_end_string = r"}"
texenv.filters["escape_tex"] = escape_tex
texenv.filters["unhyphen"] = unhyphen
texenv.trim_blocks = True
texenv.lstrip_blocks = True
texenv.filters["datify"] = date_filter
texenv.filters["datify_long"] = date_filter_long
41
texenv.filters["datify_short"] = date_filter_short
42
43
texenv.filters["datetimify"] = datetime_filter
texenv.filters["timify"] = time_filter
44
texenv.filters["class"] = class_filter
45
46
47
logo_template = getattr(config, "LATEX_LOGO_TEMPLATE", None)
if logo_template is not None:
    texenv.globals["logo_template"] = logo_template
48
49
50
latex_geometry = getattr(
    config, "LATEX_GEOMETRY",
    "vmargin=1.5cm,hmargin={1.5cm,1.2cm},bindingoffset=8mm")
51
52
53
54
55
56
57
58
59
60
texenv.globals["latex_geometry"] = latex_geometry
raw_additional_packages = getattr(config, "LATEX_ADDITIONAL_PACKAGES", None)
additional_packages = []
if raw_additional_packages is not None:
    for package in raw_additional_packages:
        if "{" not in package:
            package = "{{{}}}".format(package)
        additional_packages.append(package)
texenv.globals["additional_packages"] = additional_packages
latex_pagestyle = getattr(config, "LATEX_PAGESTYLE", None)
61
if latex_pagestyle is not None and latex_pagestyle:
62
63
64
    texenv.globals["latex_pagestyle"] = latex_pagestyle
latex_header_footer = getattr(config, "LATEX_HEADER_FOOTER", False)
texenv.globals["latex_header_footer"] = latex_header_footer
65
66
latex_templates = getattr(config, "LATEX_TEMPLATES", None)

67

68
def provide_latex_template(template, documenttype):
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
    _DOCUMENTTYPE_FILENAME_MAP = {
        "class": "protokoll2.cls",
        "protocol": "protocol.tex",
        "decision": "decision.tex"
    }
    _PROVIDES = "provides"
    _LOGO_TEMPLATE = "logo_template"
    _LOGO = "logo"
    _LATEX_GEOMETRY = "latex_geometry"
    _GEOMETRY = "geometry"
    _ADDITIONAL_PACKAGES = "additional_packages"
    _LATEX_PAGESTYLE = "latex_pagestyle"
    _PAGESTYLE = "pagestyle"
    _LATEX_HEADER_FOOTER = "latex_header_footer"
    _HEADER_FOOTER = "headerfooter"
    _latex_template_filename = _DOCUMENTTYPE_FILENAME_MAP[documenttype]
    _latex_template_foldername = ""
    if logo_template is not None:
        texenv.globals[_LOGO_TEMPLATE] = logo_template
    texenv.globals[_LATEX_GEOMETRY] = latex_geometry
    texenv.globals[_ADDITIONAL_PACKAGES] = additional_packages
    if latex_pagestyle:
        texenv.globals[_LATEX_PAGESTYLE] = latex_pagestyle
    elif _LATEX_PAGESTYLE in texenv.globals:
        del texenv.globals[_LATEX_PAGESTYLE]
    texenv.globals[_LATEX_HEADER_FOOTER] = latex_header_footer
    if latex_templates is not None and template != "":
        if template in latex_templates:
            template_data = latex_templates[template]
            if _PROVIDES in template_data:
                if documenttype in template_data[_PROVIDES]:
                    _latex_template_foldername = template
            if _LOGO in template_data:
                texenv.globals[_LOGO_TEMPLATE] = os.path.join(
                    template, template_data[_LOGO])
            if _GEOMETRY in template_data:
                texenv.globals[_LATEX_GEOMETRY] = template_data[_GEOMETRY]
            if _PAGESTYLE in template_data:
                if template_data[_PAGESTYLE]:
                    texenv.globals[_LATEX_PAGESTYLE] = (
                        template_data[_PAGESTYLE])
            if _ADDITIONAL_PACKAGES in template_data:
                _raw_additional_packages = template_data[_ADDITIONAL_PACKAGES]
                _additional_packages = []
                if _raw_additional_packages is not None:
                    for _package in _raw_additional_packages:
                        if "{" not in _package:
                            _package = "{{{}}}".format(_package)
                        _additional_packages.append(_package)
                texenv.globals[_ADDITIONAL_PACKAGES] = _additional_packages
            if _HEADER_FOOTER in latex_templates[template]:
                texenv.globals[_LATEX_HEADER_FOOTER] = (
                    template_data[_HEADER_FOOTER])
    return os.path.join(_latex_template_foldername, _latex_template_filename)

124
125
126
127
128
129
130

mailenv = app.create_jinja_environment()
mailenv.trim_blocks = True
mailenv.lstrip_blocks = True
mailenv.filters["datify"] = date_filter
mailenv.filters["datetimify"] = datetime_filter

131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
wikienv = app.create_jinja_environment()
wikienv.trim_blocks = True
wikienv.lstrip_blocks = True
wikienv.block_start_string = "<env>"
wikienv.block_end_string = "</env>"
wikienv.variable_start_string = "<var>"
wikienv.variable_end_string = "</var>"
wikienv.comment_start_string = "<comment>"
wikienv.comment_end_string = "</comment>"
wikienv.filters["datify"] = date_filter
wikienv.filters["datify_long"] = date_filter_long
wikienv.filters["datify_short"] = date_filter_short
wikienv.filters["datetimify"] = datetime_filter
wikienv.filters["timify"] = time_filter
wikienv.filters["class"] = class_filter


148
149
150
151
152
153
def _make_error(protocol, *args):
    error = protocol.create_error(*args)
    db.session.add(error)
    db.session.commit()


154
155
ID_FIELD_BEGINNING = "id "

156
157
158
159

def parse_protocol(protocol):
    parse_protocol_async.delay(protocol.id)

160
161

@celery.task
162
def parse_protocol_async(protocol_id):
163
164
165
    with app.app_context():
        with app.test_request_context("/"):
            try:
166
                protocol = Protocol.first_by_id(protocol_id)
167
168
                if protocol is None:
                    raise Exception("No protocol given. Aborting parsing.")
169
                parse_protocol_async_inner(protocol)
170
            except Exception as exc:
Robin Sonnabend's avatar
Robin Sonnabend committed
171
                stacktrace = traceback.format_exc()
172
173
                return _make_error(
                    protocol, "Parsing", "Exception",
Robin Sonnabend's avatar
Robin Sonnabend committed
174
                    "{}\n\n{}".format(str(exc), stacktrace))
175

176
177

def parse_protocol_async_inner(protocol):
178
179
180
181
    old_errors = list(protocol.errors)
    for error in old_errors:
        protocol.errors.remove(error)
    db.session.commit()
182
    if protocol.source is None or len(protocol.source.strip()) == 0:
183
        return _make_error(protocol, "Parsing", "Protocol source is empty", "")
184
    if protocol.source == config.EMPTY_ETHERPAD:
185
186
187
        return _make_error(
            protocol, "Parsing", "The etherpad is unmodified and does not "
            "contain a protocol.", protocol.source)
188
189
190
191
192
193
194
195
    tree = None
    try:
        tree = parse(protocol.source)
    except ParserException as exc:
        context = ""
        if exc.linenumber is not None:
            source_lines = protocol.source.splitlines()
            start_index = max(0, exc.linenumber - config.ERROR_CONTEXT_LINES)
196
197
198
            end_index = min(
                len(source_lines) - 1,
                exc.linenumber + config.ERROR_CONTEXT_LINES)
199
200
201
            context = "\n".join(source_lines[start_index:end_index])
        if exc.tree is not None:
            context += "\n\nParsed syntax tree was:\n" + str(exc.tree.dump())
202
203
204
205
206
207
        return _make_error(protocol, "Parsing", str(exc), context)
    remarks = {
        element.name: element
        for element in tree.children
        if isinstance(element, Remark)
    }
208
    required_fields = copy(KNOWN_KEYS)
Robin Sonnabend's avatar
Robin Sonnabend committed
209
210
    for default_meta in protocol.protocoltype.metas:
        required_fields.append(default_meta.key)
211
    if not config.PARSER_LAZY:
212
213
214
215
216
        missing_fields = [
            field
            for field in required_fields
            if field not in remarks
        ]
217
        if len(missing_fields) > 0:
218
219
220
            return _make_error(
                protocol, "Parsing", "Du hast vergessen, Metadaten anzugeben.",
                ", ".join(missing_fields))
221
222
223
    try:
        protocol.fill_from_remarks(remarks)
    except ValueError:
224
225
        return _make_error(
            protocol, "Parsing", "Invalid fields",
226
227
            "Date or time fields are not '%d.%m.%Y' respectively '%H:%M', "
            "but rather {}".format(
228
229
230
231
232
                ", ".join([
                    remarks["Datum"].value.strip(),
                    remarks["Beginn"].value.strip(),
                    remarks["Ende"].value.strip()
                ])))
233
    except DateNotMatchingException as exc:
234
235
236
237
238
239
240
241
242
243
244
        return _make_error(
            protocol, "Parsing", "Date not matching",
            "This protocol's date should be {}, but the protocol source "
            "says {}.".format(
                date_filter(exc.original_date)
                if exc.original_date is not None
                else "not present",
                date_filter(exc.protocol_date)
                if exc.protocol_date is not None
                else "not present"))
    # tags
245
    tags = tree.get_tags()
Robin Sonnabend's avatar
Robin Sonnabend committed
246
    public_elements = tree.get_visible_elements(show_private=False)
247
248
    for tag in tags:
        if tag.name not in Tag.KNOWN_TAGS:
249
250
            return _make_error(
                protocol, "Parsing", "Invalid tag",
251
252
253
                "The tag in line {} has the kind '{}', which is "
                "not defined. This is probably an error mit a missing "
                "semicolon.".format(tag.linenumber, tag.name))
254
    # todos
255
256
257
    old_todo_number_map = {}
    for todo in protocol.todos:
        old_todo_number_map[todo.description] = todo.get_id()
258
259
260
261
    protocol.delete_orphan_todos()
    db.session.commit()
    old_todos = list(protocol.todos)
    todo_tags = [tag for tag in tags if tag.name == "todo"]
262
    raw_todos = []
263
264
    for todo_tag in todo_tags:
        if len(todo_tag.values) < 2:
265
266
            return _make_error(
                protocol, "Parsing", "Invalid todo-tag",
267
268
                "The todo tag in line {} needs at least "
                "information on who and what, "
269
270
                "but has less than that. This is probably "
                "a missing semicolon.".format(todo_tag.linenumber))
271
272
273
274
275
276
277
278
279
280
281
282
283
        who = todo_tag.values[0]
        what = todo_tag.values[1]
        field_id = None
        field_state = None
        field_date = None
        for other_field in todo_tag.values[2:]:
            other_field = other_field.strip()
            if len(other_field) == 0:
                continue
            if other_field.startswith(ID_FIELD_BEGINNING):
                try:
                    field_id = int(other_field[len(ID_FIELD_BEGINNING):])
                except ValueError:
284
285
286
287
288
                    return _make_error(
                        protocol, "Parsing", "Non-numerical todo ID",
                        "The todo in line {} has a nonnumerical ID, but needs "
                        "something like \"id 1234\"".format(
                            todo_tag.linenumber))
289
290
291
            else:
                try:
                    field_state = TodoState.from_name(other_field)
292
293
294
295
296
297
                    continue
                except ValueError:
                    pass
                try:
                    field_date = datetime.strptime(other_field, "%d.%m.%Y")
                    continue
298
                except ValueError:
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
                    pass
                try:
                    field_state, field_date = TodoState.from_name_with_date(
                        other_field.strip(), protocol=protocol)
                    continue
                except ValueError:
                    pass
                try:
                    field_state = TodoState.from_name_lazy(other_field)
                except ValueError:
                    return _make_error(
                        protocol, "Parsing", "Invalid field",
                        "The todo in line {} has the field '{}', but "
                        "this does neither match a date (\"%d.%m.%Y\") "
                        "nor a state.".format(
                            todo_tag.linenumber, other_field))
        raw_todos.append(
            (who, what, field_id, field_state, field_date, todo_tag))
317
318
    for (_, _, field_id, _, _, _) in raw_todos:
        if field_id is not None:
319
320
321
322
            old_todos = [
                todo for todo in old_todos
                if todo.id != field_id
            ]
323
324
325
326
    for todo in old_todos:
        protocol.todos.remove(todo)
    db.session.commit()
    for (who, what, field_id, field_state, field_date, todo_tag) in raw_todos:
327
328
329
        if field_state is None:
            field_state = TodoState.open
        if field_state.needs_date() and field_date is None:
330
331
            return _make_error(
                protocol, "Parsing", "Todo missing date",
332
333
334
335
336
337
338
339
                "The todo in line {} has a state that needs a date, "
                "but the todo does not have one.".format(todo_tag.linenumber))
        who = who.strip()
        what = what.strip()
        todo = None
        if field_id is not None:
            todo = Todo.query.filter_by(number=field_id).first()
            if todo is None and not config.PARSER_LAZY:
340
341
342
343
                return _make_error(
                    protocol, "Parsing", "Invalid Todo ID",
                    "The todo in line {} has the ID {}, but there is no "
                    "Todo with that ID.".format(todo_tag.linenumber, field_id))
344
        if todo is None and field_id is None and what in old_todo_number_map:
345
346
            todo = Todo(
                protocoltype_id=protocol.protocoltype.id,
347
348
349
350
                who=who, description=what, state=field_state,
                date=field_date, number=old_todo_number_map[what])
            db.session.add(todo)
            db.session.commit()
351
352
353
354
355
356
        if todo is None:
            protocol_key = protocol.get_identifier()
            old_candidates = OldTodo.query.filter(
                OldTodo.protocol_key == protocol_key).all()
            if len(old_candidates) == 0:
                # new protocol
357
358
                todo = Todo(
                    protocoltype_id=protocol.protocoltype.id,
359
360
361
362
363
364
365
366
367
368
369
                    who=who, description=what, state=field_state,
                    date=field_date)
                db.session.add(todo)
                db.session.commit()
                todo.number = field_id or todo.id
                db.session.commit()
            else:
                # old protocol
                number = field_id or lookup_todo_id(old_candidates, who, what)
                todo = Todo.query.filter_by(number=number).first()
                if todo is None:
370
371
                    todo = Todo(
                        protocoltype_id=protocol.protocoltype.id,
372
373
374
375
376
                        who=who, description=what, state=field_state,
                        date=field_date, number=number)
                    db.session.add(todo)
                    db.session.commit()
        todo.protocols.append(protocol)
377
378
379
380
381
382
383
384
385
386
        is_newest_protocol = True
        for other_protocol in todo.protocols:
            if other_protocol.date > protocol.date:
                is_newest_protocol = False
                break
        if is_newest_protocol:
            todo.state = field_state
            todo.date = field_date
            todo.who = who
            todo.description = what
387
388
389
        db.session.commit()
        todo_tag.todo = todo
    # Decisions
Robin Sonnabend's avatar
Robin Sonnabend committed
390
391
392
    decision_tags = [tag for tag in tags if tag.name == "beschluss"]
    for decision_tag in decision_tags:
        if decision_tag not in public_elements:
393
394
395
396
397
398
            return _make_error(
                protocol, "Parsing", "Decision in private context.",
                "The decision in line {} is in a private context, but "
                "decisions are and have to be public. "
                "Please move it to a public spot.".format(
                    decision_tag.linenumber))
399
400
401
402
    old_decisions = list(protocol.decisions)
    for decision in old_decisions:
        protocol.decisions.remove(decision)
    db.session.commit()
403
    decisions_to_render = []
404
405
    for decision_tag in decision_tags:
        if len(decision_tag.values) == 0:
406
407
408
409
            return _make_error(
                protocol, "Parsing", "Empty decision found.",
                "The decision in line {} is empty.".format(
                    decision_tag.linenumber))
410
        decision_content = decision_tag.values[0]
411
412
        decision_categories = []
        for decision_category_name in decision_tag.values[1:]:
413
414
415
            decision_category = DecisionCategory.query.filter_by(
                protocoltype_id=protocol.protocoltype.id,
                name=decision_category_name).first()
416
            if decision_category is None:
417
418
                category_candidates = DecisionCategory.query.filter_by(
                    protocoltype_id=protocol.protocoltype.id).all()
419
420
421
422
                category_names = [
                    "'{}'".format(category.name)
                    for category in category_candidates
                ]
423
424
                return _make_error(
                    protocol, "Parsing", "Unknown decision category",
425
426
427
428
429
430
431
                    "The decision in line {} has the category {}, "
                    "but there is no such category. "
                    "Known categories are {}".format(
                        decision_tag.linenumber,
                        decision_category_name,
                        ", ".join(category_names)))
            else:
432
                decision_categories.append(decision_category)
433
434
        decision = Decision(
            protocol_id=protocol.id, content=decision_content)
435
436
        db.session.add(decision)
        db.session.commit()
437
438
        for decision_category in decision_categories:
            decision.categories.append(decision_category)
439
440
441
        decision_tag.decision = decision
        decisions_to_render.append((decision, decision_tag))
    for decision, decision_tag in decisions_to_render:
442
        decision_top = decision_tag.fork.get_top()
443
444
445
446
        decision_content = texenv.get_template(provide_latex_template(
            protocol.protocoltype.latex_template, "decision")).render(
                render_type=RenderType.latex, decision=decision,
                protocol=protocol, top=decision_top, show_private=True)
447
448
        maxdepth = decision_top.get_maxdepth()
        compile_decision(decision_content, decision, maxdepth=maxdepth)
Robin Sonnabend's avatar
Robin Sonnabend committed
449
450

    # Footnotes
451
452
453
454
455
456
457
458
    footnote_tags = [
        tag for tag in tags
        if tag.name == "footnote"
    ]
    public_footnote_tags = [
        tag for tag in footnote_tags
        if tag in public_elements
    ]
Robin Sonnabend's avatar
Robin Sonnabend committed
459

460
461
462
463
    # new Protocols
    protocol_tags = [tag for tag in tags if tag.name == "sitzung"]
    for protocol_tag in protocol_tags:
        if len(protocol_tag.values) not in {1, 2}:
464
465
            return _make_error(
                protocol, "Parsing", "Falsche Verwendung von [sitzung;…].",
466
467
468
                "Der Tag \"sitzung\" benötigt immer ein Datum "
                "und optional eine Uhrzeit, also ein bis zwei Argumente. "
                "Stattdessen wurden {} übergeben, nämlich {}".format(
469
470
                    len(protocol_tag.values),
                    protocol_tag.values))
471
472
        else:
            try:
473
                parse_datetime_from_string(protocol_tag.values[0])
474
            except ValueError as exc:
475
476
                return _make_error(
                    protocol, "Parsing", "Invalides Datum",
477
478
479
480
                    "'{}' ist kein valides Datum.".format(
                        protocol_tag.values[0]))
            if len(protocol_tag.values) > 1:
                try:
481
                    datetime.strptime(protocol_tag.values[1], "%H:%M")
482
                except ValueError:
483
484
                    return _make_error(
                        protocol, "Parsing", "Invalide Uhrzeit",
485
486
487
488
489
490
                        "'{}' ist keine valide Uhrzeit.".format(
                            protocol_tag.values[1]))
    for protocol_tag in protocol_tags:
        new_protocol_date = parse_datetime_from_string(protocol_tag.values[0])
        new_protocol_time = None
        if len(protocol_tag.values) > 1:
491
492
            new_protocol_time = datetime.strptime(
                protocol_tag.values[1], "%H:%M")
493
494
495
        if not protocol.protocoltype.get_protocols_on_date(new_protocol_date):
            Protocol.create_new_protocol(
                protocol.protocoltype, new_protocol_date, new_protocol_time)
496

Robin Sonnabend's avatar
Robin Sonnabend committed
497
    # TOPs
498
    old_tops = list(protocol.tops)
499
    tops = []
500
501
502
503
    for index, fork in enumerate(
            (child for child in tree.children if isinstance(child, Fork))):
        top = TOP(
            protocol_id=protocol.id, name=fork.name, number=index,
504
            planned=False)
505
506
507
508
509
510
511
512
        if top.name is None:
            return _make_error(
                protocol, "Parsing", "TOP-Name fehlt",
                "'{Name' sollte '{TOP Name' lauten.")
        tops.append(top)
    for top in old_tops:
        protocol.tops.remove(top)
    for top in tops:
513
514
        db.session.add(top)
    db.session.commit()
515

Robin Sonnabend's avatar
Robin Sonnabend committed
516
517
    # render
    private_render_kwargs = {
518
        "protocol": protocol,
Robin Sonnabend's avatar
Robin Sonnabend committed
519
520
        "tree": tree,
        "footnotes": footnote_tags,
521
    }
Robin Sonnabend's avatar
Robin Sonnabend committed
522
523
524
    public_render_kwargs = copy(private_render_kwargs)
    public_render_kwargs["footnotes"] = public_footnote_tags
    render_kwargs = {True: private_render_kwargs, False: public_render_kwargs}
525

526
527
    maxdepth = tree.get_maxdepth()
    privacy_states = [False]
528
529
530
531
532
533
    content_private = render_template(
        "protocol.txt", render_type=RenderType.plaintext, show_private=True,
        **private_render_kwargs)
    content_public = render_template(
        "protocol.txt", render_type=RenderType.plaintext, show_private=False,
        **public_render_kwargs)
534
535
536
537
    if content_private != content_public:
        privacy_states.append(True)
    protocol.content_private = content_private
    protocol.content_public = content_public
538
539
540
541
542
543
    protocol.content_html_private = render_template(
        "protocol.html", render_type=RenderType.html, show_private=True,
        **private_render_kwargs)
    protocol.content_html_public = render_template(
        "protocol.html", render_type=RenderType.html, show_private=False,
        **public_render_kwargs)
544

545
    for show_private in privacy_states:
546
547
548
549
550
551
552
553
        latex_source = texenv.get_template(provide_latex_template(
            protocol.protocoltype.latex_template, "protocol")).render(
                render_type=RenderType.latex,
                show_private=show_private,
                **render_kwargs[show_private])
        compile(
            latex_source, protocol, show_private=show_private,
            maxdepth=maxdepth)
Robin Sonnabend's avatar
Robin Sonnabend committed
554

555
    if protocol.protocoltype.use_wiki:
556
557
558
559
560
561
562
563
564
        wiki_type = WikiType[getattr(config, "WIKI_TYPE", "MEDIAWIKI")]
        wiki_template = {
            WikiType.MEDIAWIKI: "protocol.wiki",
            WikiType.DOKUWIKI: "protocol.dokuwiki",
        }
        wiki_render_type = {
            WikiType.MEDIAWIKI: RenderType.wikitext,
            WikiType.DOKUWIKI: RenderType.dokuwiki,
        }
Robin Sonnabend's avatar
Robin Sonnabend committed
565
        show_private = not protocol.protocoltype.wiki_only_public
566
567
        wiki_source = wikienv.get_template(wiki_template[wiki_type]).render(
            render_type=wiki_render_type[wiki_type],
Robin Sonnabend's avatar
Robin Sonnabend committed
568
569
570
            show_private=show_private,
            **render_kwargs[show_private]
        ).replace("\n\n\n", "\n\n")
571
572
573
        if wiki_type == WikiType.MEDIAWIKI:
            wiki_infobox_source = wikienv.get_template("infobox.wiki").render(
                protocoltype=protocol.protocoltype)
574
575
            push_to_wiki(
                protocol, wiki_source, wiki_infobox_source,
576
577
                "Automatisch generiert vom Protokollsystem 3.0")
        elif wiki_type == WikiType.DOKUWIKI:
578
579
            push_to_dokuwiki(
                protocol, wiki_source,
580
                "Automatisch generiert vom Protokollsystem 3.0")
581
582
    protocol.done = True
    db.session.commit()
583

584

585
586
def push_to_wiki(protocol, content, infobox_content, summary):
    push_to_wiki_async.delay(protocol.id, content, infobox_content, summary)
Robin Sonnabend's avatar
Robin Sonnabend committed
587

588

Robin Sonnabend's avatar
Robin Sonnabend committed
589
@celery.task
590
def push_to_wiki_async(protocol_id, content, infobox_content, summary):
591
    with app.app_context():
Robin Sonnabend's avatar
Robin Sonnabend committed
592
593
        protocol = Protocol.query.filter_by(id=protocol_id).first()
        try:
594
595
596
597
598
599
600
601
602
            with WikiClient() as wiki_client:
                wiki_client.edit_page(
                    title=protocol.protocoltype.get_wiki_infobox_title(),
                    content=infobox_content,
                    summary=summary)
                wiki_client.edit_page(
                    title=protocol.get_wiki_title(),
                    content=content,
                    summary=summary)
Robin Sonnabend's avatar
Robin Sonnabend committed
603
        except WikiException as exc:
604
605
606
607
            return _make_error(
                protocol, "Pushing to Wiki", "Pushing to Wiki failed.",
                str(exc))

608
609
610
611

def push_to_dokuwiki(protocol, content, summary):
    push_to_dokuwiki_async.delay(protocol.id, content, summary)

612

Robin Sonnabend's avatar
Robin Sonnabend committed
613
@celery.task
614
def push_to_dokuwiki_async(protocol_id, content, summary):
Robin Sonnabend's avatar
Robin Sonnabend committed
615
616
617
618
    with app.app_context():
        protocol = Protocol.query.filter_by(id=protocol_id).first()
        with xmlrpc.client.ServerProxy(config.WIKI_API_URL) as proxy:
            try:
619
620
621
622
623
624
                if not proxy.wiki.putPage(
                    protocol.get_wiki_title(), content,
                    {"sum":
                        "Automatisch generiert vom Protokollsystem 3."}):
                    return _make_error(
                        protocol, "Pushing to Wiki",
Robin Sonnabend's avatar
Robin Sonnabend committed
625
626
                        "Pushing to Wiki failed." "")
            except xmlrpc.client.Error as exception:
627
628
                return _make_error(
                    protocol, "Pushing to Wiki", "XML RPC Exception",
Robin Sonnabend's avatar
Robin Sonnabend committed
629
                    str(exception))
630

Robin Sonnabend's avatar
Robin Sonnabend committed
631

632
def compile(content, protocol, show_private, maxdepth):
633
634
635
    compile_async.delay(
        content, protocol.id, show_private=show_private, maxdepth=maxdepth)

Robin Sonnabend's avatar
Robin Sonnabend committed
636

637
def compile_decision(content, decision, maxdepth):
638
639
640
    compile_async.delay(
        content, decision.id, use_decision=True, maxdepth=maxdepth)

641
642

@celery.task
643
644
645
def compile_async(
        content, protocol_id, show_private=False, use_decision=False,
        maxdepth=5):
646
    with tempfile.TemporaryDirectory() as compile_dir, app.app_context():
Robin Sonnabend's avatar
Robin Sonnabend committed
647
648
649
650
651
652
653
        decision = None
        protocol = None
        if use_decision:
            decision = Decision.query.filter_by(id=protocol_id).first()
            protocol = decision.protocol
        else:
            protocol = Protocol.query.filter_by(id=protocol_id).first()
654
655
656
        try:
            current = os.getcwd()
            protocol_source_filename = "protocol.tex"
657
            protocol_target_filename = "protocol.pdf"
658
            protocol_class_filename = "protokoll2.cls"
659
            log_filename = "protocol.log"
660
661
662
            with open(
                    os.path.join(compile_dir, protocol_source_filename),
                    "w") as source_file:
663
                source_file.write(content)
664
665
666
667
668
669
670
671
672
            protocol2_class_source = texenv.get_template(
                provide_latex_template(
                    protocol.protocoltype.latex_template,
                    "class")).render(
                fonts=config.FONTS, maxdepth=maxdepth,
                bulletpoints=config.LATEX_BULLETPOINTS)
            with open(
                os.path.join(compile_dir, protocol_class_filename),
                    "w") as protocol2_class_file:
673
                protocol2_class_file.write(protocol2_class_source)
674
675
676
677
678
679
680
            os.chdir(compile_dir)
            command = [
                "/usr/bin/xelatex",
                "-halt-on-error",
                "-file-line-error",
                protocol_source_filename
            ]
681
682
683
684
685
686
            subprocess.check_call(
                command, universal_newlines=True, stdout=subprocess.DEVNULL,
                stderr=subprocess.DEVNULL)
            subprocess.check_call(
                command, universal_newlines=True, stdout=subprocess.DEVNULL,
                stderr=subprocess.DEVNULL)
687
            os.chdir(current)
Robin Sonnabend's avatar
Robin Sonnabend committed
688
689
            document = None
            if not use_decision:
690
691
692
693
                for old_document in [
                    document for document in protocol.documents
                    if document.is_compiled
                        and document.is_private == show_private]:
Robin Sonnabend's avatar
Robin Sonnabend committed
694
695
                    protocol.documents.remove(old_document)
                db.session.commit()
696
697
                document = Document(
                    protocol_id=protocol.id,
698
699
700
701
702
703
704
                    name="protokoll{}_{}_{}.pdf".format(
                        "_intern" if show_private else "",
                        protocol.protocoltype.short_name,
                        date_filter_short(protocol.date)),
                    filename="",
                    is_compiled=True,
                    is_private=show_private)
Robin Sonnabend's avatar
Robin Sonnabend committed
705
            else:
706
707
                document = DecisionDocument(
                    decision_id=decision.id,
708
709
710
711
712
                    name="beschluss_{}_{}_{}.pdf".format(
                        protocol.protocoltype.short_name,
                        date_filter_short(protocol.date),
                        decision.id),
                    filename="")
713
714
            db.session.add(document)
            db.session.commit()
715
716
            target_filename = "compiled-{}-{}.pdf".format(
                document.id, "internal" if show_private else "public")
Robin Sonnabend's avatar
Robin Sonnabend committed
717
            if use_decision:
718
719
                target_filename = "decision-{}-{}-{}.pdf".format(
                    protocol.id, decision.id, document.id)
720
            document.filename = target_filename
721
722
723
            shutil.copy(
                os.path.join(compile_dir, protocol_target_filename),
                os.path.join(config.DOCUMENTS_PATH, target_filename))
724
            db.session.commit()
725
726
727
            shutil.copy(
                os.path.join(compile_dir, log_filename),
                "/tmp/proto-tex.log")
728
729
730
        except subprocess.SubprocessError:
            log = ""
            total_log_filename = os.path.join(compile_dir, log_filename)
731
732
            total_source_filename = os.path.join(
                compile_dir, protocol_source_filename)
Robin Sonnabend's avatar
Robin Sonnabend committed
733
            log = ""
734
735
            if os.path.isfile(total_source_filename):
                with open(total_source_filename, "r") as source_file:
Robin Sonnabend's avatar
Robin Sonnabend committed
736
                    log += "Source:\n\n" + add_line_numbers(source_file.read())
737
738
            total_class_filename = os.path.join(
                compile_dir, protocol_class_filename)
739
740
            if os.path.isfile(total_class_filename):
                with open(total_class_filename, "r") as class_file:
741
742
                    log += "\n\nClass:\n\n" + add_line_numbers(
                        class_file.read())
Robin Sonnabend's avatar
Robin Sonnabend committed
743
744
745
746
747
            if os.path.isfile(total_log_filename):
                with open(total_log_filename, "r") as log_file:
                    log += "\n\nLog:\n\n" + add_line_numbers(log_file.read())
            else:
                log += "\n\nLogfile not found."
748
            _make_error(protocol, "Compiling", "Compiling LaTeX failed", log)
749
750
        finally:
            os.chdir(current)
751

752

Robin Sonnabend's avatar
Robin Sonnabend committed
753
754
755
756
def print_file(filename, protocol):
    if config.PRINTING_ACTIVE:
        print_file_async.delay(filename, protocol.id)

757

Robin Sonnabend's avatar
Robin Sonnabend committed
758
759
760
761
762
@celery.task
def print_file_async(filename, protocol_id):
    with app.app_context():
        protocol = Protocol.query.filter_by(id=protocol_id).first()
        if protocol.protocoltype.printer is None:
763
764
765
766
767
768
            return _make_error(
                protocol, "Printing", "No printer configured.",
                "You don't have any printer configured for the "
                "protocoltype {}. "
                "Please do so before printing a protocol.".format(
                    protocol.protocoltype.name))
Robin Sonnabend's avatar
Robin Sonnabend committed
769
770
771
772
773
774
775
776
        try:
            command = [
                "/usr/bin/lpr",
                "-H", config.PRINTING_SERVER,
                "-P", protocol.protocoltype.printer,
                "-U", config.PRINTING_USER,
                "-T", protocol.get_identifier(),
            ]
777
778
779
780
781
            for option in config.PRINTING_PRINTERS[
                    protocol.protocoltype.printer]:
                command.extend([
                    "-o", '"{}"'.format(option)
                    if " " in option else option])
Robin Sonnabend's avatar
Robin Sonnabend committed
782
            command.append(filename)
783
784
            subprocess.check_output(
                command, universal_newlines=True, stderr=subprocess.STDOUT)
785
        except subprocess.SubprocessError as exception:
786
787
788
789
            return _make_error(
                protocol, "Printing", "Printing {} failed.".format(
                    protocol.get_identifier()), exception.stdout)

Robin Sonnabend's avatar
Robin Sonnabend committed
790

Robin Sonnabend's avatar
Robin Sonnabend committed
791
792
def send_reminder(reminder, protocol):
    send_reminder_async.delay(reminder.id, protocol.id)
793

794

795
@celery.task
Robin Sonnabend's avatar
Robin Sonnabend committed
796
def send_reminder_async(reminder_id, protocol_id):
797
    with app.app_context():
Robin Sonnabend's avatar
Robin Sonnabend committed
798
799
        reminder = MeetingReminder.query.filter_by(id=reminder_id).first()
        protocol = Protocol.query.filter_by(id=protocol_id).first()
800
801
        reminder_text = render_template(
            "reminder-mail.txt", reminder=reminder, protocol=protocol)
Robin Sonnabend's avatar
Robin Sonnabend committed
802
        if reminder.send_public:
803
804
            send_mail(
                protocol, protocol.protocoltype.public_mail,
805
806
                "Tagesordnung der {}".format(protocol.protocoltype.name),
                reminder_text, reply_to=protocol.protocoltype.public_mail)
Robin Sonnabend's avatar
Robin Sonnabend committed
807
        if reminder.send_private:
808
809
            send_mail(
                protocol, protocol.protocoltype.private_mail,
810
811
                "Tagesordnung der {}".format(protocol.protocoltype.name),
                reminder_text, reply_to=protocol.protocoltype.private_mail)
Robin Sonnabend's avatar
Robin Sonnabend committed
812

813

814
def remind_finishing(protocol, delay_days, min_delay_days):
Robin Sonnabend's avatar
Robin Sonnabend committed
815
    remind_finishing_async.delay(protocol.id, delay_days, min_delay_days)
816

817

818
@celery.task
Robin Sonnabend's avatar
Robin Sonnabend committed
819
def remind_finishing_async(protocol_id, delay_days, min_delay_days):
820
    with app.app_context():
821
822
823
        protocol = Protocol.first_by_id(protocol_id)
        mail_text = render_template(
            "remind-finishing-mail.txt",
824
825
            protocol=protocol, delay_days=delay_days,
            min_delay_days=min_delay_days)
826
827
        send_mail(
            protocol, protocol.protocoltype.private_mail,
828
829
830
            "Unfertiges Protokoll der {}".format(protocol.protocoltype.name),
            mail_text, reply_to=protocol.protocoltype.private_mail)

831

832
def send_protocol_private(protocol):
833
    send_protocol_async.delay(protocol.id, show_private=True)
Robin Sonnabend's avatar
Robin Sonnabend committed
834
    send_todomails_async.delay(protocol.id)
835

836

837
838
839
def send_protocol_public(protocol):
    send_protocol_async.delay(protocol.id, show_private=False)

840

841
842
843
844
@celery.task
def send_protocol_async(protocol_id, show_private):
    with app.app_context():
        protocol = Protocol.query.filter_by(id=protocol_id).first()
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
        next_protocol = Protocol.query.filter_by(
            protocoltype_id=protocol.protocoltype.id).filter_by(
            done=False).filter(
            Protocol.date > datetime.now()).order_by(Protocol.date).first()
        to_addr = (
            protocol.protocoltype.private_mail
            if show_private
            else protocol.protocoltype.public_mail)
        subject = "{}{}-Protokoll vom {}".format(
            "Internes " if show_private else "",
            protocol.protocoltype.short_name, date_filter(protocol.date))
        mail_content = render_template(
            "protocol-mail.txt", protocol=protocol, show_private=show_private,
            next_protocol=next_protocol)
        appendix = [
            (document.name, document.as_file_like())
861
862
863
864
865
            for document in protocol.documents
            if show_private or not document.is_private
        ]
        send_mail(protocol, to_addr, subject, mail_content, appendix)

866

Robin Sonnabend's avatar
Robin Sonnabend committed
867
868
869
870
@celery.task
def send_todomails_async(protocol_id):
    with app.app_context():
        protocol = Protocol.query.filter_by(id=protocol_id).first()
871
        all_todos = [
Administrator's avatar
Administrator committed
872
            todo for todo in Todo.query.all()
873
            if not todo.is_done()
Administrator's avatar
Administrator committed
874
            and todo.protocoltype == protocol.protocoltype
875
        ]
Robin Sonnabend's avatar
Robin Sonnabend committed
876
877
878
879
880
881
        users = {user for todo in all_todos for user in todo.get_users()}
        grouped_todos = {
            user: [todo for todo in all_todos if user in todo.get_users()]
            for user in users
        }
        subject = "Du hast noch was zu tun!"
882
883
        todomail_providers = getattr(
            config, "ADDITIONAL_TODOMAIL_PROVIDERS", None)
884
885
886
887
888
889
890
891
        additional_todomails = {}
        if todomail_providers:
            for provider in todomail_providers:
                todomail_dict = provider()
                for key in todomail_dict:
                    if key not in additional_todomails:
                        name, mail = todomail_dict[key]
                        additional_todomails[key] = TodoMail(name, mail)
Robin Sonnabend's avatar
Robin Sonnabend committed
892
893
        for user in users:
            todomail = TodoMail.query.filter(TodoMail.name.ilike(user)).first()
894
895
896
            if todomail is None:
                if user in additional_todomails:
                    todomail = additional_todomails[user]
Robin Sonnabend's avatar
Robin Sonnabend committed
897
            if todomail is None:
898
899
900
                _make_error(
                    protocol, "Sending Todomail", "Sending Todomail failed.",
                    "User {} has no Todo-Mail-Assignment.".format(user))
Robin Sonnabend's avatar
Robin Sonnabend committed
901
902
                continue
            to_addr = todomail.get_formatted_mail()
903
904
905
906
907
            mail_content = render_template(
                "todo-mail.txt", protocol=protocol, todomail=todomail,
                todos=grouped_todos[user])
            send_mail(
                protocol, to_addr, subject, mail_content,
908
                reply_to=protocol.protocoltype.private_mail)
909

910
911
912

def send_mail(protocol, to_addr, subject, content, appendix=None,
              reply_to=None):
Robin Sonnabend's avatar
Robin Sonnabend committed
913
    if to_addr is not None and len(to_addr.strip()) > 0:
914
915
916
        send_mail_async.delay(
            protocol.id, to_addr, subject, content, appendix, reply_to)

917

Robin Sonnabend's avatar
Robin Sonnabend committed
918
@celery.task
919
920
def send_mail_async(protocol_id, to_addr, subject, content, appendix,
                    reply_to):
Robin Sonnabend's avatar
Robin Sonnabend committed
921
922
923
    with app.app_context():
        protocol = Protocol.query.filter_by(id=protocol_id).first()
        try:
924
            mail_manager.send(to_addr, subject, content, appendix, reply_to)
Robin Sonnabend's avatar
Robin Sonnabend committed
925
        except Exception as exc:
926
927
928
            return _make_error(
                protocol, "Sending Mail", "Sending mail failed", str(exc))

929

Robin Sonnabend's avatar
Robin Sonnabend committed
930
931
932
def push_tops_to_calendar(protocol):
    push_tops_to_calendar_async.delay(protocol.id)

933

Robin Sonnabend's avatar
Robin Sonnabend committed
934
935
936
937
938
939
940
941
942
943
944
@celery.task
def push_tops_to_calendar_async(protocol_id):
    if not config.CALENDAR_ACTIVE:
        return
    with app.app_context():
        protocol = Protocol.query.filter_by(id=protocol_id).first()
        if protocol.protocoltype.calendar == "":
            return
        description = render_template("calendar-tops.txt", protocol=protocol)
        try:
            client = CalendarClient(protocol.protocoltype.calendar)
945
946
            client.set_event_at(
                begin=protocol.get_datetime(),
Robin Sonnabend's avatar
Robin Sonnabend committed
947
948
                name=protocol.protocoltype.short_name, description=description)
        except CalendarException as exc:
949
950
            return _make_error(
                protocol, "Calendar",
Robin Sonnabend's avatar
Robin Sonnabend committed
951
                "Pushing TOPs to Calendar failed", str(exc))
952

953
954

def set_etherpad_content(protocol):
Robin Sonnabend's avatar
Robin Sonnabend committed
955
    set_etherpad_content_async.delay(protocol.id)
956

957

958
959
960
961
@celery.task
def set_etherpad_content_async(protocol_id):
    with app.app_context():
        protocol = Protocol.query.filter_by(id=protocol_id).first()
Administrator's avatar
Administrator committed
962
963
        identifier = protocol.get_identifier()
        return set_etherpad_text(identifier, protocol.get_template())