tasks.py 39 KB
Newer Older
1
2
3
4
5
from flask import render_template

import os
import subprocess
import shutil
6
import tempfile
7
from datetime import datetime
Robin Sonnabend's avatar
Robin Sonnabend committed
8
import traceback
9
from copy import copy
10
import xmlrpc.client
11

12
13
14
from models.database import (
    Document, Protocol, Todo, Decision, TOP, MeetingReminder,
    TodoMail, DecisionDocument, TodoState, OldTodo, DecisionCategory)
Robin Sonnabend's avatar
Robin Sonnabend committed
15
from models.errors import DateNotMatchingException
16
from server import celery, app
17
18
from shared import (
    db, escape_tex, unhyphen, date_filter, datetime_filter, date_filter_long,
19
    date_filter_short, time_filter, class_filter, KNOWN_KEYS, WikiType, config)
20
21
22
23
from utils import (
    mail_manager, add_line_numbers,
    set_etherpad_text, parse_datetime_from_string)
from protoparser import parse, ParserException, Tag, Remark, Fork, RenderType
Robin Sonnabend's avatar
Robin Sonnabend committed
24
from wiki import WikiClient, WikiException
Robin Sonnabend's avatar
Robin Sonnabend committed
25
from calendarpush import Client as CalendarClient, CalendarException
26
from legacy import lookup_todo_id
27
28
29
30
31
32
33
34
35
36
37
38
39
40

texenv = app.create_jinja_environment()
texenv.block_start_string = r"\ENV{"
texenv.block_end_string = r"}"
texenv.variable_start_string = r"\VAR{"
texenv.variable_end_string = r"}"
texenv.comment_start_string = r"\COMMENT{"
texenv.comment_end_string = r"}"
texenv.filters["escape_tex"] = escape_tex
texenv.filters["unhyphen"] = unhyphen
texenv.trim_blocks = True
texenv.lstrip_blocks = True
texenv.filters["datify"] = date_filter
texenv.filters["datify_long"] = date_filter_long
41
texenv.filters["datify_short"] = date_filter_short
42
43
texenv.filters["datetimify"] = datetime_filter
texenv.filters["timify"] = time_filter
44
texenv.filters["class"] = class_filter
45
46
47
logo_template = getattr(config, "LATEX_LOGO_TEMPLATE", None)
if logo_template is not None:
    texenv.globals["logo_template"] = logo_template
48
49
50
latex_geometry = getattr(
    config, "LATEX_GEOMETRY",
    "vmargin=1.5cm,hmargin={1.5cm,1.2cm},bindingoffset=8mm")
51
52
53
54
55
56
57
58
59
60
texenv.globals["latex_geometry"] = latex_geometry
raw_additional_packages = getattr(config, "LATEX_ADDITIONAL_PACKAGES", None)
additional_packages = []
if raw_additional_packages is not None:
    for package in raw_additional_packages:
        if "{" not in package:
            package = "{{{}}}".format(package)
        additional_packages.append(package)
texenv.globals["additional_packages"] = additional_packages
latex_pagestyle = getattr(config, "LATEX_PAGESTYLE", None)
61
if latex_pagestyle is not None and latex_pagestyle:
62
63
64
    texenv.globals["latex_pagestyle"] = latex_pagestyle
latex_header_footer = getattr(config, "LATEX_HEADER_FOOTER", False)
texenv.globals["latex_header_footer"] = latex_header_footer
65
66
latex_templates = getattr(config, "LATEX_TEMPLATES", None)

67

68
def provide_latex_template(template, documenttype):
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
    _DOCUMENTTYPE_FILENAME_MAP = {
        "class": "protokoll2.cls",
        "protocol": "protocol.tex",
        "decision": "decision.tex"
    }
    _PROVIDES = "provides"
    _LOGO_TEMPLATE = "logo_template"
    _LOGO = "logo"
    _LATEX_GEOMETRY = "latex_geometry"
    _GEOMETRY = "geometry"
    _ADDITIONAL_PACKAGES = "additional_packages"
    _LATEX_PAGESTYLE = "latex_pagestyle"
    _PAGESTYLE = "pagestyle"
    _LATEX_HEADER_FOOTER = "latex_header_footer"
    _HEADER_FOOTER = "headerfooter"
    _latex_template_filename = _DOCUMENTTYPE_FILENAME_MAP[documenttype]
    _latex_template_foldername = ""
    if logo_template is not None:
        texenv.globals[_LOGO_TEMPLATE] = logo_template
    texenv.globals[_LATEX_GEOMETRY] = latex_geometry
    texenv.globals[_ADDITIONAL_PACKAGES] = additional_packages
    if latex_pagestyle:
        texenv.globals[_LATEX_PAGESTYLE] = latex_pagestyle
    elif _LATEX_PAGESTYLE in texenv.globals:
        del texenv.globals[_LATEX_PAGESTYLE]
    texenv.globals[_LATEX_HEADER_FOOTER] = latex_header_footer
    if latex_templates is not None and template != "":
        if template in latex_templates:
            template_data = latex_templates[template]
            if _PROVIDES in template_data:
                if documenttype in template_data[_PROVIDES]:
                    _latex_template_foldername = template
            if _LOGO in template_data:
                texenv.globals[_LOGO_TEMPLATE] = os.path.join(
                    template, template_data[_LOGO])
            if _GEOMETRY in template_data:
                texenv.globals[_LATEX_GEOMETRY] = template_data[_GEOMETRY]
            if _PAGESTYLE in template_data:
                if template_data[_PAGESTYLE]:
                    texenv.globals[_LATEX_PAGESTYLE] = (
                        template_data[_PAGESTYLE])
            if _ADDITIONAL_PACKAGES in template_data:
                _raw_additional_packages = template_data[_ADDITIONAL_PACKAGES]
                _additional_packages = []
                if _raw_additional_packages is not None:
                    for _package in _raw_additional_packages:
                        if "{" not in _package:
                            _package = "{{{}}}".format(_package)
                        _additional_packages.append(_package)
                texenv.globals[_ADDITIONAL_PACKAGES] = _additional_packages
            if _HEADER_FOOTER in latex_templates[template]:
                texenv.globals[_LATEX_HEADER_FOOTER] = (
                    template_data[_HEADER_FOOTER])
    return os.path.join(_latex_template_foldername, _latex_template_filename)

124
125
126
127
128
129
130

mailenv = app.create_jinja_environment()
mailenv.trim_blocks = True
mailenv.lstrip_blocks = True
mailenv.filters["datify"] = date_filter
mailenv.filters["datetimify"] = datetime_filter

131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
wikienv = app.create_jinja_environment()
wikienv.trim_blocks = True
wikienv.lstrip_blocks = True
wikienv.block_start_string = "<env>"
wikienv.block_end_string = "</env>"
wikienv.variable_start_string = "<var>"
wikienv.variable_end_string = "</var>"
wikienv.comment_start_string = "<comment>"
wikienv.comment_end_string = "</comment>"
wikienv.filters["datify"] = date_filter
wikienv.filters["datify_long"] = date_filter_long
wikienv.filters["datify_short"] = date_filter_short
wikienv.filters["datetimify"] = datetime_filter
wikienv.filters["timify"] = time_filter
wikienv.filters["class"] = class_filter


148
149
150
151
152
153
def _make_error(protocol, *args):
    error = protocol.create_error(*args)
    db.session.add(error)
    db.session.commit()


154
155
ID_FIELD_BEGINNING = "id "

156
157
158
159

def parse_protocol(protocol):
    parse_protocol_async.delay(protocol.id)

160
161

@celery.task
162
def parse_protocol_async(protocol_id):
163
164
165
    with app.app_context():
        with app.test_request_context("/"):
            try:
166
                protocol = Protocol.first_by_id(protocol_id)
167
168
                if protocol is None:
                    raise Exception("No protocol given. Aborting parsing.")
169
                parse_protocol_async_inner(protocol)
170
            except Exception as exc:
Robin Sonnabend's avatar
Robin Sonnabend committed
171
                stacktrace = traceback.format_exc()
172
173
                return _make_error(
                    protocol, "Parsing", "Exception",
Robin Sonnabend's avatar
Robin Sonnabend committed
174
                    "{}\n\n{}".format(str(exc), stacktrace))
175

176
177

def parse_protocol_async_inner(protocol):
178
179
180
181
    old_errors = list(protocol.errors)
    for error in old_errors:
        protocol.errors.remove(error)
    db.session.commit()
182
    if protocol.source is None or len(protocol.source.strip()) == 0:
183
        return _make_error(protocol, "Parsing", "Protocol source is empty", "")
184
    if protocol.source == config.EMPTY_ETHERPAD:
185
186
187
        return _make_error(
            protocol, "Parsing", "The etherpad is unmodified and does not "
            "contain a protocol.", protocol.source)
188
189
190
191
192
193
194
195
    tree = None
    try:
        tree = parse(protocol.source)
    except ParserException as exc:
        context = ""
        if exc.linenumber is not None:
            source_lines = protocol.source.splitlines()
            start_index = max(0, exc.linenumber - config.ERROR_CONTEXT_LINES)
196
197
198
            end_index = min(
                len(source_lines) - 1,
                exc.linenumber + config.ERROR_CONTEXT_LINES)
199
200
201
            context = "\n".join(source_lines[start_index:end_index])
        if exc.tree is not None:
            context += "\n\nParsed syntax tree was:\n" + str(exc.tree.dump())
202
203
204
205
206
207
        return _make_error(protocol, "Parsing", str(exc), context)
    remarks = {
        element.name: element
        for element in tree.children
        if isinstance(element, Remark)
    }
208
    required_fields = copy(KNOWN_KEYS)
Robin Sonnabend's avatar
Robin Sonnabend committed
209
210
    for default_meta in protocol.protocoltype.metas:
        required_fields.append(default_meta.key)
211
    if not config.PARSER_LAZY:
212
213
214
215
216
        missing_fields = [
            field
            for field in required_fields
            if field not in remarks
        ]
217
        if len(missing_fields) > 0:
218
219
220
            return _make_error(
                protocol, "Parsing", "Du hast vergessen, Metadaten anzugeben.",
                ", ".join(missing_fields))
221
222
223
    try:
        protocol.fill_from_remarks(remarks)
    except ValueError:
224
225
        return _make_error(
            protocol, "Parsing", "Invalid fields",
226
227
            "Date or time fields are not '%d.%m.%Y' respectively '%H:%M', "
            "but rather {}".format(
228
229
230
231
232
                ", ".join([
                    remarks["Datum"].value.strip(),
                    remarks["Beginn"].value.strip(),
                    remarks["Ende"].value.strip()
                ])))
233
    except DateNotMatchingException as exc:
234
235
236
237
238
239
240
241
242
243
244
        return _make_error(
            protocol, "Parsing", "Date not matching",
            "This protocol's date should be {}, but the protocol source "
            "says {}.".format(
                date_filter(exc.original_date)
                if exc.original_date is not None
                else "not present",
                date_filter(exc.protocol_date)
                if exc.protocol_date is not None
                else "not present"))
    # tags
245
    tags = tree.get_tags()
Robin Sonnabend's avatar
Robin Sonnabend committed
246
    public_elements = tree.get_visible_elements(show_private=False)
247
248
    for tag in tags:
        if tag.name not in Tag.KNOWN_TAGS:
249
250
            return _make_error(
                protocol, "Parsing", "Invalid tag",
251
252
253
                "The tag in line {} has the kind '{}', which is "
                "not defined. This is probably an error mit a missing "
                "semicolon.".format(tag.linenumber, tag.name))
254
    # todos
255
256
257
    old_todo_number_map = {}
    for todo in protocol.todos:
        old_todo_number_map[todo.description] = todo.get_id()
258
259
260
261
    protocol.delete_orphan_todos()
    db.session.commit()
    old_todos = list(protocol.todos)
    todo_tags = [tag for tag in tags if tag.name == "todo"]
262
    raw_todos = []
263
264
    for todo_tag in todo_tags:
        if len(todo_tag.values) < 2:
265
266
            return _make_error(
                protocol, "Parsing", "Invalid todo-tag",
267
268
                "The todo tag in line {} needs at least "
                "information on who and what, "
269
270
                "but has less than that. This is probably "
                "a missing semicolon.".format(todo_tag.linenumber))
271
272
273
274
275
276
277
278
279
280
281
282
283
        who = todo_tag.values[0]
        what = todo_tag.values[1]
        field_id = None
        field_state = None
        field_date = None
        for other_field in todo_tag.values[2:]:
            other_field = other_field.strip()
            if len(other_field) == 0:
                continue
            if other_field.startswith(ID_FIELD_BEGINNING):
                try:
                    field_id = int(other_field[len(ID_FIELD_BEGINNING):])
                except ValueError:
284
285
286
287
288
                    return _make_error(
                        protocol, "Parsing", "Non-numerical todo ID",
                        "The todo in line {} has a nonnumerical ID, but needs "
                        "something like \"id 1234\"".format(
                            todo_tag.linenumber))
289
290
291
            else:
                try:
                    field_state = TodoState.from_name(other_field)
292
293
294
295
296
297
                    continue
                except ValueError:
                    pass
                try:
                    field_date = datetime.strptime(other_field, "%d.%m.%Y")
                    continue
298
                except ValueError:
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
                    pass
                try:
                    field_state, field_date = TodoState.from_name_with_date(
                        other_field.strip(), protocol=protocol)
                    continue
                except ValueError:
                    pass
                try:
                    field_state = TodoState.from_name_lazy(other_field)
                except ValueError:
                    return _make_error(
                        protocol, "Parsing", "Invalid field",
                        "The todo in line {} has the field '{}', but "
                        "this does neither match a date (\"%d.%m.%Y\") "
                        "nor a state.".format(
                            todo_tag.linenumber, other_field))
        raw_todos.append(
            (who, what, field_id, field_state, field_date, todo_tag))
317
318
    for (_, _, field_id, _, _, _) in raw_todos:
        if field_id is not None:
319
320
321
322
            old_todos = [
                todo for todo in old_todos
                if todo.id != field_id
            ]
323
324
325
326
    for todo in old_todos:
        protocol.todos.remove(todo)
    db.session.commit()
    for (who, what, field_id, field_state, field_date, todo_tag) in raw_todos:
327
328
329
        if field_state is None:
            field_state = TodoState.open
        if field_state.needs_date() and field_date is None:
330
331
            return _make_error(
                protocol, "Parsing", "Todo missing date",
332
333
334
335
336
337
338
339
                "The todo in line {} has a state that needs a date, "
                "but the todo does not have one.".format(todo_tag.linenumber))
        who = who.strip()
        what = what.strip()
        todo = None
        if field_id is not None:
            todo = Todo.query.filter_by(number=field_id).first()
            if todo is None and not config.PARSER_LAZY:
340
341
342
343
                return _make_error(
                    protocol, "Parsing", "Invalid Todo ID",
                    "The todo in line {} has the ID {}, but there is no "
                    "Todo with that ID.".format(todo_tag.linenumber, field_id))
344
        if todo is None and field_id is None and what in old_todo_number_map:
345
346
            todo = Todo(
                protocoltype_id=protocol.protocoltype.id,
347
348
349
350
                who=who, description=what, state=field_state,
                date=field_date, number=old_todo_number_map[what])
            db.session.add(todo)
            db.session.commit()
351
352
353
354
355
356
        if todo is None:
            protocol_key = protocol.get_identifier()
            old_candidates = OldTodo.query.filter(
                OldTodo.protocol_key == protocol_key).all()
            if len(old_candidates) == 0:
                # new protocol
357
358
                todo = Todo(
                    protocoltype_id=protocol.protocoltype.id,
359
360
361
362
363
364
365
366
367
368
369
                    who=who, description=what, state=field_state,
                    date=field_date)
                db.session.add(todo)
                db.session.commit()
                todo.number = field_id or todo.id
                db.session.commit()
            else:
                # old protocol
                number = field_id or lookup_todo_id(old_candidates, who, what)
                todo = Todo.query.filter_by(number=number).first()
                if todo is None:
370
371
                    todo = Todo(
                        protocoltype_id=protocol.protocoltype.id,
372
373
374
375
376
                        who=who, description=what, state=field_state,
                        date=field_date, number=number)
                    db.session.add(todo)
                    db.session.commit()
        todo.protocols.append(protocol)
377
378
379
380
381
382
383
384
385
386
        is_newest_protocol = True
        for other_protocol in todo.protocols:
            if other_protocol.date > protocol.date:
                is_newest_protocol = False
                break
        if is_newest_protocol:
            todo.state = field_state
            todo.date = field_date
            todo.who = who
            todo.description = what
387
388
389
        db.session.commit()
        todo_tag.todo = todo
    # Decisions
Robin Sonnabend's avatar
Robin Sonnabend committed
390
391
392
    decision_tags = [tag for tag in tags if tag.name == "beschluss"]
    for decision_tag in decision_tags:
        if decision_tag not in public_elements:
393
394
395
396
397
398
            return _make_error(
                protocol, "Parsing", "Decision in private context.",
                "The decision in line {} is in a private context, but "
                "decisions are and have to be public. "
                "Please move it to a public spot.".format(
                    decision_tag.linenumber))
399
400
401
402
    old_decisions = list(protocol.decisions)
    for decision in old_decisions:
        protocol.decisions.remove(decision)
    db.session.commit()
403
    decisions_to_render = []
404
405
    for decision_tag in decision_tags:
        if len(decision_tag.values) == 0:
406
407
408
409
            return _make_error(
                protocol, "Parsing", "Empty decision found.",
                "The decision in line {} is empty.".format(
                    decision_tag.linenumber))
410
        decision_content = decision_tag.values[0]
411
412
        decision_categories = []
        for decision_category_name in decision_tag.values[1:]:
413
414
415
            decision_category = DecisionCategory.query.filter_by(
                protocoltype_id=protocol.protocoltype.id,
                name=decision_category_name).first()
416
            if decision_category is None:
417
418
                category_candidates = DecisionCategory.query.filter_by(
                    protocoltype_id=protocol.protocoltype.id).all()
419
420
421
422
                category_names = [
                    "'{}'".format(category.name)
                    for category in category_candidates
                ]
423
424
                return _make_error(
                    protocol, "Parsing", "Unknown decision category",
425
426
427
428
429
430
431
                    "The decision in line {} has the category {}, "
                    "but there is no such category. "
                    "Known categories are {}".format(
                        decision_tag.linenumber,
                        decision_category_name,
                        ", ".join(category_names)))
            else:
432
                decision_categories.append(decision_category)
433
434
        decision = Decision(
            protocol_id=protocol.id, content=decision_content)
435
436
        db.session.add(decision)
        db.session.commit()
437
438
        for decision_category in decision_categories:
            decision.categories.append(decision_category)
439
440
441
        decision_tag.decision = decision
        decisions_to_render.append((decision, decision_tag))
    for decision, decision_tag in decisions_to_render:
442
        decision_top = decision_tag.fork.get_top()
443
444
445
446
        decision_content = texenv.get_template(provide_latex_template(
            protocol.protocoltype.latex_template, "decision")).render(
                render_type=RenderType.latex, decision=decision,
                protocol=protocol, top=decision_top, show_private=True)
447
448
        maxdepth = decision_top.get_maxdepth()
        compile_decision(decision_content, decision, maxdepth=maxdepth)
Robin Sonnabend's avatar
Robin Sonnabend committed
449
450

    # Footnotes
451
452
453
454
455
456
457
458
    footnote_tags = [
        tag for tag in tags
        if tag.name == "footnote"
    ]
    public_footnote_tags = [
        tag for tag in footnote_tags
        if tag in public_elements
    ]
Robin Sonnabend's avatar
Robin Sonnabend committed
459

460
461
462
463
    # new Protocols
    protocol_tags = [tag for tag in tags if tag.name == "sitzung"]
    for protocol_tag in protocol_tags:
        if len(protocol_tag.values) not in {1, 2}:
464
465
            return _make_error(
                protocol, "Parsing", "Falsche Verwendung von [sitzung;…].",
466
467
468
                "Der Tag \"sitzung\" benötigt immer ein Datum "
                "und optional eine Uhrzeit, also ein bis zwei Argumente. "
                "Stattdessen wurden {} übergeben, nämlich {}".format(
469
470
                    len(protocol_tag.values),
                    protocol_tag.values))
471
472
        else:
            try:
473
                parse_datetime_from_string(protocol_tag.values[0])
474
            except ValueError as exc:
475
476
                return _make_error(
                    protocol, "Parsing", "Invalides Datum",
477
478
479
480
                    "'{}' ist kein valides Datum.".format(
                        protocol_tag.values[0]))
            if len(protocol_tag.values) > 1:
                try:
481
                    datetime.strptime(protocol_tag.values[1], "%H:%M")
482
                except ValueError:
483
484
                    return _make_error(
                        protocol, "Parsing", "Invalide Uhrzeit",
485
486
487
488
489
490
                        "'{}' ist keine valide Uhrzeit.".format(
                            protocol_tag.values[1]))
    for protocol_tag in protocol_tags:
        new_protocol_date = parse_datetime_from_string(protocol_tag.values[0])
        new_protocol_time = None
        if len(protocol_tag.values) > 1:
491
492
            new_protocol_time = datetime.strptime(
                protocol_tag.values[1], "%H:%M")
493
494
495
        if not protocol.protocoltype.get_protocols_on_date(new_protocol_date):
            Protocol.create_new_protocol(
                protocol.protocoltype, new_protocol_date, new_protocol_time)
496

Robin Sonnabend's avatar
Robin Sonnabend committed
497
    # TOPs
498
499
500
    old_tops = list(protocol.tops)
    for top in old_tops:
        protocol.tops.remove(top)
501
502
503
504
    for index, fork in enumerate(
            (child for child in tree.children if isinstance(child, Fork))):
        top = TOP(
            protocol_id=protocol.id, name=fork.name, number=index,
505
            planned=False)
506
507
        db.session.add(top)
    db.session.commit()
508

Robin Sonnabend's avatar
Robin Sonnabend committed
509
510
    # render
    private_render_kwargs = {
511
        "protocol": protocol,
Robin Sonnabend's avatar
Robin Sonnabend committed
512
513
        "tree": tree,
        "footnotes": footnote_tags,
514
    }
Robin Sonnabend's avatar
Robin Sonnabend committed
515
516
517
    public_render_kwargs = copy(private_render_kwargs)
    public_render_kwargs["footnotes"] = public_footnote_tags
    render_kwargs = {True: private_render_kwargs, False: public_render_kwargs}
518

519
520
    maxdepth = tree.get_maxdepth()
    privacy_states = [False]
521
522
523
524
525
526
    content_private = render_template(
        "protocol.txt", render_type=RenderType.plaintext, show_private=True,
        **private_render_kwargs)
    content_public = render_template(
        "protocol.txt", render_type=RenderType.plaintext, show_private=False,
        **public_render_kwargs)
527
528
529
530
    if content_private != content_public:
        privacy_states.append(True)
    protocol.content_private = content_private
    protocol.content_public = content_public
531
532
533
534
535
536
    protocol.content_html_private = render_template(
        "protocol.html", render_type=RenderType.html, show_private=True,
        **private_render_kwargs)
    protocol.content_html_public = render_template(
        "protocol.html", render_type=RenderType.html, show_private=False,
        **public_render_kwargs)
537

538
    for show_private in privacy_states:
539
540
541
542
543
544
545
546
        latex_source = texenv.get_template(provide_latex_template(
            protocol.protocoltype.latex_template, "protocol")).render(
                render_type=RenderType.latex,
                show_private=show_private,
                **render_kwargs[show_private])
        compile(
            latex_source, protocol, show_private=show_private,
            maxdepth=maxdepth)
Robin Sonnabend's avatar
Robin Sonnabend committed
547

548
    if protocol.protocoltype.use_wiki:
549
550
551
552
553
554
555
556
557
        wiki_type = WikiType[getattr(config, "WIKI_TYPE", "MEDIAWIKI")]
        wiki_template = {
            WikiType.MEDIAWIKI: "protocol.wiki",
            WikiType.DOKUWIKI: "protocol.dokuwiki",
        }
        wiki_render_type = {
            WikiType.MEDIAWIKI: RenderType.wikitext,
            WikiType.DOKUWIKI: RenderType.dokuwiki,
        }
Robin Sonnabend's avatar
Robin Sonnabend committed
558
        show_private = not protocol.protocoltype.wiki_only_public
559
560
        wiki_source = wikienv.get_template(wiki_template[wiki_type]).render(
            render_type=wiki_render_type[wiki_type],
Robin Sonnabend's avatar
Robin Sonnabend committed
561
562
563
            show_private=show_private,
            **render_kwargs[show_private]
        ).replace("\n\n\n", "\n\n")
564
565
566
        if wiki_type == WikiType.MEDIAWIKI:
            wiki_infobox_source = wikienv.get_template("infobox.wiki").render(
                protocoltype=protocol.protocoltype)
567
568
            push_to_wiki(
                protocol, wiki_source, wiki_infobox_source,
569
570
                "Automatisch generiert vom Protokollsystem 3.0")
        elif wiki_type == WikiType.DOKUWIKI:
571
572
            push_to_dokuwiki(
                protocol, wiki_source,
573
                "Automatisch generiert vom Protokollsystem 3.0")
574
575
    protocol.done = True
    db.session.commit()
576

577

578
579
def push_to_wiki(protocol, content, infobox_content, summary):
    push_to_wiki_async.delay(protocol.id, content, infobox_content, summary)
Robin Sonnabend's avatar
Robin Sonnabend committed
580

581

Robin Sonnabend's avatar
Robin Sonnabend committed
582
@celery.task
583
def push_to_wiki_async(protocol_id, content, infobox_content, summary):
584
    with app.app_context():
Robin Sonnabend's avatar
Robin Sonnabend committed
585
586
        protocol = Protocol.query.filter_by(id=protocol_id).first()
        try:
587
588
589
590
591
592
593
594
595
            with WikiClient() as wiki_client:
                wiki_client.edit_page(
                    title=protocol.protocoltype.get_wiki_infobox_title(),
                    content=infobox_content,
                    summary=summary)
                wiki_client.edit_page(
                    title=protocol.get_wiki_title(),
                    content=content,
                    summary=summary)
Robin Sonnabend's avatar
Robin Sonnabend committed
596
        except WikiException as exc:
597
598
599
600
            return _make_error(
                protocol, "Pushing to Wiki", "Pushing to Wiki failed.",
                str(exc))

601
602
603
604

def push_to_dokuwiki(protocol, content, summary):
    push_to_dokuwiki_async.delay(protocol.id, content, summary)

605

Robin Sonnabend's avatar
Robin Sonnabend committed
606
@celery.task
607
def push_to_dokuwiki_async(protocol_id, content, summary):
Robin Sonnabend's avatar
Robin Sonnabend committed
608
609
610
611
    with app.app_context():
        protocol = Protocol.query.filter_by(id=protocol_id).first()
        with xmlrpc.client.ServerProxy(config.WIKI_API_URL) as proxy:
            try:
612
613
614
615
616
617
                if not proxy.wiki.putPage(
                    protocol.get_wiki_title(), content,
                    {"sum":
                        "Automatisch generiert vom Protokollsystem 3."}):
                    return _make_error(
                        protocol, "Pushing to Wiki",
Robin Sonnabend's avatar
Robin Sonnabend committed
618
619
                        "Pushing to Wiki failed." "")
            except xmlrpc.client.Error as exception:
620
621
                return _make_error(
                    protocol, "Pushing to Wiki", "XML RPC Exception",
Robin Sonnabend's avatar
Robin Sonnabend committed
622
                    str(exception))
623

Robin Sonnabend's avatar
Robin Sonnabend committed
624

625
def compile(content, protocol, show_private, maxdepth):
626
627
628
    compile_async.delay(
        content, protocol.id, show_private=show_private, maxdepth=maxdepth)

Robin Sonnabend's avatar
Robin Sonnabend committed
629

630
def compile_decision(content, decision, maxdepth):
631
632
633
    compile_async.delay(
        content, decision.id, use_decision=True, maxdepth=maxdepth)

634
635

@celery.task
636
637
638
def compile_async(
        content, protocol_id, show_private=False, use_decision=False,
        maxdepth=5):
639
    with tempfile.TemporaryDirectory() as compile_dir, app.app_context():
Robin Sonnabend's avatar
Robin Sonnabend committed
640
641
642
643
644
645
646
        decision = None
        protocol = None
        if use_decision:
            decision = Decision.query.filter_by(id=protocol_id).first()
            protocol = decision.protocol
        else:
            protocol = Protocol.query.filter_by(id=protocol_id).first()
647
648
649
        try:
            current = os.getcwd()
            protocol_source_filename = "protocol.tex"
650
            protocol_target_filename = "protocol.pdf"
651
            protocol_class_filename = "protokoll2.cls"
652
            log_filename = "protocol.log"
653
654
655
            with open(
                    os.path.join(compile_dir, protocol_source_filename),
                    "w") as source_file:
656
                source_file.write(content)
657
658
659
660
661
662
663
664
665
            protocol2_class_source = texenv.get_template(
                provide_latex_template(
                    protocol.protocoltype.latex_template,
                    "class")).render(
                fonts=config.FONTS, maxdepth=maxdepth,
                bulletpoints=config.LATEX_BULLETPOINTS)
            with open(
                os.path.join(compile_dir, protocol_class_filename),
                    "w") as protocol2_class_file:
666
                protocol2_class_file.write(protocol2_class_source)
667
668
669
670
671
672
673
            os.chdir(compile_dir)
            command = [
                "/usr/bin/xelatex",
                "-halt-on-error",
                "-file-line-error",
                protocol_source_filename
            ]
674
675
676
677
678
679
            subprocess.check_call(
                command, universal_newlines=True, stdout=subprocess.DEVNULL,
                stderr=subprocess.DEVNULL)
            subprocess.check_call(
                command, universal_newlines=True, stdout=subprocess.DEVNULL,
                stderr=subprocess.DEVNULL)
680
            os.chdir(current)
Robin Sonnabend's avatar
Robin Sonnabend committed
681
682
            document = None
            if not use_decision:
683
684
685
686
                for old_document in [
                    document for document in protocol.documents
                    if document.is_compiled
                        and document.is_private == show_private]:
Robin Sonnabend's avatar
Robin Sonnabend committed
687
688
                    protocol.documents.remove(old_document)
                db.session.commit()
689
690
                document = Document(
                    protocol_id=protocol.id,
691
692
693
694
695
696
697
                    name="protokoll{}_{}_{}.pdf".format(
                        "_intern" if show_private else "",
                        protocol.protocoltype.short_name,
                        date_filter_short(protocol.date)),
                    filename="",
                    is_compiled=True,
                    is_private=show_private)
Robin Sonnabend's avatar
Robin Sonnabend committed
698
            else:
699
700
                document = DecisionDocument(
                    decision_id=decision.id,
701
702
703
704
705
                    name="beschluss_{}_{}_{}.pdf".format(
                        protocol.protocoltype.short_name,
                        date_filter_short(protocol.date),
                        decision.id),
                    filename="")
706
707
            db.session.add(document)
            db.session.commit()
708
709
            target_filename = "compiled-{}-{}.pdf".format(
                document.id, "internal" if show_private else "public")
Robin Sonnabend's avatar
Robin Sonnabend committed
710
            if use_decision:
711
712
                target_filename = "decision-{}-{}-{}.pdf".format(
                    protocol.id, decision.id, document.id)
713
            document.filename = target_filename
714
715
716
            shutil.copy(
                os.path.join(compile_dir, protocol_target_filename),
                os.path.join(config.DOCUMENTS_PATH, target_filename))
717
            db.session.commit()
718
719
720
            shutil.copy(
                os.path.join(compile_dir, log_filename),
                "/tmp/proto-tex.log")
721
722
723
        except subprocess.SubprocessError:
            log = ""
            total_log_filename = os.path.join(compile_dir, log_filename)
724
725
            total_source_filename = os.path.join(
                compile_dir, protocol_source_filename)
Robin Sonnabend's avatar
Robin Sonnabend committed
726
            log = ""
727
728
            if os.path.isfile(total_source_filename):
                with open(total_source_filename, "r") as source_file:
Robin Sonnabend's avatar
Robin Sonnabend committed
729
                    log += "Source:\n\n" + add_line_numbers(source_file.read())
730
731
            total_class_filename = os.path.join(
                compile_dir, protocol_class_filename)
732
733
            if os.path.isfile(total_class_filename):
                with open(total_class_filename, "r") as class_file:
734
735
                    log += "\n\nClass:\n\n" + add_line_numbers(
                        class_file.read())
Robin Sonnabend's avatar
Robin Sonnabend committed
736
737
738
739
740
            if os.path.isfile(total_log_filename):
                with open(total_log_filename, "r") as log_file:
                    log += "\n\nLog:\n\n" + add_line_numbers(log_file.read())
            else:
                log += "\n\nLogfile not found."
741
            _make_error(protocol, "Compiling", "Compiling LaTeX failed", log)
742
743
        finally:
            os.chdir(current)
744

745

Robin Sonnabend's avatar
Robin Sonnabend committed
746
747
748
749
def print_file(filename, protocol):
    if config.PRINTING_ACTIVE:
        print_file_async.delay(filename, protocol.id)

750

Robin Sonnabend's avatar
Robin Sonnabend committed
751
752
753
754
755
@celery.task
def print_file_async(filename, protocol_id):
    with app.app_context():
        protocol = Protocol.query.filter_by(id=protocol_id).first()
        if protocol.protocoltype.printer is None:
756
757
758
759
760
761
            return _make_error(
                protocol, "Printing", "No printer configured.",
                "You don't have any printer configured for the "
                "protocoltype {}. "
                "Please do so before printing a protocol.".format(
                    protocol.protocoltype.name))
Robin Sonnabend's avatar
Robin Sonnabend committed
762
763
764
765
766
767
768
769
        try:
            command = [
                "/usr/bin/lpr",
                "-H", config.PRINTING_SERVER,
                "-P", protocol.protocoltype.printer,
                "-U", config.PRINTING_USER,
                "-T", protocol.get_identifier(),
            ]
770
771
772
773
774
            for option in config.PRINTING_PRINTERS[
                    protocol.protocoltype.printer]:
                command.extend([
                    "-o", '"{}"'.format(option)
                    if " " in option else option])
Robin Sonnabend's avatar
Robin Sonnabend committed
775
            command.append(filename)
776
777
            subprocess.check_output(
                command, universal_newlines=True, stderr=subprocess.STDOUT)
778
        except subprocess.SubprocessError as exception:
779
780
781
782
            return _make_error(
                protocol, "Printing", "Printing {} failed.".format(
                    protocol.get_identifier()), exception.stdout)

Robin Sonnabend's avatar
Robin Sonnabend committed
783

Robin Sonnabend's avatar
Robin Sonnabend committed
784
785
def send_reminder(reminder, protocol):
    send_reminder_async.delay(reminder.id, protocol.id)
786

787

788
@celery.task
Robin Sonnabend's avatar
Robin Sonnabend committed
789
def send_reminder_async(reminder_id, protocol_id):
790
    with app.app_context():
Robin Sonnabend's avatar
Robin Sonnabend committed
791
792
        reminder = MeetingReminder.query.filter_by(id=reminder_id).first()
        protocol = Protocol.query.filter_by(id=protocol_id).first()
793
794
        reminder_text = render_template(
            "reminder-mail.txt", reminder=reminder, protocol=protocol)
Robin Sonnabend's avatar
Robin Sonnabend committed
795
        if reminder.send_public:
796
797
            send_mail(
                protocol, protocol.protocoltype.public_mail,
798
799
                "Tagesordnung der {}".format(protocol.protocoltype.name),
                reminder_text, reply_to=protocol.protocoltype.public_mail)
Robin Sonnabend's avatar
Robin Sonnabend committed
800
        if reminder.send_private:
801
802
            send_mail(
                protocol, protocol.protocoltype.private_mail,
803
804
                "Tagesordnung der {}".format(protocol.protocoltype.name),
                reminder_text, reply_to=protocol.protocoltype.private_mail)
Robin Sonnabend's avatar
Robin Sonnabend committed
805

806

807
def remind_finishing(protocol, delay_days, min_delay_days):
Robin Sonnabend's avatar
Robin Sonnabend committed
808
    remind_finishing_async.delay(protocol.id, delay_days, min_delay_days)
809

810

811
@celery.task
Robin Sonnabend's avatar
Robin Sonnabend committed
812
def remind_finishing_async(protocol_id, delay_days, min_delay_days):
813
    with app.app_context():
814
815
816
        protocol = Protocol.first_by_id(protocol_id)
        mail_text = render_template(
            "remind-finishing-mail.txt",
817
818
            protocol=protocol, delay_days=delay_days,
            min_delay_days=min_delay_days)
819
820
        send_mail(
            protocol, protocol.protocoltype.private_mail,
821
822
823
            "Unfertiges Protokoll der {}".format(protocol.protocoltype.name),
            mail_text, reply_to=protocol.protocoltype.private_mail)

824

825
def send_protocol_private(protocol):
826
    send_protocol_async.delay(protocol.id, show_private=True)
Robin Sonnabend's avatar
Robin Sonnabend committed
827
    send_todomails_async.delay(protocol.id)
828

829

830
831
832
def send_protocol_public(protocol):
    send_protocol_async.delay(protocol.id, show_private=False)

833

834
835
836
837
@celery.task
def send_protocol_async(protocol_id, show_private):
    with app.app_context():
        protocol = Protocol.query.filter_by(id=protocol_id).first()
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
        next_protocol = Protocol.query.filter_by(
            protocoltype_id=protocol.protocoltype.id).filter_by(
            done=False).filter(
            Protocol.date > datetime.now()).order_by(Protocol.date).first()
        to_addr = (
            protocol.protocoltype.private_mail
            if show_private
            else protocol.protocoltype.public_mail)
        subject = "{}{}-Protokoll vom {}".format(
            "Internes " if show_private else "",
            protocol.protocoltype.short_name, date_filter(protocol.date))
        mail_content = render_template(
            "protocol-mail.txt", protocol=protocol, show_private=show_private,
            next_protocol=next_protocol)
        appendix = [
            (document.name, document.as_file_like())
854
855
856
857
858
            for document in protocol.documents
            if show_private or not document.is_private
        ]
        send_mail(protocol, to_addr, subject, mail_content, appendix)

859

Robin Sonnabend's avatar
Robin Sonnabend committed
860
861
862
863
@celery.task
def send_todomails_async(protocol_id):
    with app.app_context():
        protocol = Protocol.query.filter_by(id=protocol_id).first()
864
        all_todos = [
Administrator's avatar
Administrator committed
865
            todo for todo in Todo.query.all()
866
            if not todo.is_done()
Administrator's avatar
Administrator committed
867
            and todo.protocoltype == protocol.protocoltype
868
        ]
Robin Sonnabend's avatar
Robin Sonnabend committed
869
870
871
872
873
874
        users = {user for todo in all_todos for user in todo.get_users()}
        grouped_todos = {
            user: [todo for todo in all_todos if user in todo.get_users()]
            for user in users
        }
        subject = "Du hast noch was zu tun!"
875
876
        todomail_providers = getattr(
            config, "ADDITIONAL_TODOMAIL_PROVIDERS", None)
877
878
879
880
881
882
883
884
        additional_todomails = {}
        if todomail_providers:
            for provider in todomail_providers:
                todomail_dict = provider()
                for key in todomail_dict:
                    if key not in additional_todomails:
                        name, mail = todomail_dict[key]
                        additional_todomails[key] = TodoMail(name, mail)
Robin Sonnabend's avatar
Robin Sonnabend committed
885
886
        for user in users:
            todomail = TodoMail.query.filter(TodoMail.name.ilike(user)).first()
887
888
889
            if todomail is None:
                if user in additional_todomails:
                    todomail = additional_todomails[user]
Robin Sonnabend's avatar
Robin Sonnabend committed
890
            if todomail is None:
891
892
893
                _make_error(
                    protocol, "Sending Todomail", "Sending Todomail failed.",
                    "User {} has no Todo-Mail-Assignment.".format(user))
Robin Sonnabend's avatar
Robin Sonnabend committed
894
895
                continue
            to_addr = todomail.get_formatted_mail()
896
897
898
899
900
            mail_content = render_template(
                "todo-mail.txt", protocol=protocol, todomail=todomail,
                todos=grouped_todos[user])
            send_mail(
                protocol, to_addr, subject, mail_content,
901
                reply_to=protocol.protocoltype.private_mail)
902

903
904
905

def send_mail(protocol, to_addr, subject, content, appendix=None,
              reply_to=None):
Robin Sonnabend's avatar
Robin Sonnabend committed
906
    if to_addr is not None and len(to_addr.strip()) > 0:
907
908
909
        send_mail_async.delay(
            protocol.id, to_addr, subject, content, appendix, reply_to)

910

Robin Sonnabend's avatar
Robin Sonnabend committed
911
@celery.task
912
913
def send_mail_async(protocol_id, to_addr, subject, content, appendix,
                    reply_to):
Robin Sonnabend's avatar
Robin Sonnabend committed
914
915
916
    with app.app_context():
        protocol = Protocol.query.filter_by(id=protocol_id).first()
        try:
917
            mail_manager.send(to_addr, subject, content, appendix, reply_to)
Robin Sonnabend's avatar
Robin Sonnabend committed
918
        except Exception as exc:
919
920
921
            return _make_error(
                protocol, "Sending Mail", "Sending mail failed", str(exc))

922

Robin Sonnabend's avatar
Robin Sonnabend committed
923
924
925
def push_tops_to_calendar(protocol):
    push_tops_to_calendar_async.delay(protocol.id)

926

Robin Sonnabend's avatar
Robin Sonnabend committed
927
928
929
930
931
932
933
934
935
936
937
@celery.task
def push_tops_to_calendar_async(protocol_id):
    if not config.CALENDAR_ACTIVE:
        return
    with app.app_context():
        protocol = Protocol.query.filter_by(id=protocol_id).first()
        if protocol.protocoltype.calendar == "":
            return
        description = render_template("calendar-tops.txt", protocol=protocol)
        try:
            client = CalendarClient(protocol.protocoltype.calendar)
938
939
            client.set_event_at(
                begin=protocol.get_datetime(),
Robin Sonnabend's avatar
Robin Sonnabend committed
940
941
                name=protocol.protocoltype.short_name, description=description)
        except CalendarException as exc:
942
943
            return _make_error(
                protocol, "Calendar",
Robin Sonnabend's avatar
Robin Sonnabend committed
944
                "Pushing TOPs to Calendar failed", str(exc))
945

946
947

def set_etherpad_content(protocol):
Robin Sonnabend's avatar
Robin Sonnabend committed
948
    set_etherpad_content_async.delay(protocol.id)
949

950

951
952
953
954
@celery.task
def set_etherpad_content_async(protocol_id):
    with app.app_context():
        protocol = Protocol.query.filter_by(id=protocol_id).first()
Administrator's avatar
Administrator committed
955
956
        identifier = protocol.get_identifier()
        return set_etherpad_text(identifier, protocol.get_template())