tasks.py 39 KB
Newer Older
1
2
3
4
5
from flask import render_template

import os
import subprocess
import shutil
6
import tempfile
7
from datetime import datetime
Robin Sonnabend's avatar
Robin Sonnabend committed
8
import traceback
9
from copy import copy
10
import xmlrpc.client
11

12
13
14
from models.database import (
    Document, Protocol, Todo, Decision, TOP, MeetingReminder,
    TodoMail, DecisionDocument, TodoState, OldTodo, DecisionCategory)
Robin Sonnabend's avatar
Robin Sonnabend committed
15
from models.errors import DateNotMatchingException
16
from server import celery, app
17
18
19
20
21
22
23
from shared import (
    db, escape_tex, unhyphen, date_filter, datetime_filter, date_filter_long,
    date_filter_short, time_filter, class_filter, KNOWN_KEYS, WikiType)
from utils import (
    mail_manager, add_line_numbers,
    set_etherpad_text, parse_datetime_from_string)
from protoparser import parse, ParserException, Tag, Remark, Fork, RenderType
Robin Sonnabend's avatar
Robin Sonnabend committed
24
from wiki import WikiClient, WikiException
Robin Sonnabend's avatar
Robin Sonnabend committed
25
from calendarpush import Client as CalendarClient, CalendarException
26
from legacy import lookup_todo_id
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

import config

texenv = app.create_jinja_environment()
texenv.block_start_string = r"\ENV{"
texenv.block_end_string = r"}"
texenv.variable_start_string = r"\VAR{"
texenv.variable_end_string = r"}"
texenv.comment_start_string = r"\COMMENT{"
texenv.comment_end_string = r"}"
texenv.filters["escape_tex"] = escape_tex
texenv.filters["unhyphen"] = unhyphen
texenv.trim_blocks = True
texenv.lstrip_blocks = True
texenv.filters["datify"] = date_filter
texenv.filters["datify_long"] = date_filter_long
43
texenv.filters["datify_short"] = date_filter_short
44
45
texenv.filters["datetimify"] = datetime_filter
texenv.filters["timify"] = time_filter
46
texenv.filters["class"] = class_filter
47
48
49
logo_template = getattr(config, "LATEX_LOGO_TEMPLATE", None)
if logo_template is not None:
    texenv.globals["logo_template"] = logo_template
50
51
52
latex_geometry = getattr(
    config, "LATEX_GEOMETRY",
    "vmargin=1.5cm,hmargin={1.5cm,1.2cm},bindingoffset=8mm")
53
54
55
56
57
58
59
60
61
62
texenv.globals["latex_geometry"] = latex_geometry
raw_additional_packages = getattr(config, "LATEX_ADDITIONAL_PACKAGES", None)
additional_packages = []
if raw_additional_packages is not None:
    for package in raw_additional_packages:
        if "{" not in package:
            package = "{{{}}}".format(package)
        additional_packages.append(package)
texenv.globals["additional_packages"] = additional_packages
latex_pagestyle = getattr(config, "LATEX_PAGESTYLE", None)
63
if latex_pagestyle is not None and latex_pagestyle:
64
65
66
    texenv.globals["latex_pagestyle"] = latex_pagestyle
latex_header_footer = getattr(config, "LATEX_HEADER_FOOTER", False)
texenv.globals["latex_header_footer"] = latex_header_footer
67
68
latex_templates = getattr(config, "LATEX_TEMPLATES", None)

69

70
def provide_latex_template(template, documenttype):
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
    _DOCUMENTTYPE_FILENAME_MAP = {
        "class": "protokoll2.cls",
        "protocol": "protocol.tex",
        "decision": "decision.tex"
    }
    _PROVIDES = "provides"
    _LOGO_TEMPLATE = "logo_template"
    _LOGO = "logo"
    _LATEX_GEOMETRY = "latex_geometry"
    _GEOMETRY = "geometry"
    _ADDITIONAL_PACKAGES = "additional_packages"
    _LATEX_PAGESTYLE = "latex_pagestyle"
    _PAGESTYLE = "pagestyle"
    _LATEX_HEADER_FOOTER = "latex_header_footer"
    _HEADER_FOOTER = "headerfooter"
    _latex_template_filename = _DOCUMENTTYPE_FILENAME_MAP[documenttype]
    _latex_template_foldername = ""
    if logo_template is not None:
        texenv.globals[_LOGO_TEMPLATE] = logo_template
    texenv.globals[_LATEX_GEOMETRY] = latex_geometry
    texenv.globals[_ADDITIONAL_PACKAGES] = additional_packages
    if latex_pagestyle:
        texenv.globals[_LATEX_PAGESTYLE] = latex_pagestyle
    elif _LATEX_PAGESTYLE in texenv.globals:
        del texenv.globals[_LATEX_PAGESTYLE]
    texenv.globals[_LATEX_HEADER_FOOTER] = latex_header_footer
    if latex_templates is not None and template != "":
        if template in latex_templates:
            template_data = latex_templates[template]
            if _PROVIDES in template_data:
                if documenttype in template_data[_PROVIDES]:
                    _latex_template_foldername = template
            if _LOGO in template_data:
                texenv.globals[_LOGO_TEMPLATE] = os.path.join(
                    template, template_data[_LOGO])
            if _GEOMETRY in template_data:
                texenv.globals[_LATEX_GEOMETRY] = template_data[_GEOMETRY]
            if _PAGESTYLE in template_data:
                if template_data[_PAGESTYLE]:
                    texenv.globals[_LATEX_PAGESTYLE] = (
                        template_data[_PAGESTYLE])
            if _ADDITIONAL_PACKAGES in template_data:
                _raw_additional_packages = template_data[_ADDITIONAL_PACKAGES]
                _additional_packages = []
                if _raw_additional_packages is not None:
                    for _package in _raw_additional_packages:
                        if "{" not in _package:
                            _package = "{{{}}}".format(_package)
                        _additional_packages.append(_package)
                texenv.globals[_ADDITIONAL_PACKAGES] = _additional_packages
            if _HEADER_FOOTER in latex_templates[template]:
                texenv.globals[_LATEX_HEADER_FOOTER] = (
                    template_data[_HEADER_FOOTER])
    return os.path.join(_latex_template_foldername, _latex_template_filename)

126
127
128
129
130
131
132

mailenv = app.create_jinja_environment()
mailenv.trim_blocks = True
mailenv.lstrip_blocks = True
mailenv.filters["datify"] = date_filter
mailenv.filters["datetimify"] = datetime_filter

133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
wikienv = app.create_jinja_environment()
wikienv.trim_blocks = True
wikienv.lstrip_blocks = True
wikienv.block_start_string = "<env>"
wikienv.block_end_string = "</env>"
wikienv.variable_start_string = "<var>"
wikienv.variable_end_string = "</var>"
wikienv.comment_start_string = "<comment>"
wikienv.comment_end_string = "</comment>"
wikienv.filters["datify"] = date_filter
wikienv.filters["datify_long"] = date_filter_long
wikienv.filters["datify_short"] = date_filter_short
wikienv.filters["datetimify"] = datetime_filter
wikienv.filters["timify"] = time_filter
wikienv.filters["class"] = class_filter


150
151
152
153
154
155
def _make_error(protocol, *args):
    error = protocol.create_error(*args)
    db.session.add(error)
    db.session.commit()


156
157
ID_FIELD_BEGINNING = "id "

158
159
160
161

def parse_protocol(protocol):
    parse_protocol_async.delay(protocol.id)

162
163

@celery.task
164
def parse_protocol_async(protocol_id):
165
166
167
    with app.app_context():
        with app.test_request_context("/"):
            try:
168
                protocol = Protocol.first_by_id(protocol_id)
169
170
                if protocol is None:
                    raise Exception("No protocol given. Aborting parsing.")
171
                parse_protocol_async_inner(protocol)
172
            except Exception as exc:
Robin Sonnabend's avatar
Robin Sonnabend committed
173
                stacktrace = traceback.format_exc()
174
175
                return _make_error(
                    protocol, "Parsing", "Exception",
Robin Sonnabend's avatar
Robin Sonnabend committed
176
                    "{}\n\n{}".format(str(exc), stacktrace))
177

178
179

def parse_protocol_async_inner(protocol):
180
181
182
183
    old_errors = list(protocol.errors)
    for error in old_errors:
        protocol.errors.remove(error)
    db.session.commit()
184
    if protocol.source is None or len(protocol.source.strip()) == 0:
185
        return _make_error(protocol, "Parsing", "Protocol source is empty", "")
186
    if protocol.source == config.EMPTY_ETHERPAD:
187
188
189
        return _make_error(
            protocol, "Parsing", "The etherpad is unmodified and does not "
            "contain a protocol.", protocol.source)
190
191
192
193
194
195
196
197
    tree = None
    try:
        tree = parse(protocol.source)
    except ParserException as exc:
        context = ""
        if exc.linenumber is not None:
            source_lines = protocol.source.splitlines()
            start_index = max(0, exc.linenumber - config.ERROR_CONTEXT_LINES)
198
199
200
            end_index = min(
                len(source_lines) - 1,
                exc.linenumber + config.ERROR_CONTEXT_LINES)
201
202
203
            context = "\n".join(source_lines[start_index:end_index])
        if exc.tree is not None:
            context += "\n\nParsed syntax tree was:\n" + str(exc.tree.dump())
204
205
206
207
208
209
        return _make_error(protocol, "Parsing", str(exc), context)
    remarks = {
        element.name: element
        for element in tree.children
        if isinstance(element, Remark)
    }
210
    required_fields = copy(KNOWN_KEYS)
Robin Sonnabend's avatar
Robin Sonnabend committed
211
212
    for default_meta in protocol.protocoltype.metas:
        required_fields.append(default_meta.key)
213
    if not config.PARSER_LAZY:
214
215
216
217
218
        missing_fields = [
            field
            for field in required_fields
            if field not in remarks
        ]
219
        if len(missing_fields) > 0:
220
221
222
            return _make_error(
                protocol, "Parsing", "Du hast vergessen, Metadaten anzugeben.",
                ", ".join(missing_fields))
223
224
225
    try:
        protocol.fill_from_remarks(remarks)
    except ValueError:
226
227
        return _make_error(
            protocol, "Parsing", "Invalid fields",
228
229
            "Date or time fields are not '%d.%m.%Y' respectively '%H:%M', "
            "but rather {}".format(
230
231
232
233
234
                ", ".join([
                    remarks["Datum"].value.strip(),
                    remarks["Beginn"].value.strip(),
                    remarks["Ende"].value.strip()
                ])))
235
    except DateNotMatchingException as exc:
236
237
238
239
240
241
242
243
244
245
246
        return _make_error(
            protocol, "Parsing", "Date not matching",
            "This protocol's date should be {}, but the protocol source "
            "says {}.".format(
                date_filter(exc.original_date)
                if exc.original_date is not None
                else "not present",
                date_filter(exc.protocol_date)
                if exc.protocol_date is not None
                else "not present"))
    # tags
247
    tags = tree.get_tags()
Robin Sonnabend's avatar
Robin Sonnabend committed
248
    public_elements = tree.get_visible_elements(show_private=False)
249
250
    for tag in tags:
        if tag.name not in Tag.KNOWN_TAGS:
251
252
            return _make_error(
                protocol, "Parsing", "Invalid tag",
253
254
255
                "The tag in line {} has the kind '{}', which is "
                "not defined. This is probably an error mit a missing "
                "semicolon.".format(tag.linenumber, tag.name))
256
    # todos
257
258
259
    old_todo_number_map = {}
    for todo in protocol.todos:
        old_todo_number_map[todo.description] = todo.get_id()
260
261
262
263
    protocol.delete_orphan_todos()
    db.session.commit()
    old_todos = list(protocol.todos)
    todo_tags = [tag for tag in tags if tag.name == "todo"]
264
    raw_todos = []
265
266
    for todo_tag in todo_tags:
        if len(todo_tag.values) < 2:
267
268
            return _make_error(
                protocol, "Parsing", "Invalid todo-tag",
269
270
                "The todo tag in line {} needs at least "
                "information on who and what, "
271
272
                "but has less than that. This is probably "
                "a missing semicolon.".format(todo_tag.linenumber))
273
274
275
276
277
278
279
280
281
282
283
284
285
        who = todo_tag.values[0]
        what = todo_tag.values[1]
        field_id = None
        field_state = None
        field_date = None
        for other_field in todo_tag.values[2:]:
            other_field = other_field.strip()
            if len(other_field) == 0:
                continue
            if other_field.startswith(ID_FIELD_BEGINNING):
                try:
                    field_id = int(other_field[len(ID_FIELD_BEGINNING):])
                except ValueError:
286
287
288
289
290
                    return _make_error(
                        protocol, "Parsing", "Non-numerical todo ID",
                        "The todo in line {} has a nonnumerical ID, but needs "
                        "something like \"id 1234\"".format(
                            todo_tag.linenumber))
291
292
293
            else:
                try:
                    field_state = TodoState.from_name(other_field)
294
295
296
297
298
299
                    continue
                except ValueError:
                    pass
                try:
                    field_date = datetime.strptime(other_field, "%d.%m.%Y")
                    continue
300
                except ValueError:
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
                    pass
                try:
                    field_state, field_date = TodoState.from_name_with_date(
                        other_field.strip(), protocol=protocol)
                    continue
                except ValueError:
                    pass
                try:
                    field_state = TodoState.from_name_lazy(other_field)
                except ValueError:
                    return _make_error(
                        protocol, "Parsing", "Invalid field",
                        "The todo in line {} has the field '{}', but "
                        "this does neither match a date (\"%d.%m.%Y\") "
                        "nor a state.".format(
                            todo_tag.linenumber, other_field))
        raw_todos.append(
            (who, what, field_id, field_state, field_date, todo_tag))
319
320
    for (_, _, field_id, _, _, _) in raw_todos:
        if field_id is not None:
321
322
323
324
            old_todos = [
                todo for todo in old_todos
                if todo.id != field_id
            ]
325
326
327
328
    for todo in old_todos:
        protocol.todos.remove(todo)
    db.session.commit()
    for (who, what, field_id, field_state, field_date, todo_tag) in raw_todos:
329
330
331
        if field_state is None:
            field_state = TodoState.open
        if field_state.needs_date() and field_date is None:
332
333
            return _make_error(
                protocol, "Parsing", "Todo missing date",
334
335
336
337
338
339
340
341
                "The todo in line {} has a state that needs a date, "
                "but the todo does not have one.".format(todo_tag.linenumber))
        who = who.strip()
        what = what.strip()
        todo = None
        if field_id is not None:
            todo = Todo.query.filter_by(number=field_id).first()
            if todo is None and not config.PARSER_LAZY:
342
343
344
345
                return _make_error(
                    protocol, "Parsing", "Invalid Todo ID",
                    "The todo in line {} has the ID {}, but there is no "
                    "Todo with that ID.".format(todo_tag.linenumber, field_id))
346
        if todo is None and field_id is None and what in old_todo_number_map:
347
348
            todo = Todo(
                protocoltype_id=protocol.protocoltype.id,
349
350
351
352
                who=who, description=what, state=field_state,
                date=field_date, number=old_todo_number_map[what])
            db.session.add(todo)
            db.session.commit()
353
354
355
356
357
358
        if todo is None:
            protocol_key = protocol.get_identifier()
            old_candidates = OldTodo.query.filter(
                OldTodo.protocol_key == protocol_key).all()
            if len(old_candidates) == 0:
                # new protocol
359
360
                todo = Todo(
                    protocoltype_id=protocol.protocoltype.id,
361
362
363
364
365
366
367
368
369
370
371
                    who=who, description=what, state=field_state,
                    date=field_date)
                db.session.add(todo)
                db.session.commit()
                todo.number = field_id or todo.id
                db.session.commit()
            else:
                # old protocol
                number = field_id or lookup_todo_id(old_candidates, who, what)
                todo = Todo.query.filter_by(number=number).first()
                if todo is None:
372
373
                    todo = Todo(
                        protocoltype_id=protocol.protocoltype.id,
374
375
376
377
378
                        who=who, description=what, state=field_state,
                        date=field_date, number=number)
                    db.session.add(todo)
                    db.session.commit()
        todo.protocols.append(protocol)
379
380
381
382
383
384
385
386
387
388
        is_newest_protocol = True
        for other_protocol in todo.protocols:
            if other_protocol.date > protocol.date:
                is_newest_protocol = False
                break
        if is_newest_protocol:
            todo.state = field_state
            todo.date = field_date
            todo.who = who
            todo.description = what
389
390
391
        db.session.commit()
        todo_tag.todo = todo
    # Decisions
Robin Sonnabend's avatar
Robin Sonnabend committed
392
393
394
    decision_tags = [tag for tag in tags if tag.name == "beschluss"]
    for decision_tag in decision_tags:
        if decision_tag not in public_elements:
395
396
397
398
399
400
            return _make_error(
                protocol, "Parsing", "Decision in private context.",
                "The decision in line {} is in a private context, but "
                "decisions are and have to be public. "
                "Please move it to a public spot.".format(
                    decision_tag.linenumber))
401
402
403
404
    old_decisions = list(protocol.decisions)
    for decision in old_decisions:
        protocol.decisions.remove(decision)
    db.session.commit()
405
    decisions_to_render = []
406
407
    for decision_tag in decision_tags:
        if len(decision_tag.values) == 0:
408
409
410
411
            return _make_error(
                protocol, "Parsing", "Empty decision found.",
                "The decision in line {} is empty.".format(
                    decision_tag.linenumber))
412
        decision_content = decision_tag.values[0]
413
414
        decision_categories = []
        for decision_category_name in decision_tag.values[1:]:
415
416
417
            decision_category = DecisionCategory.query.filter_by(
                protocoltype_id=protocol.protocoltype.id,
                name=decision_category_name).first()
418
            if decision_category is None:
419
420
                category_candidates = DecisionCategory.query.filter_by(
                    protocoltype_id=protocol.protocoltype.id).all()
421
422
423
424
                category_names = [
                    "'{}'".format(category.name)
                    for category in category_candidates
                ]
425
426
                return _make_error(
                    protocol, "Parsing", "Unknown decision category",
427
428
429
430
431
432
433
                    "The decision in line {} has the category {}, "
                    "but there is no such category. "
                    "Known categories are {}".format(
                        decision_tag.linenumber,
                        decision_category_name,
                        ", ".join(category_names)))
            else:
434
                decision_categories.append(decision_category)
435
436
        decision = Decision(
            protocol_id=protocol.id, content=decision_content)
437
438
        db.session.add(decision)
        db.session.commit()
439
440
        for decision_category in decision_categories:
            decision.categories.append(decision_category)
441
442
443
        decision_tag.decision = decision
        decisions_to_render.append((decision, decision_tag))
    for decision, decision_tag in decisions_to_render:
444
        decision_top = decision_tag.fork.get_top()
445
446
447
448
        decision_content = texenv.get_template(provide_latex_template(
            protocol.protocoltype.latex_template, "decision")).render(
                render_type=RenderType.latex, decision=decision,
                protocol=protocol, top=decision_top, show_private=True)
449
450
        maxdepth = decision_top.get_maxdepth()
        compile_decision(decision_content, decision, maxdepth=maxdepth)
Robin Sonnabend's avatar
Robin Sonnabend committed
451
452

    # Footnotes
453
454
455
456
457
458
459
460
    footnote_tags = [
        tag for tag in tags
        if tag.name == "footnote"
    ]
    public_footnote_tags = [
        tag for tag in footnote_tags
        if tag in public_elements
    ]
Robin Sonnabend's avatar
Robin Sonnabend committed
461

462
463
464
465
    # new Protocols
    protocol_tags = [tag for tag in tags if tag.name == "sitzung"]
    for protocol_tag in protocol_tags:
        if len(protocol_tag.values) not in {1, 2}:
466
467
            return _make_error(
                protocol, "Parsing", "Falsche Verwendung von [sitzung;…].",
468
469
470
                "Der Tag \"sitzung\" benötigt immer ein Datum "
                "und optional eine Uhrzeit, also ein bis zwei Argumente. "
                "Stattdessen wurden {} übergeben, nämlich {}".format(
471
472
                    len(protocol_tag.values),
                    protocol_tag.values))
473
474
        else:
            try:
475
                parse_datetime_from_string(protocol_tag.values[0])
476
            except ValueError as exc:
477
478
                return _make_error(
                    protocol, "Parsing", "Invalides Datum",
479
480
481
482
                    "'{}' ist kein valides Datum.".format(
                        protocol_tag.values[0]))
            if len(protocol_tag.values) > 1:
                try:
483
                    datetime.strptime(protocol_tag.values[1], "%H:%M")
484
                except ValueError:
485
486
                    return _make_error(
                        protocol, "Parsing", "Invalide Uhrzeit",
487
488
489
490
491
492
                        "'{}' ist keine valide Uhrzeit.".format(
                            protocol_tag.values[1]))
    for protocol_tag in protocol_tags:
        new_protocol_date = parse_datetime_from_string(protocol_tag.values[0])
        new_protocol_time = None
        if len(protocol_tag.values) > 1:
493
494
            new_protocol_time = datetime.strptime(
                protocol_tag.values[1], "%H:%M")
495
496
497
        if not protocol.protocoltype.get_protocols_on_date(new_protocol_date):
            Protocol.create_new_protocol(
                protocol.protocoltype, new_protocol_date, new_protocol_time)
498

Robin Sonnabend's avatar
Robin Sonnabend committed
499
    # TOPs
500
501
502
    old_tops = list(protocol.tops)
    for top in old_tops:
        protocol.tops.remove(top)
503
504
505
506
    for index, fork in enumerate(
            (child for child in tree.children if isinstance(child, Fork))):
        top = TOP(
            protocol_id=protocol.id, name=fork.name, number=index,
507
            planned=False)
508
509
        db.session.add(top)
    db.session.commit()
510

Robin Sonnabend's avatar
Robin Sonnabend committed
511
512
    # render
    private_render_kwargs = {
513
        "protocol": protocol,
Robin Sonnabend's avatar
Robin Sonnabend committed
514
515
        "tree": tree,
        "footnotes": footnote_tags,
516
    }
Robin Sonnabend's avatar
Robin Sonnabend committed
517
518
519
    public_render_kwargs = copy(private_render_kwargs)
    public_render_kwargs["footnotes"] = public_footnote_tags
    render_kwargs = {True: private_render_kwargs, False: public_render_kwargs}
520

521
522
    maxdepth = tree.get_maxdepth()
    privacy_states = [False]
523
524
525
526
527
528
    content_private = render_template(
        "protocol.txt", render_type=RenderType.plaintext, show_private=True,
        **private_render_kwargs)
    content_public = render_template(
        "protocol.txt", render_type=RenderType.plaintext, show_private=False,
        **public_render_kwargs)
529
530
531
532
    if content_private != content_public:
        privacy_states.append(True)
    protocol.content_private = content_private
    protocol.content_public = content_public
533
534
535
536
537
538
    protocol.content_html_private = render_template(
        "protocol.html", render_type=RenderType.html, show_private=True,
        **private_render_kwargs)
    protocol.content_html_public = render_template(
        "protocol.html", render_type=RenderType.html, show_private=False,
        **public_render_kwargs)
539

540
    for show_private in privacy_states:
541
542
543
544
545
546
547
548
        latex_source = texenv.get_template(provide_latex_template(
            protocol.protocoltype.latex_template, "protocol")).render(
                render_type=RenderType.latex,
                show_private=show_private,
                **render_kwargs[show_private])
        compile(
            latex_source, protocol, show_private=show_private,
            maxdepth=maxdepth)
Robin Sonnabend's avatar
Robin Sonnabend committed
549

550
    if protocol.protocoltype.use_wiki:
551
552
553
554
555
556
557
558
559
        wiki_type = WikiType[getattr(config, "WIKI_TYPE", "MEDIAWIKI")]
        wiki_template = {
            WikiType.MEDIAWIKI: "protocol.wiki",
            WikiType.DOKUWIKI: "protocol.dokuwiki",
        }
        wiki_render_type = {
            WikiType.MEDIAWIKI: RenderType.wikitext,
            WikiType.DOKUWIKI: RenderType.dokuwiki,
        }
Robin Sonnabend's avatar
Robin Sonnabend committed
560
        show_private = not protocol.protocoltype.wiki_only_public
561
562
        wiki_source = wikienv.get_template(wiki_template[wiki_type]).render(
            render_type=wiki_render_type[wiki_type],
Robin Sonnabend's avatar
Robin Sonnabend committed
563
564
565
            show_private=show_private,
            **render_kwargs[show_private]
        ).replace("\n\n\n", "\n\n")
566
567
568
        if wiki_type == WikiType.MEDIAWIKI:
            wiki_infobox_source = wikienv.get_template("infobox.wiki").render(
                protocoltype=protocol.protocoltype)
569
570
            push_to_wiki(
                protocol, wiki_source, wiki_infobox_source,
571
572
                "Automatisch generiert vom Protokollsystem 3.0")
        elif wiki_type == WikiType.DOKUWIKI:
573
574
            push_to_dokuwiki(
                protocol, wiki_source,
575
                "Automatisch generiert vom Protokollsystem 3.0")
576
577
    protocol.done = True
    db.session.commit()
578

579

580
581
def push_to_wiki(protocol, content, infobox_content, summary):
    push_to_wiki_async.delay(protocol.id, content, infobox_content, summary)
Robin Sonnabend's avatar
Robin Sonnabend committed
582

583

Robin Sonnabend's avatar
Robin Sonnabend committed
584
@celery.task
585
def push_to_wiki_async(protocol_id, content, infobox_content, summary):
586
    with app.app_context():
Robin Sonnabend's avatar
Robin Sonnabend committed
587
588
        protocol = Protocol.query.filter_by(id=protocol_id).first()
        try:
589
590
591
592
593
594
595
596
597
            with WikiClient() as wiki_client:
                wiki_client.edit_page(
                    title=protocol.protocoltype.get_wiki_infobox_title(),
                    content=infobox_content,
                    summary=summary)
                wiki_client.edit_page(
                    title=protocol.get_wiki_title(),
                    content=content,
                    summary=summary)
Robin Sonnabend's avatar
Robin Sonnabend committed
598
        except WikiException as exc:
599
600
601
602
            return _make_error(
                protocol, "Pushing to Wiki", "Pushing to Wiki failed.",
                str(exc))

603
604
605
606

def push_to_dokuwiki(protocol, content, summary):
    push_to_dokuwiki_async.delay(protocol.id, content, summary)

607

Robin Sonnabend's avatar
Robin Sonnabend committed
608
@celery.task
609
def push_to_dokuwiki_async(protocol_id, content, summary):
Robin Sonnabend's avatar
Robin Sonnabend committed
610
611
612
613
    with app.app_context():
        protocol = Protocol.query.filter_by(id=protocol_id).first()
        with xmlrpc.client.ServerProxy(config.WIKI_API_URL) as proxy:
            try:
614
615
616
617
618
619
                if not proxy.wiki.putPage(
                    protocol.get_wiki_title(), content,
                    {"sum":
                        "Automatisch generiert vom Protokollsystem 3."}):
                    return _make_error(
                        protocol, "Pushing to Wiki",
Robin Sonnabend's avatar
Robin Sonnabend committed
620
621
                        "Pushing to Wiki failed." "")
            except xmlrpc.client.Error as exception:
622
623
                return _make_error(
                    protocol, "Pushing to Wiki", "XML RPC Exception",
Robin Sonnabend's avatar
Robin Sonnabend committed
624
                    str(exception))
625

Robin Sonnabend's avatar
Robin Sonnabend committed
626

627
def compile(content, protocol, show_private, maxdepth):
628
629
630
    compile_async.delay(
        content, protocol.id, show_private=show_private, maxdepth=maxdepth)

Robin Sonnabend's avatar
Robin Sonnabend committed
631

632
def compile_decision(content, decision, maxdepth):
633
634
635
    compile_async.delay(
        content, decision.id, use_decision=True, maxdepth=maxdepth)

636
637

@celery.task
638
639
640
def compile_async(
        content, protocol_id, show_private=False, use_decision=False,
        maxdepth=5):
641
    with tempfile.TemporaryDirectory() as compile_dir, app.app_context():
Robin Sonnabend's avatar
Robin Sonnabend committed
642
643
644
645
646
647
648
        decision = None
        protocol = None
        if use_decision:
            decision = Decision.query.filter_by(id=protocol_id).first()
            protocol = decision.protocol
        else:
            protocol = Protocol.query.filter_by(id=protocol_id).first()
649
650
651
        try:
            current = os.getcwd()
            protocol_source_filename = "protocol.tex"
652
            protocol_target_filename = "protocol.pdf"
653
            protocol_class_filename = "protokoll2.cls"
654
            log_filename = "protocol.log"
655
656
657
            with open(
                    os.path.join(compile_dir, protocol_source_filename),
                    "w") as source_file:
658
                source_file.write(content)
659
660
661
662
663
664
665
666
667
            protocol2_class_source = texenv.get_template(
                provide_latex_template(
                    protocol.protocoltype.latex_template,
                    "class")).render(
                fonts=config.FONTS, maxdepth=maxdepth,
                bulletpoints=config.LATEX_BULLETPOINTS)
            with open(
                os.path.join(compile_dir, protocol_class_filename),
                    "w") as protocol2_class_file:
668
                protocol2_class_file.write(protocol2_class_source)
669
670
671
672
673
674
675
            os.chdir(compile_dir)
            command = [
                "/usr/bin/xelatex",
                "-halt-on-error",
                "-file-line-error",
                protocol_source_filename
            ]
676
677
678
679
680
681
            subprocess.check_call(
                command, universal_newlines=True, stdout=subprocess.DEVNULL,
                stderr=subprocess.DEVNULL)
            subprocess.check_call(
                command, universal_newlines=True, stdout=subprocess.DEVNULL,
                stderr=subprocess.DEVNULL)
682
            os.chdir(current)
Robin Sonnabend's avatar
Robin Sonnabend committed
683
684
            document = None
            if not use_decision:
685
686
687
688
                for old_document in [
                    document for document in protocol.documents
                    if document.is_compiled
                        and document.is_private == show_private]:
Robin Sonnabend's avatar
Robin Sonnabend committed
689
690
                    protocol.documents.remove(old_document)
                db.session.commit()
691
692
                document = Document(
                    protocol_id=protocol.id,
693
694
695
696
697
698
699
                    name="protokoll{}_{}_{}.pdf".format(
                        "_intern" if show_private else "",
                        protocol.protocoltype.short_name,
                        date_filter_short(protocol.date)),
                    filename="",
                    is_compiled=True,
                    is_private=show_private)
Robin Sonnabend's avatar
Robin Sonnabend committed
700
            else:
701
702
                document = DecisionDocument(
                    decision_id=decision.id,
703
704
705
706
707
                    name="beschluss_{}_{}_{}.pdf".format(
                        protocol.protocoltype.short_name,
                        date_filter_short(protocol.date),
                        decision.id),
                    filename="")
708
709
            db.session.add(document)
            db.session.commit()
710
711
            target_filename = "compiled-{}-{}.pdf".format(
                document.id, "internal" if show_private else "public")
Robin Sonnabend's avatar
Robin Sonnabend committed
712
            if use_decision:
713
714
                target_filename = "decision-{}-{}-{}.pdf".format(
                    protocol.id, decision.id, document.id)
715
            document.filename = target_filename
716
717
718
            shutil.copy(
                os.path.join(compile_dir, protocol_target_filename),
                os.path.join(config.DOCUMENTS_PATH, target_filename))
719
            db.session.commit()
720
721
722
            shutil.copy(
                os.path.join(compile_dir, log_filename),
                "/tmp/proto-tex.log")
723
724
725
        except subprocess.SubprocessError:
            log = ""
            total_log_filename = os.path.join(compile_dir, log_filename)
726
727
            total_source_filename = os.path.join(
                compile_dir, protocol_source_filename)
Robin Sonnabend's avatar
Robin Sonnabend committed
728
            log = ""
729
730
            if os.path.isfile(total_source_filename):
                with open(total_source_filename, "r") as source_file:
Robin Sonnabend's avatar
Robin Sonnabend committed
731
                    log += "Source:\n\n" + add_line_numbers(source_file.read())
732
733
            total_class_filename = os.path.join(
                compile_dir, protocol_class_filename)
734
735
            if os.path.isfile(total_class_filename):
                with open(total_class_filename, "r") as class_file:
736
737
                    log += "\n\nClass:\n\n" + add_line_numbers(
                        class_file.read())
Robin Sonnabend's avatar
Robin Sonnabend committed
738
739
740
741
742
            if os.path.isfile(total_log_filename):
                with open(total_log_filename, "r") as log_file:
                    log += "\n\nLog:\n\n" + add_line_numbers(log_file.read())
            else:
                log += "\n\nLogfile not found."
743
            _make_error(protocol, "Compiling", "Compiling LaTeX failed", log)
744
745
        finally:
            os.chdir(current)
746

747

Robin Sonnabend's avatar
Robin Sonnabend committed
748
749
750
751
def print_file(filename, protocol):
    if config.PRINTING_ACTIVE:
        print_file_async.delay(filename, protocol.id)

752

Robin Sonnabend's avatar
Robin Sonnabend committed
753
754
755
756
757
@celery.task
def print_file_async(filename, protocol_id):
    with app.app_context():
        protocol = Protocol.query.filter_by(id=protocol_id).first()
        if protocol.protocoltype.printer is None:
758
759
760
761
762
763
            return _make_error(
                protocol, "Printing", "No printer configured.",
                "You don't have any printer configured for the "
                "protocoltype {}. "
                "Please do so before printing a protocol.".format(
                    protocol.protocoltype.name))
Robin Sonnabend's avatar
Robin Sonnabend committed
764
765
766
767
768
769
770
771
        try:
            command = [
                "/usr/bin/lpr",
                "-H", config.PRINTING_SERVER,
                "-P", protocol.protocoltype.printer,
                "-U", config.PRINTING_USER,
                "-T", protocol.get_identifier(),
            ]
772
773
774
775
776
            for option in config.PRINTING_PRINTERS[
                    protocol.protocoltype.printer]:
                command.extend([
                    "-o", '"{}"'.format(option)
                    if " " in option else option])
Robin Sonnabend's avatar
Robin Sonnabend committed
777
            command.append(filename)
778
779
            subprocess.check_output(
                command, universal_newlines=True, stderr=subprocess.STDOUT)
780
        except subprocess.SubprocessError as exception:
781
782
783
784
            return _make_error(
                protocol, "Printing", "Printing {} failed.".format(
                    protocol.get_identifier()), exception.stdout)

Robin Sonnabend's avatar
Robin Sonnabend committed
785

Robin Sonnabend's avatar
Robin Sonnabend committed
786
787
def send_reminder(reminder, protocol):
    send_reminder_async.delay(reminder.id, protocol.id)
788

789

790
@celery.task
Robin Sonnabend's avatar
Robin Sonnabend committed
791
def send_reminder_async(reminder_id, protocol_id):
792
    with app.app_context():
Robin Sonnabend's avatar
Robin Sonnabend committed
793
794
        reminder = MeetingReminder.query.filter_by(id=reminder_id).first()
        protocol = Protocol.query.filter_by(id=protocol_id).first()
795
796
        reminder_text = render_template(
            "reminder-mail.txt", reminder=reminder, protocol=protocol)
Robin Sonnabend's avatar
Robin Sonnabend committed
797
        if reminder.send_public:
798
799
            send_mail(
                protocol, protocol.protocoltype.public_mail,
800
801
                "Tagesordnung der {}".format(protocol.protocoltype.name),
                reminder_text, reply_to=protocol.protocoltype.public_mail)
Robin Sonnabend's avatar
Robin Sonnabend committed
802
        if reminder.send_private:
803
804
            send_mail(
                protocol, protocol.protocoltype.private_mail,
805
806
                "Tagesordnung der {}".format(protocol.protocoltype.name),
                reminder_text, reply_to=protocol.protocoltype.private_mail)
Robin Sonnabend's avatar
Robin Sonnabend committed
807

808

809
def remind_finishing(protocol, delay_days, min_delay_days):
Robin Sonnabend's avatar
Robin Sonnabend committed
810
    remind_finishing_async.delay(protocol.id, delay_days, min_delay_days)
811

812

813
@celery.task
Robin Sonnabend's avatar
Robin Sonnabend committed
814
def remind_finishing_async(protocol_id, delay_days, min_delay_days):
815
    with app.app_context():
816
817
818
        protocol = Protocol.first_by_id(protocol_id)
        mail_text = render_template(
            "remind-finishing-mail.txt",
819
820
            protocol=protocol, delay_days=delay_days,
            min_delay_days=min_delay_days)
821
822
        send_mail(
            protocol, protocol.protocoltype.private_mail,
823
824
825
            "Unfertiges Protokoll der {}".format(protocol.protocoltype.name),
            mail_text, reply_to=protocol.protocoltype.private_mail)

826

827
def send_protocol_private(protocol):
828
    send_protocol_async.delay(protocol.id, show_private=True)
Robin Sonnabend's avatar
Robin Sonnabend committed
829
    send_todomails_async.delay(protocol.id)
830

831

832
833
834
def send_protocol_public(protocol):
    send_protocol_async.delay(protocol.id, show_private=False)

835

836
837
838
839
@celery.task
def send_protocol_async(protocol_id, show_private):
    with app.app_context():
        protocol = Protocol.query.filter_by(id=protocol_id).first()
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
        next_protocol = Protocol.query.filter_by(
            protocoltype_id=protocol.protocoltype.id).filter_by(
            done=False).filter(
            Protocol.date > datetime.now()).order_by(Protocol.date).first()
        to_addr = (
            protocol.protocoltype.private_mail
            if show_private
            else protocol.protocoltype.public_mail)
        subject = "{}{}-Protokoll vom {}".format(
            "Internes " if show_private else "",
            protocol.protocoltype.short_name, date_filter(protocol.date))
        mail_content = render_template(
            "protocol-mail.txt", protocol=protocol, show_private=show_private,
            next_protocol=next_protocol)
        appendix = [
            (document.name, document.as_file_like())
856
857
858
859
860
            for document in protocol.documents
            if show_private or not document.is_private
        ]
        send_mail(protocol, to_addr, subject, mail_content, appendix)

861

Robin Sonnabend's avatar
Robin Sonnabend committed
862
863
864
865
@celery.task
def send_todomails_async(protocol_id):
    with app.app_context():
        protocol = Protocol.query.filter_by(id=protocol_id).first()
866
        all_todos = [
Administrator's avatar
Administrator committed
867
            todo for todo in Todo.query.all()
868
            if not todo.is_done()
Administrator's avatar
Administrator committed
869
            and todo.protocoltype == protocol.protocoltype
870
        ]
Robin Sonnabend's avatar
Robin Sonnabend committed
871
872
873
874
875
876
        users = {user for todo in all_todos for user in todo.get_users()}
        grouped_todos = {
            user: [todo for todo in all_todos if user in todo.get_users()]
            for user in users
        }
        subject = "Du hast noch was zu tun!"
877
878
        todomail_providers = getattr(
            config, "ADDITIONAL_TODOMAIL_PROVIDERS", None)
879
880
881
882
883
884
885
886
        additional_todomails = {}
        if todomail_providers:
            for provider in todomail_providers:
                todomail_dict = provider()
                for key in todomail_dict:
                    if key not in additional_todomails:
                        name, mail = todomail_dict[key]
                        additional_todomails[key] = TodoMail(name, mail)
Robin Sonnabend's avatar
Robin Sonnabend committed
887
888
        for user in users:
            todomail = TodoMail.query.filter(TodoMail.name.ilike(user)).first()
889
890
891
            if todomail is None:
                if user in additional_todomails:
                    todomail = additional_todomails[user]
Robin Sonnabend's avatar
Robin Sonnabend committed
892
            if todomail is None:
893
894
895
                _make_error(
                    protocol, "Sending Todomail", "Sending Todomail failed.",
                    "User {} has no Todo-Mail-Assignment.".format(user))
Robin Sonnabend's avatar
Robin Sonnabend committed
896
897
                continue
            to_addr = todomail.get_formatted_mail()
898
899
900
901
902
            mail_content = render_template(
                "todo-mail.txt", protocol=protocol, todomail=todomail,
                todos=grouped_todos[user])
            send_mail(
                protocol, to_addr, subject, mail_content,
903
                reply_to=protocol.protocoltype.private_mail)
904

905
906
907

def send_mail(protocol, to_addr, subject, content, appendix=None,
              reply_to=None):
Robin Sonnabend's avatar
Robin Sonnabend committed
908
    if to_addr is not None and len(to_addr.strip()) > 0:
909
910
911
        send_mail_async.delay(
            protocol.id, to_addr, subject, content, appendix, reply_to)

912

Robin Sonnabend's avatar
Robin Sonnabend committed
913
@celery.task
914
915
def send_mail_async(protocol_id, to_addr, subject, content, appendix,
                    reply_to):
Robin Sonnabend's avatar
Robin Sonnabend committed
916
917
918
    with app.app_context():
        protocol = Protocol.query.filter_by(id=protocol_id).first()
        try:
919
            mail_manager.send(to_addr, subject, content, appendix, reply_to)
Robin Sonnabend's avatar
Robin Sonnabend committed
920
        except Exception as exc:
921
922
923
            return _make_error(
                protocol, "Sending Mail", "Sending mail failed", str(exc))

924

Robin Sonnabend's avatar
Robin Sonnabend committed
925
926
927
def push_tops_to_calendar(protocol):
    push_tops_to_calendar_async.delay(protocol.id)

928

Robin Sonnabend's avatar
Robin Sonnabend committed
929
930
931
932
933
934
935
936
937
938
939
@celery.task
def push_tops_to_calendar_async(protocol_id):
    if not config.CALENDAR_ACTIVE:
        return
    with app.app_context():
        protocol = Protocol.query.filter_by(id=protocol_id).first()
        if protocol.protocoltype.calendar == "":
            return
        description = render_template("calendar-tops.txt", protocol=protocol)
        try:
            client = CalendarClient(protocol.protocoltype.calendar)
940
941
            client.set_event_at(
                begin=protocol.get_datetime(),
Robin Sonnabend's avatar
Robin Sonnabend committed
942
943
                name=protocol.protocoltype.short_name, description=description)
        except CalendarException as exc:
944
945
            return _make_error(
                protocol, "Calendar",
Robin Sonnabend's avatar
Robin Sonnabend committed
946
                "Pushing TOPs to Calendar failed", str(exc))
947

948
949

def set_etherpad_content(protocol):
Robin Sonnabend's avatar
Robin Sonnabend committed
950
    set_etherpad_content_async.delay(protocol.id)
951

952

953
954
955
956
@celery.task
def set_etherpad_content_async(protocol_id):
    with app.app_context():
        protocol = Protocol.query.filter_by(id=protocol_id).first()
Administrator's avatar
Administrator committed
957
958
        identifier = protocol.get_identifier()
        return set_etherpad_text(identifier, protocol.get_template())