diff --git a/TODO b/TODO new file mode 100644 index 0000000000000000000000000000000000000000..1d8020cd8cfd3da65faa3b765a408eab08720893 --- /dev/null +++ b/TODO @@ -0,0 +1,19 @@ +- [x] Markdown Rendering +- [x] Markdown Template (Struktur, Child-Render, Fußnoten; siehe auch HTML) +- [x] Markdown "Compile" + Datenbank (-> Datenbank sorgt für Mailversand) +- [x] PDF global in config(proxy).py deaktivieren +- [x] Markdown compile global in config(proxy).py deaktivieren +- [x] Dokumente löschen auf file extension (pdf vs md) begrenzen +- [x] `python parse_protocol_async_inner` soll Mardkown "Compile" aufrufen +- [ ] GitLab-Wiki-Pusher +- [ ] GitLab-Wiki-Verlinker +- [ ] GitLab-Wiki Config vs Markdown Compile Config +- [ ] Markdown Rendering prüfen (siehe auch: todos in protoparser) + + +- [ ] Sitzungs-Tag: Ort für nächste Sitzung +- [ ] uwsgi Config wegen performance checken +- [ ] Todo-Versand +- [ ] Markdown-Input +- [ ] Abfrage vor Löschen von Protokollen +- [ ] Error handling: sinnvolle fehlermeldung wenn endzeit nicht angegeben diff --git a/configproxy.py b/configproxy.py index 6629da2d03986be27cedbdd1ed7597cf566185ea..19a4e1c7acb8099c9ff03965a3d37c28eb96a4ca 100755 --- a/configproxy.py +++ b/configproxy.py @@ -664,6 +664,20 @@ CONFIG_SECTIONS = [ ConfigSection( name="Rendering", entries=[ + ConfigEntry( + name="RENDERING_PDF", + default=True, + required=False, + internal=False, + description="Enable rendering to PDF via XeLaTeX", + ), + ConfigEntry( + name="RENDERING_MD", + default=False, + required=False, + internal=False, + description="Enable rendering to (GitLab-flavoured) Markdown", + ), ConfigEntry( name="FONTS", default={ diff --git a/models/database.py b/models/database.py index 6d98e438eb843b9d6e789706878938375b6e32d4..12505151b25be5fe6344f4a8d97d8f1edf6687f2 100644 --- a/models/database.py +++ b/models/database.py @@ -752,6 +752,14 @@ class Todo(DatabaseModel): ] return " ".join(parts) + def render_md(self, current_protocol=None): + parts = [ + self.get_state(), + "**{}:**".format(self.who), + self.description + ] + return " ".join(parts) + def render_latex(self, current_protocol=None): return r"\Todo{{{}}}{{{}}}{{{}}}{{{}}}".format( "Neuer Todo" if self.is_new(current_protocol) else "Todo", diff --git a/protoparser.py b/protoparser.py index c755c59f31c015d23b2d16a3d5b08d17c1396274..e3966d454739701fdf78641e07eade687c462ebe 100644 --- a/protoparser.py +++ b/protoparser.py @@ -38,6 +38,7 @@ class RenderType(Enum): plaintext = 2 html = 3 dokuwiki = 4 + markdown = 5 def _not_implemented(self, render_type): @@ -197,6 +198,8 @@ class Text: return self.text elif render_type == RenderType.dokuwiki: return self.text + elif render_type == RenderType.markdown: + return self.text else: raise _not_implemented(self, render_type) @@ -305,7 +308,7 @@ class Tag: return ( '<sup id="#fnref{0}"><a href="#fn{0}">Fn</a></sup>'.format( footnote_hash(self.values[0]))) - return "[{}: {}]".format(self.name, ";".join(self.values)) + return "<b>{}:</b> {}".format(self.name, ";".join(self.values)) elif render_type == RenderType.dokuwiki: if self.name == "url": return self.values[0] @@ -322,6 +325,28 @@ class Tag: else: return "**{}:** {}".format( self.name.capitalize(), ";".join(self.values)) + elif render_type == RenderType.markdown: + if self.name == "url": + return "[{0}]({0})".format(self.values[0]) + elif self.name == "todo": + if not show_private: + return "" + if getattr(self, "todo", None) is not None: + return self.todo.render_md(current_protocol=protocol) + else: + return "**Todo:** {}".format(";".join(self.values)) + elif self.name == "beschluss": + if getattr(self, "decision", None) is not None: + parts = ["**Beschluss:**", self.decision.content] + if len(self.decision.categories) > 0: + parts.append("*{}*".format( + self.decision.get_categories_str())) + return " ".join(parts) + else: + return "**Beschluss:** {}".format(self.values[0]) + elif self.name == "footnote": + return '[^{}]'.format(footnote_hash(self.values[0])) + return "**{}**: {}".format(self.name, ";".join(self.values)) else: raise _not_implemented(self, render_type) @@ -378,11 +403,13 @@ class Remark(Element): elif render_type == RenderType.wikitext: return "{}: {}".format(self.name, self.value) elif render_type == RenderType.plaintext: - return "{}: {}".format(RenderType.plaintex) + return "{}: {}".format(self.name, self.value) elif render_type == RenderType.html: return "<p>{}: {}</p>".format(self.name, self.value) elif render_type == RenderType.dokuwiki: return r"{}: {}\\".format(self.name, self.value) + elif render_type == RenderType.markdown: + return "**{}**: {}".format(self.name, self.value) else: raise _not_implemented(self, render_type) @@ -566,6 +593,56 @@ class Fork(Element): return "" else: return content_lines + elif render_type == RenderType.markdown: + depth = level + 1 + content_lines = "" + if depth < 2: + title_line = "{} {}".format("#" * (depth + 1), name_line) + content_parts = [] + for child in self.children: + part = child.render( + render_type, show_private, level=level + 1, + protocol=protocol, decision_render=decision_render, top_render=top_render) + if len(part.strip()) == 0: + continue + content_parts.append(part) + if self.test_private(self.name) and not show_private: + return "An dieser Stelle wurde intern protokolliert." + else: + content_lines = "{}\n{}{}{}".format( + title_line, + "*Beginn interner Abschnitt* \n" if self.test_private(self.name) else "", + "\n".join(content_parts), + "*Ende interner Abschnitt* \n" if self.test_private(self.name) else "", + ) #todo: check whether we need extra newline here + return content_lines + else: + is_private = self.test_private(self.name) + content_parts = [] + for child in self.children: + part = child.render( + render_type, show_private, level=level + 1 if not is_private else level, + protocol=protocol, decision_render=decision_render, top_render=top_render) + if len(part.strip()) == 0: + continue + if is_private: + level -= 1 + if level < 2: + content_parts.append(part) + content_parts.append("\n") + else: + content_parts.append((" " * level) + "* {}".format(part)) + + if is_private and not show_private: + return "An dieser Stelle wurde intern protokolliert." + else: + content_lines = "{}\n{}{}{}".format( + name_line if not is_private else "", + "*Beginn interner Abschnitt* \n" if is_private else "", + "\n".join(content_parts), + "*Ende interner Abschnitt* \n" if is_private else "", + ) # todo: check whether we need extra newline here + return content_lines else: raise _not_implemented(self, render_type) diff --git a/requirements-2025.txt b/requirements-2025.txt new file mode 100644 index 0000000000000000000000000000000000000000..b863dad22523951de1a1ea3ede558e9f80ba1809 --- /dev/null +++ b/requirements-2025.txt @@ -0,0 +1,100 @@ +alembic==1.8.1 +amqp==5.1.1 +appdirs==1.4.4 +APScheduler==3.9.1 +argh==0.26.2 +async-timeout==4.0.2 +bandit==1.7.4 +billiard==3.6.4.0 +blessed==1.19.1 +blessings==1.7 +blinker==1.5 +bpython==0.23 +caldav==0.10.0 +celery==5.2.7 +certifi==2022.9.24 +chardet==5.0.0 +charset-normalizer==2.1.1 +click==8.1.3 +click-didyoumean==0.3.0 +click-plugins==1.1.1 +click-repl==0.2.0 +colorama==0.4.6 +coverage==6.5.0 +curtsies==0.4.1 +cwcwidth==0.1.8 +Deprecated==1.2.13 +dnspython==2.2.1 +enum-compat==0.0.3 +etherpad-lite==0.5 +eventlet==0.33.1 +feedgen==0.9.0 +Flask==2.2.2 +Flask-Migrate==3.1.0 +Flask-SQLAlchemy==3.0.2 +Flask-WTF==1.0.1 +fuzzywuzzy==0.18.0 +gitdb==4.0.9 +gitdb2==4.0.2 +GitPython==3.1.29 +greenlet==3.1.1 +icalendar==5.0.1 +idna==3.4 +itsdangerous==2.1.2 +Jinja2==3.1.2 +kombu==5.2.4 +ldap3==2.9.1 +Levenshtein==0.27.1 +lxml==5.3.1 +Mako==1.2.3 +mando==0.7.1 +MarkupSafe==2.1.1 +mccabe==0.7.0 +monotonic==1.6 +nose==1.3.7 +packaging==21.3 +pbr==5.11.0 +prompt-toolkit==3.0.31 +psycopg2==2.9.10 +pyasn1==0.4.8 +pyasn1-modules==0.2.8 +pycodestyle==2.9.1 +Pygments==2.13.0 +pyldap==3.0.0.post1 +pyparsing==3.0.9 +python-dateutil==2.8.2 +python-dotenv==0.21.0 +python-editor==1.0.4 +python-engineio==4.3.4 +python-ldap==3.4.3 +python-Levenshtein==0.27.1 +python-pam==2.0.2 +pytz==2022.5 +pytz-deprecation-shim==0.1.0.post0 +pyxdg==0.28 +PyYAML==6.0.2 +RapidFuzz==3.12.2 +raven==6.10.0 +redis==4.3.4 +regex==2022.9.13 +requests==2.28.1 +setuptools==76.0.0 +six==1.16.0 +smmap==5.0.0 +smmap2==3.0.1 +SQLAlchemy==1.4.42 +SQLAlchemy-Utils==0.38.3 +stevedore==4.1.0 +typing==3.7.4.3 +tzdata==2022.5 +tzlocal==4.2 +urllib3==1.26.12 +uwsgidecorators==1.1.0 +vine==5.0.0 +vobject==0.9.6.1 +watchdog==2.1.9 +wcwidth==0.2.5 +Werkzeug==2.2.2 +wheel==0.45.1 +wrapt==1.14.1 +WTForms==3.0.1 diff --git a/requirements.txt b/requirements.txt index 57271353d070318274e6d64ff41ea36b5012be17..d0485344b01d9e022d9578026f0392627413bdbf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -37,15 +37,14 @@ fuzzywuzzy==0.18.0 gitdb==4.0.9 gitdb2==4.0.2 GitPython==3.1.29 -greenlet==1.1.3.post0 +greenlet==3.1.1 icalendar==5.0.1 idna==3.4 itsdangerous==2.1.2 Jinja2==3.1.2 kombu==5.2.4 ldap3==2.9.1 -Levenshtein==0.20.7 -lxml==4.9.1 +lxml>=5.0.0 Mako==1.2.3 mando==0.7.1 MarkupSafe==2.1.1 @@ -53,10 +52,9 @@ mccabe==0.7.0 monotonic==1.6 nose==1.3.7 packaging==21.3 -pathtools==0.1.2 pbr==5.11.0 prompt-toolkit==3.0.31 -psycopg2-binary==2.9.5 +psycopg2 pyasn1==0.4.8 pyasn1-modules==0.2.8 pycodestyle==2.9.1 @@ -68,12 +66,11 @@ python-dotenv==0.21.0 python-editor==1.0.4 python-engineio==4.3.4 python-ldap==3.4.3 -python-Levenshtein==0.20.7 python-pam==2.0.2 pytz==2022.5 pytz-deprecation-shim==0.1.0.post0 pyxdg==0.28 -PyYAML==6.0 +PyYAML==6.0.2 rapidfuzz==2.12.0 raven==6.10.0 redis==4.3.4 diff --git a/server.py b/server.py index 0997e7ca654d2119ca65594664ec86f3bd2d2e3d..be08fb9ca8b6335545c81b6dc0400ecd2489d0ca 100755 --- a/server.py +++ b/server.py @@ -5,6 +5,7 @@ locale.setlocale(locale.LC_TIME, "de_DE.utf8") from flask import ( Flask, request, session, flash, redirect, url_for, abort, render_template, Response, Markup) +import flask import click from werkzeug.utils import secure_filename from flask_migrate import Migrate @@ -929,9 +930,12 @@ def recompile_protocol(protocol): @require_modify_right() def get_protocol_source(protocol): file_like = BytesIO(protocol.source.encode("utf-8")) - return send_file( - file_like, cache_timeout=1, as_attachment=True, - attachment_filename="{}.txt".format(protocol.get_short_identifier())) + return flask.send_file( + file_like, #cache_timeout=1, + # as_attachment=True, + download_name="{}.txt".format(protocol.get_short_identifier()) + #attachment_filename="{}.txt".format(protocol.get_short_identifier()) + ) @app.route("/protocol/template/<int:protocol_id>") @@ -940,10 +944,13 @@ def get_protocol_source(protocol): @require_modify_right() def get_protocol_template(protocol): file_like = BytesIO(protocol.get_template().encode("utf-8")) - return send_file( - file_like, cache_timeout=1, as_attachment=True, - attachment_filename="{}-template.txt".format( - protocol.get_short_identifier())) + return flask.send_file( + file_like, #cache_timeout=1, + as_attachment=True, + download_name="{}.txt".format(protocol.get_short_identifier()) + #attachment_filename="{}-template.txt".format( + # protocol.get_short_identifier()) + ) @app.route("/protocol/etherpush/<int:protocol_id>") @@ -1402,9 +1409,11 @@ def download_document(document): and not document.protocol.has_public_view_right(user))): flash("Dir fehlen die nötigen Zugriffsrechte.", "alert-error") return back.redirect() - return send_file( - document.as_file_like(), cache_timeout=1, - as_attachment=True, attachment_filename=document.name) + return flask.send_file( + document.as_file_like(), #cache_timeout=1, + as_attachment=True, #attachment_filename=document.name + download_name=document.name, + ) @app.route("/document/upload/<int:protocol_id>", methods=["POST"]) @@ -1505,9 +1514,11 @@ def print_decision(decisiondocument): @db_lookup(DecisionDocument) @require_private_view_right() def download_decision(decisiondocument): - return send_file( - decisiondocument.as_file_like(), cache_timeout=1, - as_attachment=True, attachment_filename=decisiondocument.name) + return flask.send_file( + decisiondocument.as_file_like(), #cache_timeout=1, + as_attachment=True, #attachment_filename=decisiondocument.name + download_name=decisiondocument.name, + ) @app.route("/errors/list") diff --git a/tasks.py b/tasks.py index d894769faa617cef4fe44395e9aaadc48960ec90..9240e3cd8a9b7237be3aca3e1323e7df260d276c 100644 --- a/tasks.py +++ b/tasks.py @@ -451,8 +451,12 @@ def parse_protocol_async_inner(protocol, ignore_old_date=False): protocol.protocoltype.latex_template, "decision")).render( render_type=RenderType.latex, decision=decision, protocol=protocol, top=decision_top, show_private=True) + decision_md_content = render_template("decision.md", + render_type=RenderType.markdown, decision=decision, + protocol=protocol, top=decision_top, show_private=True) maxdepth = decision_top.get_maxdepth() compile_decision(decision_content, decision, maxdepth=maxdepth) + compile_decision_md(decision_md_content, decision, maxdepth=maxdepth) # Footnotes footnote_tags = [ @@ -563,6 +567,13 @@ def parse_protocol_async_inner(protocol, ignore_old_date=False): compile( latex_source, protocol, show_private=show_private, maxdepth=maxdepth) + md_source = render_template("protocol.md", + render_type=RenderType.markdown, + show_private=show_private, + **render_kwargs[show_private]) + compile_md( + md_source, protocol, show_private=show_private, + maxdepth=maxdepth) # Export extra TOPs extra_tops = [child for child in tree.children if isinstance(child, Fork) and child.is_extra] @@ -570,13 +581,21 @@ def parse_protocol_async_inner(protocol, ignore_old_date=False): for show_private in privacy_states: latex_source = texenv.get_template(provide_latex_template( protocol.protocoltype.latex_template, "top")).render( - render_type=RenderType.latex, - top=top, - show_private=show_private, - **render_kwargs[show_private]) + render_type=RenderType.latex, + top=top, + show_private=show_private, + **render_kwargs[show_private]) + md_source = render_template("top.md", + render_type=RenderType.markdown, + top=top, + show_private=show_private, + **render_kwargs[show_private]) compile_extra( latex_source, protocol, show_private=show_private, extra_name=top.name, maxdepth=maxdepth) + compile_extra_md( + md_source, protocol, show_private=show_private, extra_name=top.name, + maxdepth=maxdepth) if protocol.protocoltype.use_wiki: @@ -663,18 +682,101 @@ def push_to_dokuwiki_async(protocol_id, content, summary): def compile(content, protocol, show_private, maxdepth): + if not getattr(config, "RENDERING_PDF", True): + return compile_async.delay( content, protocol.id, show_private=show_private, maxdepth=maxdepth) - def compile_decision(content, decision, maxdepth): + if not getattr(config, "RENDERING_PDF", True): + return compile_async.delay( content, decision.id, use_decision=True, maxdepth=maxdepth) def compile_extra(content, protocol, show_private, maxdepth, extra_name): + if not getattr(config, "RENDERING_PDF", True): + return compile_async.delay( content, protocol.id, use_decision=False, show_private=show_private, maxdepth=maxdepth, is_extra=True, extra_name=extra_name) +def compile_md(content, protocol, show_private, maxdepth): + if not getattr(config, "RENDERING_MD", False): + return + compile_md_async.delay( + content, protocol.id, show_private=show_private, maxdepth=maxdepth) + +def compile_decision_md(content, decision, maxdepth): + if not getattr(config, "RENDERING_MD", False): + return + compile_md_async.delay( + content, decision.id, use_decision=True, maxdepth=maxdepth) + +def compile_extra_md(content, protocol, show_private, maxdepth, extra_name): + if not getattr(config, "RENDERING_MD", False): + return + compile_md_async.delay( + content, protocol.id, use_decision=False, show_private=show_private, maxdepth=maxdepth, is_extra=True, extra_name=extra_name) + +@celery.task +def compile_md_async(content, protocol_id, show_private=False, use_decision=False, is_extra=False, extra_name="", + maxdepth=5): + with app.app_context(): + decision = None + protocol = None + if use_decision: + decision = Decision.query.filter_by(id=protocol_id).first() + protocol = decision.protocol + else: + protocol = Protocol.query.filter_by(id=protocol_id).first() + if not use_decision and not is_extra: + for old_document in protocol.documents: + if old_document.is_compiled and old_document.is_private == show_private \ + and old_document.filename.endswith(".md"): + protocol.documents.remove(old_document) + db.session.commit() + document = Document( + protocol_id=protocol.id, + name="protokoll{}_{}_{}.md".format( + "_intern" if show_private else "", + protocol.protocoltype.short_name, + date_filter_short(protocol.date)), + filename="", + is_compiled=True, + is_private=show_private) + elif use_decision and not is_extra: + document = DecisionDocument( + decision_id=decision.id, + name="beschluss_{}_{}_{}.md".format( + protocol.protocoltype.short_name, + date_filter_short(protocol.date), + decision.id), + filename="") + elif is_extra and not use_decision: + document = Document( + protocol_id=protocol.id, + name="extra-{}{}_{}_{}.md".format( + extra_name, + "_intern" if show_private else "", + protocol.protocoltype.short_name, + date_filter_short(protocol.date)), + filename="", + is_compiled=True, + is_extra=True, + is_private=show_private) + else: + raise NotImplementedError("Unknown type.") + db.session.add(document) + db.session.commit() + target_filename = "compiled-{}-{}.md".format( + document.id, "internal" if show_private else "public") + if use_decision: + target_filename = "decision-{}-{}-{}.md".format( + protocol.id, decision.id, document.id) + document.filename = target_filename + with open(os.path.join(config.DOCUMENTS_PATH, target_filename), "w") as fp: + fp.write(content) + db.session.commit() + @celery.task def compile_async( content, protocol_id, show_private=False, use_decision=False, is_extra=False, extra_name="", @@ -723,11 +825,10 @@ def compile_async( os.chdir(current) document = None if not use_decision and not is_extra: - for old_document in [ - document for document in protocol.documents - if document.is_compiled - and document.is_private == show_private]: - protocol.documents.remove(old_document) + for old_document in protocol.documents: + if document.is_compiled and document.is_private == show_private \ + and old_document.filename.endswith(".pdf"): + protocol.documents.remove(old_document) db.session.commit() document = Document( protocol_id=protocol.id, @@ -796,6 +897,8 @@ def compile_async( else: log += "\n\nLogfile not found." _make_error(protocol, "Compiling", "Compiling LaTeX failed", log) + except FileNotFoundError: + _make_error(protocol, "No XeLaTeX found", "") finally: os.chdir(current) diff --git a/templates/decision.md b/templates/decision.md new file mode 100644 index 0000000000000000000000000000000000000000..d6b8d937a18ba39cdf9724193c0a32c4d75bb4db --- /dev/null +++ b/templates/decision.md @@ -0,0 +1,11 @@ +# Protokoll: {{ protocol.protocoltype.name }} +{{ protocol.protocoltype.organization }} +{% if protocol.date %} **Datum: ** {{ protocol.date|datify_long }} {% endif %} +{% for meta in protocol.metas %} +**{{ meta.name }}:** {{meta.value}} +{% endfor %} + +## Beschluss +{{ decision.content }} + +{{top.render(render_type=render_type, level=0, show_private=show_private, protocol=protocol, decision_render=True)}} diff --git a/templates/protocol.md b/templates/protocol.md new file mode 100644 index 0000000000000000000000000000000000000000..e7ddff7260d39abc3ac68f4c7b6229aa6c689d6c --- /dev/null +++ b/templates/protocol.md @@ -0,0 +1,40 @@ +# Protokoll: {{ protocol.protocoltype.name }} +der *{{ protocol.protocoltype.organization }}* +{% if protocol.date %}**Datum:** {{ protocol.date|datify_long }} +{% endif %} +{% for meta in protocol.metas %} +{% if not meta.internal or show_private %} +**{{ meta.name }}:** {{meta.value}} +{% endif %} +{% endfor %} + +## Beschlüsse +{% if protocol.decisions|length > 0 %} +{% for decision in protocol.decisions %} +- {{ decision.content }} {% if decision.categories|length > 0 and show_private %} *({{decision.get_categories_str()}})*{% endif %} +{% endfor %} +{% else %} +- Keine Beschlüsse +{% endif %} + +{% if protocol.start_time is not none %} +**Beginn der Sitzung:** {{protocol.start_time|timify}} +{% endif %} + +{% for top in tree.children %} +{% if top|class == "Fork" %} +{{top.render(render_type=render_type, level=0, show_private=show_private, protocol=protocol)}} + +{% endif %} +{% endfor %} + +{% if protocol.end_time is not none %} +**Ende der Sitzung:** {{protocol.end_time|timify}} +{% endif %} + +{% if footnotes|length > 0 %} +--- +{% for footnote in footnotes %} +[{{footnote.values[0]|footnote_hash}}]: {{footnote.values[0]}} +{% endfor %} +{% endif %} diff --git a/templates/top.md b/templates/top.md new file mode 100644 index 0000000000000000000000000000000000000000..b021475f3757ffd3af800f7c19c50243e465e587 --- /dev/null +++ b/templates/top.md @@ -0,0 +1,17 @@ +# {{top.name}} +**zur {{protocol.protocoltype.name}}** +am {{protocol.date|datify_long}} +{% if show_private %}(:locked: intern){% endif %} + +{% if top|class == "Fork" %} +{{top.render(render_type=render_type, level=0, show_private=show_private, protocol=protocol)}} + +{% endif %} + +{% if footnotes|length > 0 %} +--- +{% for footnote in footnotes %} + +[{{footnote.values[0]|footnote_hash}}]: {{footnote.values[0]}} +{% endfor %} +{% endif %}