Data structure and integrated parsing

config.py.example

+SQLALCHEMY_DATABASE_URI = "postgresql://proto3:@/proto3"
+SQLALCHEMY_TRACK_MODIFICATIONS = False
+SECRET_KEY = "abc"
+DEBUG = False
+MAIL_ACTIVE = True
+MAIL_FROM = "protokolle@example.com"
+MAIL_HOST = "mail.example.com:465"
+MAIL_USER = "user"
+MAIL_PASSWORD = "password"
+MAIL_PREFIX = "protokolle"
+CELERY_BROKER_URL = "redis://localhost:6379/0"
+CELERY_TASK_SERIALIZER = "pickle"
+CELERY_ACCEPT_CONTENT = ["pickle"]
+URL_ROOT = "protokolle.example.com"
+URL_PROTO = "https"
+URL_PATH = "/"
+URL_PARAMS = ""
+ERROR_CONTEXT_LINES = 3

models/database.py

+from flask import render_template, send_file, url_for, redirect, flash, request
+
+from datetime import datetime, date, timedelta
+import time
+import math
+
+from shared import db
+from utils import random_string, url_manager
+
+#from models.tables import TexResponsiblesTable, TexSupportersTable
+
+from sqlalchemy.orm import relationship, backref
+
+class ProtocolType(db.Model):
+    __tablename__ = "protocoltypes"
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.String, unique=True)
+    short_name = db.Column(db.String, unique=True)
+    organization = db.Column(db.String)
+    is_public = db.Column(db.Boolean)
+    private_group = db.Column(db.String)
+    public_group = db.Column(db.String)
+    private_mail = db.Column(db.String)
+    public_mail = db.Column(db.String)
+
+    protocols = relationship("Protocol", backref=backref("protocoltype"), cascade="all, delete-orphan", order_by="Protocol.id")
+    default_tops = relationship("DefaultTOP", backref=backref("protocoltype"), cascade="all, delete-orphan", order_by="DefaultTOP.number")
+    reminders = relationship("MeetingReminder", backref=backref("protocoltype"), cascade="all, delete-orphan", order_by="MeetingReminder.time_before")
+
+    def __init__(self, name, short_name, organization,
+        is_public, private_group, public_group, private_mail, public_mail):
+        self.name = name
+        self.short_name = short_name
+        self.organization = organization
+        self.is_public = is_public
+        self.private_group = private_group
+        self.public_group = public_group
+        self.private_mail = private_mail
+        self.public_mail = public_mail
+
+    def __repr__(self):
+        return "<ProtocolType(id={}, short_name={}, name={}, organization={})>".format(
+            self.id, self.short_name, self.name, self.organization)
+
+class Protocol(db.Model):
+    __tablename__ = "protocols"
+    id = db.Column(db.Integer, primary_key=True)
+    protocoltype_id = db.Column(db.Integer, db.ForeignKey("protocoltypes.id"))
+    source = db.Column(db.String, nullable=True)
+    date = db.Column(db.Date)
+    start_time = db.Column(db.Time)
+    end_time = db.Column(db.Time)
+    author = db.Column(db.String)
+    participants = db.Column(db.String)
+    location = db.Column(db.String)
+
+    tops = relationship("TOP", backref=backref("protocol"), cascade="all, delete-orphan", order_by="TOP.number")
+    decisions = relationship("Decision", backref=backref("protocol"), cascade="all, delete-orphan", order_by="Decision.id")
+    documents = relationship("Document", backref=backref("protocol"), cascade="all, delete-orphan", order_by="Document.is_compiled")
+    errors = relationship("Error", backref=backref("protocol"), cascade="all, delete-orphan", order_by="Error.id")
+
+    def __init__(self, protocoltype_id, date, source=None, start_time=None, end_time=None, author=None, participants=None, location=None):
+        self.protocoltype_id = protocoltype_id
+        self.date = date
+        self.source = source
+        self.start_time = start_time
+        self.end_time = end_time
+        self.author = author
+        self.participants = participants
+        self.location = location
+
+    def __repr__(self):
+        return "<Protocol(id={}, protocoltype_id={})>".format(
+            self.id, self.protocoltype_id)
+
+    def create_error(self, action, name, description):
+        now = datetime.now()
+        return Error(self.id, action, name, now, description)
+
+    def fill_from_remarks(self, remarks):
+        self.date = datetime.strptime(remarks["Datum"].value, "%d.%m.%Y")
+        self.start_time = time.strptime(remarks["Beginn"].value, "%H:%M")
+        self.end_time = time.strptime(remarks["Ende"].value, "%H:%M")
+        self.author = remarks["Autor"].value
+        self.participants = remarks["Anwesende"].value
+        self.location = remarks["Ort"].value
+
+
+class DefaultTOP(db.Model):
+    __tablename__ = "defaulttops"
+    id = db.Column(db.Integer, primary_key=True)
+    protocoltype_id = db.Column(db.Integer, db.ForeignKey("protocoltypes.id"))
+    name = db.Column(db.String)
+    number = db.Column(db.Integer)
+
+    def __init__(self, protocoltype_id, name, number):
+        self.protocoltype_id = protocoltype_id
+        self.name = name
+        self.number = number
+
+    def __repr__(self):
+        return "<DefaultTOP(id={}, protocoltype_id={}, name={}, number={})>".format(
+            self.id, self.protocoltype_id, self.name, self.number)
+
+    def is_at_end(self):
+        return self.number < 0
+
+class TOP(db.Model):
+    __tablename__ = "tops"
+    id = db.Column(db.Integer, primary_key=True)
+    protocol_id = db.Column(db.Integer, db.ForeignKey("protocols.id"))
+    name = db.Column(db.String)
+    number = db.Column(db.String)
+    planned = db.Column(db.Boolean)
+
+    def __init__(self, protocol_id, name, number, planned):
+        self.protocol_id = protocol_id
+        self.name = name
+        self.number = number
+        self.planned = planned
+
+    def __repr__(self):
+        return "<TOP(id={}, protocol_id={}, name={}, number={}, planned={})>".format(
+            self.id, self.protocol_id, self.name, self.number, self.planned)
+
+class Document(db.Model):
+    __tablename__ = "documents"
+    id = db.Column(db.Integer, primary_key=True)
+    protocol_id = db.Column(db.Integer, db.ForeignKey("protocols.id"))
+    name = db.Column(db.String)
+    filename = db.Column(db.String, unique=True)
+    is_compiled = db.Column(db.Boolean)
+
+    def __init__(self, protocol_id, name, filename, is_compiled):
+        self.protocol_id = protocol_id
+        self.name = name
+        self.filename = filename
+        self.is_compiled = is_compiled
+
+    def __repr__(self):
+        return "<Document(id={}, protocol_id={}, name={}, filename={}, is_compiled={})>".format(
+            self.id, self.protocol_id, self.name, self.filename, self.is_compiled)
+
+class Todo(db.Model):
+    __tablename__ = "todos"
+    id = db.Column(db.Integer, primary_key=True)
+    who = db.Column(db.String)
+    description = db.Column(db.String)
+    tags = db.Column(db.String)
+    done = db.Column(db.Boolean)
+
+    protocols = relationship("Protocol", secondary="todoprotocolassociations", backref="todos")
+
+    def __init__(self, who, description, tags, done):
+        self.who = who
+        self.description = description
+        self.tags = tags
+        self.done = done
+
+    def __repr__(self):
+        return "<Todo(id={}, who={}, description={}, tags={}, done={})>".format(
+            self.id, self.who, self.description, self.tags, self.done)
+
+class TodoProtocolAssociation(db.Model):
+    __tablename__ = "todoprotocolassociations"
+    todo_id = db.Column(db.Integer, db.ForeignKey("todos.id"), primary_key=True)
+    protocol_id = db.Column(db.Integer, db.ForeignKey("protocols.id"), primary_key=True)
+
+class Decision(db.Model):
+    __tablename__ = "decisions"
+    id = db.Column(db.Integer, primary_key=True)
+    protocol_id = db.Column(db.Integer, db.ForeignKey("protocols.id"))
+    content = db.Column(db.String)
+
+    def __init__(self, protocol_id, content):
+        self.protocol_id = protocol_id
+        self.content = content
+
+    def __repr__(self):
+        return "<Decision(id={}, protocol_id={}, content='{}')>".format(
+            self.id, self.protocol_id, self.content)
+
+class MeetingReminder(db.Model):
+    __tablename__ = "meetingreminders"
+    id = db.Column(db.Integer, primary_key=True)
+    protocoltype_id = db.Column(db.Integer, db.ForeignKey("protocoltypes.id"))
+    time_before = db.Column(db.Interval)
+    send_public = db.Column(db.Boolean)
+    send_private = db.Column(db.Boolean)
+
+    def __init__(self, protocoltype_id, time_before, send_public, send_private):
+        self.protocoltype_id = protocoltype_id
+        self.time_before = time_before
+        self.send_public = send_public
+        self.send_private = send_private
+
+    def __repr__(self):
+        return "<MeetingReminder(id={}, protocoltype_id={}, time_before={}, send_public={}, send_private={})>".format(
+            self.id, self.protocoltype_id, self.time_before, self.send_public, self.send_private)
+
+class Error(db.Model):
+    __tablename__ = "errors"
+    id = db.Column(db.Integer, primary_key=True)
+    protocol_id = db.Column(db.Integer, db.ForeignKey("protocols.id"))
+    action = db.Column(db.String)
+    name = db.Column(db.String)
+    datetime = db.Column(db.DateTime)
+    description = db.Column(db.String)
+
+    def __init__(self, protocol_id, action, name, datetime, description):
+        self.protocol_id = protocol_id
+        self.action = action
+        self.name = name
+        self.datetime = datetime
+        self.description = description
+
+    def __repr__(self):
+        return "<Error(id={}, protocol_id={}, action={}, name={}, datetime={})>".format(
+            self.id, self.protocol_id, self.action, self.name, self.datetime)

parser.py

@@ -100,6 +100,10 @@ class Content(Element):
         for child in self.children:
             child.dump(level + 1)
 
+    def get_tags(self, tags):
+        tags.extend([child for child in self.children if isinstance(child, Tag)])
+        return tags
+
     @staticmethod
     def parse(match, current, linenumber=None):
         linenumber = Element.parse_inner(match, current, linenumber)
@@ -254,6 +258,13 @@ class Fork(Element):
             + "\n".join(map(lambda e: r"\item {}".format(e.render()), self.children)) + "\n"
             + r"\end{itemize}" + "\n")
 
+    def get_tags(self, tags=None):
+        if tags is None:
+            tags = []
+        for child in self.children:
+            child.get_tags(tags)
+        return tags
+
     def is_anonymous(self):
         return self.environment == None
 

requirements.txt

+alembic
+blinker
+Flask
+Flask-Migrate
+Flask-Script
+Flask-SQLAlchemy
+Flask-WTF
+Flask-SocketIO
+Jinja2
+Mako
+MarkupSafe
+psycopg2
+python-editor
+SQLAlchemy
+Werkzeug
+wheel
+WTForms
+celery[redis]
+regex
+watchdog

server.py

+#!/usr/bin/env python3
+import locale
+locale.setlocale(locale.LC_TIME, "de_DE.utf8")
+
+from flask import Flask, g, current_app, request, session, flash, redirect, url_for, abort, render_template, Response
+from flask_script import Manager, prompt
+from flask_migrate import Migrate, MigrateCommand
+#from flask_socketio import SocketIO
+from celery import Celery
+
+import config
+from shared import db, date_filter, datetime_filter
+from utils import is_past, mail_manager, url_manager
+
+app = Flask(__name__)
+app.config.from_object(config)
+db.init_app(app)
+migrate = Migrate(app, db)
+manager = Manager(app)
+manager.add_command("db", MigrateCommand)
+
+def make_celery(app, config):
+    celery = Celery(app.import_name, broker=config.CELERY_BROKER_URL)
+    celery.conf.update(app.config)
+    return celery
+celery = make_celery(app, config)
+
+#def make_socketio(app, config):
+#    socketio = SocketIO(app)
+#    return socketio
+#socketio = make_socketio(app, config)
+
+app.jinja_env.trim_blocks = True
+app.jinja_env.lstrip_blocks = True
+app.jinja_env.filters["datify"] = date_filter
+app.jinja_env.filters["datetimify"] = datetime_filter
+app.jinja_env.filters["url_complete"] = url_manager.complete
+
+import tasks
+
+# blueprints here
+
+@app.route("/")
+def index():
+    return render_template("index.html")
+
+@app.route("/imprint/")
+def imprint():
+    return render_template("imprint.html")
+
+@app.route("/contact/")
+def contact():
+    return render_template("contact.html")
+
+@app.route("/privacy/")
+def privacy():
+    return render_template("privacy.html")
+
+if __name__ == "__main__":
+    manager.run()

shared.py

+from flask_sqlalchemy import SQLAlchemy
+
+import re
+
+import config
+
+db = SQLAlchemy()
+
+# the following code is written by Lars Beckers and not to be published without permission
+latex_chars = [
+    ("\\", "\\backslash"), # this needs to be first
+    ("$", "\$"),
+    ('%', '\\%'),
+    ('&', '\\&'),
+    ('#', '\\#'),
+    ('_', '\\_'),
+    ('{', '\\{'),
+    ('}', '\\}'),
+    #('[', '\\['),
+    #(']', '\\]'),
+    #('"', '"\''),
+    ('~', '$\\sim{}$'),
+    ('^', '\\textasciicircum{}'),
+    ('Ë„', '\\textasciicircum{}'),
+    ('`', '{}`'),
+    ('-->', '$\longrightarrow$'),
+    ('->', '$\rightarrow$'),
+    ('==>', '$\Longrightarrow$'),
+    ('=>', '$\Rightarrow$'),
+    ('>=', '$\geq$'),
+    ('=<', '$\leq$'),
+    ('<', '$<$'),
+    ('>', '$>$'),
+    ('\\backslashin', '$\\in$'),
+    ('\\backslash', '$\\backslash$') # this needs to be last
+]
+
+def escape_tex(text):
+    out = text
+    for old, new in latex_chars:
+        out = out.replace(old, new)
+    # beware, the following is carefully crafted code
+    res = ''
+    k, l = (0, -1)
+    while k >= 0:
+        k = out.find('"', l+1)
+        if k >= 0:
+            res += out[l+1:k]
+            l = out.find('"', k+1)
+            if l >= 0:
+                res += '\\enquote{' + out[k+1:l] + '}'
+            else:
+                res += '"\'' + out[k+1:]
+            k = l
+        else:
+            res += out[l+1:]
+    # yes, this is not quite escaping latex chars, but anyway...
+    res = re.sub('([a-z])\(', '\\1 (', res)
+    res = re.sub('\)([a-z])', ') \\1', res)
+    #logging.debug('escape latex ({0}/{1}): {2} --> {3}'.format(len(text), len(res), text.split('\n')[0], res.split('\n')[0]))
+    return res
+
+def unhyphen(text):
+    return " ".join([r"\mbox{" + word + "}" for word in text.split(" ")])
+
+def date_filter(date):
+    return date.strftime("%d. %B %Y")
+def datetime_filter(date):
+    return date.strftime("%d. %B %Y, %H:%M")
+def date_filter_long(date):
+    return date.strftime("%A, %d.%m.%Y, Kalenderwoche %W")
+def time_filter(time):
+    return time.strftime("%H:%m")

start_celery.sh

+#!/bin/bash
+celery -A server.celery worker --loglevel=debug --concurrency=4

static/css/style.css

+body {
+    font-family: sans-serif;
+}
+
+.header {
+    border-bottom: 1px solid black;
+    padding: 4px;
+    font-size: 16pt;
+    margin-bottom: 1pc;
+}
+
+.header > p {
+    display: inline;
+}
+
+.header-title {
+    font-weight: bold;
+}
+
+.header-link {
+    
+}
+
+.footer {
+    border-top: 1px solid black;
+    padding: 4px;
+    font-size: 10pt;
+    margin-top: 1pc;
+    width: 100%;
+}
+
+.footer > p {
+    display: inline;
+}
+
+.footer-text {
+}
+
+.footer-link {
+}
+
+.main {
+    max-width: 940px;
+    margin: 0px auto;
+}
+
+.main > p {
+    margin: 0px;
+    padding: 0.5ex;
+    display: inline-block;
+}
+
+.title {
+    font-size: 16pt;
+}
+
+.form {
+    background-color: #eee;
+    padding: 1ex;
+    border: 1px dashed gray;
+}
+
+.form-title {
+    font-size: 16pt;
+    margin-bottom: 3pt;
+    border-bottom: 1px solid black;
+}
+
+.form-field {
+    font-size: 12pt;
+    margin: 2pt;
+}
+
+.form-label {
+    font-size: 12pt;
+    display: inline-block;
+    min-width: 100px;
+}
+
+.form-textareafield > textarea {
+    width: 50%;
+    min-width: 350px;
+    max-width: 100%;
+    height: 150px;
+}
+
+.form-errors {
+    font-size: 10pt;
+    color: darkred;
+}
+
+.form-errors-entry {
+    margin: 0px;
+}
+
+.section {
+    margin-top: 1ex;
+    margin-bottom: 1ex;
+    background-color: #eee;
+    padding: 1ex;
+    border: 1px dashed gray;
+}
+
+.section > p {
+    display: inline;
+}
+
+.section-title {
+    font-weight: bold;
+    font-size: 14pt;
+}
+
+.table {
+    border-spacing: 0px;
+    border-collapse: collapse;
+    text-align: center;
+}
+
+.table-header {
+    font-size: 13pt;
+}
+
+.table-body > tr > td {
+    border-right: 1px solid gray;
+    border-left: 1px solid gray;
+    padding-left: 5px;
+    padding-right: 5px;
+}
+
+.table-body > tr:first-child {
+    border-top: 1px solid gray;
+}
+
+.table-body > tr:last-child {
+    border-bottom: 1px solid gray;
+}
+
+.table-body > tr:nth-child(odd) {
+    background-color: #fefefe;
+}
+
+.table-body > tr:nth-child(even) {
+    background-color: #f0f0f0;
+}
+
+.alert {
+    font-weight: bold;
+    text-align: center;
+}
+
+.alert-error {
+    color: #800000;
+}
+
+.alert-success {
+    color: #008000;
+}
+
+.alert-warning {
+    color: #808000;
+}
+
+.mail-content {
+    white-space: pre;
+}
+
+.imprint-header {
+    font-weight: bold;
+}
+
+.imprint > div {
+    
+}
+
+.privacy-title {
+    font-weight: bold;
+    font-size: 14pt;
+}
+
+.privacy-content {
+    
+}
+
+.pad-content {
+    width: 100%;
+}

tasks.py

+from flask import render_template
+
+import os
+import subprocess
+import shutil
+
+from models.database import Document, Protocol, Error, Todo, Decision
+from server import celery, app
+from shared import db, escape_tex, unhyphen, date_filter, datetime_filter, date_filter_long, time_filter
+from utils import mail_manager, url_manager, encode_kwargs, decode_kwargs
+from parser import parse, ParserException, Element, Content, Text, Tag, Remark, Fork
+
+import config
+
+texenv = app.create_jinja_environment()
+texenv.block_start_string = r"\ENV{"
+texenv.block_end_string = r"}"
+texenv.variable_start_string = r"\VAR{"
+texenv.variable_end_string = r"}"
+texenv.comment_start_string = r"\COMMENT{"
+texenv.comment_end_string = r"}"
+texenv.filters["escape_tex"] = escape_tex
+texenv.filters["unhyphen"] = unhyphen
+texenv.trim_blocks = True
+texenv.lstrip_blocks = True
+texenv.filters["url_complete"] = url_manager.complete
+texenv.filters["datify"] = date_filter
+texenv.filters["datify_long"] = date_filter_long
+texenv.filters["datetimify"] = datetime_filter
+texenv.filters["timify"] = time_filter
+
+mailenv = app.create_jinja_environment()
+mailenv.trim_blocks = True
+mailenv.lstrip_blocks = True
+mailenv.filters["url_complete"] = url_manager.complete
+mailenv.filters["datify"] = date_filter
+mailenv.filters["datetimify"] = datetime_filter
+
+ID_FIELD_BEGINNING = "id "
+
+def parse_protocol(protocol, **kwargs):
+    parse_protocol_async.delay(protocol.id, encode_kwargs(kwargs))
+
+@celery.task
+def parse_protocol_async(protocol_id, encoded_kwargs):
+    with app.app_context():
+        with app.test_request_context("/"):
+            kwargs = decode_kwargs(encoded_kwargs)
+            protocol = Protocol.query.filter_by(id=protocol_id).first()
+            if protocol is None:
+                raise Exception("No protocol given. Aborting parsing.")
+            if protocol.source is None:
+                error = protocol.create_error("Parsing", "Protocol source is None", "")
+                db.session.add(error)
+                db.session.commit()
+                return
+            tree = None
+            try:
+                tree = parse(protocol.source)
+            except ParserException as exc:
+                context = ""
+                if exc.linenumber is not None:
+                    source_lines = source.splitlines()
+                    start_index = max(0, exc.linenumber - config.ERROR_CONTEXT_LINES)
+                    end_index = min(len(source_lines) - 1, exc.linenumber + config.ERROR_CONTEXT_LINES)
+                    context = "\n".join(source_lines[start_index:end_index])
+                error = protocol.create_error("Parsing", str(exc), context)
+                db.session.add(error)