Commit b5e133fc authored by Robin Sonnabend's avatar Robin Sonnabend

Data structure and integrated parsing

parent 7fbe71df
SQLALCHEMY_DATABASE_URI = "postgresql://proto3:@/proto3"
SQLALCHEMY_TRACK_MODIFICATIONS = False
SECRET_KEY = "abc"
DEBUG = False
MAIL_ACTIVE = True
MAIL_FROM = "protokolle@example.com"
MAIL_HOST = "mail.example.com:465"
MAIL_USER = "user"
MAIL_PASSWORD = "password"
MAIL_PREFIX = "protokolle"
CELERY_BROKER_URL = "redis://localhost:6379/0"
CELERY_TASK_SERIALIZER = "pickle"
CELERY_ACCEPT_CONTENT = ["pickle"]
URL_ROOT = "protokolle.example.com"
URL_PROTO = "https"
URL_PATH = "/"
URL_PARAMS = ""
ERROR_CONTEXT_LINES = 3
from flask import render_template, send_file, url_for, redirect, flash, request
from datetime import datetime, date, timedelta
import time
import math
from shared import db
from utils import random_string, url_manager
#from models.tables import TexResponsiblesTable, TexSupportersTable
from sqlalchemy.orm import relationship, backref
class ProtocolType(db.Model):
__tablename__ = "protocoltypes"
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String, unique=True)
short_name = db.Column(db.String, unique=True)
organization = db.Column(db.String)
is_public = db.Column(db.Boolean)
private_group = db.Column(db.String)
public_group = db.Column(db.String)
private_mail = db.Column(db.String)
public_mail = db.Column(db.String)
protocols = relationship("Protocol", backref=backref("protocoltype"), cascade="all, delete-orphan", order_by="Protocol.id")
default_tops = relationship("DefaultTOP", backref=backref("protocoltype"), cascade="all, delete-orphan", order_by="DefaultTOP.number")
reminders = relationship("MeetingReminder", backref=backref("protocoltype"), cascade="all, delete-orphan", order_by="MeetingReminder.time_before")
def __init__(self, name, short_name, organization,
is_public, private_group, public_group, private_mail, public_mail):
self.name = name
self.short_name = short_name
self.organization = organization
self.is_public = is_public
self.private_group = private_group
self.public_group = public_group
self.private_mail = private_mail
self.public_mail = public_mail
def __repr__(self):
return "<ProtocolType(id={}, short_name={}, name={}, organization={})>".format(
self.id, self.short_name, self.name, self.organization)
class Protocol(db.Model):
__tablename__ = "protocols"
id = db.Column(db.Integer, primary_key=True)
protocoltype_id = db.Column(db.Integer, db.ForeignKey("protocoltypes.id"))
source = db.Column(db.String, nullable=True)
date = db.Column(db.Date)
start_time = db.Column(db.Time)
end_time = db.Column(db.Time)
author = db.Column(db.String)
participants = db.Column(db.String)
location = db.Column(db.String)
tops = relationship("TOP", backref=backref("protocol"), cascade="all, delete-orphan", order_by="TOP.number")
decisions = relationship("Decision", backref=backref("protocol"), cascade="all, delete-orphan", order_by="Decision.id")
documents = relationship("Document", backref=backref("protocol"), cascade="all, delete-orphan", order_by="Document.is_compiled")
errors = relationship("Error", backref=backref("protocol"), cascade="all, delete-orphan", order_by="Error.id")
def __init__(self, protocoltype_id, date, source=None, start_time=None, end_time=None, author=None, participants=None, location=None):
self.protocoltype_id = protocoltype_id
self.date = date
self.source = source
self.start_time = start_time
self.end_time = end_time
self.author = author
self.participants = participants
self.location = location
def __repr__(self):
return "<Protocol(id={}, protocoltype_id={})>".format(
self.id, self.protocoltype_id)
def create_error(self, action, name, description):
now = datetime.now()
return Error(self.id, action, name, now, description)
def fill_from_remarks(self, remarks):
self.date = datetime.strptime(remarks["Datum"].value, "%d.%m.%Y")
self.start_time = time.strptime(remarks["Beginn"].value, "%H:%M")
self.end_time = time.strptime(remarks["Ende"].value, "%H:%M")
self.author = remarks["Autor"].value
self.participants = remarks["Anwesende"].value
self.location = remarks["Ort"].value
class DefaultTOP(db.Model):
__tablename__ = "defaulttops"
id = db.Column(db.Integer, primary_key=True)
protocoltype_id = db.Column(db.Integer, db.ForeignKey("protocoltypes.id"))
name = db.Column(db.String)
number = db.Column(db.Integer)
def __init__(self, protocoltype_id, name, number):
self.protocoltype_id = protocoltype_id
self.name = name
self.number = number
def __repr__(self):
return "<DefaultTOP(id={}, protocoltype_id={}, name={}, number={})>".format(
self.id, self.protocoltype_id, self.name, self.number)
def is_at_end(self):
return self.number < 0
class TOP(db.Model):
__tablename__ = "tops"
id = db.Column(db.Integer, primary_key=True)
protocol_id = db.Column(db.Integer, db.ForeignKey("protocols.id"))
name = db.Column(db.String)
number = db.Column(db.String)
planned = db.Column(db.Boolean)
def __init__(self, protocol_id, name, number, planned):
self.protocol_id = protocol_id
self.name = name
self.number = number
self.planned = planned
def __repr__(self):
return "<TOP(id={}, protocol_id={}, name={}, number={}, planned={})>".format(
self.id, self.protocol_id, self.name, self.number, self.planned)
class Document(db.Model):
__tablename__ = "documents"
id = db.Column(db.Integer, primary_key=True)
protocol_id = db.Column(db.Integer, db.ForeignKey("protocols.id"))
name = db.Column(db.String)
filename = db.Column(db.String, unique=True)
is_compiled = db.Column(db.Boolean)
def __init__(self, protocol_id, name, filename, is_compiled):
self.protocol_id = protocol_id
self.name = name
self.filename = filename
self.is_compiled = is_compiled
def __repr__(self):
return "<Document(id={}, protocol_id={}, name={}, filename={}, is_compiled={})>".format(
self.id, self.protocol_id, self.name, self.filename, self.is_compiled)
class Todo(db.Model):
__tablename__ = "todos"
id = db.Column(db.Integer, primary_key=True)
who = db.Column(db.String)
description = db.Column(db.String)
tags = db.Column(db.String)
done = db.Column(db.Boolean)
protocols = relationship("Protocol", secondary="todoprotocolassociations", backref="todos")
def __init__(self, who, description, tags, done):
self.who = who
self.description = description
self.tags = tags
self.done = done
def __repr__(self):
return "<Todo(id={}, who={}, description={}, tags={}, done={})>".format(
self.id, self.who, self.description, self.tags, self.done)
class TodoProtocolAssociation(db.Model):
__tablename__ = "todoprotocolassociations"
todo_id = db.Column(db.Integer, db.ForeignKey("todos.id"), primary_key=True)
protocol_id = db.Column(db.Integer, db.ForeignKey("protocols.id"), primary_key=True)
class Decision(db.Model):
__tablename__ = "decisions"
id = db.Column(db.Integer, primary_key=True)
protocol_id = db.Column(db.Integer, db.ForeignKey("protocols.id"))
content = db.Column(db.String)
def __init__(self, protocol_id, content):
self.protocol_id = protocol_id
self.content = content
def __repr__(self):
return "<Decision(id={}, protocol_id={}, content='{}')>".format(
self.id, self.protocol_id, self.content)
class MeetingReminder(db.Model):
__tablename__ = "meetingreminders"
id = db.Column(db.Integer, primary_key=True)
protocoltype_id = db.Column(db.Integer, db.ForeignKey("protocoltypes.id"))
time_before = db.Column(db.Interval)
send_public = db.Column(db.Boolean)
send_private = db.Column(db.Boolean)
def __init__(self, protocoltype_id, time_before, send_public, send_private):
self.protocoltype_id = protocoltype_id
self.time_before = time_before
self.send_public = send_public
self.send_private = send_private
def __repr__(self):
return "<MeetingReminder(id={}, protocoltype_id={}, time_before={}, send_public={}, send_private={})>".format(
self.id, self.protocoltype_id, self.time_before, self.send_public, self.send_private)
class Error(db.Model):
__tablename__ = "errors"
id = db.Column(db.Integer, primary_key=True)
protocol_id = db.Column(db.Integer, db.ForeignKey("protocols.id"))
action = db.Column(db.String)
name = db.Column(db.String)
datetime = db.Column(db.DateTime)
description = db.Column(db.String)
def __init__(self, protocol_id, action, name, datetime, description):
self.protocol_id = protocol_id
self.action = action
self.name = name
self.datetime = datetime
self.description = description
def __repr__(self):
return "<Error(id={}, protocol_id={}, action={}, name={}, datetime={})>".format(
self.id, self.protocol_id, self.action, self.name, self.datetime)
......@@ -100,6 +100,10 @@ class Content(Element):
for child in self.children:
child.dump(level + 1)
def get_tags(self, tags):
tags.extend([child for child in self.children if isinstance(child, Tag)])
return tags
@staticmethod
def parse(match, current, linenumber=None):
linenumber = Element.parse_inner(match, current, linenumber)
......@@ -254,6 +258,13 @@ class Fork(Element):
+ "\n".join(map(lambda e: r"\item {}".format(e.render()), self.children)) + "\n"
+ r"\end{itemize}" + "\n")
def get_tags(self, tags=None):
if tags is None:
tags = []
for child in self.children:
child.get_tags(tags)
return tags
def is_anonymous(self):
return self.environment == None
......
#!/usr/bin/env python3
import locale
locale.setlocale(locale.LC_TIME, "de_DE.utf8")
from flask import Flask, g, current_app, request, session, flash, redirect, url_for, abort, render_template, Response
from flask_script import Manager, prompt
from flask_migrate import Migrate, MigrateCommand
#from flask_socketio import SocketIO
from celery import Celery
import config
from shared import db, date_filter, datetime_filter
from utils import is_past, mail_manager, url_manager
app = Flask(__name__)
app.config.from_object(config)
db.init_app(app)
migrate = Migrate(app, db)
manager = Manager(app)
manager.add_command("db", MigrateCommand)
def make_celery(app, config):
celery = Celery(app.import_name, broker=config.CELERY_BROKER_URL)
celery.conf.update(app.config)
return celery
celery = make_celery(app, config)
#def make_socketio(app, config):
# socketio = SocketIO(app)
# return socketio
#socketio = make_socketio(app, config)
app.jinja_env.trim_blocks = True
app.jinja_env.lstrip_blocks = True
app.jinja_env.filters["datify"] = date_filter
app.jinja_env.filters["datetimify"] = datetime_filter
app.jinja_env.filters["url_complete"] = url_manager.complete
import tasks
# blueprints here
@app.route("/")
def index():
return render_template("index.html")
@app.route("/imprint/")
def imprint():
return render_template("imprint.html")
@app.route("/contact/")
def contact():
return render_template("contact.html")
@app.route("/privacy/")
def privacy():
return render_template("privacy.html")
if __name__ == "__main__":
manager.run()
from flask_sqlalchemy import SQLAlchemy
import re
import config
db = SQLAlchemy()
# the following code is written by Lars Beckers and not to be published without permission
latex_chars = [
("\\", "\\backslash"), # this needs to be first
("$", "\$"),
('%', '\\%'),
('&', '\\&'),
('#', '\\#'),
('_', '\\_'),
('{', '\\{'),
('}', '\\}'),
#('[', '\\['),
#(']', '\\]'),
#('"', '"\''),
('~', '$\\sim{}$'),
('^', '\\textasciicircum{}'),
('Ë„', '\\textasciicircum{}'),
('`', '{}`'),
('-->', '$\longrightarrow$'),
('->', '$\rightarrow$'),
('==>', '$\Longrightarrow$'),
('=>', '$\Rightarrow$'),
('>=', '$\geq$'),
('=<', '$\leq$'),
('<', '$<$'),
('>', '$>$'),
('\\backslashin', '$\\in$'),
('\\backslash', '$\\backslash$') # this needs to be last
]
def escape_tex(text):
out = text
for old, new in latex_chars:
out = out.replace(old, new)
# beware, the following is carefully crafted code
res = ''
k, l = (0, -1)
while k >= 0:
k = out.find('"', l+1)
if k >= 0:
res += out[l+1:k]
l = out.find('"', k+1)
if l >= 0:
res += '\\enquote{' + out[k+1:l] + '}'
else:
res += '"\'' + out[k+1:]
k = l
else:
res += out[l+1:]
# yes, this is not quite escaping latex chars, but anyway...
res = re.sub('([a-z])\(', '\\1 (', res)
res = re.sub('\)([a-z])', ') \\1', res)
#logging.debug('escape latex ({0}/{1}): {2} --> {3}'.format(len(text), len(res), text.split('\n')[0], res.split('\n')[0]))
return res
def unhyphen(text):
return " ".join([r"\mbox{" + word + "}" for word in text.split(" ")])
def date_filter(date):
return date.strftime("%d. %B %Y")
def datetime_filter(date):
return date.strftime("%d. %B %Y, %H:%M")
def date_filter_long(date):
return date.strftime("%A, %d.%m.%Y, Kalenderwoche %W")
def time_filter(time):
return time.strftime("%H:%m")
#!/bin/bash
celery -A server.celery worker --loglevel=debug --concurrency=4
body {
font-family: sans-serif;
}
.header {
border-bottom: 1px solid black;
padding: 4px;
font-size: 16pt;
margin-bottom: 1pc;
}
.header > p {
display: inline;
}
.header-title {
font-weight: bold;
}
.header-link {
}
.footer {
border-top: 1px solid black;
padding: 4px;
font-size: 10pt;
margin-top: 1pc;
width: 100%;
}
.footer > p {
display: inline;
}
.footer-text {
}
.footer-link {
}
.main {
max-width: 940px;
margin: 0px auto;
}
.main > p {
margin: 0px;
padding: 0.5ex;
display: inline-block;
}
.title {
font-size: 16pt;
}
.form {
background-color: #eee;
padding: 1ex;
border: 1px dashed gray;
}
.form-title {
font-size: 16pt;
margin-bottom: 3pt;
border-bottom: 1px solid black;
}
.form-field {
font-size: 12pt;
margin: 2pt;
}
.form-label {
font-size: 12pt;
display: inline-block;
min-width: 100px;
}
.form-textareafield > textarea {
width: 50%;
min-width: 350px;
max-width: 100%;
height: 150px;
}
.form-errors {
font-size: 10pt;
color: darkred;
}
.form-errors-entry {
margin: 0px;
}
.section {
margin-top: 1ex;
margin-bottom: 1ex;
background-color: #eee;
padding: 1ex;
border: 1px dashed gray;
}
.section > p {
display: inline;
}
.section-title {
font-weight: bold;
font-size: 14pt;
}
.table {
border-spacing: 0px;
border-collapse: collapse;
text-align: center;
}
.table-header {
font-size: 13pt;
}
.table-body > tr > td {
border-right: 1px solid gray;
border-left: 1px solid gray;
padding-left: 5px;
padding-right: 5px;
}
.table-body > tr:first-child {
border-top: 1px solid gray;
}
.table-body > tr:last-child {
border-bottom: 1px solid gray;
}
.table-body > tr:nth-child(odd) {
background-color: #fefefe;
}
.table-body > tr:nth-child(even) {
background-color: #f0f0f0;
}
.alert {
font-weight: bold;
text-align: center;
}
.alert-error {
color: #800000;
}
.alert-success {
color: #008000;
}
.alert-warning {
color: #808000;
}
.mail-content {
white-space: pre;
}
.imprint-header {
font-weight: bold;
}
.imprint > div {
}
.privacy-title {
font-weight: bold;
font-size: 14pt;
}
.privacy-content {
}
.pad-content {
width: 100%;
}
from flask import render_template
import os
import subprocess
import shutil
from models.database import Document, Protocol, Error, Todo, Decision
from server import celery, app
from shared import db, escape_tex, unhyphen, date_filter, datetime_filter, date_filter_long, time_filter
from utils import mail_manager, url_manager, encode_kwargs, decode_kwargs
from parser import parse, ParserException, Element, Content, Text, Tag, Remark, Fork
import config
texenv = app.create_jinja_environment()
texenv.block_start_string = r"\ENV{"
texenv.block_end_string = r"}"
texenv.variable_start_string = r"\VAR{"
texenv.variable_end_string = r"}"
texenv.comment_start_string = r"\COMMENT{"
texenv.comment_end_string = r"}"
texenv.filters["escape_tex"] = escape_tex
texenv.filters["unhyphen"] = unhyphen
texenv.trim_blocks = True
texenv.lstrip_blocks = True
texenv.filters["url_complete"] = url_manager.complete
texenv.filters["datify"] = date_filter
texenv.filters["datify_long"] = date_filter_long
texenv.filters["datetimify"] = datetime_filter
texenv.filters["timify"] = time_filter
mailenv = app.create_jinja_environment()
mailenv.trim_blocks = True
mailenv.lstrip_blocks = True
mailenv.filters["url_complete"] = url_manager.complete
mailenv.filters["datify"] = date_filter
mailenv.filters["datetimify"] = datetime_filter
ID_FIELD_BEGINNING = "id "
def parse_protocol(protocol, **kwargs):
parse_protocol_async.delay(protocol.id, encode_kwargs(kwargs))
@celery.task
def parse_protocol_async(protocol_id, encoded_kwargs):
with app.app_context():
with app.test_request_context("/"):
kwargs = decode_kwargs(encoded_kwargs)
protocol = Protocol.query.filter_by(id=protocol_id).first()
if protocol is None:
raise Exception("No protocol given. Aborting parsing.")
if protocol.source is None:
error = protocol.create_error("Parsing", "Protocol source is None", "")
db.session.add(error)
db.session.commit()
return
tree = None
try:
tree = parse(protocol.source)
except ParserException as exc:
context = ""
if exc.linenumber is not None:
source_lines = source.splitlines()
start_index = max(0, exc.linenumber - config.ERROR_CONTEXT_LINES)
end_index = min(len(source_lines) - 1, exc.linenumber + config.ERROR_CONTEXT_LINES)
context = "\n".join(source_lines[start_index:end_index])
error = protocol.create_error("Parsing", str(exc), context)
db.session.add(error)
db.session.commit()
return
remarks = {element.name: element for element in tree.children is isinstance(element, Remark)}
required_fields = ["Datum", "Anwesende", "Beginn", "Ende", "Autor", "Ort"]
missing_fields = [field for field in required_files if field not in remarks]
if len(missing_fields) > 0:
error = protocol.create_error("Parsing", "Missing fields", ", ".join(missing_fields))
db.session.add(error)
db.session.commit()
return
try:
protocol.fill_from_remarks(remarks)
except ValueError:
error = protocol.create_error(
"Parsing", "Invalid fields",
"Date or time fields are not '%d.%m.%Y' respectively '%H:%M', "
"but rather {}".format(
", ".join([remarks["Datum"], remarks["Beginn"], remarks["Ende"]])))
db.session.add(error)
db.session.commit()
return
old_todos = list(protocol.todos)
for todo in old_todos
protocol.todos.remove(todo)
db.session.commit()
tags = tree.get_tags()
todo_tags = [tag for tag in tags if tag.name == "todo"]
for todo_tag in todo_tags:
if len(todo_tag.values) < 2:
error = protocol.create_error("Parsing", "Invalid todo-tag",
"The todo tag in line {} needs at least "
"information on who and what, "
"but has less than that.".format(todo_tag.linenumber))
db.session.add(error)
db.session.commit()
return
who = todo_tag.values[0]
what = todo_tag.values[1]
todo = None
for other_field in todo_tag.values[2:]:
if other_field.startswith(ID_FIELD_BEGINNING):
field_id = 0
try:
field_id = int(other_field[len(ID_FIELD_BEGINNING):])
except ValueError:
error = protocol.create_error("Parsing", "Non-numerical todo ID",
"The todo in line {} has a nonnumerical ID, but needs "