Commit 2cdf562e authored by Robin Sonnabend's avatar Robin Sonnabend
Browse files

More legacy, can now import every old protocol

parent 58a13bf0
......@@ -3,6 +3,7 @@ import random
import quopri
from caldav import DAVClient, Principal, Calendar, Event
from caldav.lib.error import PropfindError
from vobject.base import ContentLine
import config
......@@ -14,17 +15,33 @@ class Client:
def __init__(self, calendar=None, url=None):
self.url = url if url is not None else config.CALENDAR_URL
self.client = DAVClient(self.url)
self.principal = self.client.principal()
self.principal = None
for _ in range(config.CALENDAR_MAX_REQUESTS):
try:
self.principal = self.client.principal()
break
except PropfindError as exc:
print(exc)
if self.principal is None:
raise CalendarException("Got {} PropfindErrors from the CalDAV server.".format(config.CALENDAR_MAX_REQUESTS))
if calendar is not None:
self.calendar = self.get_calendar(calendar)
else:
self.calendar = calendar
def get_calendars(self):
return [
calendar.name
for calendar in self.principal.calendars()
]
if not config.CALENDAR_ACTIVE:
return
for _ in range(config.CALENDAR_MAX_REQUESTS):
try:
return [
calendar.name
for calendar in self.principal.calendars()
]
except PropfindError as exc:
print(exc)
raise CalendarException("Got {} PropfindErrors from the CalDAV server.".format(config.CALENDAR_MAX_REQUESTS))
def get_calendar(self, calendar_name):
candidates = self.principal.calendars()
......@@ -34,6 +51,8 @@ class Client:
raise CalendarException("No calendar named {}.".format(calendar_name))
def set_event_at(self, begin, name, description):
if not config.CALENDAR_ACTIVE:
return
candidates = [
Event.from_raw_event(raw_event)
for raw_event in self.calendar.date_search(begin)
......@@ -126,10 +145,3 @@ def get_timezone_offset():
def encode_quopri(text):
return quopri.encodestring(text.encode("utf-8")).replace(b"\n", b"=0D=0A").decode("utf-8")
def main():
client = Client("Protokolltest")
client.set_event_at(datetime(2017, 2, 27, 19, 0), "FSS", "Tagesordnung\nTOP 1")
if __name__ == "__main__":
if config.CALENDAR_ACTIVE:
main()
......@@ -85,7 +85,7 @@ ADMIN_MAIL = "admin@example.com"
PARSER_LAZY = False
# minimum similarity (0-100) todos need to have to be considered equal
FUZZY_MIN_SCORE = 50
FUZZY_MIN_SCORE = 90
# choose something nice from fc-list
# Nimbus Sans looks very much like Computer Modern
......@@ -122,3 +122,9 @@ DOCUMENTS_PATH = "documents"
# keywords indicating private protocol parts
PRIVATE_KEYWORDS = ["private", "internal", "privat", "intern"]
LATEX_BULLETPOINTS = [
r"\textbullet",
r"\normalfont \bfseries \textendash",
r"\textasteriskcentered",
r"\textperiodcentered"
]
from models.database import Todo, OldTodo
from datetime import datetime
from fuzzywuzzy import fuzz, process
import tempfile
from models.database import Todo, OldTodo, Protocol, ProtocolType
from shared import db
import config
def log_fuzzy(text):
with tempfile.NamedTemporaryFile(delete=False, mode="w") as tmpfile:
tmpfile.write(text + "\n\n")
print(text)
def lookup_todo_id(old_candidates, new_who, new_description):
# Check for perfect matches
for candidate in old_candidates:
......@@ -22,25 +29,62 @@ def lookup_todo_id(old_candidates, new_who, new_description):
best_match, best_match_score = process.extractOne(
new_description, content_to_number.keys())
if best_match_score >= config.FUZZY_MIN_SCORE:
print("Used fuzzy matching on '{}', got '{}' with score {}.".format(
log_fuzzy("Used fuzzy matching on '{}', got '{}' with score {}.".format(
new_description, best_match, best_match_score))
return content_to_number[best_match]
else:
print("Best match for '{}' is '{}' with score {}, rejecting.".format(
log_fuzzy("Best match for '{}' is '{}' with score {}, rejecting.".format(
new_description, best_match, best_match_score))
return None
INSERT_PROTOCOLTYPE = "INSERT INTO `protocolManager_protocoltype`"
INSERT_PROTOCOL = "INSERT INTO `protocolManager_protocol`"
INSERT_TODO = "INSERT INTO `protocolManager_todo`"
def import_old_protocols(sql_text):
protocoltype_lines = []
protocol_lines = []
for line in sql_text.splitlines():
if line.startswith(INSERT_PROTOCOLTYPE):
protocoltype_lines.append(line)
elif line.startswith(INSERT_PROTOCOL):
protocol_lines.append(line)
if (len(protocoltype_lines) == 0
or len(protocol_lines) == 0):
raise ValueError("Necessary lines not found.")
type_id_to_handle = {}
for type_line in protocoltype_lines:
for id, handle, name, mail, protocol_id in _split_insert_line(type_line):
type_id_to_handle[int(id)] = handle.lower()
protocols = []
for protocol_line in protocol_lines:
for (protocol_id, old_type_id, date, source, textsummary, htmlsummary,
deleted, sent, document_id) in _split_insert_line(protocol_line):
date = datetime.strptime(date, "%Y-%m-%d")
handle = type_id_to_handle[int(old_type_id)]
type = ProtocolType.query.filter(ProtocolType.short_name.ilike(handle)).first()
if type is None:
raise KeyError("No protocoltype for handle '{}'.".format(handle))
protocol = Protocol(type.id, date, source=source)
db.session.add(protocol)
db.session.commit()
import tasks
protocols.append(protocol)
for protocol in sorted(protocols, key=lambda p: p.date):
print(protocol.date)
tasks.parse_protocol(protocol)
def import_old_todos(sql_text):
protocoltype_lines = []
protocol_lines = []
todo_lines = []
for line in sql_text.splitlines():
if line.startswith("INSERT INTO `protocolManager_protocoltype`"):
if line.startswith(INSERT_PROTOCOLTYPE):
protocoltype_lines.append(line)
elif line.startswith("INSERT INTO `protocolManager_protocol`"):
elif line.startswith(INSERT_PROTOCOL):
protocol_lines.append(line)
elif line.startswith("INSERT INTO `protocolManager_todo`"):
elif line.startswith(INSERT_TODO):
todo_lines.append(line)
if (len(protocoltype_lines) == 0
or len(protocol_lines) == 0
......@@ -125,7 +169,16 @@ def _split_base_level(text, begin="(", end=")", separator=",", string_terminator
escaped = False
for char in part:
if escaped:
current_field += char
if char == "n":
current_field += "\n"
elif char == "r":
current_field += "\r"
elif char == "t":
current_field += "\t"
else:
if char not in "\"'\\":
print("escaped char: '{}'".format(char))
current_field += char
escaped = False
elif in_string:
if char == escape:
......
......@@ -43,7 +43,7 @@ class ProtocolType(db.Model):
def __init__(self, name, short_name, organization, usual_time,
is_public, private_group, public_group, private_mail, public_mail,
use_wiki, wiki_category, wiki_only_public, printer):
use_wiki, wiki_category, wiki_only_public, printer, calendar):
self.name = name
self.short_name = short_name
self.organization = organization
......@@ -416,6 +416,44 @@ class TodoState(Enum):
raise ValueError("Unknown state: '{}'".format(name))
return NAME_TO_STATE[name]
@staticmethod
def from_name_lazy(name):
name = name.strip().lower()
STATE_TO_NAME, NAME_TO_STATE = make_states(TodoState)
for key in NAME_TO_STATE:
if name.startswith(key):
return NAME_TO_STATE[key]
raise ValueError("{} does not start with a state.".format(name))
@staticmethod
def from_name_with_date(name, protocol=None):
name = name.strip().lower()
if not " " in name:
raise ValueError("{} does definitely not contain a state and a date".format(name))
name_part, date_part = name.split(" ", 1)
state = TodoState.from_name(name_part)
date = None
last_exc = None
formats = [("%d.%m.%Y", False)]
if config.PARSER_LAZY:
formats.extend([("%d.%m.", True), ("%d.%m", True)])
for format, year_missing in formats:
try:
date = datetime.strptime(date_part.strip(), format).date()
if year_missing:
year = datetime.now().year
if protocol is not None:
year = protocol.date.year
date = datetime(year=year, month=date.month, day=date.day).date()
break
except ValueError as exc:
last_exc = exc
continue
if date is None:
raise last_exc
return state, date
class Todo(db.Model):
__tablename__ = "todos"
id = db.Column(db.Integer, primary_key=True)
......@@ -443,9 +481,9 @@ class Todo(db.Model):
def is_done(self):
if self.state.needs_date():
if self.state == TodoState.after:
return datetime.now().date() >= self.date
elif self.state == TodoState.before:
return datetime.now().date() <= self.date
elif self.state == TodoState.before:
return datetime.now().date() >= self.date
return self.state.is_done()
def get_id(self):
......
......@@ -156,7 +156,7 @@ class Content(Element):
# v2: does not require the semicolon, but the newline
#PATTERN = r"\s*(?<content>(?:[^\[\];\r\n]+)?(?:\[[^\]\r\n]+\][^;\[\]\r\n]*)*);?"
# v3: does not allow braces in the content
PATTERN = r"\s*(?<content>(?:[^\[\];\r\n{}]+)?(?:\[[^\]\r\n]+\][^;\[\]\r\n]*)*);?"
PATTERN = r"\s*(?<content>(?:[^\[\];\r\n{}]+)?(?:\[[^\]\r\n{}]+\][^;\[\]\r\n{}]*)*);?"
class Text:
def __init__(self, text, linenumber, fork):
......@@ -188,7 +188,10 @@ class Text:
raise ParserException("Text is empty!", linenumber)
return Text(content, linenumber, current)
PATTERN = r"(?<text>[^\[]+)(?:(?=\[)|$)"
# v1: does not allow any [, as that is part of a tag
# PATTERN = r"(?<text>[^\[]+)(?:(?=\[)|$)"
# v2: does allow one [ at the beginning, which is used if it did not match a tag
PATTERN = r"(?<text>\[?[^\[{}]+)(?:(?=\[)|$)"
class Tag:
......@@ -206,7 +209,7 @@ class Tag:
if not show_private:
return ""
return self.todo.render_latex(current_protocol=protocol)
return r"\textbf{{{}:}} {}".format(escape_tex(self.name.capitalize()), escape_tex(self.values[0]))
return r"\textbf{{{}:}} {}".format(escape_tex(self.name.capitalize()), escape_tex(";".join(self.values)))
elif render_type == RenderType.plaintext:
if self.name == "url":
return self.values[0]
......@@ -214,7 +217,7 @@ class Tag:
if not show_private:
return ""
return self.values[0]
return "{}: {}".format(self.name.capitalize(), self.values[0])
return "{}: {}".format(self.name.capitalize(), ";".join(self.values))
elif render_type == RenderType.wikitext:
if self.name == "url":
return "[{0} {0}]".format(self.values[0])
......@@ -222,7 +225,7 @@ class Tag:
if not show_private:
return ""
return self.todo.render_wikitext(current_protocol=protocol)
return "'''{}:''' {}".format(self.name.capitalize(), self.values[0])
return "'''{}:''' {}".format(self.name.capitalize(), ";".join(self.values))
else:
raise _not_implemented(self, render_type)
......@@ -240,8 +243,12 @@ class Tag:
raise ParserException("Tag is empty!", linenumber)
parts = content.split(";")
return Tag(parts[0], parts[1:], linenumber, current)
# v1: matches [text without semicolons]
#PATTERN = r"\[(?<content>(?:[^;\]]*;)*(?:[^;\]]*))\]"
# v2: needs at least two parts separated by a semicolon
PATTERN = r"\[(?<content>(?:[^;\]]*;)+(?:[^;\]]*))\]"
PATTERN = r"\[(?<content>(?:[^;\]]*;)*(?:[^;\]]*))\]"
class Empty(Element):
def __init__(self, linenumber):
......@@ -301,8 +308,8 @@ class Remark(Element):
PATTERN = r"\s*\#(?<content>[^\n]+)"
class Fork(Element):
def __init__(self, environment, name, parent, linenumber, children=None):
self.environment = environment if environment is None or len(environment) > 0 else None
def __init__(self, is_top, name, parent, linenumber, children=None):
self.is_top = is_top
self.name = name.strip() if (name is not None and len(name) > 0) else None
self.parent = parent
self.linenumber = linenumber
......@@ -311,22 +318,19 @@ class Fork(Element):
def dump(self, level=None):
if level is None:
level = 0
result_lines = ["{}fork: {}".format(INDENT_LETTER * level, self.name)]
result_lines = ["{}fork: {}'{}'".format(INDENT_LETTER * level, "TOP " if self.is_top else "", self.name)]
for child in self.children:
result_lines.append(child.dump(level + 1))
return "\n".join(result_lines)
def test_private(self, name):
if name is None:
return False
stripped_name = name.replace(":", "").strip()
return stripped_name in config.PRIVATE_KEYWORDS
def render(self, render_type, show_private, level, protocol=None):
name_parts = []
if self.environment is not None:
name_parts.append(self.environment)
if self.name is not None:
name_parts.append(self.name)
name_line = " ".join(name_parts)
name_line = escape_tex(self.name if self.name is not None else "")
if level == 0 and self.name == "Todos" and not show_private:
return ""
if render_type == RenderType.latex:
......@@ -341,6 +345,8 @@ class Fork(Element):
part = r"\item {}".format(part)
content_parts.append(part)
content_lines = "\n".join(content_parts)
if len(content_lines.strip()) == 0:
content_lines = "\\item Nichts\n"
if level == 0:
return "\n".join([begin_line, content_lines, end_line])
elif self.test_private(self.name):
......@@ -388,7 +394,7 @@ class Fork(Element):
return tags
def is_anonymous(self):
return self.environment == None
return self.name == None
def is_root(self):
return self.parent is None
......@@ -398,6 +404,17 @@ class Fork(Element):
return self
return self.parent.get_top()
def get_maxdepth(self):
child_depths = [
child.get_maxdepth()
for child in self.children
if isinstance(child, Fork)
]
if len(child_depths) > 0:
return max(child_depths) + 1
else:
return 1
@staticmethod
def create_root():
return Fork(None, None, None, 0)
......@@ -405,18 +422,13 @@ class Fork(Element):
@staticmethod
def parse(match, current, linenumber=None):
linenumber = Element.parse_inner(match, current, linenumber)
environment = match.group("environment")
name1 = match.group("name1")
name2 = match.group("name2")
name = ""
if name1 is not None:
name = name1
if name2 is not None:
if len(name) > 0:
name += " {}".format(name2)
else:
name = name2
element = Fork(environment, name, current, linenumber)
topname = match.group("topname")
name = match.group("name")
is_top = False
if topname is not None:
is_top = True
name = topname
element = Fork(is_top, name, current, linenumber)
current = Element.parse_outer(element, current)
return current, linenumber
......@@ -434,7 +446,11 @@ class Fork(Element):
# v1: has a problem with old protocols that do not use a lot of semicolons
#PATTERN = r"\s*(?<name1>[^{};]+)?{(?<environment>\S+)?\h*(?<name2>[^\n]+)?"
# v2: do not allow newlines in name1 or semicolons in name2
PATTERN = r"\s*(?<name1>[^{};\n]+)?{(?<environment>\S+)?\h*(?<name2>[^;\n]+)?"
#PATTERN = r"\s*(?<name1>[^{};\n]+)?{(?<environment>[^\s{};]+)?\h*(?<name2>[^;{}\n]+)?"
# v3: no environment/name2 for normal lists, only for tops
#PATTERN = r"\s*(?<name>[^{};\n]+)?{(?:TOP\h*(?<topname>[^;{}\n]+))?"
# v4: do allow one newline between name and {
PATTERN = r"\s*(?<name>(?:[^{};\n])+)?\n?\s*{(?:TOP\h*(?<topname>[^;{}\n]+))?"
END_PATTERN = r"\s*};?"
PATTERNS = OrderedDict([
......@@ -446,8 +462,8 @@ PATTERNS = OrderedDict([
])
TEXT_PATTERNS = OrderedDict([
(re.compile(Text.PATTERN), Text.parse),
(re.compile(Tag.PATTERN), Tag.parse)
(re.compile(Tag.PATTERN), Tag.parse),
(re.compile(Text.PATTERN), Text.parse)
])
def parse(source):
......@@ -460,11 +476,15 @@ def parse(source):
match = pattern.match(source)
if match is not None:
source = source[len(match.group()):]
current, linenumber = PATTERNS[pattern](match, current, linenumber)
try:
current, linenumber = PATTERNS[pattern](match, current, linenumber)
except ParserException as exc:
exc.tree = tree
raise exc
found = True
break
if not found:
raise ParserException("No matching syntax element found!", linenumber)
raise ParserException("No matching syntax element found!", linenumber, tree=tree)
if current is not tree:
raise ParserException("Source ended within fork! (started at line {})".format(current.linenumber), linenumber=current.linenumber, tree=tree)
return tree
......
......@@ -24,7 +24,7 @@ from utils import is_past, mail_manager, url_manager, get_first_unused_int, set_
from models.database import ProtocolType, Protocol, DefaultTOP, TOP, Document, Todo, Decision, MeetingReminder, Error, TodoMail, DecisionDocument, TodoState
from views.forms import LoginForm, ProtocolTypeForm, DefaultTopForm, MeetingReminderForm, NewProtocolForm, DocumentUploadForm, KnownProtocolSourceUploadForm, NewProtocolSourceUploadForm, ProtocolForm, TopForm, SearchForm, NewProtocolFileUploadForm, NewTodoForm, TodoForm, TodoMailForm
from views.tables import ProtocolsTable, ProtocolTypesTable, ProtocolTypeTable, DefaultTOPsTable, MeetingRemindersTable, ErrorsTable, TodosTable, DocumentsTable, DecisionsTable, TodoTable, ErrorTable, TodoMailsTable
from legacy import import_old_todos
from legacy import import_old_todos, import_old_protocols
app = Flask(__name__)
app.config.from_object(config)
......@@ -81,10 +81,15 @@ app.jinja_env.globals.update(dir=dir)
@manager.command
def import_legacy():
"""Import the old todos from an sql dump"""
"""Import the old todos and protocols from an sql dump"""
filename = prompt("SQL-file")
#filename = "legacy.sql"
with open(filename, "r") as sqlfile:
import_old_todos(sqlfile.read())
content = sqlfile.read()
import_old_todos(content)
import_old_protocols(content)
@app.route("/")
def index():
......@@ -93,7 +98,7 @@ def index():
protocol for protocol in Protocol.query.all()
if protocol.protocoltype.has_public_view_right(user)
]
def _sort_key(protocol):
def _protocol_sort_key(protocol):
if protocol.date is not None:
return protocol.date
return datetime.now().date()
......@@ -104,7 +109,7 @@ def index():
if not protocol.done
and (protocol.date - current_day).days < config.MAX_INDEX_DAYS
],
key=_sort_key
key=_protocol_sort_key
)
finished_protocols = sorted(
[
......@@ -113,7 +118,7 @@ def index():
and (protocol.has_public_view_right(user)
or protocol.has_private_view_right(user))
],
key=_sort_key
key=_protocol_sort_key
)
protocol = finished_protocols[0] if len(finished_protocols) > 0 else None
todos = None
......@@ -123,6 +128,10 @@ def index():
if todo.protocoltype.has_private_view_right(user)
and not todo.is_done()
]
def _todo_sort_key(todo):
protocol = todo.get_first_protocol()
return protocol.date if protocol.date is not None else datetime.now().date()
todos = sorted(todos, key=_todo_sort_key, reverse=True)
todos_table = TodosTable(todos) if todos is not None else None
return render_template("index.html", open_protocols=open_protocols, protocol=protocol, todos=todos, todos_table=todos_table)
......@@ -143,6 +152,7 @@ def list_types():
if (protocoltype.public_group in user.groups
or protocoltype.private_group in user.groups
or protocoltype.is_public)]
types = sorted(types, key=lambda t: t.short_name)
types_table = ProtocolTypesTable(types)
return render_template("types-list.html", types=types, types_table=types_table)
......@@ -160,7 +170,8 @@ def new_type():
form.private_group.data, form.public_group.data,
form.private_mail.data, form.public_mail.data,
form.use_wiki.data, form.wiki_category.data,
form.wiki_only_public.data, form.printer.data)
form.wiki_only_public.data, form.printer.data,
form.calendar.data)
db.session.add(protocoltype)
db.session.commit()
flash("Der Protokolltyp {} wurde angelegt.".format(protocoltype.name), "alert-success")
......@@ -204,6 +215,20 @@ def show_type(type_id):
reminders_table = MeetingRemindersTable(protocoltype.reminders, protocoltype)
return render_template("type-show.html", protocoltype=protocoltype, protocoltype_table=protocoltype_table, default_tops_table=default_tops_table, reminders_table=reminders_table, mail_active=config.MAIL_ACTIVE)
@app.route("/type/delete/<int:type_id>")
@login_required
def delete_type(type_id):
user = current_user()
protocoltype = ProtocolType.query.filter_by(id=type_id).first()
if protocoltype is None or not protocoltype.has_modify_right(user):
flash("Invalider Protokolltyp oder fehlende Zugriffsrechte.", "alert-error")
return redirect(request.args.get("next") or url_for("index"))
name = protocoltype.name
db.session.delete(protocoltype)
db.session.commit()
flash("Der Protokolltype {} wurde gelöscht.".format(name), "alert-success")
return redirect(request.args.get("next") or url_for("list_types"))
@app.route("/type/reminders/new/<int:type_id>", methods=["GET", "POST"])
@login_required
def new_reminder(type_id):
......@@ -436,7 +461,7 @@ def list_protocols():
search_results[protocol] = "<br />\n".join(formatted_lines)
protocols = sorted(protocols, key=lambda protocol: protocol.date, reverse=True)
page = _get_page()
page_count = int(math.ceil(len(protocols)) / config.PAGE_LENGTH)
page_count = int(math.ceil(len(protocols) / config.PAGE_LENGTH))
if page >= page_count:
page = 0
begin_index = page * config.PAGE_LENGTH
......@@ -637,11 +662,9 @@ def etherpush_protocol(protocol_id):
if not config.ETHERPAD_ACTIVE:
flash("Die Etherpadfunktion ist nicht aktiviert.", "alert-error")
return redirect(request.args.get("next") or url_for("show_protocol", protocol_id=protocol_id))
if set_etherpad_text(protocol.get_identifier(), protocol.get_template()):
flash("Vorlage von {} in Etherpad hochgeladen.".format(protocol.get_identifier()), "alert-success")
else:
flash("Das Etherpad wurde bereits bearbeitet.", "alert-error")
return redirect(request.args.get("next") or url_for("show_protocol", protocol_id=protocol.id))
if not protocol.is_done():
tasks.set_etherpad_content(protocol)
return redirect(request.args.get("next") or protocol.get_etherpad_link())
@app.route("/protocol/update/<int:protocol_id>", methods=["GET", "POST"])
@login_required
......
#!/bin/bash
celery -A server.celery worker --loglevel=debug --concurrency=4
celery -A server.celery worker --loglevel=debug --concurrency=1
This diff is collapsed.
......@@ -23,10 +23,18 @@
\\\normalsize \VAR{protocol.protocoltype.organization|escape_tex}
}{}
\begin{tabular}{rp{14cm}}
\ENV{if protocol.date is not none}
{\bf Datum:} & \VAR{protocol.date|datify_long|escape_tex}\\
\ENV{endif}
\ENV{if protocol.location is not none}
{\bf Ort:} & \VAR{protocol.location|escape_tex}\\
{\bf Protokollant:} & \VAR{protocol.author|escape_tex}\\
\ENV{endif}
\ENV{if protocol.author is not none}
{\bf Protokoll:} & \VAR{protocol.author|escape_tex}\\
\ENV{endif}
\ENV{if protocol.participants is not none}
{\bf Anwesend:} & \VAR{protocol.participants|escape_tex}\\
\ENV{endif}
\end{tabular}
\normalsize
......
......@@ -58,7 +58,7 @@
<p><strong>Ort:</strong> {{protocol.location}}</p>
{% endif %}
{% if protocol.author is not none %}
<p><strong>Protokollant:</strong> {{protocol.author}}</p>
<p><strong>Protokoll:</strong> {{protocol.author}}</p>
{% endif %}