Commit 607b9aae authored by Robin Sonnabend's avatar Robin Sonnabend

Lazy parser, todo states, import old todo IDs

parent 09666c23
......@@ -17,3 +17,4 @@ documents/
.rediscli_history
.viminfo
test/
*.sql
from models.database import Todo, OldTodo
from fuzzywuzzy import fuzz, process
from shared import db
import config
def lookup_todo_id(old_candidates, new_who, new_description):
# Check for perfect matches
for candidate in old_candidates:
if candidate.who == new_who and candidate.description == new_description:
return candidate.old_id
# Accept if who has been changed
for candidate in old_candidates:
if candidate.description == new_description:
return candidate.old_id
# Do fuzzy matching on description
content_to_number = {
candidate.description: candidate.old_id
for candidate in old_candidates
}
best_match, best_match_score = process.extractOne(
new_description, content_to_number.keys())
if best_match_score >= config.FUZZY_MIN_SCORE:
print("Used fuzzy matching on '{}', got '{}' with score {}.".format(
new_description, best_match, best_match_score))
return content_to_number[best_match]
else:
print("Best match for '{}' is '{}' with score {}, rejecting.".format(
new_description, best_match, best_match_score))
return None
def import_old_todos(sql_text):
protocoltype_lines = []
protocol_lines = []
todo_lines = []
for line in sql_text.splitlines():
if line.startswith("INSERT INTO `protocolManager_protocoltype`"):
protocoltype_lines.append(line)
elif line.startswith("INSERT INTO `protocolManager_protocol`"):
protocol_lines.append(line)
elif line.startswith("INSERT INTO `protocolManager_todo`"):
todo_lines.append(line)
if (len(protocoltype_lines) == 0
or len(protocol_lines) == 0
or len(todo_lines) == 0):
raise ValueError("Necessary lines not found.")
type_id_to_handle = {}
for type_line in protocoltype_lines:
for id, handle, name, mail, protocol_id in _split_insert_line(type_line):
type_id_to_handle[int(id)] = handle.lower()
protocol_id_to_key = {}
for protocol_line in protocol_lines:
for (protocol_id, type_id, date, source, textsummary, htmlsummary,
deleted, sent, document_id) in _split_insert_line(protocol_line):
handle = type_id_to_handle[int(type_id)]
date_string = date [2:]
protocol_id_to_key[int(protocol_id)] = "{}-{}".format(handle, date_string)
todos = []
for todo_line in todo_lines:
for old_id, protocol_id, who, what, start_time, end_time, done in _split_insert_line(todo_line):
protocol_id = int(protocol_id)
if protocol_id not in protocol_id_to_key:
print("Missing protocol with ID {} for Todo {}".format(protocol_id, what))
continue
todo = OldTodo(old_id=old_id, who=who, description=what,
protocol_key=protocol_id_to_key[protocol_id])
todos.append(todo)
OldTodo.query.delete()
db.session.commit()
for todo in todos:
db.session.add(todo)
db.session.commit()
def _split_insert_line(line):
insert_part, values_part = line.split("VALUES", 1)
return _split_base_level(values_part)
def _split_base_level(text, begin="(", end=")", separator=",", string_terminator="'", line_end=";", ignore=" ", escape="\\"):
raw_parts = []
current_part = None
index = 0
in_string = False
escaped = False
for char in text:
if escaped:
current_part += char
escaped = False
elif current_part is None:
if char == ignore:
continue
elif char == begin:
current_part = ""
elif char == line_end:
break
elif char == separator:
pass
else:
raise ValueError(
"Found invalid char '{}' at position {}".format(
char, index))
else:
if in_string:
current_part += char
if char == escape:
escaped = True
elif char == string_terminator:
in_string = False
else:
if char == string_terminator:
current_part += char
in_string = True
elif char == end:
raw_parts.append(current_part)
current_part = None
else:
current_part += char
index += 1
parts = []
for part in raw_parts:
fields = []
current_field = ""
in_string = False
escaped = False
for char in part:
if escaped:
current_field += char
escaped = False
elif in_string:
if char == escape:
escaped = True
elif char == string_terminator:
in_string = False
else:
current_field += char
else:
if char == string_terminator:
in_string = True
elif char == separator:
fields.append(current_field)
current_field = ""
else:
current_field += char
if len(current_field) > 0:
fields.append(current_field)
parts.append(fields)
return parts
"""empty message
Revision ID: 0131d5776f8d
Revises: 24bd2198a626
Create Date: 2017-02-24 21:03:34.294388
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '0131d5776f8d'
down_revision = '24bd2198a626'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('todos', sa.Column('protocoltype_id', sa.Integer(), nullable=True))
op.create_foreign_key(None, 'todos', 'protocoltypes', ['protocoltype_id'], ['id'])
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_constraint(None, 'todos', type_='foreignkey')
op.drop_column('todos', 'protocoltype_id')
# ### end Alembic commands ###
"""empty message
Revision ID: 0e5220a9f169
Revises: 188f389b2286
Create Date: 2017-02-26 15:53:41.410353
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '0e5220a9f169'
down_revision = '188f389b2286'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('decisiondocuments',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('decision_id', sa.Integer(), nullable=True),
sa.Column('name', sa.String(), nullable=True),
sa.Column('filename', sa.String(), nullable=True),
sa.Column('is_compiled', sa.Boolean(), nullable=True),
sa.Column('is_private', sa.Boolean(), nullable=True),
sa.ForeignKeyConstraint(['decision_id'], ['decisions.id'], ),
sa.PrimaryKeyConstraint('id')
)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('decisiondocuments')
# ### end Alembic commands ###
"""empty message
Revision ID: 162da8aeeb71
Revises: a3d9d1b87ba0
Create Date: 2017-02-22 16:52:08.142214
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = '162da8aeeb71'
down_revision = 'a3d9d1b87ba0'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('meetingreminders', sa.Column('days_before', sa.Integer(), nullable=True))
op.drop_column('meetingreminders', 'time_before')
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('meetingreminders', sa.Column('time_before', postgresql.INTERVAL(), autoincrement=False, nullable=True))
op.drop_column('meetingreminders', 'days_before')
# ### end Alembic commands ###
"""empty message
Revision ID: 188f389b2286
Revises: 515d261a624b
Create Date: 2017-02-26 12:55:43.761405
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '188f389b2286'
down_revision = '515d261a624b'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('todomails',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(), nullable=True),
sa.Column('mail', sa.String(), nullable=True),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name')
)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('todomails')
# ### end Alembic commands ###
"""empty message
Revision ID: 24bd2198a626
Revises: 2e2682dfac21
Create Date: 2017-02-24 17:20:07.135782
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '24bd2198a626'
down_revision = '2e2682dfac21'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('todos', sa.Column('number', sa.Integer(), nullable=True))
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('todos', 'number')
# ### end Alembic commands ###
"""empty message
Revision ID: 2e2682dfac21
Revises: aebae2c4523d
Create Date: 2017-02-24 16:31:01.729972
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '2e2682dfac21'
down_revision = 'aebae2c4523d'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('todos', sa.Column('is_id_fixed', sa.Boolean(), nullable=True))
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('todos', 'is_id_fixed')
# ### end Alembic commands ###
"""empty message
Revision ID: 310d9ab321b8
Revises: 0131d5776f8d
Create Date: 2017-02-25 17:26:34.663460
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '310d9ab321b8'
down_revision = '0131d5776f8d'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('protocols', sa.Column('plain_text_private', sa.String(), nullable=True))
op.add_column('protocols', sa.Column('plain_text_public', sa.String(), nullable=True))
op.drop_column('todos', 'is_id_fixed')
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('todos', sa.Column('is_id_fixed', sa.BOOLEAN(), autoincrement=False, nullable=True))
op.drop_column('protocols', 'plain_text_public')
op.drop_column('protocols', 'plain_text_private')
# ### end Alembic commands ###
"""empty message
Revision ID: 495509e8f49a
Revises: 310d9ab321b8
Create Date: 2017-02-25 17:34:03.830014
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '495509e8f49a'
down_revision = '310d9ab321b8'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('protocols', sa.Column('content_private', sa.String(), nullable=True))
op.add_column('protocols', sa.Column('content_public', sa.String(), nullable=True))
op.drop_column('protocols', 'plain_text_private')
op.drop_column('protocols', 'plain_text_public')
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('protocols', sa.Column('plain_text_public', sa.VARCHAR(), autoincrement=False, nullable=True))
op.add_column('protocols', sa.Column('plain_text_private', sa.VARCHAR(), autoincrement=False, nullable=True))
op.drop_column('protocols', 'content_public')
op.drop_column('protocols', 'content_private')
# ### end Alembic commands ###
"""empty message
Revision ID: 515d261a624b
Revises: 77bf71eef07f
Create Date: 2017-02-26 00:33:13.555804
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '515d261a624b'
down_revision = '77bf71eef07f'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('protocoltypes', sa.Column('usual_time', sa.Time(), nullable=True))
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('protocoltypes', 'usual_time')
# ### end Alembic commands ###
"""empty message
Revision ID: 77bf71eef07f
Revises: f91d760158dc
Create Date: 2017-02-26 00:26:48.499578
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '77bf71eef07f'
down_revision = 'f91d760158dc'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('meetingreminders', sa.Column('additional_text', sa.String(), nullable=True))
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('meetingreminders', 'additional_text')
# ### end Alembic commands ###
"""empty message
Revision ID: 97cf1913e60d
Revises: bbc1782c0999
Create Date: 2017-02-22 23:36:29.467493
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '97cf1913e60d'
down_revision = 'bbc1782c0999'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('tops', sa.Column('number', sa.Integer(), nullable=True))
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('tops', 'number')
# ### end Alembic commands ###
"""empty message
Revision ID: a3d9d1b87ba0
Revises: efaa3b4fd3e8
Create Date: 2017-02-22 16:00:02.816515
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'a3d9d1b87ba0'
down_revision = 'efaa3b4fd3e8'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('protocols', sa.Column('done', sa.Boolean(), nullable=True))
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('protocols', 'done')
# ### end Alembic commands ###
"""empty message
Revision ID: aebae2c4523d
Revises: b114754024fb
Create Date: 2017-02-24 05:58:37.240601
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'aebae2c4523d'
down_revision = 'b114754024fb'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_constraint('documents_filename_key', 'documents', type_='unique')
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_unique_constraint('documents_filename_key', 'documents', ['filename'])
# ### end Alembic commands ###
"""empty message
Revision ID: b114754024fb
Revises: 97cf1913e60d
Create Date: 2017-02-23 20:33:56.446729
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'b114754024fb'
down_revision = '97cf1913e60d'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('documents', sa.Column('is_private', sa.Boolean(), nullable=True))
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('documents', 'is_private')
# ### end Alembic commands ###
"""empty message
Revision ID: bbc1782c0999
Revises: 162da8aeeb71
Create Date: 2017-02-22 23:36:11.613892
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'bbc1782c0999'
down_revision = '162da8aeeb71'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('tops', 'number')
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('tops', sa.Column('number', sa.VARCHAR(), autoincrement=False, nullable=True))
# ### end Alembic commands ###
"""empty message
Revision ID: efaa3b4fd3e8
Revision ID: d543c6a2ea6e
Revises:
Create Date: 2017-02-22 05:27:41.905321
Create Date: 2017-02-27 20:41:51.001496
"""
from alembic import op
......@@ -10,7 +10,7 @@ import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'efaa3b4fd3e8'
revision = 'd543c6a2ea6e'
down_revision = None
branch_labels = None
depends_on = None
......@@ -18,27 +18,39 @@ depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('oldtodos',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('old_id', sa.Integer(), nullable=True),
sa.Column('who', sa.String(), nullable=True),
sa.Column('description', sa.String(), nullable=True),
sa.Column('protocol_key', sa.String(), nullable=True),
sa.PrimaryKeyConstraint('id')
)
op.create_table('protocoltypes',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(), nullable=True),
sa.Column('short_name', sa.String(), nullable=True),
sa.Column('organization', sa.String(), nullable=True),
sa.Column('usual_time', sa.Time(), nullable=True),
sa.Column('is_public', sa.Boolean(), nullable=True),
sa.Column('private_group', sa.String(), nullable=True),
sa.Column('public_group', sa.String(), nullable=True),
sa.Column('private_mail', sa.String(), nullable=True),
sa.Column('public_mail', sa.String(), nullable=True),
sa.Column('use_wiki', sa.Boolean(), nullable=True),
sa.Column('wiki_category', sa.String(), nullable=True),
sa.Column('wiki_only_public', sa.Boolean(), nullable=True),
sa.Column('printer', sa.String(), nullable=True),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name'),
sa.UniqueConstraint('short_name')
)
op.create_table('todos',
op.create_table('todomails',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('who', sa.String(), nullable=True),
sa.Column('description', sa.String(), nullable=True),
sa.Column('tags', sa.String(), nullable=True),
sa.Column('done', sa.Boolean(), nullable=True),
sa.PrimaryKeyConstraint('id')
sa.Column('name', sa.String(), nullable=True),
sa.Column('mail', sa.String(), nullable=True),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name')
)
op.create_table('defaulttops',
sa.Column('id', sa.Integer(), nullable=False),
......@@ -51,9 +63,10 @@ def upgrade():
op.create_table('meetingreminders',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('protocoltype_id', sa.Integer(), nullable=True),
sa.Column('time_before', sa.Interval(), nullable=True),
sa.Column('days_before', sa.Integer(), nullable=True),
sa.Column('send_public', sa.Boolean(), nullable=True),
sa.Column('send_private', sa.Boolean(), nullable=True),
sa.Column('additional_text', sa.String(), nullable=True),
sa.ForeignKeyConstraint(['protocoltype_id'], ['protocoltypes.id'], ),
sa.PrimaryKeyConstraint('id')
)
......@@ -61,12 +74,26 @@ def upgrade():
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('protocoltype_id', sa.Integer(), nullable=True),
sa.Column('source', sa.String(), nullable=True),
sa.Column('content_public', sa.String(), nullable=True),
sa.Column('content_private', sa.String(), nullable=True),
sa.Column('date', sa.Date(), nullable=True),
sa.Column('start_time', sa.Time(), nullable=True),
sa.Column('end_time', sa.Time(), nullable=True),
sa.Column('author', sa.String(), nullable=True),
sa.Column('participants', sa.String(), nullable=True),
sa.Column('location', sa.String(), nullable=True),
sa.Column('done', sa.Boolean(), nullable=True),
sa.ForeignKeyConstraint(['protocoltype_id'], ['protocoltypes.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_table('todos',
sa.Column('id', sa.Integer(), nullable=False),