Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
Roman Karwacik
website
Commits
080cdbc5
Commit
080cdbc5
authored
Oct 05, 2016
by
Andreas Valder
Browse files
extended sorter to work with old folder strukture
parent
1adfa144
Changes
2
Hide whitespace changes
Inline
Side-by-side
config.py.example
View file @
080cdbc5
# Defaults for development ,do not use in production!
DEBUG = False
VIDEOPREFIX = 'https://videoag.fsmpi.rwth-aachen.de'
VIDEOMOUNT = 'files/
'
VIDEOMOUNT =
[
'files/
protected/','files/pub/','files/vpnonline/']
#SECRET_KEY = 'something random'
DB_SCHEMA = 'db_schema.sql'
...
...
sorter.py
View file @
080cdbc5
...
...
@@ -19,105 +19,116 @@ def sort_log():
'''
))
@
app
.
route
(
'/sort/now'
)
@
sched_func
(
60
)
@
mod_required
def
sort_now
():
return
sort_wraper
()
@
sched_func
(
60
)
def
sort_auto
():
pass
# return sort_wraper()
def
sort_wraper
():
courses
=
query
(
'SELECT * FROM courses'
)
formats
=
query
(
'SELECT * FROM formats ORDER BY prio'
)
for
c
in
courses
:
existingvideos
=
query
(
'SELECT videos.path FROM videos JOIN lectures ON (videos.lecture_id = lectures.id) WHERE lectures.course_id = ?'
,
c
[
'id'
])
lectures
=
query
(
'SELECT * from lectures where course_id = ?'
,
c
[
'id'
])
coursepath
=
config
[
'VIDEOMOUNT'
]
+
c
[
'handle'
]
try
:
files
=
os
.
listdir
(
coursepath
)
except
FileNotFoundError
:
files
=
[]
for
f
in
files
:
# if the video is in the table "videos" already, skip it
exists
=
False
for
e
in
existingvideos
:
# vpnonline/08ws-swt/08ws-swt-081118.mp4
e_filename
=
e
[
'path'
].
split
(
'/'
,
2
)[
1
]
if
f
==
e_filename
:
exists
=
True
break
if
exists
:
continue
filepath
=
coursepath
+
'/'
+
f
# filenames: <handle>-<sorter>-<format>.mp4
# sorter musst be found with fuzzy matching. musst be one or more of the following: (inside the loop)
splitfilename
=
f
.
replace
(
'_'
,
'-'
).
replace
(
' '
,
'-'
).
split
(
'-'
)
if
not
os
.
path
.
splitext
(
f
)[
1
]
==
'.mp4'
:
continue
data
=
{
'keywords'
:
[]}
# parse the file name and save all data in 'data'
for
s
in
splitfilename
:
s
=
s
.
replace
(
'.mp4'
,
''
)
#-<YYMMDD> (date)
#-<HHMM> (time)
#-<keyword>
# Looking for keywords in: id,title,speaker,comment, comma seperated list in internal
for
basepath
in
config
[
'VIDEOMOUNT'
]:
existingvideos
=
query
(
'SELECT videos.path FROM videos JOIN lectures ON (videos.lecture_id = lectures.id) WHERE lectures.course_id = ?'
,
c
[
'id'
])
lectures
=
query
(
'SELECT * from lectures where course_id = ?'
,
c
[
'id'
])
coursepath
=
basepath
+
c
[
'handle'
]
try
:
files
=
os
.
listdir
(
coursepath
)
except
FileNotFoundError
:
files
=
[]
for
f
in
files
:
try
:
if
len
(
s
)
==
6
:
data
[
'date'
]
=
datetime
.
strptime
(
s
,
'%y%m%d'
).
date
()
elif
len
(
s
)
==
4
:
data
[
'time'
]
=
datetime
.
strptime
(
s
,
'%H%M'
).
time
()
else
:
data
[
'keywords'
].
append
(
s
)
except
ValueError
:
data
[
'keywords'
].
append
(
s
)
# try to match the file on a single lecture
matches
=
[]
# first try date and time (if one of them is set)
if
'date'
in
data
:
for
l
in
lectures
:
if
((
l
[
'time'
].
date
()
==
data
[
'date'
])
and
not
(
'time'
in
data
))
or
((
'time'
in
data
)
and
l
[
'time'
]
==
datetime
.
combine
(
data
[
'date'
],
data
[
'time'
]))
:
matches
.
append
(
l
)
# if we can't match based on date and time, we have to match keywords
if
((
len
(
matches
)
!=
1
)
and
(
len
(
data
[
'keywords'
])
>
0
)):
found
=
False
for
field
in
[
'id'
,
'title'
,
'speaker'
,
'comment'
,
'internal'
]:
#only test lectures with the correct date/time
if
len
(
matches
)
==
0
:
matches
.
extend
(
lectures
)
for
l
in
matches
:
for
k
in
data
[
'keywords'
]:
# first test for exact match, else make it asci and try substring test
if
(
k
==
l
[
field
])
or
(
str
(
k
).
lower
()
in
str
(
l
[
field
]).
lower
().
replace
(
'ä'
,
'ae'
).
replace
(
'ü'
,
'ue'
).
replace
(
'ö'
,
'oe'
).
replace
(
'ß'
,
'ss'
)
):
found
=
True
matches
=
[
l
]
# if the video is in the table "videos" already, skip it
exists
=
False
for
e
in
existingvideos
:
# vpnonline/08ws-swt/08ws-swt-081118.mp4
e_filename
=
e
[
'path'
].
split
(
'/'
,
2
)[
1
]
if
f
==
e_filename
:
exists
=
True
break
if
exists
:
continue
filepath
=
coursepath
+
'/'
+
f
# filenames: <handle>-<sorter>-<format>.mp4
# sorter musst be found with fuzzy matching. musst be one or more of the following: (inside the loop)
splitfilename
=
f
.
replace
(
'_'
,
'-'
).
replace
(
' '
,
'-'
).
split
(
'-'
)
if
not
os
.
path
.
splitext
(
f
)[
1
]
==
'.mp4'
:
continue
data
=
{
'keywords'
:
[]}
# parse the file name and save all data in 'data'
for
s
in
splitfilename
:
s
=
s
.
replace
(
'.mp4'
,
''
)
#-<YYMMDD> (date)
#-<HHMM> (time)
#-<keyword>
# Looking for keywords in: id,title,speaker,comment, comma seperated list in internal
try
:
if
len
(
s
)
==
6
:
data
[
'date'
]
=
datetime
.
strptime
(
s
,
'%y%m%d'
).
date
()
elif
len
(
s
)
==
4
:
data
[
'time'
]
=
datetime
.
strptime
(
s
,
'%H%M'
).
time
()
else
:
data
[
'keywords'
].
append
(
s
)
except
ValueError
:
data
[
'keywords'
].
append
(
s
)
# try to match the file on a single lecture
matches
=
[]
# first try date and time (if one of them is set)
if
'date'
in
data
:
for
l
in
lectures
:
if
((
l
[
'time'
].
date
()
==
data
[
'date'
])
and
not
(
'time'
in
data
))
or
((
'time'
in
data
)
and
l
[
'time'
]
==
datetime
.
combine
(
data
[
'date'
],
data
[
'time'
]))
:
matches
.
append
(
l
)
# if we can't match based on date and time, we have to match keywords
if
((
len
(
matches
)
!=
1
)
and
(
len
(
data
[
'keywords'
])
>
0
)):
found
=
False
for
field
in
[
'id'
,
'title'
,
'speaker'
,
'comment'
,
'internal'
]:
#only test lectures with the correct date/time
if
len
(
matches
)
==
0
:
matches
.
extend
(
lectures
)
for
l
in
matches
:
for
k
in
data
[
'keywords'
]:
# first test for exact match, else make it asci and try substring test
if
(
k
==
l
[
field
])
or
(
str
(
k
).
lower
()
in
str
(
l
[
field
]).
lower
().
replace
(
'ä'
,
'ae'
).
replace
(
'ü'
,
'ue'
).
replace
(
'ö'
,
'oe'
).
replace
(
'ß'
,
'ss'
)
):
found
=
True
matches
=
[
l
]
if
found
:
break
if
found
:
break
if
found
:
break
if
found
:
break
if
found
:
break
# now we should have found exactly one match
if
len
(
matches
)
==
1
:
# now match the format
for
i
in
formats
:
#we match the last part of the file name without the extension
formatstring
=
splitfilename
[
-
1
].
split
(
'.'
,
1
)[
0
].
lower
()
if
formatstring
in
i
[
'keywords'
].
replace
(
','
,
' '
).
split
(
' '
):
data
[
'format'
]
=
i
[
'id'
]
break
# if we found the format, insert the video
if
not
'format'
in
data
:
data
[
'format'
]
=
0
modify
(
'BEGIN'
)
video_id
=
modify
(
'INSERT INTO videos_data (lecture_id,visible,path,video_format,title,comment,internal,file_modified,time_created,time_updated,created_by,hash,file_size) VALUES (?,0,?,?,"","","",?,?,?,?,"",?)'
,
matches
[
0
][
'id'
],
c
[
'handle'
]
+
'/'
+
f
,
data
[
'format'
],
datetime
.
now
(),
datetime
.
now
(),
datetime
.
now
(),
session
[
'user'
][
'givenName'
],
os
.
stat
(
coursepath
+
'/'
+
f
).
st_size
)
query
(
'INSERT INTO sortlog (lecture_id,video_id,path,`when`) VALUES (?,?,?,?)'
,
matches
[
0
][
'id'
],
video_id
,
c
[
'handle'
]
+
'/'
+
f
,
datetime
.
now
())
modify
(
'COMMIT'
)
# for debuging only
# else:
# d = []
# for m in matches:
# d.append(m['id'])
# print('failed',{"data":data,"path":f,"results":d})
# now we should have found exactly one match
if
len
(
matches
)
==
1
:
# now match the format
for
i
in
formats
:
#we match the last part of the file name without the extension
formatstring
=
splitfilename
[
-
1
].
split
(
'.'
,
1
)[
0
].
lower
()
if
formatstring
in
i
[
'keywords'
].
replace
(
','
,
' '
).
split
(
' '
):
data
[
'format'
]
=
i
[
'id'
]
break
# if we found the format, insert the video
if
not
'format'
in
data
:
data
[
'format'
]
=
0
modify
(
'BEGIN'
)
video_id
=
modify
(
'INSERT INTO videos_data (lecture_id,visible,path,video_format,title,comment,internal,file_modified,time_created,time_updated,created_by,hash,file_size) VALUES (?,0,?,?,"","","",?,?,?,?,"",?)'
,
matches
[
0
][
'id'
],
c
[
'handle'
]
+
'/'
+
f
,
data
[
'format'
],
datetime
.
now
(),
datetime
.
now
(),
datetime
.
now
(),
-
1
,
os
.
stat
(
coursepath
+
'/'
+
f
).
st_size
)
query
(
'INSERT INTO sortlog (lecture_id,video_id,path,`when`) VALUES (?,?,?,?)'
,
matches
[
0
][
'id'
],
video_id
,
c
[
'handle'
]
+
'/'
+
f
,
datetime
.
now
())
modify
(
'COMMIT'
)
# for debuging only
# else:
# d = []
# for m in matches:
# d.append(m['id'])
# print('failed',{"data":data,"path":f,"results":d})
except
Exception
:
pass
if
'ref'
in
request
.
values
:
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment