Skip to content
Snippets Groups Projects
Commit 53be1719 authored by Julian Rother's avatar Julian Rother
Browse files

Implemented autoencoding

parent ed056ea1
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
db_example.sql
+ 6
− 3
View file @ 53be1719
......@@ -14303,13 +14303,13 @@ INSERT INTO `formats` (`id`,`description`,`keywords`,`resolution`,`aspect`,`prio
INSERT INTO `formats` (`id`,`description`,`keywords`,`resolution`,`aspect`,`prio`,`player_prio`,`mimetype`) VALUES (1,'SDV','sd,pal','720x576','4:3',1,0,'video/mp4');
INSERT INTO `formats` (`id`,`description`,`keywords`,`resolution`,`aspect`,`prio`,`player_prio`,`mimetype`) VALUES (2,'SDV','ws,palws','720x576','16:9',1,0,'video/mp4');
INSERT INTO `formats` (`id`,`description`,`keywords`,`resolution`,`aspect`,`prio`,`player_prio`,`mimetype`) VALUES (3,'HDV','hdv','1440x1080','16:9',9,6,'video/mp4');
INSERT INTO `formats` (`id`,`description`,`keywords`,`resolution`,`aspect`,`prio`,`player_prio`,`mimetype`) VALUES (4,'1080p','1080p','1920x1080','16:9',10,7,'video/mp4');
INSERT INTO `formats` (`id`,`description`,`keywords`,`resolution`,`aspect`,`prio`,`player_prio`,`mimetype`) VALUES (5,'720p','720p','1280x720','16:9',5,10,'video/mp4');
INSERT INTO `formats` (`id`,`description`,`keywords`,`resolution`,`aspect`,`prio`,`player_prio`,`mimetype`,`options`,`suffix`) VALUES (4,'1080p','1080p','1920x1080','16:9',10,7,'video/mp4','{"format": "mp4", "options": {"movflags": "faststart"}, "streams": [{"name": "video", "codec": "libx264", "options": {"b": 2000000, "g": 125, "profile": "high", "preset": "veryslow"}, "filters": ["scale=-1:1080,fps=fps=25"]}, {"name": "audio", "codec": "aac", "options": {"b": 128000}, "filters": ["aformat=sample_rates=48000Hz"]}]}','-1080p.mp4');
INSERT INTO `formats` (`id`,`description`,`keywords`,`resolution`,`aspect`,`prio`,`player_prio`,`mimetype`,`options`,`suffix`) VALUES (5,'720p','720p','1280x720','16:9',5,10,'video/mp4','{"format": "mp4", "options": {"movflags": "faststart"}, "streams": [{"name": "video", "codec": "libx264", "options": {"b": 1000000, "g": 125, "profile": "main", "preset": "veryslow"}, "filters": ["scale=-1:720,fps=fps=25"]}, {"name": "audio", "codec": "aac", "options": {"b": 128000}, "filters": ["aformat=sample_rates=48000Hz"]}]}','-720p.mp4');
INSERT INTO `formats` (`id`,`description`,`keywords`,`resolution`,`aspect`,`prio`,`player_prio`,`mimetype`) VALUES (6,'640x512','','640x512','16:9',-1,0,'video/mp4');
INSERT INTO `formats` (`id`,`description`,`keywords`,`resolution`,`aspect`,`prio`,`player_prio`,`mimetype`) VALUES (7,'MP3','mp3','','',-10,0,'video/mp4');
INSERT INTO `formats` (`id`,`description`,`keywords`,`resolution`,`aspect`,`prio`,`player_prio`,`mimetype`) VALUES (8,'720p F4V','','1280x720','16:9',4,10,'video/mp4');
INSERT INTO `formats` (`id`,`description`,`keywords`,`resolution`,`aspect`,`prio`,`player_prio`,`mimetype`) VALUES (9,'480p','ipod,480p','640x480','4:3',2,8,'video/mp4');
INSERT INTO `formats` (`id`,`description`,`keywords`,`resolution`,`aspect`,`prio`,`player_prio`,`mimetype`) VALUES (10,'360p','ipod,360p','640x360','16:9',3,8,'video/mp4');
INSERT INTO `formats` (`id`,`description`,`keywords`,`resolution`,`aspect`,`prio`,`player_prio`,`mimetype`,`options`,`suffix`) VALUES (10,'360p','ipod,360p','640x360','16:9',3,8,'video/mp4','{"format": "mp4", "options": {"movflags": "faststart"}, "streams": [{"name": "video", "codec": "libx264", "options": {"b": 500000, "g": 125, "profile": "baseline", "preset": "veryslow"}, "filters": ["scale=-1:360,fps=fps=25"]}, {"name": "audio", "codec": "aac", "options": {"b": 128000}, "filters": ["aformat=sample_rates=48000Hz"]}]}','-360p.mp4');
INSERT INTO `formats` (`id`,`description`,`keywords`,`resolution`,`aspect`,`prio`,`player_prio`,`mimetype`) VALUES (11,'AAC','','','',-2,0,'video/mp4');
INSERT INTO `formats` (`id`,`description`,`keywords`,`resolution`,`aspect`,`prio`,`player_prio`,`mimetype`) VALUES (12,'270p','ipodsmall,270p','480x270','16:9',1,3,'video/mp4');
INSERT INTO `formats` (`id`,`description`,`keywords`,`resolution`,`aspect`,`prio`,`player_prio`,`mimetype`) VALUES (13,'Screencast','','1024x768','4:3',4,5,'video/mp4');
......@@ -14643,4 +14643,7 @@ INSERT INTO `areas` (`area`,`abbreviation`,`default`,`rank`,`coordinates`) VALUE
INSERT INTO `areas` (`area`,`abbreviation`,`default`,`rank`,`coordinates`) VALUES ('Melaten','Melaten',1,1,'50.7819866,6.0486125');
INSERT INTO `areas` (`area`,`abbreviation`,`default`,`rank`,`coordinates`) VALUES ('Posthof','Posthof',0,NULL,'50.7702924,6.086709');
INSERT INTO `areas` (`area`,`abbreviation`,`default`,`rank`,`coordinates`) VALUES ('Templergraben','Templer',1,4,'50.7777117,6.0756922');
INSERT INTO `profiles` (`name`,`format`) VALUES ('default',4);
INSERT INTO `profiles` (`name`,`format`) VALUES ('default',5);
INSERT INTO `profiles` (`name`,`format`) VALUES ('default',10);
COMMIT;
db_schema.sql
+ 5
− 3
View file @ 53be1719
......@@ -83,7 +83,8 @@ CREATE TABLE IF NOT EXISTS `formats` (
`prio` INTEGER NOT NULL DEFAULT '0',
`player_prio` INTEGER NOT NULL DEFAULT '0',
`mimetype` varchar(32) NOT NULL,
`options` text
`options` text,
`suffix` varchar(32)
);
CREATE TABLE IF NOT EXISTS `lectures_data` (
`id` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
......@@ -138,8 +139,9 @@ CREATE TABLE IF NOT EXISTS `sources` (
`id` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
`lecture_id` INTEGER NOT NULL,
`path` text NOT NULL,
`profile` text NOT NULL,
`hash` varchar(32) NOT NULL
`type` text NOT NULL,
`hash` varchar(32) NOT NULL,
`time_created` datetime NOT NULL
);
CREATE TABLE IF NOT EXISTS `log` (
`id` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
......
encoding.py
+ 76
− 8
View file @ 53be1719
from server import *
import os.path
def schedule_remux(lectureid):
lecture = query('SELECT * FROM lectures WHERE id = ?', lectureid)[0]
course = query('SELECT * FROM courses WHERE id = ?', lecture['course_id'])[0]
chapters = query('SELECT text, time FROM chapters WHERE lecture_id = ?', lectureid)
videos = query('SELECT videos.*, sources.path AS srcpath, sources.hash AS srchash FROM videos JOIN sources ON videos.source = sources.id WHERE videos.lecture_id = ?', lectureid)
def set_metadata(dest, course, lecture):
chapters = query('SELECT text, time FROM chapters WHERE lecture_id = ? AND visible', lecture['id'])
metadata = {'title': lecture['title'], 'album': course['title'],
'description': lecture['comment'],
'date': lecture['time'].strftime('%m/%d/%Y'),
'artist': lecture['speaker'] if lecture['speaker'] else course['organizer']}
dest['metadata'] = metadata
dest['chapters'] = chapters
def schedule_remux(lectureid):
lecture = query('SELECT * FROM lectures WHERE id = ?', lectureid)[0]
course = query('SELECT * FROM courses WHERE id = ?', lecture['course_id'])[0]
videos = query('''SELECT videos.*, sources.path AS srcpath, sources.hash AS srchash, formats.options AS fmtopts
FROM videos
JOIN sources ON videos.source = sources.id
JOIN formats ON videos.video_format = formats.id
WHERE videos.lecture_id = ?''', lectureid)
for video in videos:
if not video['source']:
continue
data = {'video_id': video['id'], 'path': video['path'],
'srcpath': video['srcpath'], 'srchash': video['srchash'],
'chapters': chapters, 'metadata': metadata}
data = {'path': video['path'], 'srcpath': video['srcpath'], 'srchash': video['srchash']}
fmt = json.loads(video['fmtopts'])
if 'format' in fmt:
data['format'] = fmt['format']
data['options'] = fmt.get('options', {})
set_metadata(data, course, lecture)
schedule_job('remux', data)
def schedule_transcode(source, fmt_id=None, video=None):
if video:
fmt_id = video['video_format']
assert(video['lecture_id'] == source['lecture_id'])
assert(fmt_id != None)
fmt = query('SELECT * FROM formats WHERE id = ?', fmt_id)[0]
lecture = query('SELECT * FROM lectures WHERE id = ?', source['lecture_id'])[0]
course = query('SELECT * FROM courses WHERE id = ?', lecture['course_id'])[0]
data = {'input': {'path': source['path'], 'streams': []}, 'output': json.loads(fmt['options']), 'filters': []}
if source['type'] == 'plain':
stream = {'name': 'video', 'type': 'video'}
data['input']['streams'].append(stream)
stream = {'name': 'audio', 'type': 'audio'}
data['input']['streams'].append(stream)
else:
assert(False)
set_metadata(data['output'], course, lecture)
basename = os.path.basename(source['path']).rsplit('.', 1)[0]
data['output']['path'] = 'pub/'+course['handle']+'/'+basename+fmt['suffix']
if video:
old_source = query('SELECT * FROM sources WHERE id = ?', video['source'])
data['output']['path'] = video['path']
data['video_id'] = video['id']
data['srcpath'] = old_source['path']
data['srchash'] = old_source['hash']
else:
data['lecture_id'] = lecture['id']
data['format_id'] = fmt['id']
data['source_id'] = source['id']
schedule_job('transcode', data)
@job_handler('probe-raw')
def update_lecture_videos(jobid, jobtype, data, state, status):
if 'lecture_id' not in data:
return
if 'source_id' not in data:
modify('INSERT INTO sources (lecture_id, path, type, hash, time_created) VALUES (?, ?, ?, ?, ?)',
data['lecture_id'], data['path'], 'plain', status['hash'], datetime.now())
sources = query('SELECT * FROM sources WHERE sources.lecture_id = ? ORDER BY time_created', data['lecture_id'])
if not sources:
return
latest = sources[-1]
videos = query('SELECT * FROM videos WHERE videos.lecture_id = ?', data['lecture_id'])
current_fmts = [v['video_format'] for v in videos]
formats = query('''SELECT formats.* FROM formats
JOIN profiles ON formats.id = profiles.format
JOIN courses ON profiles.name = courses.profile
JOIN lectures ON courses.id = lectures.course_id
WHERE lectures.id = ?''', data['lecture_id'])
for fmt in formats:
if fmt['id'] not in current_fmts:
schedule_transcode(latest, fmt_id=fmt['id'])
for video in videos:
if video['source'] != latest['id']:
schedule_transcode(latest, video=video)
@edit_handler('chapters')
def chapter_changed(table, column, value, id):
print('chapter_changed')
......
sorter.py
+ 29
− 0
View file @ 53be1719
from server import *
import traceback
import os.path
@app.route('/internal/sort/log')
@register_navbar('Sortierlog', icon='sort-by-attributes-alt')
......@@ -63,6 +64,21 @@ def schedule_thumbnail(lectureid, filePath=None):
data = '{"lectureid": "'+str(lectureid)+'", "path": "'+path+'"}'
query('INSERT INTO jobs (type, data, time_created) VALUES ("thumbnail", ?, ?)', data, datetime.now());
@job_handler('transcode')
def insert_transcoded_video(jobid, jobtype, data, state, status):
if 'lecture_id' not in data or 'source_id' not in data or 'format_id' not in data:
return
if 'video_id' in data:
return
video_id = modify('''INSERT INTO videos_data
(lecture_id, visible, path, video_format, title, comment, internal, file_modified, time_created, time_updated, created_by, hash, file_size, source)
VALUES
(?, 0, ?, ?, "", "", "", ?, ?, ?, ?, ?, ?, ?)''',
data['lecture_id'], data['output']['path'], data['format_id'],
datetime.now(), datetime.now(), datetime.now(), -1, status['hash'],
status['filesize'], data['source_id'])
schedule_thumbnail(data['lecture_id'])
def sort_file(filename, course=None, lectures=None):
# filenames: <handle>-<sorter>-<format>.mp4
# "sorter" musst be found with fuzzy matching. "sorter" musst be one or more of the following types: (inside the loop)
......@@ -176,6 +192,19 @@ def sort_encoded(filename):
schedule_job('publish_video', {'source': filename, 'path': 'pub/'+course['handle']+'/'+filename, 'lecture_id': lecture['id'], 'format_id': fmt})
return 'OK', 200
@app.route('/internal/sort/autoencode')
@sort_api_token_required
def sort_autoencode():
filename = request.values['path']
path = 'autoencode/'+filename
matches, fmt = sort_file(filename)
if len(matches) != 1:
log_sort_error(-1, 'raw/'+path, matches)
return "Could not match filename", 400
lecture = matches[0]
schedule_job('probe-raw', {'path': path, 'lecture_id': lecture['id'], 'import-chapters': True})
return 'OK', 200
@job_handler('publish_video')
def handle_published_video(jobid, jobtype, data, state, status):
if 'lecture_id' not in data or 'format_id' not in data:
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment