Commit 53be1719 authored by Julian Rother's avatar Julian Rother

Implemented autoencoding

parent ed056ea1
......@@ -14303,13 +14303,13 @@ INSERT INTO `formats` (`id`,`description`,`keywords`,`resolution`,`aspect`,`prio
INSERT INTO `formats` (`id`,`description`,`keywords`,`resolution`,`aspect`,`prio`,`player_prio`,`mimetype`) VALUES (1,'SDV','sd,pal','720x576','4:3',1,0,'video/mp4');
INSERT INTO `formats` (`id`,`description`,`keywords`,`resolution`,`aspect`,`prio`,`player_prio`,`mimetype`) VALUES (2,'SDV','ws,palws','720x576','16:9',1,0,'video/mp4');
INSERT INTO `formats` (`id`,`description`,`keywords`,`resolution`,`aspect`,`prio`,`player_prio`,`mimetype`) VALUES (3,'HDV','hdv','1440x1080','16:9',9,6,'video/mp4');
INSERT INTO `formats` (`id`,`description`,`keywords`,`resolution`,`aspect`,`prio`,`player_prio`,`mimetype`) VALUES (4,'1080p','1080p','1920x1080','16:9',10,7,'video/mp4');
INSERT INTO `formats` (`id`,`description`,`keywords`,`resolution`,`aspect`,`prio`,`player_prio`,`mimetype`) VALUES (5,'720p','720p','1280x720','16:9',5,10,'video/mp4');
INSERT INTO `formats` (`id`,`description`,`keywords`,`resolution`,`aspect`,`prio`,`player_prio`,`mimetype`,`options`,`suffix`) VALUES (4,'1080p','1080p','1920x1080','16:9',10,7,'video/mp4','{"format": "mp4", "options": {"movflags": "faststart"}, "streams": [{"name": "video", "codec": "libx264", "options": {"b": 2000000, "g": 125, "profile": "high", "preset": "veryslow"}, "filters": ["scale=-1:1080,fps=fps=25"]}, {"name": "audio", "codec": "aac", "options": {"b": 128000}, "filters": ["aformat=sample_rates=48000Hz"]}]}','-1080p.mp4');
INSERT INTO `formats` (`id`,`description`,`keywords`,`resolution`,`aspect`,`prio`,`player_prio`,`mimetype`,`options`,`suffix`) VALUES (5,'720p','720p','1280x720','16:9',5,10,'video/mp4','{"format": "mp4", "options": {"movflags": "faststart"}, "streams": [{"name": "video", "codec": "libx264", "options": {"b": 1000000, "g": 125, "profile": "main", "preset": "veryslow"}, "filters": ["scale=-1:720,fps=fps=25"]}, {"name": "audio", "codec": "aac", "options": {"b": 128000}, "filters": ["aformat=sample_rates=48000Hz"]}]}','-720p.mp4');
INSERT INTO `formats` (`id`,`description`,`keywords`,`resolution`,`aspect`,`prio`,`player_prio`,`mimetype`) VALUES (6,'640x512','','640x512','16:9',-1,0,'video/mp4');
INSERT INTO `formats` (`id`,`description`,`keywords`,`resolution`,`aspect`,`prio`,`player_prio`,`mimetype`) VALUES (7,'MP3','mp3','','',-10,0,'video/mp4');
INSERT INTO `formats` (`id`,`description`,`keywords`,`resolution`,`aspect`,`prio`,`player_prio`,`mimetype`) VALUES (8,'720p F4V','','1280x720','16:9',4,10,'video/mp4');
INSERT INTO `formats` (`id`,`description`,`keywords`,`resolution`,`aspect`,`prio`,`player_prio`,`mimetype`) VALUES (9,'480p','ipod,480p','640x480','4:3',2,8,'video/mp4');
INSERT INTO `formats` (`id`,`description`,`keywords`,`resolution`,`aspect`,`prio`,`player_prio`,`mimetype`) VALUES (10,'360p','ipod,360p','640x360','16:9',3,8,'video/mp4');
INSERT INTO `formats` (`id`,`description`,`keywords`,`resolution`,`aspect`,`prio`,`player_prio`,`mimetype`,`options`,`suffix`) VALUES (10,'360p','ipod,360p','640x360','16:9',3,8,'video/mp4','{"format": "mp4", "options": {"movflags": "faststart"}, "streams": [{"name": "video", "codec": "libx264", "options": {"b": 500000, "g": 125, "profile": "baseline", "preset": "veryslow"}, "filters": ["scale=-1:360,fps=fps=25"]}, {"name": "audio", "codec": "aac", "options": {"b": 128000}, "filters": ["aformat=sample_rates=48000Hz"]}]}','-360p.mp4');
INSERT INTO `formats` (`id`,`description`,`keywords`,`resolution`,`aspect`,`prio`,`player_prio`,`mimetype`) VALUES (11,'AAC','','','',-2,0,'video/mp4');
INSERT INTO `formats` (`id`,`description`,`keywords`,`resolution`,`aspect`,`prio`,`player_prio`,`mimetype`) VALUES (12,'270p','ipodsmall,270p','480x270','16:9',1,3,'video/mp4');
INSERT INTO `formats` (`id`,`description`,`keywords`,`resolution`,`aspect`,`prio`,`player_prio`,`mimetype`) VALUES (13,'Screencast','','1024x768','4:3',4,5,'video/mp4');
......@@ -14643,4 +14643,7 @@ INSERT INTO `areas` (`area`,`abbreviation`,`default`,`rank`,`coordinates`) VALUE
INSERT INTO `areas` (`area`,`abbreviation`,`default`,`rank`,`coordinates`) VALUES ('Melaten','Melaten',1,1,'50.7819866,6.0486125');
INSERT INTO `areas` (`area`,`abbreviation`,`default`,`rank`,`coordinates`) VALUES ('Posthof','Posthof',0,NULL,'50.7702924,6.086709');
INSERT INTO `areas` (`area`,`abbreviation`,`default`,`rank`,`coordinates`) VALUES ('Templergraben','Templer',1,4,'50.7777117,6.0756922');
INSERT INTO `profiles` (`name`,`format`) VALUES ('default',4);
INSERT INTO `profiles` (`name`,`format`) VALUES ('default',5);
INSERT INTO `profiles` (`name`,`format`) VALUES ('default',10);
COMMIT;
......@@ -83,7 +83,8 @@ CREATE TABLE IF NOT EXISTS `formats` (
`prio` INTEGER NOT NULL DEFAULT '0',
`player_prio` INTEGER NOT NULL DEFAULT '0',
`mimetype` varchar(32) NOT NULL,
`options` text
`options` text,
`suffix` varchar(32)
);
CREATE TABLE IF NOT EXISTS `lectures_data` (
`id` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
......@@ -138,8 +139,9 @@ CREATE TABLE IF NOT EXISTS `sources` (
`id` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
`lecture_id` INTEGER NOT NULL,
`path` text NOT NULL,
`profile` text NOT NULL,
`hash` varchar(32) NOT NULL
`type` text NOT NULL,
`hash` varchar(32) NOT NULL,
`time_created` datetime NOT NULL
);
CREATE TABLE IF NOT EXISTS `log` (
`id` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
......
from server import *
import os.path
def schedule_remux(lectureid):
lecture = query('SELECT * FROM lectures WHERE id = ?', lectureid)[0]
course = query('SELECT * FROM courses WHERE id = ?', lecture['course_id'])[0]
chapters = query('SELECT text, time FROM chapters WHERE lecture_id = ?', lectureid)
videos = query('SELECT videos.*, sources.path AS srcpath, sources.hash AS srchash FROM videos JOIN sources ON videos.source = sources.id WHERE videos.lecture_id = ?', lectureid)
def set_metadata(dest, course, lecture):
chapters = query('SELECT text, time FROM chapters WHERE lecture_id = ? AND visible', lecture['id'])
metadata = {'title': lecture['title'], 'album': course['title'],
'description': lecture['comment'],
'date': lecture['time'].strftime('%m/%d/%Y'),
'artist': lecture['speaker'] if lecture['speaker'] else course['organizer']}
dest['metadata'] = metadata
dest['chapters'] = chapters
def schedule_remux(lectureid):
lecture = query('SELECT * FROM lectures WHERE id = ?', lectureid)[0]
course = query('SELECT * FROM courses WHERE id = ?', lecture['course_id'])[0]
videos = query('''SELECT videos.*, sources.path AS srcpath, sources.hash AS srchash, formats.options AS fmtopts
FROM videos
JOIN sources ON videos.source = sources.id
JOIN formats ON videos.video_format = formats.id
WHERE videos.lecture_id = ?''', lectureid)
for video in videos:
if not video['source']:
continue
data = {'video_id': video['id'], 'path': video['path'],
'srcpath': video['srcpath'], 'srchash': video['srchash'],
'chapters': chapters, 'metadata': metadata}
data = {'path': video['path'], 'srcpath': video['srcpath'], 'srchash': video['srchash']}
fmt = json.loads(video['fmtopts'])
if 'format' in fmt:
data['format'] = fmt['format']
data['options'] = fmt.get('options', {})
set_metadata(data, course, lecture)
schedule_job('remux', data)
def schedule_transcode(source, fmt_id=None, video=None):
if video:
fmt_id = video['video_format']
assert(video['lecture_id'] == source['lecture_id'])
assert(fmt_id != None)
fmt = query('SELECT * FROM formats WHERE id = ?', fmt_id)[0]
lecture = query('SELECT * FROM lectures WHERE id = ?', source['lecture_id'])[0]
course = query('SELECT * FROM courses WHERE id = ?', lecture['course_id'])[0]
data = {'input': {'path': source['path'], 'streams': []}, 'output': json.loads(fmt['options']), 'filters': []}
if source['type'] == 'plain':
stream = {'name': 'video', 'type': 'video'}
data['input']['streams'].append(stream)
stream = {'name': 'audio', 'type': 'audio'}
data['input']['streams'].append(stream)
else:
assert(False)
set_metadata(data['output'], course, lecture)
basename = os.path.basename(source['path']).rsplit('.', 1)[0]
data['output']['path'] = 'pub/'+course['handle']+'/'+basename+fmt['suffix']
if video:
old_source = query('SELECT * FROM sources WHERE id = ?', video['source'])
data['output']['path'] = video['path']
data['video_id'] = video['id']
data['srcpath'] = old_source['path']
data['srchash'] = old_source['hash']
else:
data['lecture_id'] = lecture['id']
data['format_id'] = fmt['id']
data['source_id'] = source['id']
schedule_job('transcode', data)
@job_handler('probe-raw')
def update_lecture_videos(jobid, jobtype, data, state, status):
if 'lecture_id' not in data:
return
if 'source_id' not in data:
modify('INSERT INTO sources (lecture_id, path, type, hash, time_created) VALUES (?, ?, ?, ?, ?)',
data['lecture_id'], data['path'], 'plain', status['hash'], datetime.now())
sources = query('SELECT * FROM sources WHERE sources.lecture_id = ? ORDER BY time_created', data['lecture_id'])
if not sources:
return
latest = sources[-1]
videos = query('SELECT * FROM videos WHERE videos.lecture_id = ?', data['lecture_id'])
current_fmts = [v['video_format'] for v in videos]
formats = query('''SELECT formats.* FROM formats
JOIN profiles ON formats.id = profiles.format
JOIN courses ON profiles.name = courses.profile
JOIN lectures ON courses.id = lectures.course_id
WHERE lectures.id = ?''', data['lecture_id'])
for fmt in formats:
if fmt['id'] not in current_fmts:
schedule_transcode(latest, fmt_id=fmt['id'])
for video in videos:
if video['source'] != latest['id']:
schedule_transcode(latest, video=video)
@edit_handler('chapters')
def chapter_changed(table, column, value, id):
print('chapter_changed')
......
from server import *
import traceback
import os.path
@app.route('/internal/sort/log')
@register_navbar('Sortierlog', icon='sort-by-attributes-alt')
......@@ -63,6 +64,21 @@ def schedule_thumbnail(lectureid, filePath=None):
data = '{"lectureid": "'+str(lectureid)+'", "path": "'+path+'"}'
query('INSERT INTO jobs (type, data, time_created) VALUES ("thumbnail", ?, ?)', data, datetime.now());
@job_handler('transcode')
def insert_transcoded_video(jobid, jobtype, data, state, status):
if 'lecture_id' not in data or 'source_id' not in data or 'format_id' not in data:
return
if 'video_id' in data:
return
video_id = modify('''INSERT INTO videos_data
(lecture_id, visible, path, video_format, title, comment, internal, file_modified, time_created, time_updated, created_by, hash, file_size, source)
VALUES
(?, 0, ?, ?, "", "", "", ?, ?, ?, ?, ?, ?, ?)''',
data['lecture_id'], data['output']['path'], data['format_id'],
datetime.now(), datetime.now(), datetime.now(), -1, status['hash'],
status['filesize'], data['source_id'])
schedule_thumbnail(data['lecture_id'])
def sort_file(filename, course=None, lectures=None):
# filenames: <handle>-<sorter>-<format>.mp4
# "sorter" musst be found with fuzzy matching. "sorter" musst be one or more of the following types: (inside the loop)
......@@ -176,6 +192,19 @@ def sort_encoded(filename):
schedule_job('publish_video', {'source': filename, 'path': 'pub/'+course['handle']+'/'+filename, 'lecture_id': lecture['id'], 'format_id': fmt})
return 'OK', 200
@app.route('/internal/sort/autoencode')
@sort_api_token_required
def sort_autoencode():
filename = request.values['path']
path = 'autoencode/'+filename
matches, fmt = sort_file(filename)
if len(matches) != 1:
log_sort_error(-1, 'raw/'+path, matches)
return "Could not match filename", 400
lecture = matches[0]
schedule_job('probe-raw', {'path': path, 'lecture_id': lecture['id'], 'import-chapters': True})
return 'OK', 200
@job_handler('publish_video')
def handle_published_video(jobid, jobtype, data, state, status):
if 'lecture_id' not in data or 'format_id' not in data:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment