Commit effd6235 authored by Andreas Valder's avatar Andreas Valder

Merge branch 'master' of git.fsmpi.rwth-aachen.de:videoagwebsite/videoagwebsite

parents 4be17653 336c92d7
This diff is collapsed.
......@@ -63,7 +63,10 @@ CREATE TABLE IF NOT EXISTS `courses_data` (
`responsible` text NOT NULL DEFAULT '',
`feed_url` text NOT NULL DEFAULT '',
`external` INTEGER NOT NULL DEFAULT 0,
`coursechapters` INTEGER NOT NULL DEFAULT 0
`coursechapters` INTEGER NOT NULL DEFAULT 0,
`autopublish` INTEGER NOT NULL DEFAULT 0,
`autovisible` INTEGER NOT NULL DEFAULT 0,
`profile` varchar(64) NOT NULL DEFAULT 'default'
);
CREATE TABLE IF NOT EXISTS `filesizes` (
`path` varchar(255) NOT NULL PRIMARY KEY,
......@@ -79,7 +82,8 @@ CREATE TABLE IF NOT EXISTS `formats` (
`aspect` varchar(16) NOT NULL,
`prio` INTEGER NOT NULL DEFAULT '0',
`player_prio` INTEGER NOT NULL DEFAULT '0',
`mimetype` varchar(32) NOT NULL
`mimetype` varchar(32) NOT NULL,
`options` text
);
CREATE TABLE IF NOT EXISTS `lectures_data` (
`id` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
......@@ -101,7 +105,8 @@ CREATE TABLE IF NOT EXISTS `lectures_data` (
`jumplist` text NOT NULL DEFAULT '',
`titlefile` varchar(255) NOT NULL DEFAULT '',
`live` INTEGER NOT NULL DEFAULT 0,
`norecording` INTEGER NOT NULL DEFAULT 0
`norecording` INTEGER NOT NULL DEFAULT 0,
`profile` varchar(64)
);
CREATE TABLE IF NOT EXISTS `places` (
`place` varchar(20) NOT NULL PRIMARY KEY,
......@@ -129,6 +134,13 @@ CREATE TABLE IF NOT EXISTS `site_texts` (
`modified_when` datetime NOT NULL,
`modified_by` text NOT NULL
);
CREATE TABLE IF NOT EXISTS `sources` (
`id` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
`lecture_id` INTEGER NOT NULL,
`path` text NOT NULL,
`profile` text NOT NULL,
`hash` varchar(32) NOT NULL
);
CREATE TABLE IF NOT EXISTS `log` (
`id` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
`time` datetime NOT NULL,
......@@ -151,6 +163,11 @@ CREATE TABLE IF NOT EXISTS `logcache` (
`date` datetime NOT NULL,
`value` INTEGER
);
CREATE TABLE IF NOT EXISTS `profiles` (
`id` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
`name` varchar(64) NOT NULL,
`format` INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS `streams` (
`handle` varchar(32) NOT NULL PRIMARY KEY,
`active` INTEGER NOT NULL,
......@@ -204,7 +221,8 @@ CREATE TABLE IF NOT EXISTS `videos_data` (
`created_by` INTEGER NOT NULL,
`file_size` bigINTEGER NOT NULL DEFAULT '-1',
`video_format` INTEGER NOT NULL,
`hash` varchar(32) NOT NULL
`hash` varchar(32) NOT NULL,
`source` INTEGER
);
CREATE TABLE IF NOT EXISTS `announcements` (
`id` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
......
......@@ -141,6 +141,10 @@ def edit(prefix='', ignore=[]):
VALUES (?,?,?,?,?,(SELECT `%s` FROM %s WHERE %s = ?),?,?,1)'%(path['column'], path['tableinfo']['table'], path['tableinfo']['idcolumn']),
path['table'], path['id'], path['tableinfo']['idcolumn'], path['column'], val, path['id'], datetime.now(), session['user']['dbid'])
modify('UPDATE %s SET `%s` = ?, time_updated = ? WHERE `%s` = ?'%(path['tableinfo']['table'], path['column'], path['tableinfo']['idcolumn']), val, datetime.now(),path['id'])
for func in edit_handlers.get(path['table'], {}).get(None, []):
func(path['table'], path['column'], val, path['id'])
for func in edit_handlers.get(path['table'], {}).get(path['column'], []):
func(path['table'], path['column'], val, path['id'])
if 'ref' in request.values:
return redirect(request.values['ref'])
return "OK", 200
......
from server import *
@job_handler('probe', 'probe-raw')
def import_xmp_chapters(jobid, jobtype, data, state, status):
if 'lecture_id' not in data and data.get('import-chapters', False):
return
times = set()
# Only add new chapters, deleted chapters are taken into account here
for chapter in query('SELECT * FROM chapters WHERE lecture_id = ?', data['lecture_id']):
for offset in range(5):
times.add(chapter['time']-offset)
times.add(chapter['time']+offset)
for chapter in status.get('xmp_chapters', []):
if int(chapter['time']) in times:
continue
modify('INSERT INTO chapters (lecture_id, time, text, visible, time_created, time_updated) VALUES (?, ?, ?, 0, ?, ?)',
data['lecture_id'], int(chapter['time']), chapter['text'],
datetime.now(), datetime.now())
@job_handler('probe', 'remux', 'transcode')
def update_video_metadata(jobid, jobtype, data, state, status):
if 'video_id' not in data:
return
if jobtype not in ['remux', 'transcode']:
video = query('SELECT * FROM videos WHERE id = ?', data['video_id'])[0]
if video['hash'] and video['hash'] != status['hash']:
print('Hash mismatch for video', data['video_id'])
return
modify('UPDATE videos_data SET hash = ?, file_size = ? WHERE id = ?',
status['hash'], status['filesize'], data['video_id'])
def schedule_remux(lectureid):
lecture = query('SELECT * FROM lectures WHERE id = ?', lectureid)[0]
course = query('SELECT * FROM courses WHERE id = ?', lecture['course_id'])[0]
chapters = query('SELECT text, time FROM chapters WHERE lecture_id = ?', lectureid)
videos = query('SELECT videos.*, sources.path AS srcpath, sources.hash AS srchash FROM videos JOIN sources ON videos.source = sources.id WHERE videos.lecture_id = ?', lectureid)
metadata = {'title': lecture['title'], 'album': course['title'],
'description': lecture['comment'],
'date': lecture['time'].strftime('%m/%d/%Y'),
'artist': lecture['speaker'] if lecture['speaker'] else course['organizer']}
for video in videos:
if not video['source']:
continue
data = {'video_id': video['id'], 'path': video['path'],
'srcpath': video['srcpath'], 'srchash': video['srchash'],
'chapters': chapters, 'metadata': metadata}
schedule_job('remux', data)
@edit_handler('chapters')
def chapter_changed(table, column, value, id):
print('chapter_changed')
chapters = query('SELECT * FROM chapters WHERE id = ?', id)
if not chapters:
return
chapter = chapters[0]
if column in ['visible', 'deleted'] or (chapter['visible'] and not chapter['deleted']):
schedule_remux(chapter['lecture_id'])
@edit_handler('courses')
def course_changed(table, column, value, id):
if column not in ['title', 'organizer']:
return
lectures = query('SELECT * FROM lectures WHERE course_id = ?', id)
for lecture in lectures:
schedule_remux(lecture['id'])
@edit_handler('lectures')
def lecture_changed(table, column, value, id):
if column in ['title', 'comment', 'time', 'speaker']:
schedule_remux(id)
......@@ -67,15 +67,12 @@ def jobs_api_token_required(func):
else:
token = None
if not token == config['JOBS_API_KEY']:
if not token == config.get('JOBS_API_KEY', [None]):
return 'Permission denied', 403
else:
return func(*args, **kwargs)
return decorator
def date_json_handler(obj):
return obj.isoformat() if hasattr(obj, 'isoformat') else obj
@sched_func(10)
def jobs_catch_broken():
# scheduled but never pinged
......@@ -103,13 +100,19 @@ def jobs_worker_ping(hostname):
@jobs_api_token_required
def jobs_ping(id):
hostname = request.values['host']
status = json.dumps(request.values['status'], default=date_json_handler)
status = json.dumps(json.loads(request.values['status']), default=date_json_handler)
state = request.values['state']
if state == 'finished':
query('UPDATE jobs SET time_finished = ?, status = ?, state = "finished" where id = ?', datetime.now(), status, id)
else:
query('UPDATE jobs SET worker = ?, last_ping = ?, status = ?, state = ? where id = ?', hostname, datetime.now(), status, state, id)
return 'OK',200
job = query('SELECT * FROM jobs WHERE id = ?', id)[0]
for func in job_handlers.get(job['type'], {}).get(state, []):
try:
func(id, job['type'], json.loads(job['data']), state, json.loads(job['status']))
except Exception:
traceback.print_exc()
return 'OK', 200
@app.route('/internal/jobs/api/worker/<hostname>/schedule', methods=['POST'])
@jobs_api_token_required
......
......@@ -461,16 +461,51 @@ def dbstatus():
clusters[cluster].append(host)
return render_template('dbstatus.html', clusters=clusters, statuses=status, vars=variables), 200
job_handlers = {}
def job_handler(*types, state='finished'):
def wrapper(func):
for jobtype in types:
if jobtype not in job_handlers:
job_handlers[jobtype] = {}
if jobtype not in job_handlers[jobtype]:
job_handlers[jobtype][state] = []
job_handlers[jobtype][state].append(func)
return func
return wrapper
def date_json_handler(obj):
return obj.isoformat() if hasattr(obj, 'isoformat') else obj
def schedule_job(jobtype, data=None, priority=0):
if not data:
data = {}
modify('INSERT INTO jobs (type, priority, data, time_created) VALUES (?, ?, ?, ?)',
jobtype, priority, json.dumps(data, default=date_json_handler), datetime.now())
edit_handlers = {}
def edit_handler(*tables, field=None):
def wrapper(func):
for table in tables:
if table not in edit_handlers:
edit_handlers[table] = {}
if field not in edit_handlers[table]:
edit_handlers[table][field] = []
edit_handlers[table][field].append(func)
return func
return wrapper
import edit
import feeds
import importer
import stats
import sorter
if 'ICAL_URL' in config:
import meetings
import l2pauth
if 'JOBS_API_KEY' in config:
import jobs
import jobs
import sorter
import encoding
import timetable
import chapters
import icalexport
......
......@@ -34,6 +34,7 @@ def insert_video(lectureid,dbfilepath,filepath,fileformatid):
lectureid, dbfilepath, fileformatid, datetime.now(), datetime.now(), datetime.now(), -1, os.stat(filepath).st_size)
query('INSERT INTO sortlog (lecture_id,video_id,path,`when`) VALUES (?,?,?,?)', lectureid, video_id, dbfilepath, datetime.now())
schedule_thumbnail(lectureid)
schedule_job('probe', {'path': dbfilepath, 'lecture_id': lectureid, 'video_id': video_id, 'import-chapters': True})
def schedule_thumbnail(lectureid, filePath=None):
videos = query('''
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment