From 989a7eaee5f5f46143d79d16e5fa786efbf99d83 Mon Sep 17 00:00:00 2001
From: Andreas <andreasv@fsmpi.rwth-aachen.de>
Date: Thu, 3 Nov 2016 15:01:00 +0100
Subject: [PATCH] added basics for a jobs api
---
db_schema.sql | 18 ++++++++
jobs.py | 50 ++++++++++++++++++---
server.py | 1 -
sorter.py | 18 ++++++++
templates/jobs_overview.html | 87 ++++++++++++++++++++++++++++++++++++
5 files changed, 166 insertions(+), 8 deletions(-)
create mode 100644 templates/jobs_overview.html
diff --git a/db_schema.sql b/db_schema.sql
index 43ad82e..c1570e4 100644
--- a/db_schema.sql
+++ b/db_schema.sql
@@ -230,6 +230,24 @@ CREATE TABLE IF NOT EXISTS `sorterrorlog_data` (
`time_created` datetime NOT NULL
);
+CREATE TABLE IF NOT EXISTS `worker` (
+`hostname` text NOT NULL PRIMARY KEY,
+ `last_ping` datetime NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS `jobs` (
+`id` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+ `type` text NOT NULL,
+ `priority` INTEGER NOT NULL DEFAULT 0,
+ `state` text NOT NULL DEFAULT 'ready',
+ `time_finished` datetime DEFAULT '',
+ `time_created` datetime NOT NULL,
+ `last_ping` datetime NOT NULL DEFAULT '',
+ `worker` text DEFAULT NULL,
+ `data` text NOT NULL DEFAULT '{}',
+ `status` text NOT NULL DEFAULT '{}'
+);
+
CREATE VIEW IF NOT EXISTS `courses` AS select * from `courses_data` where (not(`courses_data`.`deleted`));
CREATE VIEW IF NOT EXISTS `lectures` AS select `lectures_data`.* from `lectures_data` join `courses_data` on (`courses_data`.`id` = `course_id`) where (not(`lectures_data`.`deleted` or `courses_data`.`deleted`));
CREATE VIEW IF NOT EXISTS `videos` AS select `videos_data`.* from `videos_data` join `lectures_data` on (`lectures_data`.`id` = `lecture_id`) join `courses_data` on (`courses_data`.`id` = `course_id`) where (not(`videos_data`.`deleted` or `lectures_data`.`deleted` or `courses_data`.`deleted`));
diff --git a/jobs.py b/jobs.py
index 8fea7a8..c5d6442 100644
--- a/jobs.py
+++ b/jobs.py
@@ -1,15 +1,51 @@
from server import *
import traceback
+import json
@app.route('/jobs/overview')
@register_navbar('Jobs', iconlib='fa', icon='suitcase')
@mod_required
def jobs_overview():
- # todo
- return render_template('base.html')
+ worker = query('SELECT * FROM worker ORDER BY last_ping DESC')
+ jobs = query('SELECT * FROM jobs')
+ return render_template('jobs_overview.html',worker=worker,jobs=jobs)
-@app.route('/jobs/api', methods=['GET', 'POST'])
-@mod_required
-def jobs_api():
- # todo
- return render_template('base.html')
+def jobs_api_token_required(func):
+ @wraps(func)
+ def decorator(*args, **kwargs):
+ if 'apikey' in request.values:
+ token = request.values['apikey']
+ elif request.get_json() and ('apikey' in request.get_json()):
+ token = request.get_json()['apikey']
+ else:
+ token = None
+
+ if not token == config['JOBS_API_KEY']:
+ return 'Permission denied', 403
+ else:
+ return func(*args, **kwargs)
+ return decorator
+
+def date_json_handler(obj):
+ return obj.isoformat() if hasattr(obj, 'isoformat') else obj
+
+@app.route('/jobs/api/ping', methods=['GET', 'POST'])
+@jobs_api_token_required
+def jobs_ping():
+ hostname = request.values['host']
+ query('INSERT OR REPLACE INTO worker (hostname, last_ping) values (?, ?)',hostname,datetime.now())
+ return 'OK',200
+
+@app.route('/jobs/api/schedule', methods=['POST'])
+@jobs_api_token_required
+def jobs_schedule():
+ hostdata = request.get_json()
+ print(hostdata)
+ if not hostdata:
+ return 'no data', 500
+ jobtypes = hostdata['jobtypes'] if 'jobtypes' in hostdata else []
+ for i in query('SELECT * FROM jobs WHERE state = "ready" ORDER BY priority DESC'):
+ if i['type'] in hostdata['jobtypes'].split(','):
+ job=i
+ break
+ return Response(json.dumps(job, default=date_json_handler), mimetype='application/json')
diff --git a/server.py b/server.py
index 1bf0958..62918f7 100644
--- a/server.py
+++ b/server.py
@@ -10,7 +10,6 @@ import random
import sched
import traceback
import string
-import traceback
app = Flask(__name__)
diff --git a/sorter.py b/sorter.py
index f7473b1..b17ccb4 100644
--- a/sorter.py
+++ b/sorter.py
@@ -33,7 +33,25 @@ def insert_video(lectureid,dbfilepath,filepath,fileformatid):
(?,0,?,?,"","","",?,?,?,?,"",?)''',
lectureid, dbfilepath, fileformatid, datetime.now(), datetime.now(), datetime.now(), -1, os.stat(filepath).st_size)
query('INSERT INTO sortlog (lecture_id,video_id,path,`when`) VALUES (?,?,?,?)', lectureid, video_id, dbfilepath, datetime.now())
+ schedule_thumbnail(lectureid)
+def schedule_thumbnail(lectureid):
+ exists = query('SELECT * FROM jobs WHERE data LIKE ?','%"lectureid": "'+str(lectureid)+'"%' )
+ if exists:
+ return
+ path = query('SELECT path FROM videos WHERE lecture_id')
+ videos = query('''
+ SELECT videos.path
+ FROM videos
+ JOIN formats ON (videos.video_format = formats.id)
+ WHERE videos.lecture_id = ?
+ ORDER BY formats.prio DESC''', lectureid )
+ if videos:
+ path = videos[0]['path']
+ else:
+ return
+ data = '{"lectureid": "'+str(lectureid)+'", "path": "'+path+'"}'
+ query('INSERT INTO jobs (type, data, time_created) VALUES ("thumbnail", ?, ?)', data, datetime.now());
@app.route('/sort/now')
@mod_required
diff --git a/templates/jobs_overview.html b/templates/jobs_overview.html
new file mode 100644
index 0000000..a5af33b
--- /dev/null
+++ b/templates/jobs_overview.html
@@ -0,0 +1,87 @@
+{% extends "base.html" %}
+{% block content %}
+<div class="panel-group">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ <h1 class="panel-title">
+ Worker
+ </h1>
+ </div>
+ <div class="panel-collapse collapse in">
+ <div class="panel-body">
+ <table class="table">
+ <tr>
+ <th>Hostname</th>
+ <th>letzter Ping</th>
+ </tr>
+ {% for i in worker %}
+ {% set td = (datetime.now()-(i.last_ping) ) %}
+
+ {% if td < timedelta(seconds=10) %}
+ <tr class="success">
+ {% elif td < timedelta(seconds=30) %}
+ <tr class="warning">
+ {% else %}
+ <tr class="danger">
+ {% endif %}
+ <td>{{i.hostname}}</td>
+ <td>{{i.last_ping}}</td>
+ </tr>
+ {% endfor %}
+ </table>
+ </div>
+ </div>
+ </div>
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ <h1 class="panel-title">
+ Jobs
+ </h1>
+ </div>
+ <div class="panel-collapse collapse in">
+ <div class="panel-body">
+ <table class="table">
+ <tr>
+ <th>ID</th>
+ <th>Type</th>
+ <th>Priority</th>
+ <th>Worker</th>
+ <th>letzter Ping</th>
+ <th>State</th>
+ <th>Eingereiht am</th>
+ <th>Fertig geworden am</th>
+ <th>Daten</th>
+ </tr>
+ {% for i in jobs %}
+ {% if i.last_ping %}
+ {% set td = (datetime.now()-(i.last_ping) ) %}
+ {% else %}
+ {% set td = -1 %}
+ {% endif %}
+
+ {% if td == -1 %}
+ <tr>
+ {% elif td < timedelta(seconds=20) %}
+ <tr class="success">
+ {% elif td < timedelta(seconds=60) %}
+ <tr class="warning">
+ {% else %}
+ <tr class="danger">
+ {% endif %}
+ <td>{{i.id}}</td>
+ <td>{{i.type}}</td>
+ <td>{{i.priority}}</td>
+ <td>{{i.worker}}</td>
+ <td>{{i.last_ping}}</td>
+ <td>{{i.state}}</td>
+ <td>{{i.time_created}}</td>
+ <td>{{i.time_finished}}</td>
+ <td>{{i.data}}</td>
+ </tr>
+ {% endfor %}
+ </table>
+ </div>
+ </div>
+ </div>
+</div>
+{% endblock %}
--
GitLab