From f841a3a6a6fe191cde00471ea9759c3f16de2105 Mon Sep 17 00:00:00 2001
From: Julian Rother <julianr@fsmpi.rwth-aachen.de>
Date: Sun, 19 Feb 2017 00:55:22 +0100
Subject: [PATCH] Optimized time-based stats
---
db_schema.sql | 1 +
server.py | 2 +-
stats.py | 67 ++++++++++++++++++------------------------
templates/course.html | 2 +-
templates/lecture.html | 2 +-
templates/macros.html | 25 ++++++++++++----
templates/stats.html | 4 ++-
7 files changed, 54 insertions(+), 49 deletions(-)
diff --git a/db_schema.sql b/db_schema.sql
index f85b8c5..27af157 100644
--- a/db_schema.sql
+++ b/db_schema.sql
@@ -128,6 +128,7 @@ CREATE TABLE IF NOT EXISTS `log` (
`ip` varchar(64),
`id` varchar(64),
`time` datetime NOT NULL,
+ `date` datetime NOT NULL,
`source` varchar(8),
`object` varchar(10),
`obj_id` INTEGER,
diff --git a/server.py b/server.py
index 9bd4889..1f04049 100644
--- a/server.py
+++ b/server.py
@@ -552,7 +552,7 @@ def auth(): # For use with nginx auth_request
password = auth.password
if checkperm(perms, username=username, password=password):
try:
- modify('INSERT INTO log (ip, id, `time`, object, obj_id, path) VALUES (?, ?, ?, "video", ?, ?)', ip, cookie, datetime.now(), perms[0]['vid'], url)
+ modify('INSERT INTO log (ip, id, `time`, `date`, object, obj_id, path) VALUES (?, ?, ?, "video", ?, ?)', ip, cookie, datetime.now(), datetime.combine(date.today(), time()), perms[0]['vid'], url)
except:
pass
r = make_response('OK', 200)
diff --git a/stats.py b/stats.py
index 0f188af..e0b3e4e 100644
--- a/stats.py
+++ b/stats.py
@@ -36,46 +36,35 @@ def stats_generic(req, param=None):
res['y'].append(row['y'])
return Response(json.dumps(res, default=plotly_date_handler), mimetype='application/json')
-@app.route('/stats/viewsperday')
-@mod_required
-def stats_viewsperday():
- start = None
- end = None
- courseid = request.args.get('course', None)
- lectureid = request.args.get('lecture', None)
- videoid = request.args.get('video', None)
- rows = query('''SELECT log.id AS id, log.time AS time, formats.description AS fmt
- FROM log
- JOIN videos ON (videos.id = log.obj_id)
- JOIN lectures ON (lectures.id = videos.lecture_id)
- JOIN courses ON (courses.id = lectures.course_id)
- JOIN formats ON (formats.id = videos.video_format)
- WHERE (? OR courses.id = ?) AND (? OR lectures.id = ?) AND (? OR videos.id = ?)''',
- courseid == None, courseid, lectureid == None, lectureid, videoid == None, videoid)
+@app.route('/stats/viewsperday/<req>')
+@app.route('/stats/viewsperday/<req>/<param>')
+def stats_viewsperday(req, param=None):
+ queries = {
+ 'lecture': '(SELECT log.date AS date, formats.description AS trace, COUNT(DISTINCT log.id) AS y FROM log JOIN videos ON videos.id = log.obj_id JOIN formats ON formats.id = videos.video_format WHERE videos.lecture_id = ? GROUP BY log.date, videos.video_format) UNION (SELECT log.date AS date, "total" AS trace, COUNT(DISTINCT log.id) AS y FROM log JOIN videos ON videos.id = log.obj_id WHERE videos.lecture_id = ? GROUP BY log.date)',
+ 'course': '(SELECT log.date AS date, formats.description AS trace, COUNT(DISTINCT log.id) AS y FROM log JOIN videos ON videos.id = log.obj_id JOIN lectures ON lectures.id = videos.lecture_id JOIN formats ON formats.id = videos.video_format WHERE lectures.course_id = ? GROUP BY log.date, videos.video_format) UNION (SELECT log.date AS date, "total" AS trace, COUNT(DISTINCT log.id) AS y FROM log JOIN videos ON videos.id = log.obj_id JOIN lectures ON lectures.id = videos.lecture_id WHERE lectures.course_id = ? GROUP BY log.date)',
+ 'global': '(SELECT log.date AS date, formats.description AS trace, COUNT(DISTINCT log.id) AS y FROM log JOIN videos ON videos.id = log.obj_id JOIN formats ON formats.id = videos.video_format GROUP BY log.date, videos.video_format) UNION (SELECT log.date AS date, "total" AS trace, COUNT(DISTINCT log.id) AS y FROM log GROUP BY log.date)',
+ 'courses': 'SELECT log.date AS date, courses.handle AS trace, COUNT(DISTINCT log.id) AS y FROM log JOIN videos ON videos.id = log.obj_id JOIN lectures ON lectures.id = videos.lecture_id JOIN courses ON courses.id = lectures.course_id GROUP BY log.date, courses.id'
+ }
+ if req not in queries:
+ return 404, 'Not found!'
+ rows = query(queries[req], *(queries[req].count('?')*[param]))
+ traces = set()
data = {}
- if not end:
- end = date.today()
- first = None
- fmts = ['total']
- for e in rows:
- key = e['time'].date()
- if not first or key < first:
- first = key
- if key not in data:
- data[key] = {'total': set()}
- if e['fmt'] not in data[key]:
- data[key][e['fmt']] = set()
- data[key]['total'].add(e['id'])
- data[key][e['fmt']].add(e['id'])
- if e['fmt'] not in fmts:
- fmts.append(e['fmt'])
- if not start:
- start = first
- res = {'times': [], 'views': [{'name': name, 'vals': []} for name in fmts]}
+ start = None
+ for row in rows:
+ if not start or row['date'] < start:
+ start = row['date']
+ traces.add(row['trace'])
+ if row['date'] not in data:
+ data[row['date']] = {}
+ data[row['date']][row['trace']] = row['y']
+ end = date.today()
+ x = []
+ y = [{'name': trace, 'vals': []} for trace in traces]
while start and start <= end:
- res['times'].append(start.isoformat())
- for d in res['views']:
- d['vals'].append(len(data.get(start, {}).get(d['name'], set())))
+ x.append(start)
+ for trace in y:
+ trace['vals'].append(data.get(start, {}).get(trace['name'], 0))
start += timedelta(days=1)
- return Response(json.dumps(res, default=date_json_handler), mimetype='application/json')
+ return Response(json.dumps({'times': x, 'views': y}, default=plotly_date_handler), mimetype='application/json')
diff --git a/templates/course.html b/templates/course.html
index 5cb4f7c..0b02b86 100644
--- a/templates/course.html
+++ b/templates/course.html
@@ -48,7 +48,7 @@
</table>
</div>
<div id="statview" class="col-xs-6" style="height:600px"></div>
- {{stats_viewsperday("statview", "Abrufe pro Tag", course=course.id)}}
+ {{stats_viewsperday("statview", "course", "Abrufer pro Tag", param=course.id)}}
<div id="lecture_views" class="col-xs-6" style="height:600px"></div>
{{stats_generic("lecture_views", "lecture_views", "Abrufe pro Aufnahme", param=course.id, type="bar")}}
{% endif %}
diff --git a/templates/lecture.html b/templates/lecture.html
index b7589bb..d7705bf 100644
--- a/templates/lecture.html
+++ b/templates/lecture.html
@@ -60,7 +60,7 @@
</table>
</div>
<div id="statview" class="col-xs-12" style="height:600px"></div>
- {{stats_viewsperday("statview", "Abrufe pro Tag", lecture=lecture.id)}}
+ {{stats_viewsperday("statview", "lecture", "Abrufer pro Tag", param=lecture.id)}}
{% endif %}
</div>
</div>
diff --git a/templates/macros.html b/templates/macros.html
index 757ce31..8550225 100644
--- a/templates/macros.html
+++ b/templates/macros.html
@@ -246,20 +246,33 @@ $('#embedcodebtn').popover(
{% macro vtttime(time) %}{{ '%02d:%02d:%02d.000'|format( time//3600, (time//60)%60, time%60) }}{% endmacro %}
-{% macro stats_viewsperday(id, title, type="scatter", course=None, lecture=None, video=None) %}
+{% macro stats_viewsperday(id, req, title, type="scatter", param=None, maxtraces=7) %}
<script>
$.ajax({
method: "GET",
- url: "{{url_for('stats_viewsperday', course=course, lecture=lecture, video=video)}}",
+ url: "{{url_for('stats_viewsperday', req=req, param=param)}}",
dataType: "json",
error: moderator.api.handleapierror,
success: function (data) {
var traces = [];
- for (var i = 1; i < data.views.length; i++) {
- traces.push({"x": data.times, "y": data.views[i].vals, "type": "{{type}}", "name": data.views[i].name, line: {"width": 1}});
+ for (var i = 0; i < data.views.length; i++) {
+ if (data.views[i].name == 'total' && data.views.length > 2)
+ traces.push({"x": data.times, "y": data.views[i].vals, "type": "{{type}}", "name": "gesamt", line: {"color": "black", "width": 2}});
+ else
+ traces.push({"x": data.times, "y": data.views[i].vals, "type": "{{type}}", "name": data.views[i].name, line: {"width": 1}});
}
- if (data.views.length > 2)
- traces.push({"x": data.times, "y": data.views[0].vals, "type": "{{type}}", "name": "gesamt", line: {"color": "black", "width": 2}});
+ traces.sort(function (a, b) {
+ amax = 0;
+ bmax = 0;
+ for (var i = 0; i < a.y.length; i++)
+ amax = Math.max(amax, a.y[i]);
+ for (var i = 0; i < b.y.length; i++)
+ bmax = Math.max(bmax, b.y[i]);
+ return bmax-amax;
+ });
+ for (var i = 0; i < traces.length; i++)
+ if (i > {{maxtraces}})
+ traces[i].visible = "legendonly";
var layout = {
"title": "{{title}}",
"showlegend": (traces.length != 1)
diff --git a/templates/stats.html b/templates/stats.html
index b96585a..eafcd20 100644
--- a/templates/stats.html
+++ b/templates/stats.html
@@ -27,7 +27,9 @@
<div id="categories_lectures" class="col-xs-6" style="height:600px"></div>
{{stats_pie("categories_lectures", "categories_lectures", "Aufnahmen nach Kategorie", showlegend=False)}}
<div id="statview" class="col-xs-12" style="height:600px"></div>
- {{stats_viewsperday("statview", "Abrufe pro Tag")}}
+ {{stats_viewsperday("statview", "global", "Abrufer pro Tag")}}
+ <div id="courseviewsperday" class="col-xs-12" style="height:600px"></div>
+ {{stats_viewsperday("courseviewsperday", "courses", "Abrufer pro Tag")}}
</div>
i </div>
</div>
--
GitLab