From d1959b89ebe444aebe618b8e1d7c297c86459767 Mon Sep 17 00:00:00 2001
From: Julian Rother <julianr@fsmpi.rwth-aachen.de>
Date: Sun, 19 Feb 2017 16:57:58 +0100
Subject: [PATCH] Introduced logcache table to speed up time-based stats
---
db_schema.sql | 7 +++++++
stats.py | 27 ++++++++++++++++++---------
2 files changed, 25 insertions(+), 9 deletions(-)
diff --git a/db_schema.sql b/db_schema.sql
index 27af157..ef074b6 100644
--- a/db_schema.sql
+++ b/db_schema.sql
@@ -134,6 +134,13 @@ CREATE TABLE IF NOT EXISTS `log` (
`obj_id` INTEGER,
`path` varchar(255) NOT NULL
);
+CREATE TABLE IF NOT EXISTS `logcache` (
+ `req` varchar(64),
+ `param` varchar(64),
+ `trace` varchar(64),
+ `date` datetime NOT NULL,
+ `value` INTEGER
+);
CREATE TABLE IF NOT EXISTS `streams` (
`handle` varchar(32) NOT NULL PRIMARY KEY,
`active` INTEGER NOT NULL,
diff --git a/stats.py b/stats.py
index e0b3e4e..0265bf1 100644
--- a/stats.py
+++ b/stats.py
@@ -1,6 +1,7 @@
from server import *
import json
from jobs import date_json_handler
+from hashlib import md5
@app.route('/stats')
@register_navbar('Statistiken', icon='stats')
@@ -38,19 +39,27 @@ def stats_generic(req, param=None):
@app.route('/stats/viewsperday/<req>')
@app.route('/stats/viewsperday/<req>/<param>')
-def stats_viewsperday(req, param=None):
+def stats_viewsperday(req, param=""):
+ update_expr = 'INSERT INTO logcache (req, param, trace, date, value) (SELECT "%s", ?, trace, date, y FROM (%s) AS cachetmp WHERE date < ?)'
+ query_expr = '(SELECT date, trace, value AS y FROM logcache WHERE req = "%s" AND param = ?) UNION (SELECT * FROM (%s) AS cachetmp)'
+ date_subexpr = '(SELECT CASE WHEN MAX(date) IS NULL THEN "2000-00-00" ELSE MAX(date) END FROM `logcache` WHERE req = "%s" AND param = ?)'
queries = {
- 'lecture': '(SELECT log.date AS date, formats.description AS trace, COUNT(DISTINCT log.id) AS y FROM log JOIN videos ON videos.id = log.obj_id JOIN formats ON formats.id = videos.video_format WHERE videos.lecture_id = ? GROUP BY log.date, videos.video_format) UNION (SELECT log.date AS date, "total" AS trace, COUNT(DISTINCT log.id) AS y FROM log JOIN videos ON videos.id = log.obj_id WHERE videos.lecture_id = ? GROUP BY log.date)',
- 'course': '(SELECT log.date AS date, formats.description AS trace, COUNT(DISTINCT log.id) AS y FROM log JOIN videos ON videos.id = log.obj_id JOIN lectures ON lectures.id = videos.lecture_id JOIN formats ON formats.id = videos.video_format WHERE lectures.course_id = ? GROUP BY log.date, videos.video_format) UNION (SELECT log.date AS date, "total" AS trace, COUNT(DISTINCT log.id) AS y FROM log JOIN videos ON videos.id = log.obj_id JOIN lectures ON lectures.id = videos.lecture_id WHERE lectures.course_id = ? GROUP BY log.date)',
- 'global': '(SELECT log.date AS date, formats.description AS trace, COUNT(DISTINCT log.id) AS y FROM log JOIN videos ON videos.id = log.obj_id JOIN formats ON formats.id = videos.video_format GROUP BY log.date, videos.video_format) UNION (SELECT log.date AS date, "total" AS trace, COUNT(DISTINCT log.id) AS y FROM log GROUP BY log.date)',
- 'courses': 'SELECT log.date AS date, courses.handle AS trace, COUNT(DISTINCT log.id) AS y FROM log JOIN videos ON videos.id = log.obj_id JOIN lectures ON lectures.id = videos.lecture_id JOIN courses ON courses.id = lectures.course_id GROUP BY log.date, courses.id'
+ 'lecture': '(SELECT log.date AS date, formats.description AS trace, COUNT(DISTINCT log.id) AS y FROM log JOIN videos ON videos.id = log.obj_id JOIN formats ON formats.id = videos.video_format WHERE log.date > %T AND videos.lecture_id = ? GROUP BY log.date, videos.video_format) UNION (SELECT log.date AS date, "total" AS trace, COUNT(DISTINCT log.id) AS y FROM log JOIN videos ON videos.id = log.obj_id WHERE log.date > %T AND videos.lecture_id = ? GROUP BY log.date)',
+ 'course': '(SELECT log.date AS date, formats.description AS trace, COUNT(DISTINCT log.id) AS y FROM log JOIN videos ON videos.id = log.obj_id JOIN lectures ON lectures.id = videos.lecture_id JOIN formats ON formats.id = videos.video_format WHERE log.date > %T AND lectures.course_id = ? GROUP BY log.date, videos.video_format) UNION (SELECT log.date AS date, "total" AS trace, COUNT(DISTINCT log.id) AS y FROM log JOIN videos ON videos.id = log.obj_id JOIN lectures ON lectures.id = videos.lecture_id WHERE log.date > %T AND lectures.course_id = ? GROUP BY log.date)',
+ 'global': '(SELECT log.date AS date, formats.description AS trace, COUNT(DISTINCT log.id) AS y FROM log JOIN videos ON videos.id = log.obj_id JOIN formats ON formats.id = videos.video_format WHERE log.date > %T GROUP BY log.date, videos.video_format) UNION (SELECT log.date AS date, "total" AS trace, COUNT(DISTINCT log.id) AS y FROM log WHERE log.date > %T GROUP BY log.date)',
+ 'courses': 'SELECT log.date AS date, courses.handle AS trace, COUNT(DISTINCT log.id) AS y FROM log JOIN videos ON videos.id = log.obj_id JOIN lectures ON lectures.id = videos.lecture_id JOIN courses ON courses.id = lectures.course_id WHERE log.date > %T GROUP BY log.date, courses.id'
}
- if req not in queries:
- return 404, 'Not found!'
- rows = query(queries[req], *(queries[req].count('?')*[param]))
+ expr = queries[req]
+ expr = expr.replace('%T', date_subexpr%('viewsperday.'+req))
+ params = [param]*expr.count('?')
+ try:
+ modify(update_expr%('viewsperday.'+req, expr), param, *params, datetime.combine(date.today(), time()))
+ except Exception:
+ traceback.print_exc()
+ rows = query(query_expr%('viewsperday.'+req, expr), param, *params)
+ start = None
traces = set()
data = {}
- start = None
for row in rows:
if not start or row['date'] < start:
start = row['date']
--
GitLab