Merge branch 'master' of git.fsmpi.rwth-aachen.de:videoagwebsite/videoagwebsite

config.py.example

@@ -11,7 +11,7 @@ DB_DATA = 'db_example.sql'
 #DB_ENGINE = 'mysql'
 #MYSQL_HOST = 'localhost'
 #MYSQL_USER = 'videoag'
-#MYSQL_PORT = 3306
+MYSQL_PORT = 3306
 #MYSQL_PASSWD = 'somuchsecret'
 #MYSQL_DB = 'videos'
 

db_schema.sql

@@ -125,13 +125,18 @@ CREATE TABLE IF NOT EXISTS `site_texts` (
   `modified_by` text NOT NULL
 );
 CREATE TABLE IF NOT EXISTS `log` (
-	`ip` varchar(64),
-	`id` varchar(64),
+	`id` INTEGER,
 	`time` datetime NOT NULL,
-	`source` varchar(8),
-	`object` varchar(10),
-	`obj_id` INTEGER,
-	`path` varchar(255) NOT NULL
+	`date` datetime NOT NULL,
+	`source` INTEGER,
+	`video` INTEGER
+);
+CREATE TABLE IF NOT EXISTS `logcache` (
+	`req` varchar(64),
+	`param` varchar(64),
+	`trace` varchar(64),
+	`date` datetime NOT NULL,
+	`value` INTEGER
 );
 CREATE TABLE IF NOT EXISTS `streams` (
   `handle` varchar(32) NOT NULL PRIMARY KEY,

server.py

@@ -527,11 +527,10 @@ def auth(): # For use with nginx auth_request
 		return 'Internal Server Error', 500
 	url = request.headers['X-Original-Uri'].lstrip(config['VIDEOPREFIX'])
 	ip = request.headers.get('X-Real-IP', '')
-	if 'tracking' in request.cookies:
-		cookie = request.cookies['tracking']
+	if request.cookies.get('tracking', '') and request.cookies['tracking'].isdigit():
+		cookie = int(request.cookies['tracking'])
 	else:
-		cookie = ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(64))
-	#	r.set_cookie('tracking', request.cookies.get('tracking', ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(64))), max_age=2147483647)
+		cookie = random.getrandbits(8*8-1)
 	if url.endswith('jpg') or ismod():
 		return "OK", 200
 	perms = query('''SELECT videos.path, videos.id AS vid, perm.*
@@ -552,11 +551,11 @@ def auth(): # For use with nginx auth_request
 		password = auth.password
 	if checkperm(perms, username=username, password=password):
 		try:
-			modify('INSERT INTO log (ip, id, `time`, object, obj_id, path) VALUES (?, ?, ?, "video", ?, ?)', ip, cookie, datetime.now(), perms[0]['vid'], url)
+			modify('INSERT INTO log (id, `time`, `date`, video, source) VALUES (?, ?, ?, ?, 1)', cookie, datetime.now(), datetime.combine(date.today(), time()), perms[0]['vid'])
 		except:
 			pass
 		r = make_response('OK', 200)
-		r.set_cookie('tracking', cookie, max_age=2147483647) # Many many years
+		r.set_cookie('tracking', str(cookie), max_age=2147483647) # Many many years
 		return r
 	password_auth = False
 	for perm in perms:

stats.py

 from server import *
 import json
 from jobs import date_json_handler
+from hashlib import md5
 
 @app.route('/stats')
 @register_navbar('Statistiken', icon='stats')
@@ -9,13 +10,13 @@ def stats():
 	return render_template('stats.html')
 
 statsqueries = {}
-statsqueries['formats_views'] = "SELECT formats.description AS x, count(DISTINCT log.id) AS y FROM log JOIN videos ON (videos.id = log.obj_id) JOIN formats ON (formats.id = videos.video_format) GROUP BY formats.id"
+statsqueries['formats_views'] = "SELECT formats.description AS x, count(DISTINCT log.id) AS y FROM log JOIN videos ON (videos.id = log.video) JOIN formats ON (formats.id = videos.video_format) GROUP BY formats.id"
 statsqueries['course_count'] = "SELECT semester AS x, count(id) AS y FROM courses GROUP BY semester"
 statsqueries['lectures_count'] = "SELECT semester AS x, count(lectures.id) AS y FROM lectures JOIN courses ON (courses.id = lectures.course_id) GROUP BY semester"
 statsqueries['categories_courses'] = "SELECT courses.subject AS x, count(courses.id) AS y FROM courses GROUP BY courses.subject ORDER BY y DESC LIMIT 100"
 statsqueries['organizer_courses'] = "SELECT courses.organizer AS x, count(courses.id) AS y FROM courses GROUP BY courses.organizer ORDER BY y DESC LIMIT 100"
 statsqueries['categories_lectures'] = "SELECT courses.subject AS x, count(lectures.id) AS y FROM lectures JOIN courses ON (courses.id = lectures.course_id) WHERE lectures.visible GROUP BY courses.subject ORDER BY y DESC LIMIT 100"
-statsqueries['lecture_views'] = "SELECT lectures.time AS x, count(DISTINCT log.id) AS y FROM log JOIN videos ON (videos.id = log.obj_id) JOIN lectures ON (lectures.id = videos.lecture_id) WHERE (lectures.course_id = ?) GROUP BY lectures.id ORDER BY lectures.time"
+statsqueries['lecture_views'] = "SELECT lectures.time AS x, count(DISTINCT log.id) AS y FROM log JOIN videos ON (videos.id = log.video) JOIN lectures ON (lectures.id = videos.lecture_id) WHERE (lectures.course_id = ?) GROUP BY lectures.id ORDER BY lectures.time"
 
 def plotly_date_handler(obj):
 	return obj.strftime("%Y-%m-%d %H:%M:%S")
@@ -36,46 +37,44 @@ def stats_generic(req, param=None):
 		res['y'].append(row['y'])
 	return Response(json.dumps(res, default=plotly_date_handler),  mimetype='application/json')
 
-@app.route('/stats/viewsperday')
+@app.route('/stats/viewsperday/<req>')
+@app.route('/stats/viewsperday/<req>/<param>')
 @mod_required
-def stats_viewsperday():
+def stats_viewsperday(req, param=""):
+	update_expr = 'INSERT INTO logcache (req, param, trace, date, value) (SELECT "%s", ?, trace, date, y FROM (%s) AS cachetmp WHERE date < ?)'
+	query_expr = '(SELECT date, trace, value AS y FROM logcache WHERE req = "%s" AND param = ?) UNION (SELECT * FROM (%s) AS cachetmp)'
+	date_subexpr = '(SELECT CASE WHEN MAX(date) IS NULL THEN "2000-00-00" ELSE MAX(date) END FROM `logcache` WHERE req = "%s" AND param = ?)'
+	queries = {
+		'lecture': '(SELECT log.date AS date, formats.description AS trace, COUNT(DISTINCT log.id) AS y FROM log JOIN videos ON videos.id = log.video JOIN formats ON formats.id = videos.video_format WHERE log.date > %T AND videos.lecture_id = ? GROUP BY log.date, videos.video_format) UNION (SELECT log.date AS date, "total" AS trace, COUNT(DISTINCT log.id) AS y FROM log JOIN videos ON videos.id = log.video WHERE log.date > %T AND videos.lecture_id = ? GROUP BY log.date)',
+		'course': '(SELECT log.date AS date, formats.description AS trace, COUNT(DISTINCT log.id) AS y FROM log JOIN videos ON videos.id = log.video JOIN lectures ON lectures.id = videos.lecture_id JOIN formats ON formats.id = videos.video_format WHERE log.date > %T AND lectures.course_id = ? GROUP BY log.date, videos.video_format) UNION (SELECT log.date AS date, "total" AS trace, COUNT(DISTINCT log.id) AS y FROM log JOIN videos ON videos.id = log.video JOIN lectures ON lectures.id = videos.lecture_id WHERE log.date > %T AND lectures.course_id = ? GROUP BY log.date)',
+		'global': '(SELECT log.date AS date, formats.description AS trace, COUNT(DISTINCT log.id) AS y FROM log JOIN videos ON videos.id = log.video JOIN formats ON formats.id = videos.video_format WHERE log.date > %T GROUP BY log.date, videos.video_format) UNION (SELECT log.date AS date, "total" AS trace, COUNT(DISTINCT log.id) AS y FROM log WHERE log.date > %T GROUP BY log.date)',
+		'courses': 'SELECT log.date AS date, courses.handle AS trace, COUNT(DISTINCT log.id) AS y FROM log JOIN videos ON videos.id = log.video JOIN lectures ON lectures.id = videos.lecture_id JOIN courses ON courses.id = lectures.course_id WHERE log.date > %T GROUP BY log.date, courses.id'
+	}
+	expr = queries[req]
+	expr = expr.replace('%T', date_subexpr%('viewsperday.'+req))
+	params = [param]*expr.count('?')
+	try:
+		modify(update_expr%('viewsperday.'+req, expr), param, *(params+[datetime.combine(date.today(), time())]))
+	except Exception:
+		traceback.print_exc()
+	rows = query(query_expr%('viewsperday.'+req, expr), param, *params)
 	start = None
-	end = None
-	courseid = request.args.get('course', None)
-	lectureid = request.args.get('lecture', None)
-	videoid = request.args.get('video', None)
-	rows = query('''SELECT log.id AS id, log.time AS time, formats.description AS fmt
-			FROM log
-			JOIN videos ON (videos.id = log.obj_id)
-			JOIN lectures ON (lectures.id = videos.lecture_id)
-			JOIN courses ON (courses.id = lectures.course_id)
-			JOIN formats ON (formats.id = videos.video_format)
-			WHERE (? OR courses.id = ?) AND (? OR lectures.id = ?) AND (? OR videos.id = ?)''',
-			courseid == None, courseid, lectureid == None, lectureid, videoid == None, videoid)
+	traces = set()
 	data = {}
-	if not end:
+	for row in rows:
+		if not start or row['date'] < start:
+			start = row['date']
+		traces.add(row['trace'])
+		if row['date'] not in data:
+			data[row['date']] = {}
+		data[row['date']][row['trace']] = row['y']
 	end = date.today()
-	first = None
-	fmts = ['total']
-	for e in rows:
-		key = e['time'].date()
-		if not first or key < first:
-			first = key
-		if key not in data:
-			data[key] = {'total': set()}
-		if e['fmt'] not in data[key]:
-			data[key][e['fmt']] = set()
-		data[key]['total'].add(e['id'])
-		data[key][e['fmt']].add(e['id'])
-		if e['fmt'] not in fmts:
-			fmts.append(e['fmt'])
-	if not start:
-		start = first
-	res = {'times': [], 'views': [{'name': name, 'vals': []} for name in fmts]}
+	x = []
+	y = [{'name': trace, 'vals': []} for trace in traces]
 	while start and start <= end:
-		res['times'].append(start.isoformat())
-		for d in res['views']:
-			d['vals'].append(len(data.get(start, {}).get(d['name'], set())))
+		x.append(start)
+		for trace in y:
+			trace['vals'].append(data.get(start, {}).get(trace['name'], 0))
 		start += timedelta(days=1)
-	return Response(json.dumps(res, default=date_json_handler),  mimetype='application/json')
+	return Response(json.dumps({'times': x, 'views': y}, default=plotly_date_handler),  mimetype='application/json')
 

templates/course.html

@@ -48,7 +48,7 @@
 			</table>
 		</div>
 		<div id="statview" class="col-xs-6" style="height:600px"></div>
-		{{stats_viewsperday("statview", "Abrufe pro Tag", course=course.id)}}
+		{{stats_viewsperday("statview", "course", "Abrufer pro Tag", param=course.id)}}
 		<div id="lecture_views" class="col-xs-6" style="height:600px"></div>
 		{{stats_generic("lecture_views", "lecture_views", "Abrufe pro Aufnahme", param=course.id, type="bar")}}
 		{% endif %}

templates/lecture.html

@@ -60,7 +60,7 @@
 					</table>
 				</div>
 				<div id="statview" class="col-xs-12" style="height:600px"></div>
-				{{stats_viewsperday("statview", "Abrufe pro Tag", lecture=lecture.id)}}
+				{{stats_viewsperday("statview", "lecture", "Abrufer pro Tag", param=lecture.id)}}
 			{% endif %}
 		</div>
 	</div>

templates/macros.html

@@ -269,20 +269,33 @@ $('#embedcodebtn').popover(
 
 {% macro vtttime(time) %}{{ '%02d:%02d:%02d.000'|format( time//3600, (time//60)%60, time%60) }}{% endmacro %}
 
-{% macro stats_viewsperday(id, title, type="scatter", course=None, lecture=None, video=None) %}
+{% macro stats_viewsperday(id, req, title, type="scatter", param=None, maxtraces=7) %}
 <script>
 	$.ajax({
 		method: "GET",
-		url: "{{url_for('stats_viewsperday', course=course, lecture=lecture, video=video)}}",
+		url: "{{url_for('stats_viewsperday', req=req, param=param)}}",
 		dataType: "json",
 		error: moderator.api.handleapierror,
 		success: function (data) {
 			var traces = [];
-			for (var i = 1; i < data.views.length; i++) {
+			for (var i = 0; i < data.views.length; i++) {
+				if (data.views[i].name == 'total' && data.views.length > 2)
+					traces.push({"x": data.times, "y": data.views[i].vals, "type": "{{type}}", "name": "gesamt", line: {"color": "black", "width": 2}});
+				else
 					traces.push({"x": data.times, "y": data.views[i].vals, "type": "{{type}}", "name": data.views[i].name, line: {"width": 1}});
 			}
-			if (data.views.length > 2)
-				traces.push({"x": data.times, "y": data.views[0].vals, "type": "{{type}}", "name": "gesamt", line: {"color": "black", "width": 2}});
+			traces.sort(function (a, b) {
+				amax = 0;
+				bmax = 0;
+				for (var i = 0; i < a.y.length; i++)
+					amax = Math.max(amax, a.y[i]);
+				for (var i = 0; i < b.y.length; i++)
+					bmax = Math.max(bmax, b.y[i]);
+				return bmax-amax;
+			});
+			for (var i = 0; i < traces.length; i++)
+				if (i > {{maxtraces}})
+					traces[i].visible = "legendonly";
 			var layout = {
 				"title": "{{title}}",
 				"showlegend": (traces.length != 1)

templates/stats.html

@@ -27,7 +27,9 @@
 				<div id="categories_lectures" class="col-xs-6" style="height:600px"></div>
 				{{stats_pie("categories_lectures", "categories_lectures", "Aufnahmen nach Kategorie", showlegend=False)}}
 				<div id="statview" class="col-xs-12" style="height:600px"></div>
-				{{stats_viewsperday("statview", "Abrufe pro Tag")}}
+				{{stats_viewsperday("statview", "global", "Abrufer pro Tag")}}
+				<div id="courseviewsperday" class="col-xs-12" style="height:600px"></div>
+				{{stats_viewsperday("courseviewsperday", "courses", "Abrufer pro Tag")}}
 			</div>
 	i	</div>
 	</div>