Improve compatibility of sql queries

cutprogress.py

@@ -40,15 +40,21 @@ def cutprogress(user=None):
 				lectures.course_id,
 				lectures.time,
 				lectures.title,
+				video_counts.videos_total,
+				video_counts.videos_visible
+			FROM lectures
+			JOIN courses ON ( courses.id = lectures.course_id )
+			LEFT JOIN (
+					SELECT
+						videos.lecture_id,
 						COUNT(videos.id) as videos_total,
 						COUNT(videos.visible) as videos_visible
-			FROM lectures
-			JOIN courses ON courses.id = lectures.course_id
-			LEFT JOIN videos ON lectures.id = videos.lecture_id
+					FROM videos
+					GROUP BY videos.lecture_id
+				) AS video_counts ON ( video_counts.lecture_id = lectures.id )
 			WHERE courses.id = ?
 				AND lectures.time <= ?
 				AND NOT lectures.norecording
-			GROUP BY lectures.id
 			ORDER BY lectures.time ASC, lectures.id ASC
 			''', course['id'], datetime.now())
 	# Generate list of days, figure out when weeks change

db.py

@@ -148,20 +148,3 @@ def close_db(*args): #pylint: disable=unused-argument
 	if 'db' in g:
 		g.db.close()
 		del g.db
-
-def searchquery(text, columns, match, tables, suffix, *suffixparams):
-	params = []
-	subexprs = []
-	words = text.split(' ')
-	prio = len(words)+1
-	for word in words:
-		if word == '' or word.isspace():
-			continue
-		matchexpr = ' OR '.join(['%s LIKE ?'%column for column in match])
-		subexprs.append('SELECT %s, %s AS _prio FROM %s WHERE %s'%(columns, str(prio), tables, matchexpr))
-		params += ['%'+word+'%']*len(match)
-		prio -= 1
-	if subexprs == []:
-		return []
-	expr = 'SELECT *,SUM(_prio) AS _score FROM (%s) AS _tmp %s'%(' UNION '.join(subexprs), suffix)
-	return query(expr, *(list(params)+list(suffixparams)))

edit.py

@@ -255,7 +255,8 @@ def changelog():
 @csrf_protect
 def set_responsible(course_id, user_id, value):
 	if value:
-		modify('REPLACE INTO responsible (course_id, user_id) values (?, ?)', course_id, user_id)
+		if not query('SELECT id FROM responsible WHERE course_id = ? AND user_id = ?', course_id, user_id):
+			modify('INSERT INTO responsible (course_id, user_id) VALUES (?, ?)', course_id, user_id)
 	else:
 		modify('DELETE FROM responsible WHERE course_id = ? AND user_id = ?', course_id, user_id)
 	return "OK", 200

jobs.py

@@ -87,7 +87,10 @@ def jobs_ping(id):
 @app.route('/internal/jobs/api/worker/<hostname>/schedule', methods=['POST'])
 @api_token_required('JOBS_API_KEY')
 def jobs_schedule(hostname):
-	query('REPLACE INTO worker (hostname, last_ping) values (?, ?)', hostname, datetime.now())
+	if query("SELECT hostname FROM worker WHERE hostname = ?", hostname):
+		query("UPDATE worker SET last_ping = ? WHERE hostname = ?", datetime.now(), hostname)
+	else:
+		query("INSERT INTO worker (hostname, last_ping) VALUES (?, ?)", hostname, datetime.now())
 	hostdata = request.get_json()
 	if not hostdata:
 		return 'no hostdata sent', 400

livestreams.py

@@ -44,10 +44,8 @@ def streamauth_legacy(server=None):
 				break
 		if 'lecture' in request.values:
 			match = {'id': request.values['lecture']}
-		try:
+		if not query("SELECT handle FROM streams WHERE handle = ?", request.values['name']):
 			modify("INSERT INTO streams (handle, active, visible, lecture_id, description, poster) VALUES (?, false, true, -1, '', '')", request.values['name'])
-		except:
-			pass
 		if server:
 			data = {'src': 'rtmp://%s/live/%s'%(server, request.values['name']),
 				'destbase': 'rtmp://%s/hls/%s'%(server, request.values['name'])}

server.py

@@ -71,7 +71,7 @@ def evalperm(perms):
 	return [{'type': 'public'}]
 
 #pylint: disable=wrong-import-position
-from db import query, modify, show, searchquery
+from db import query, modify, show
 from template_helper import *
 from mail import notify_mods, notify_admins #pylint: disable=unused-import
 from ldap import ldapauth
@@ -171,13 +171,21 @@ def index():
 		i['date'] = i['time'].date()
 	latestvideos = query('''
 		SELECT lectures.*, \'course\' AS sep, courses.*
-		FROM lectures
-		LEFT JOIN videos ON (videos.lecture_id = lectures.id)
-		LEFT JOIN courses on (courses.id = lectures.course_id)
+		FROM (
+				SELECT
+					videos.lecture_id,
+					MAX(videos.time_created) AS _time_publish
+				FROM videos
+				JOIN lectures ON ( lectures.id = videos.lecture_id )
+				JOIN courses ON ( courses.id = lectures.course_id )
 				WHERE (? OR (courses.visible AND courses.listed AND lectures.visible AND videos.visible))
 				GROUP BY videos.lecture_id
-		ORDER BY MAX(videos.time_created) DESC
-		LIMIT 6	''', ismod())
+				ORDER BY _time_publish DESC
+				LIMIT 6
+			) AS _latest
+		JOIN lectures ON ( lectures.id = _latest.lecture_id )
+		JOIN courses ON ( courses.id = lectures.course_id )
+		''', ismod())
 	livestreams = query('''SELECT streams.handle AS livehandle, lectures.*, \'course\' AS sep, courses.*
 		FROM streams
 		JOIN lectures ON lectures.id = streams.lecture_id
@@ -247,7 +255,7 @@ def course(id=None, handle=None):
 	for i in query('SELECT lectures.id AS id, COUNT(chapters.id) AS c FROM chapters \
 			JOIN lectures ON chapters.lecture_id = lectures.id \
 			WHERE lectures.course_id = ? AND NOT chapters.visible AND NOT chapters.deleted \
-			GROUP BY chapters.lecture_id;', course['id']):
+			GROUP BY lectures.id;', course['id']):
 		chapters[i['id']] = i['c']
 	lectures = query('SELECT * FROM lectures WHERE course_id = ? AND (? OR visible) ORDER BY time, duration DESC', course['id'], ismod())
 	for lecture in lectures:
@@ -395,9 +403,37 @@ def search():
 	if 'q' not in request.args:
 		return redirect(url_for('index'))
 	searchtext = request.args['q']
-	courses = searchquery(searchtext, '*', ['title', 'short', 'organizer', 'subject', 'description'],
-			'courses', 'WHERE (? OR (visible AND listed)) GROUP BY id ORDER BY _score DESC, semester DESC LIMIT 20', ismod())
-	lectures = searchquery(searchtext, 'lectures.*, \
+	courses = _course_query_search(searchtext, ismod())
+	lectures = _lecture_query_search(searchtext, ismod())
+	for lecture in lectures:
+		lecture['course'] = {}
+		for key in lecture:
+			if key.startswith('courses_'):
+				lecture['course'][key[8:]] = lecture[key]
+	return render_template('search.html', searchtext=searchtext, courses=courses, lectures=lectures)
+
+
+# This search is basically stolen from the new api
+
+def _course_query_search(search_term: str, is_mod: bool):
+	return _query_search(
+		"courses",
+		["title", "short", "organizer", "subject", "description"],
+		None,
+		None,
+		None if is_mod else 'WHERE "courses"."visible" AND "courses"."listed"',
+		'"courses"."semester" DESC',
+		20,
+		search_term
+	)
+
+
+def _lecture_query_search(search_term: str, is_mod: bool):
+	return _query_search(
+		"lectures",
+		["title", "comment", "speaker"],
+		'JOIN "courses" ON ("lectures"."course_id" = "courses"."id")',
+		"""\
 courses.visible AS coursevisible, \
 courses.listed, \
 courses.id AS courses_id, \
@@ -416,16 +452,58 @@ def search():
 courses.downloadable AS courses_downloadable, \
 courses.embedinvisible AS courses_embedinvisible, \
 courses.description AS courses_description, \
-			courses.internal AS courses_internal',
-			['lectures.title', 'lectures.comment', 'lectures.speaker', 'courses.short'],
-			'lectures LEFT JOIN courses on (courses.id = lectures.course_id)',
-			'WHERE (? OR (coursevisible AND listed AND visible)) GROUP BY id ORDER BY _score DESC, time DESC LIMIT 30', ismod())
-	for lecture in lectures:
-		lecture['course'] = {}
-		for key in lecture:
-			if key.startswith('courses_'):
-				lecture['course'][key[8:]] = lecture[key]
-	return render_template('search.html', searchtext=searchtext, courses=courses, lectures=lectures)
+courses.internal AS courses_internal
+""",
+		None if is_mod else 'WHERE "courses"."visible" AND "courses"."listed" AND "lectures"."visible"',
+		'"lectures"."time" DESC',
+		30,
+		search_term
+	)
+
+
+def _query_search(
+		table: str,
+		search_columns: list[str],
+		join_clause: str or None,
+		extra_select_columns: str or None,
+		where_clause: str or None,
+		extra_ordering: str or None,
+		limit: int,
+		search_term: str):
+	base_sub_query = f"""
+			SELECT "{table}"."id" AS "_id", CAST(%s AS INT) AS "_priority" FROM "{table}" WHERE {" OR ".join(
+		map(lambda column: f'LOWER("{table}"."{column}") LIKE ?',
+			search_columns))}
+		"""
+	
+	words: list[str] = list(filter(lambda w: not w.isspace(), search_term.split(" ")))
+	if len(words) == 0:
+		return []
+	sub_queries: list[str] = []
+	all_values: list[DbValueType] = []
+	prio = len(words)
+	for word in words:
+		word = word.lower()
+		word = word.replace("%", "\\%").replace("_", "\\_")
+		word = "%" + word + "%"
+		sub_queries.append(base_sub_query % prio)
+		for _ in range(0, len(search_columns)):
+			all_values.append(word)
+		prio -= 1
+	
+	return query(f"""
+		SELECT "{table}".* {"" if extra_select_columns is None else "," + extra_select_columns}
+		FROM "{table}"
+		JOIN (
+			SELECT "_id", CAST(SUM("_priority") AS INT) AS "_score"
+			FROM ({"UNION ALL".join(sub_queries)}) AS "_sub_result"
+			GROUP BY "_id"
+		) AS "_data" ON ("{table}"."id" = "_data"."_id")
+		{"" if join_clause is None else join_clause}
+		{"" if where_clause is None else where_clause}
+		ORDER BY "_data"."_score" DESC{"" if extra_ordering is None else ", " + extra_ordering} LIMIT {limit}
+		""", *all_values)
+
 
 def check_mod(user, groups):
 	if not user: