Merge branch '368-terminimport-von-rwth-online' into 'master'

Resolve "Terminimport von RWTH Online"

Closes #368

See merge request !16

README.md

@@ -41,9 +41,10 @@ Notwendig:
 * git (zum Anzeigen der aktuellen Version)
 
 Optional (wird für einzelne Features benötigt):
-* python-lxml (Campus Import) 
+* python-lxml (Campus- und RO-Import) 
+* python-pytz (RO-Import) 
 * python-ldap (Login mit Fachschaftsaccount)
-* python-icalendar (SoGo-Kalenderimport für Sitzungsankündigungen)
+* python-icalendar (RO-Import, Kalenderimport für Sitzungsankündigungen)
 * python-mysql-connector (wenn MySQL als Datenbank verwendet werden soll)
 * python-coverage (Für Coverage Tests benötigt)
 

importer.py

 from server import *
 
+import urllib.request
+import urllib.parse
+
 @app.route('/internal/import/<int:id>', methods=['GET', 'POST'])
 @mod_required
 def list_import_sources(id):
@@ -25,104 +28,161 @@ def list_import_sources(id):
 
 	return render_template('import_campus.html', course=courses, import_campus=import_campus, events=[])
 
+def fetch_co_course_events(i):
+	from lxml import html
+	from lxml import etree
+	events = []
+	try:
+		remote_html = urllib.request.urlopen(i['url']).read()
+	except:
+		flash("Ungültige URL: '"+i['url']+"'")
+	tablexpath = "//td[text()='Termine und Ort']/following::table[1]"
+	basetable = html.fromstring(remote_html).xpath(tablexpath)[0]
+	parsebase = html.tostring(basetable);
+
+	#parse recurring events
+	toparse = [i['url']]
+	for j in basetable.xpath("//table[@cellpadding='5']//tr[@class='hierarchy4' and td[@name='togglePeriodApp']]"):
+		url = str(j.xpath("td[@name='togglePeriodApp']/a/@href")[0])
+		toparse.append(url)
+	events_raw = []
+	for j in toparse:
+		if j.startswith('event'):
+			url = 'https://www.campus.rwth-aachen.de/rwth/all/'+j
+		else:
+			url = j
+		text = urllib.request.urlopen(url).read()
+		dom = html.fromstring(text).xpath(tablexpath)[0]
+		#we get the "heading" row, from it extract the room and time. best way to get it is to match on the picture -.-
+		baserow = dom.xpath("//table[@cellpadding='5']//tr[@class='hierarchy4' and td[@name='togglePeriodApp']/*/img[@src='../../server/img/minus.gif']]")
+		if not baserow:
+			continue
+		baserow = baserow[0]
+		rowdata = {'dates': []}
+
+		# "kein raum vergeben" is a special case, else use campus id
+		if baserow.xpath("td[6]/text()")[0] == 'Kein Raum vergeben':
+			rowdata['place'] = ''
+		elif baserow.xpath("td[6]/a"):
+			rowdata['place'] = baserow.xpath("td[6]/a")[0].text_content()
+		else:
+			rowdata['place'] = baserow.xpath("td[6]/text()")[0].split(' ',1)[0]
+
+		rowdata['start'] = baserow.xpath("td[3]/text()")[0]
+		rowdata['end'] = baserow.xpath("td[5]/text()")[0]
+		rowdata['dates'] = baserow.getparent().xpath("tr[@class='hierarchy5']//td[@colspan='3']/text()")
+		events_raw.append(rowdata)
+
+	# parse single appointments
+	if basetable.xpath("//table[@cellpadding='3']/tr/td[text()='Einmalige Termine:']"):
+		singletable = basetable.xpath("//table[@cellpadding='3']/tr/td[text()='Einmalige Termine:']")[0].getparent().getparent()
+		for row in singletable.xpath("tr/td[2]"):
+			rowdata = {}
+			if row.xpath("text()[2]")[0] == 'Kein Raum vergeben':
+				rowdata['place'] = ''
+			elif row.xpath("a"):
+				rowdata['place'] = row.xpath("a")[0].text_content()
+			else:
+				rowdata['place'] = row.xpath("text()[2]")[0].split(' ',1)[0]
+
+			rowdata['dates'] = [row.xpath("text()[1]")[0][4:14]]
+			rowdata['start'] = row.xpath("text()[1]")[0][17:22]
+			rowdata['end'] = row.xpath("text()[1]")[0][27:32]
+			events_raw.append(rowdata)
+
+	#now we have to filter our data and do some lookups
+	for j in events_raw:
+		for k in j['dates']:
+			e = {}
+			fmt= "%d.%m.%Y %H:%M"
+			e['time'] = datetime.strptime("%s %s"%(k,j['start']) ,fmt)
+			e['duration'] = int((datetime.strptime("%s %s"%(k,j['end']) ,fmt) - e['time']).seconds/60)
+			j['place'] = str(j['place'])
+			if j['place'] != '':
+				dbplace = query("SELECT name FROM places WHERE (campus_room = ?) OR (campus_name = ?) OR ((NOT campus_name) AND name = ?)",j['place'],j['place'],j['place'])
+				if dbplace:
+					e['place'] = dbplace[0]['name']
+				else:
+					e['place'] = 'Unbekannter Ort ('+j['place']+')'
+			else:
+				e['place'] = ''
+			e['title'] = i['type']
+			events.append(e)
+	# it is parsed.
+	return events
+
+def fetch_ro_event_ical(ids):
+	data = {'pMode': 'T', 'pInclPruef': 'N', 'pInclPruefGepl': 'N', 'pOutputFormat': '99', 'pCharset': 'UTF8', 'pMaskAction': 'DOWNLOAD'}
+	data = list(data.items())
+	for id in ids:
+		data.append(('pTerminNr', id))
+	data = urllib.parse.urlencode(data).encode('utf-8')
+	r = urllib.request.Request('https://online.rwth-aachen.de/RWTHonline/pl/ui/%24ctx/wbKalender.wbExport',
+			data=data, method='POST')
+	with urllib.request.urlopen(r) as f:
+		return f.read().decode('utf-8')
+
+def fetch_ro_course_ical(id):
+	from lxml import html
+	url = 'https://online.rwth-aachen.de/RWTHonline/pl/ui/%24ctx/wbTermin_List.wbLehrveranstaltung?pStpSpNr='+'%i'%(int(id))
+	req = urllib.request.urlopen(url)
+	dom = html.fromstring(req.read())
+	event_ids = [x.value for x in dom.xpath('//input[@name="pTerminNr"]')]
+	return fetch_ro_event_ical(event_ids)
+
+def fetch_ro_course_events(item):
+	import icalendar
+	import pytz
+	localtz = pytz.timezone('Europe/Berlin')
+	# First fix crappy javascript fragment-Paths
+	url = urllib.parse.urlparse(item['url'].replace('#/', ''))
+	args = urllib.parse.parse_qs(url.query)
+	if 'pStpSpNr' in args: # Legacy URLs
+		id = args['pStpSpNr'][0]
+	elif url.path.split('/')[-2] == 'courses': # New URLs
+		id = url.path.split('/')[-1]
+	else:
+		flash("Ungültige URL: '"+i['url']+"'")
+	cal = icalendar.Calendar().from_ical(fetch_ro_course_ical(id))
+	events = []
+	for comp in cal.subcomponents:
+		if comp.name != 'VEVENT':
+			continue
+		if comp.get('STATUS') != 'CONFIRMED':
+			continue
+		e = {}
+		place = str(comp.get('LOCATION', ''))
+		if place:
+			campus_room = place.split('(')[-1].split(')')[0]
+			dbplace = query('SELECT name FROM places WHERE campus_room = ?', campus_room)
+			if dbplace:
+				e['place'] = dbplace[0]['name']
+			else:
+				e['place'] = 'Unbekannter Ort ('+place+')'
+		else:
+			e['place'] = ''
+		e['time'] = comp['DTSTART'].dt.astimezone(localtz).replace(tzinfo=None)
+		e['duration'] = int((comp['DTEND'].dt - comp['DTSTART'].dt).seconds/60)
+		e['title'] = item['type']
+		events.append(e)
+	return events
+
 @app.route('/internal/import/<int:id>/now', methods=['GET', 'POST'])
 @mod_required
 def import_from(id):
-
 	courses = query('SELECT * FROM courses WHERE id = ?', id)[0]
 	lectures = query('SELECT * FROM lectures WHERE course_id = ?', courses['id'])
-	
-	
 	import_campus = query('SELECT * FROM import_campus WHERE course_id = ?',id)
 	events = []
 	try:
-		from lxml import html
-		from lxml import etree
-		import urllib.request
 		# if u have to port this to anything new, god be with you.
 		for i in import_campus:
-			try:
-				remote_html = urllib.request.urlopen(i['url']).read()
-			except:
-				flash("Ungültige URL: '"+i['url']+"'")
-			tablexpath = "//td[text()='Termine und Ort']/following::table[1]"
-			basetable = html.fromstring(remote_html).xpath(tablexpath)[0]
-			parsebase = html.tostring(basetable);
-
-			#parse recurring events
-			toparse = [i['url']]
-			for j in basetable.xpath("//table[@cellpadding='5']//tr[@class='hierarchy4' and td[@name='togglePeriodApp']]"):
-				url = str(j.xpath("td[@name='togglePeriodApp']/a/@href")[0])
-				toparse.append(url)
-			events_raw = []
-			for j in toparse:
-				if j.startswith('event'):
-					url = 'https://www.campus.rwth-aachen.de/rwth/all/'+j
-				else:
-					url = j
-				text = urllib.request.urlopen(url).read()
-				dom = html.fromstring(text).xpath(tablexpath)[0]
-				#we get the "heading" row, from it extract the room and time. best way to get it is to match on the picture -.-
-				baserow = dom.xpath("//table[@cellpadding='5']//tr[@class='hierarchy4' and td[@name='togglePeriodApp']/*/img[@src='../../server/img/minus.gif']]")
-				if not baserow:
-					continue
-				baserow = baserow[0]
-				rowdata = {'dates': []}
-
-				# "kein raum vergeben" is a special case, else use campus id
-				if baserow.xpath("td[6]/text()")[0] == 'Kein Raum vergeben':
-					rowdata['place'] = ''
-				elif baserow.xpath("td[6]/a"):
-					rowdata['place'] = baserow.xpath("td[6]/a")[0].text_content()
-				else:
-					rowdata['place'] = baserow.xpath("td[6]/text()")[0].split(' ',1)[0]
-
-				rowdata['start'] = baserow.xpath("td[3]/text()")[0]
-				rowdata['end'] = baserow.xpath("td[5]/text()")[0]
-				rowdata['dates'] = baserow.getparent().xpath("tr[@class='hierarchy5']//td[@colspan='3']/text()")
-				events_raw.append(rowdata)
-
-			# parse single appointments
-			if basetable.xpath("//table[@cellpadding='3']/tr/td[text()='Einmalige Termine:']"):
-				singletable = basetable.xpath("//table[@cellpadding='3']/tr/td[text()='Einmalige Termine:']")[0].getparent().getparent()
-				for row in singletable.xpath("tr/td[2]"):
-					rowdata = {}
-					if row.xpath("text()[2]")[0] == 'Kein Raum vergeben':
-						rowdata['place'] = ''
-					elif row.xpath("a"):
-						rowdata['place'] = row.xpath("a")[0].text_content()
-					else:
-						rowdata['place'] = row.xpath("text()[2]")[0].split(' ',1)[0]
-
-					rowdata['dates'] = [row.xpath("text()[1]")[0][4:14]]
-					rowdata['start'] = row.xpath("text()[1]")[0][17:22]
-					rowdata['end'] = row.xpath("text()[1]")[0][27:32]
-					events_raw.append(rowdata)
-
-			#now we have to filter our data and do some lookups
-			for j in events_raw:
-				for k in j['dates']:
-					e = {}
-					fmt= "%d.%m.%Y %H:%M"
-					e['time'] = datetime.strptime("%s %s"%(k,j['start']) ,fmt)
-					e['duration'] = int((datetime.strptime("%s %s"%(k,j['end']) ,fmt) - e['time']).seconds/60)
-					j['place'] = str(j['place'])
-					if j['place'] != '':
-						dbplace = query("SELECT name FROM places WHERE (campus_room = ?) OR (campus_name = ?) OR ((NOT campus_name) AND name = ?)",j['place'],j['place'],j['place'])
-						if dbplace:
-							e['place'] = dbplace[0]['name']
-						else:
-							e['place'] = 'Unbekannter Ort ('+j['place']+')'
-					else:
-						e['place'] = ''
-					e['title'] = i['type']
-					events.append(e)
-			# it is parsed.
-
-
-
+			if 'www.campus.rwth-aachen.de' in i['url']:
+				events += fetch_co_course_events(i)
+			else:
+				events += fetch_ro_course_events(i)
 	except ImportError:
-		flash('python-lxml not found, campus import will not work.')
+		flash('python-lxml or python-pytz not found, campus and ro import will not work!')
 
 	# events to add
 	newevents = []

templates/course.html

@@ -87,7 +87,7 @@
 		<h1 class="panel-title">Videos
 			{% if ismod() %}
 				<a class="btn btn-default" style="margin-right: 5px;" href="{{ url_for('create', table='lectures', time=datetime.now(), title='Noch kein Titel', visible='0', course_id=course.id, ref=url_for('course', id=course.id)) }}">Neuer Termin</a>
-				<a class="btn btn-default" style="margin-right: 5px;" href="{{url_for('list_import_sources', id=course['id'])}}">Campus Import</a>
+				<a class="btn btn-default" style="margin-right: 5px;" href="{{url_for('list_import_sources', id=course['id'])}}">Import</a>
 			{% endif %}
 			<ul class="list-inline pull-right">
 				<li>

templates/import_campus.html

@@ -4,11 +4,11 @@
 <div class="panel-group">
 	<div class="panel panel-default">
 		<div class="panel-heading">
-			<h1 class="panel-title">Campus Import für <strong>{{course.title}}</strong> <span><a href="{{url_for('course', handle=course.handle)}}" class="btn btn-default" >Zur Veranstaltungsseite</a><span> </h1>
+			<h1 class="panel-title">Campus-/RWTHonline-Import für <strong>{{course.title}}</strong> <span><a href="{{url_for('course', handle=course.handle)}}" class="btn btn-default" >Zur Veranstaltungsseite</a><span> </h1>
 		</div>
 		<div class="panel-body">
 			<div>
-				<p>Es folgen viele Pärchen an Campus-URL und Veranstaltungstyp Pärchen. Die Campus URL bekommt man aus dem Campus-System (<a href="https://www.campus.rwth-aachen.de/rwth/all/groups.asp" target="_blank">hier</a>). Der Veranstaltungstyp ist z.B. "Vorlesung" oder "Übung" oder "Praktikum".
+				<p>Es folgen ein oder mehrere Veranstaltungs-URLs mit dem jeweiligen Veranstaltungstyp. Die Veranstaltungs-URL bekommt man aus dem Campus-System (<a href="https://www.campus.rwth-aachen.de/rwth/all/groups.asp" target="_blank">hier</a>) bzw. bei RWTHonline über die Veranstaltungssuche (<a href="https://online.rwth-aachen.de/RWTHonline/pl/ui/%24ctx/wbSuche.LVSuche" target="_blank">hier</a> oder über Durchklicken im neuen Interface). Der Veranstaltungstyp ist z.B. "Vorlesung" oder "Übung" oder "Praktikum".
 				</p>
 				<form method="post" action="{{url_for('list_import_sources', id=course['id'])}}">
 					<ul class="list-group row" style="margin-left: 0px; margin-right: 0px;">
@@ -73,7 +73,7 @@
 					<span class="col-xs-3">
 					</span>
 					<span class="pull-right">
-						<button class="btn btn-default newlecture" onclick="moderator.api.gethttp('{{ url_for('create', table='lectures', course_id=course.id, time=i.time, title=i.title, place=i.place) }}')">anlegen</a>
+						<button class="btn btn-default newlecture" onclick="moderator.api.gethttp('{{ url_for('create', table='lectures', course_id=course.id, time=i.time, title=i.title, place=i.place, duration=i.duration) }}')">anlegen</a>
 					</span>
 				</li>
 			{% endfor %}

tests/test_misc.py

@@ -268,6 +268,9 @@ class VideoTestCase(unittest.TestCase):
 			r = self.app.post('/internal/import/257', data={'campus.new.url': 'https://www.campus.rwth-aachen.de/rwth/all/event.asp?gguid=0x4664DBD60E5A02479B53089BF0EB0681&tguid=0x0B473CF286B45B4984CD02565C07D6F8', 'campus.new.type': 'Vorlesung'})
 			assert r.status_code == 200
 
+			r = self.app.post('/internal/import/257', data={'campus.new.url': 'https://online.rwth-aachen.de/RWTHonline/pl/ui/%24ctx/wbLv.wbShowLVDetail?pStpSpNr=269474&pSpracheNr=1', 'campus.new.type': 'Übung'})
+			assert r.status_code == 200
+
 			r = self.app.get('/internal/import/257/now')
 			assert r.status_code == 200