add initial cleaned implementation

.gitignore

+sshgen.cfg
+*.swp
+*.pyc
+__pycache__/*

README.md

+# SSHgen: tooling to generate SSH configs
+
+## `zonedl.py`
+
+`zonedl.py` allows downloading one or more zone files from the RWTE^2H Aachen DNS-Admin-Portal. It allows administrators to configure their zone through a web interface and is able to generate a "preview" of the zone. After authenticating via Shibboleth, this file can be downloaded (and stripped of HTML).
+
+The program has the following options and arguments:
+
+- `--list` lists available zones without actually downloading them.
+- `--zone` downloads one or more zones by its internal id.
+- `--domain` downloads one or more zones by their domain names.
+- `--passwordstore` specify the passwordstore entry to use for login via Shibboleth (if not specified, ask for username/password on the terminal).
+- `dest` destination to download zone(s), can be a directory, `-` for stdout.
+
+The URLs used for Shibboleth authentication and the DNS-Portal are specified at the beginning of the source file. They may change. Also, this program relies on parsing the HTML structure of the DNS-Portal and Login-Page to some extent. Beware of changes.
+
+An alternative to this program would be gaining AXFR access to the authorative DNS server, which has not been granted to us, yet.
+
+## `generate.py`
+
+`generate.py` takes one or more zone files as input, reads some configuration file and generates and returns a SSH configuration file. This allows management of CNAME aliases and multiple A records and at the same time use consistent host keys and configuration options.
+
+Currently, the possible options which one can configure is quite limited to the most pressing use cases of my config. This could be improved easily.
+
+The configurations file lives either at `./sshgen.cfg` or the location given by `--cfg`. Select a preset with `--preset`. A sample configuration file is available. It configures the location of the zone file(s), the domain stripping and proxy presets, and the various rewriting/exclusion/aliasing/agent settings.
+

generate.py

+#!/usr/bin/env python3
+
+import bs4
+import dns.zone
+import re
+import argparse
+from pathlib import Path
+import configparser
+
+config = configparser.ConfigParser(interpolation=configparser.ExtendedInterpolation())
+
+try:
+    with open('./sshgen.cfg') as fp:
+        config.read_file(fp)
+except Exception:
+    pass
+if not 'presets' in config:
+    config['presets'] = {}
+
+parser = argparse.ArgumentParser(description='Generates a SSH config file from some DNS zone(s).')
+parser.add_argument('--preset', choices=list(config['presets']), default=list(config['presets'].keys())[0], help='select a configuration preset')
+parser.add_argument('--cfg', action='store', default='./sshgen.cfg', help='config file')
+args = parser.parse_args()
+preset = args.preset
+
+if args.cfg != './sshgen.cfg':
+    with open(args.cfg) as fp:
+        config.read_file(fp)
+
+def get_zones():
+    all_zones = []
+    for x,y in config['zones'].items():
+        p = Path(y)
+        if p.is_dir():
+            all_zones.extend([z for z in p.iterdir()])
+        elif p.is_file():
+            all_zones.append(p)
+        else:
+            print('incorrectly configured zone {}, skipping'.format(x), file=sys.stderr)
+    return all_zones
+
+def get_zone_file(zone):
+    with open(str(zone), 'r') as fp:
+        return '\n'.join(fp.readlines())
+
+def retrieve_hosts():
+    d = get_zones()
+    h = {}
+    i = {}
+
+    for k in d:
+        z = dns.zone.from_text(get_zone_file(k), relativize=False)
+
+        # TODO AAAA records (and others)
+        for (name, ttl, rdata) in z.iterate_rdatas('A'):
+            l = h.get(name)
+            if l is None:
+                l = []
+                h[name] = l
+            m = i.get(rdata.address)
+            if m is None:
+                m = []
+                i[rdata.address] = m
+            m.append(name)
+        for (name, ttl, rdata) in z.iterate_rdatas('CNAME'):
+            l = h.get(rdata.target)
+            if l is None:
+                l = []
+                h[rdata.target] = l
+            l.append(name)
+
+    fin = False
+    while not fin:
+        fin = True
+        for key in h:
+            for k in h:
+                if key in h[k]:
+                    h[k].extend(h[key])
+                    del h[key]
+                    fin = False
+                    break
+            if not fin:
+                break
+    for key in i:
+        if len(i[key]) > 1:
+            max_len = 0
+            max_host = None
+            for j in i[key]:
+                if j in h: # TODO
+                    x = len(str(h[j]))
+                    if x > max_len:
+                        max_len = x
+                        max_host = j
+            for j in i[key]:
+                if j == max_host or j not in h: # TODO
+                    continue
+                h[max_host].append(j)
+                h[max_host].extend(h[j])
+                del h[j]
+
+    return h
+
+proxies = {}
+strip_domains = []
+preset_config = [k.strip() for k in config['presets'][preset].split(',')]
+for c in preset_config:
+    if c.startswith('proxies_'):
+        proxies.update({re.compile(k.strip()): v for v in config[c] for k in config[c][v].split(',')})
+    elif c.startswith('strip_'):
+        strip_domains.extend([re.compile('\.{}\.?'.format(k.strip())) for k in config['strips'][c[len('strip_'):]].split(',')])
+    else:
+        pass
+
+exclude_hosts = [re.compile(x.strip()) for x in config['excludes']['hosts'].split(',')]
+exclude_aliases = [re.compile(x.strip()) for x in config['excludes']['aliases'].split(',')]
+usernames = {re.compile(k.strip()): v for v in config['usernames'] for k in config['usernames'][v].split(',')}
+agents = {re.compile(k.strip()): True for k in config['agents']['enabled'].split(',')}
+agents.update({re.compile(k.strip()): False for k in config['agents']['disabled'].split(',')})
+
+h = {}
+h = retrieve_hosts()
+
+def modify_list(h):
+    for e in exclude_hosts:
+        h = {l:m for l,m in h.items() if not e.match(str(l))}
+        for k in h:
+            h[k] = [l for l in h[k] if not e.match(str(l))]
+    for e in exclude_aliases:
+        ni = {}
+        for k in h:
+            h[k] = [l for l in h[k] if not e.match(str(l))]
+            if e.match(str(k)):
+                ni[h[k][0]] = h[k][1:]
+        h.update(ni)
+        h = {l:m for l,m in h.items() if not e.match(str(l))}
+    for k in h:
+        for ak,av in config['aliases'].items():
+            if str(k) == ak or str(k)[:-1] == ak:
+                h[k].extend([x.strip() for x in av.split(',')])
+    return h
+
+h = modify_list(h)
+
+def re_suffix(pattern, text):
+    res = pattern.search(text)
+    if res and res.span()[1] == len(text) and res.span()[0] != 0:
+        return text[res.span()[0]:res.span()[1]]
+    return None
+
+for k in h:
+    c = [str(k)]
+    c.extend([str(k)[:-len(re_suffix(d, str(k)))] for d in strip_domains if re_suffix(d, str(k))])
+    c.extend(map(str,h[k]))
+    for j in map(str,h[k]):
+        c.extend([j[:-len(re_suffix(d, j))] for d in strip_domains if re_suffix(d, j)])
+    c = [x[:-1] if x.endswith('.') else x for x in c]
+
+    print('Host ' + ' '.join(c))
+    hn = str(k)
+    print('\tHostName ' + (hn[:-1] if hn.endswith('.') else hn))
+    for u in usernames:
+        if u.match(str(k)):
+            print('\tUser ' + usernames[u])
+            break
+    for a in agents:
+        if a.match(str(k)):
+            print('\tForwardAgent ' + ('yes' if agents[a] else 'no'))
+            break
+    for p in proxies:
+        if p.match(str(k)):
+            print('\tProxyJump ' + proxies[p])
+            break
+    print('')

sshgen.example.cfg

+[DEFAULT]
+
+[zones]
+zones = /path/to/zone/dir
+zonefile = /path/to/file.zone
+
+[presets]
+world = proxies_world, proxies_rwth, strip_rwth
+rwth = proxies_rwth, strip_rwth
+fsmpi = proxies_rwth, strip_rwth, strip_fsmpi
+asta = strip_rwth, strip_asta
+
+[proxies_world]
+portal.fsmpi.rwth-aachen.de = ^(?!(portal|git)\.)[a-z0-9-]*.fsmpi.rwth-aachen.de,
+	([a-z0-9-]+\.)*(fachschaften|fslogo|esp|video|vampir).rwth-aachen.de,
+	([a-z0-9-]+\.)*(noc).rwth-aachen.de
+
+[proxies_rwth]
+ts.asta.rwth-aachen.de = ^(?!ts\.)[a-z0-9-]*.asta.rwth-aachen.de,
+	^(?!git\.)[a-z0-9-]*\.stud.rwth-aachen.de,
+	([a-z0-9-]+\.)*(relatif|achso).rwth-aachen.de
+
+[strips]
+rwth = rwth-aachen.de
+fsmpi = fsmpi.rwth-aachen.de
+asta = asta.rwth-aachen.de
+
+[excludes]
+hosts = ap-[a-z0-9]+.fsmpi.rwth-aachen.de,
+	(vl1023-)?dhcp-?[a-z]*[0-9]+.(fsmpi|asta).rwth-aachen.de,
+	vpn[0-9]+.fsmpi.rwth-aachen.de,
+	pr-[a-z0-9]+.(fsmpi|asta).rwth-aachen.de,
+	ipmi-[a-z0-9]+.(fsmpi|asta).rwth-aachen.de,
+	[a-z0-9-]*\._msdcs.(fsmpi|asta).rwth-aachen.de,
+	sw-[a-z-]+.(fsmpi|asta).rwth-aachen.de,
+aliases = fsmpi.rwth-aachen.de,
+	asta.rwth-aachen.de
+
+[aliases]
+learninglinux.fsmpi.rwth-aachen.de = lls
+portal.fsmpi.rwth-aachen.de = portal, fsmpi.rwth-aachen.de
+ts.asta.rwth-aachen.de = ts, asta.rwth-aachen.de
+
+[usernames]
+root = osak-build.fsmpi.rwth-aachen.de,
+	^video-?(?!(main|backup|rohdaten|test)\.)[a-z0-9-]*\.fsmpi.rwth-aachen.de,
+	(cirrus|stratus|pileus|lenticular|nacreous|cloud).fsmpi.rwth-aachen.de,
+	(monitoring|zabbix(-(db|srv))?|metafa).fsmpi.rwth-aachen.de,
+	(vm|fw)[0-9]+.asta.rwth-aachen.de,
+	(groupware|www|db|moni|ad|ad2|nightline|dns).asta.rwth-aachen.de,
+	git\.(fsmpi|stud).rwth-aachen.de,
+	[pf]-asta-([a-z]+-)?[0-9](-vl[0-9]+)?.asta.rwth-aachen.de,
+	([a-z0-9-]+\.)*(fachschaften|fslogo|esp|achso).rwth-aachen.de
+fsadmin = learninglinux.fsmpi.rwth-aachen.de
+manager = [e][0-9]{4}-asta-([a-z]+-)?[0-9](-vl[0-9]+)?.asta.rwth-aachen.de
+lars = [c][0-9]{4}-fsmpi-[0-9](-vl[0-9]+)?.fsmpi.rwth-aachen.de,
+	([a-z0-9-]+\.)*(noc).rwth-aachen.de
+larsb = ([a-z0-9-]+\.)*(fsmpi|video|vampir).rwth-aachen.de
+lbeckers = ([a-z0-9-]+\.)*(asta|stud).rwth-aachen.de
+
+[agents]
+disabled = osak-build(-legacy)?.fsmpi.rwth-aachen.de,
+	learninglinux.fsmpi.rwth-aachen.de,
+	www9.fsmpi.rwth-aachen.de,
+	^video-?(?!(main|backup|rohdaten|test)\.)[a-z0-9-]*\.fsmpi.rwth-aachen.de,
+	[ce][0-9]{4}-(fsmpi|asta)-[0-9](-vl[0-9]+)?\.(fsmpi|asta).rwth-aachen.de
+enabled = ([a-z0-9-]+\.)*(fsmpi|fachschaften|fslogo|esp|video|vampir).rwth-aachen.de,
+	([a-z0-9-]+\.)*(asta|stud|relatif|achso).rwth-aachen.de
+

zonedl.py

+#!/usr/bin/env python3
+
+import requests
+import bs4
+import re
+import argparse
+import getpass
+import subprocess
+import sys
+from pathlib import Path
+
+parser = argparse.ArgumentParser(description='Downloads a zone file from RWTH DNS-Admin-Portal.')
+group = parser.add_mutual_exclusive_group(required=True)
+group.add_argument('--list', action='store_true', default=False, help='list available zones')
+group.add_argument('--zone', type=int, nargs='+', help='download zone by id')
+group.add_argument('--domain', nargs='+', help='download zone by name')
+parser.add_argument('--passwordstore', action='store', default=None, help='password store entry used for login to the portal')
+parser.add_argument('dest', action='store', default='-', help='destination to store downloaded zone(s), - for stdout', nargs='?')
+args = parser.parse_args()
+
+DNS_ADMIN = 'https://noc-portal.rz.rwth-aachen.de/dnsadmin'
+ZONE_FILE = 'https://noc-portal.rz.rwth-aachen.de/dnsadmin/zones/{}/print'
+SHIB_PREFIX = 'https://sso.rwth-aachen.de'
+SHIB_AUTH = 'https://sso.rwth-aachen.de/idp/profile/SAML2/Redirect/SSO'
+SHIB_REDIRECT = 'https://noc-portal.rz.rwth-aachen.de/Shibboleth.sso/SAML2/POST'
+
+if args.passwordstore:
+    prc = subprocess.run(['pass', 'show', args.passwordstore], stdout=subprocess.PIPE, check=True)
+    USERNAME = prc.splitlines()[1].strip()
+    PASSWORD = prc.splitlines()[0].strip()
+else:
+    USERNAME = input('Username: ')
+    PASSWORD = getpass.getpass()
+
+def get_zones(session):
+    r = session.get(DNS_ADMIN)
+    if r.url.startswith(SHIB_AUTH):
+        res,session,r = shib_auth(session, r)
+        if not res:
+            return {}
+    b = bs4.BeautifulSoup(r.text, 'lxml')
+    z = b.find(id='zones_table').find('tbody')
+    d = {}
+    for zone in z.find_all('tr'):
+        a = zone.find('td').find('a')
+        #d[int(a['href'].split('/')[-1])] = a.text
+        d[a.text] = int(a['href'].split('/')[-1])
+    return d
+
+def get_zone_file(session, zone):
+    r = session.get(ZONE_FILE.format(str(zone)))
+    if r.url.startswith(SHIB_AUTH):
+        res,session,r = shib_auth(session, r)
+        if not res:
+            return ''
+    b = bs4.BeautifulSoup(r.text, 'lxml')
+    t = b.find(id='extra-wrapper').find('span').text
+    return t.replace('<br>', '\n')
+
+def shib_auth(session, resp, iterations=0):
+    b = bs4.BeautifulSoup(resp.text, 'lxml')
+    form = b.find('form')
+    data = {}
+    if form['name'] == 'loginformular':
+        data={'j_username': USERNAME, 'j_password': PASSWORD, 'donotcache': 'true', '_shib_idp_revokeConsent': 'false', '_eventId_proceed': ''}
+    elif form['name'] == 'form1':
+        data={'shib_idp_ls_exception.shib_idp_session_ss': '',
+                'shib_idp_ls_success.shib_idp_session_ss': 'false',
+                'shib_idp_ls_value.shib_idp_session_ss': '',
+                'shib_idp_ls_exception.shib_idp_persistent_ss': '',
+                'shib_idp_ls_success.shib_idp_persistent_ss': 'false',
+                'shib_idp_ls_value.shib_idp_persistent_ss': '',
+                'shib_idp_ls_supported': '',
+                '_eventId_proceed': '',
+                }
+    r = session.post(SHIB_PREFIX+form['action'], data=data)
+    b = bs4.BeautifulSoup(r.text, 'lxml')
+    f = b.find('form')['action']
+    if not f.startswith(SHIB_AUTH[len(SHIB_PREFIX):]):
+        relaystate = b.find('input', attrs={'name': 'RelayState'})['value']
+        samlresponse = b.find('input', attrs={'name': 'SAMLResponse'})['value']
+        r = session.post(f, data={'RelayState': relaystate, 'SAMLResponse': samlresponse})
+        return True,session,r
+    else:
+        if iterations == 0:
+            return shib_auth(session, r, 1)
+        # authentication failed
+        return False,session,r
+
+s = requests.session()
+d = get_zones(s)
+
+if args.dest == '-':
+    fp = sys.stdout
+elif Path(args.dest).is_dir():
+    fp = None
+else:
+    fp = open(args.dest, 'w')
+
+if args.list:
+    if fp is None:
+        fp = open(str(Path(args.dest) / 'list'), 'w')
+    for k in d:
+        print('{}\t{}'.format(d[k], k), file=fp)
+    sys.exit(0)
+
+wanted = []
+if args.domain:
+    for k in d:
+        if k in args.domain:
+            wanted.append(d[k])
+    for a in args.domain:
+        if a not in d:
+            print('Domain {} is not available, skipping'.format(a), file=sys.stderr)
+else:
+    for k in d:
+        if d[k] in args.zone:
+            wanted.append(d[k])
+    for a in args.zone:
+        if a not in wanted:
+            print('Zone {} is not available, skipping'.format(a), file=sys.stderr)
+
+for w in wanted:
+    if fp is None:
+        fp = open(str(Path(args.dest) / w), 'w')
+    print(get_zone_file(s, w), file=fp)
+