new uwsgi-python role with unified deployment

missing:
* testing
* multiple instances of one software on one host

uwsgi-python/files/uwsgi.service

+[Unit]
+Description=uWSGI service unit
+After=syslog.target
+
+[Service]
+ExecStart=/usr/bin/uwsgi --ini /etc/uwsgi/apps/%I.ini
+ExecReload=/bin/kill -HUP $MAINPID
+ExecStop=/bin/kill -INT $MAINPID
+Restart=always
+Type=notify
+StandardError=syslog
+NotifyAccess=all
+KillSignal=SIGQUIT
+
+[Install]
+WantedBy=multi-user.target

uwsgi-python/files/uwsgi.socket

+[Unit]
+Description=Socket for uWSGI %I
+
+[Socket]
+# Change this to your uwsgi application port or unix socket location
+ListenStream=/run/uwsgi/%I.sock
+
+[Install]
+WantedBy=sockets.target

uwsgi-python/handlers/main.yml

@@ -3,3 +3,10 @@
 
 - name: create tmpfiles
   command: systemd-tmpfiles --create
+
+- name: reload systemd service files
+  command: systemctl daemon-reload
+
+- name: restart uwsgi instance
+  service: name="uwsgi@{{ name }}" state=restarted
+

uwsgi-python/tasks/app.yml

+---
+
+
+- name: ensure we have python 2
+  apt:
+    name: "{{ item }}"
+    state: installed
+  with_items:
+    - python
+    - python-dev
+    - python-virtualenv
+    - uwsgi-plugin-python
+    - virtualenv
+  when: uwsgi_python == 2
+  tags:
+    - uwsgi-app
+    - "{{ app_name }}"
+
+- name: ensure we have python 3
+  apt:
+    name: "{{ item }}"
+    state: installed
+  with_items:
+    - python3
+    - python3-dev
+    - python3-virtualenv
+    - uwsgi-plugin-python3
+    - virtualenv
+  when: uwsgi_python == 3
+  tags:
+    - uwsgi-app
+    - "{{ app_name }}"
+
+- name: ensure we have the necessary libraries for ldap
+  apt:
+    name: "{{ item }}"
+    state: installed
+  with_items:
+    - libsasl2-dev
+    - libssl-dev
+    - libldap2-dev
+  tags:
+    - uwsgi-app
+    - "{{ app_name }}"
+
+- include: sqlite.yml
+  when: app_db == "sqlite"
+
+- include: mysql.yml
+  when: app_db == "mysql"
+
+- include: postgres.yml
+  when: app_db == "postgres"
+
+- name: ensure we have a group
+  group:
+    name: "{{ app_group }}"
+    system: yes
+    state: present
+  tags:
+    - uwsgi-app
+    - "{{ app_name }}"
+
+- name: ensure we have a user
+  user:
+    name: "{{ app_user }}"
+    group: "{{ app_group }}"
+    system: yes
+    home: "{{ app_home }}"
+    shell: /usr/bin/nologin
+    createhome: no
+    state: present
+  tags:
+    - uwsgi-app
+    - "{{ app_name }}"
+
+- name: ensure a temporary directory exists
+  template:
+    src: tmpfiles.conf.j2
+    dest: "/etc/tmpfiles.d/10-{{ app_name }}.conf"
+    owner: root
+    group: root
+    mode: 0644
+  notify:
+    - create tmpfiles
+  tags:
+    - uwsgi-app
+    - "{{ app_name }}"
+
+- name: ensure we have our uwsgi config file
+  template:
+    src: uwsgi.ini.j2
+    dest: "/etc/uwsgi/apps/{{ app_name }}.ini"
+    owner: root
+    group: root
+    mode: 0644
+  tags:
+    - uwsgi-app
+    - "{{ app_name }}"
+
+- name: ensure additional software is installed
+  apt: name={{ item }} state=present
+  with_items: "{{ app_additional_software }}"
+  when: 
+    - app_additional_software is defined
+  tags:
+    - uwsgi-app
+    - "{{ app_name }}"
+
+- name: ensure the deploy key is available
+  copy:
+    src: "{{ app_deploy_key }}"
+    dest: "/root/.ssh/{{ app_name }}"
+    owner: root
+    group: root
+    mode: 0600
+  tags:
+    - uwsgi-app
+    - "{{ app_name }}"
+
+# https://github.com/ansible/ansible/issues/27699
+- name: ensure git module is able to clone
+  command: mount -o remount,exec /tmp
+  tags:
+    - uwsgi-app
+    - "{{ app_name }}"
+
+- name: ensure we have the program
+  git:
+    repo: "{{ app_git_url }}"
+    dest: "{{ app_path }}"
+    key_file: "/root/.ssh/{{ app_name }}"
+    version: "{{ app_git_version }}"
+  notify:
+    - restart uwsgi instance
+  register: git 
+  tags:
+    - uwsgi-app
+    - "{{ app_name }}"
+
+- name: ensure git module is not able to clone anymore
+  command: mount -o remount,noexec /tmp
+  tags:
+    - uwsgi-app
+    - "{{ app_name }}"
+
+- name: ensure we have a virtualenv
+  pip:
+    requirements: "{{ app_path }}/requirements.txt"
+    virtualenv: "{{ app_venv }}"
+    virtualenv_python: "{{ app_python_version }}"
+  notify:
+    - restart uwsgi instance
+  tags:
+    - uwsgi-app
+    - "{{ app_name }}"
+
+- name: ensure we have our config
+  template:
+    src: "apps/{{ app_name }}.j2"
+    dest: "{{ app_path }}/{{ app_config_file }}"
+    owner: "{{ app_user }}"
+    group: "{{ app_group }}"
+    mode: 0640
+  notify:
+    - restart uwsgi instance
+  tags:
+    - uwsgi-app
+    - "{{ app_name }}"
+
+- name: ensure we have our secret config
+  template:
+    src: secret_config.py.j2
+    dest: "{{ app_path }}/secret_config.py"
+    owner: "{{ app_user }}"
+    group: "{{ app_group }}"
+    mode: 0600
+    force: no
+  notify:
+    - restart uwsgi instance
+  tags:
+    - uwsgi-app
+    - "{{ app_name }}"
+  when: 
+    - secret_config is defined
+    - secret_config == True
+
+
+- include_tasks: "{{ item }}"
+  with_first_found:
+    - files:
+        - "apps/{{ app_name }}.yml"
+      skip: true 
+  tags:
+    - uwsgi-app
+    - "{{ app_name }}"
+
+- meta: flush_handlers
+
+- name: ensure the service is enabled
+  service:
+    name: "uwsgi@{{ app_name }}"
+    enabled: yes
+    state: started
+  tags:
+    - uwsgi-app
+    - "{{ app_name }}"

uwsgi-python/tasks/main.yml

 ---
 # file: uwsgi-python/tasks/main.yml
 
-- name: ensure we have python 2
-  apt:
-    name: "{{ item }}"
-    state: installed
-  with_items:
-    - python
-    - python-dev
-    - python-virtualenv
-    - uwsgi-plugin-python
-    - virtualenv
-  when: uwsgi_python == 2
-  tags:
-    - uwsgi-python
-    - webservices
-
-- name: ensure we have python 3
-  apt:
-    name: "{{ item }}"
-    state: installed
-  with_items:
-    - python3
-    - python3-dev
-    - python3-virtualenv
-    - uwsgi-plugin-python3
-    - virtualenv
-  when: uwsgi_python == 3
-  tags:
-    - uwsgi-python
-    - webservices
 
-- name: ensure we have the necessary libraries for ldap
+- name: ensure uwsgi is installed
   apt:
-    name: "{{ item }}"
+    name: uwsgi
     state: installed
-  with_items:
-    - libsasl2-dev
-    - libssl-dev
-    - libldap2-dev
-  tags:
-    - uwsgi-python
-    - webservices
-
-- include: sqlite.yml
-  when: uwsgi_db == "sqlite"
-
-- include: mysql.yml
-  when: uwsgi_db == "mysql"
-
-- include: postgres.yml
-  when: uwsgi_db == "postgres"
-
-- name: ensure we have a group
-  group:
-    name: "{{ uwsgi_group }}"
-    system: yes
-    state: present
   tags:
-    - uwsgi-python
-    - webservices
-
-- name: ensure we have a user
-  user:
-    name: "{{ uwsgi_user }}"
-    group: "{{ uwsgi_group }}"
-    system: yes
-    home: "{{ uwsgi_home }}"
-    shell: /usr/bin/nologin
-    createhome: no
-    state: present
-  tags:
-    - uwsgi-python
+    - uwsgi
     - webservices
 
 - name: ensure a temporary directory exists
-  template:
-    src: tmpfiles.conf.j2
-    dest: "/etc/tmpfiles.d/10-{{uwsgi_name}}.conf"
+  copy:
+    src: tmpfiles.conf
+    dest: /etc/tmpfiles.d/10-uwsgi.conf
     owner: root
     group: root
     mode: 0644
   notify:
     - create tmpfiles
   tags:
-    - uwsgi-python
+    - uwsgi
     - webservices
 
-- name: ensure we have our uwsgi config file
-  template:
-    src: uwsgi.ini.j2
-    dest: "/etc/uwsgi/apps-available/{{ uwsgi_name }}.ini"
+- name: ensure we have archlinux's systemd-service file
+  copy: 
+    src: uwsgi.service
+    dest: /etc/systemd/system/uwsgi.service
     owner: root
     group: root
     mode: 0644
   tags:
-    - uwsgi-python
+    - uwsgi
     - webservices
+
+# TODO
+# enthält webapps eine mehr-instanz-app mehrfach? wenn ja, ist ../vars/item.yml doof
+# wenn nein, wie realisieren wir das? bsp: schildergenerator mit schilder, boxes
+- include_tasks: app.yml
+  vars_files:
+    - "../vars/{{ item }}.yml"
+    - "{{ inventory_dir }}/vars/{{ item }}.yml"
+  with_items: "{{ webapps }}"
+

uwsgi-python/tasks/mysql.yml

@@ -15,7 +15,7 @@
 
 - name: ensure the mysql database exists
   mysql_db:
-    name: "{{ uwsgi_name }}"
+    name: "{{ app_name }}"
     state: present
     login_user: root
     login_password: "{{ lookup('passwordstore', 'db/{{ansible_hostname}}-mysql create=true length=20') }}"
@@ -26,12 +26,12 @@
 
 - name: ensure the database user for mysql exists
   mysql_user:
-    name: "{{uwsgi_user}}"
+    name: "{{ app_user }}"
     password: "{{ lookup('passwordstore', 'db/{{ansible_hostname}}-mysql-{{uwsgi_user}} create=true length=20') }}"
     state: present
     login_user: root
     login_password: "{{lookup('passwordstore', 'db/{{ansible_hostname}}-mysql create=true length=20')}}"
-    priv: "{{uwsgi_name}}.*:ALL"
+    priv: "{{ app_name }}.*:ALL"
   no_log: True
   tags:
     - uwsgi-python

uwsgi-python/tasks/postgres.yml

@@ -3,7 +3,7 @@
 
 - name: ensure we have a postgres database user
   postgresql_user:
-    name: "{{ uwsgi_user }}"
+    name: "{{ app_user }}"
     state: present
   become: yes
   become_user: postgres
@@ -13,8 +13,8 @@
 
 - name: ensure we have a postgres database
   postgresql_db:
-    name: "{{ uwsgi_name }}"
-    owner: "{{ uwsgi_user }}"
+    name: "{{ app_name }}"
+    owner: "{{ app_user }}"
     state: present
   become: yes
   become_user: postgres
@@ -24,8 +24,8 @@
 
 - name: ensure the database user has privileges
   postgresql_privs:
-    database: "{{ uwsgi_name }}"
-    roles: "{{ uwsgi_user }}"
+    database: "{{ app_name }}"
+    roles: "{{ app_user }}"
     privs: ALL
     state: present
     type: database

uwsgi-python/templates/apps/shorturl.j2

+from secret_config import secret_key as SECRET_KEY
+SQLALCHEMY_DATABASE_URI = 'postgresql://{{ app_user }}:@/{{ app_name }}'
+DEFAULT_REDIRECT = '{{ shorturl_default_redirect }}'
+DEBUG = False
+SESSION_COOKIE_SECURE = True
+
+import datetime
+REMEMBER_COOKIE_NAME = 'remember_token'
+REMEMBER_COOKIE_DURATION = datetime.timedelta(30)
+REMEMBER_COOKIE_DOMAIN = None
+REMEMBER_COOKIE_PATH = '/'
+REMEMBER_COOKIE_SECURE = True
+REMEMBER_COOKIE_HTTPONLY = True
+
+ADMIN_GROUP = '{{ shorturl_admin_group }}'
+USER_GROUP = '{{ shorturl_user_group }}'
+
+AD_HOST = '{{ shorturl_ad_host }}'
+AD_DOMAIN = '{{ shorturl_ad_domain }}'
+AD_USER_DN = '{{ shorturl_ad_user_dn }}'
+AD_GROUP_DN = '{{ shorturl_ad_group_dn }}'
+AD_CA_CERT = '{{ shorturl_ad_cert }}'
+
+TARGET_REGEX = '{{ shorturl_target_regex }}'
+
+BRANDING_NAME = '{{ shorturl_branding_name }}'
+BRANDING_DOMAIN = '{{ shorturl_branding_domain }}'
+BRANDING_DOMAIN_REGEX = '{{ shorturl_branding_domain_regex }}'
+BRANDING_CONTACT = '{{ shorturl_branding_contact }}'
+
+MAIL_SUBJECT = '{{ shorturl_mail_subject }}'
+MAIL_DOMAIN = '{{ shorturl_mail_domain }}'
+MAIL_ADMIN = '{{ shorturl_mail_admin }}'
+MAIL_HOST = '{{ shorturl_mail_host }}'

uwsgi-python/templates/tmpfiles.conf.j2

-d /run/uwsgi/app/{{uwsgi_name}} 0775 {{uwsgi_user}} {{uwsgi_group}} - -
+d /run/uwsgi/{{uwsgi_name}} 0775 {{uwsgi_user}} {{uwsgi_group}} - -

uwsgi-python/templates/uwsgi.ini.j2

 [uwsgi]
-uwsgi-socket = /run/uwsgi/app/{{uwsgi_name}}/{{uwsgi_name}}.sock
+uwsgi-socket = /run/uwsgi/{{app_name}}/{{app_name}}.sock
 #http = localhost:5000
 chmod-socket = 660
-chown-socket = {{uwsgi_user}}:www-data
+chown-socket = {{app_user}}:www-data
 autoload =
 master =
 processes = 4
@@ -23,19 +23,16 @@ enable-threads =
 mule = 
 {% endfor %}
 #umask = 227
-chdir = {{uwsgi_path}}
-uid = {{uwsgi_user}}
-gid = {{uwsgi_group}}
-logto = /var/log/uwsgi/{{uwsgi_name}}.log
-logfile-chown = {{uwsgi_user}}:{{uwsgi_group}}
-logfile-chmod = 664
+chdir = {{app_path}}
+uid = {{app_user}}
+gid = {{app_group}}
 log-date =
 log-4xx = 
 log-5xx = 
 log-x-forwarded-for =
-{% if uwsgi_python == 2 %}
+{% if app_python_version == 2 %}
 plugin = python27
-{% elif uwsgi_python == 3 %}
+{% elif app_python_version == 3 %}
 {% if debian_version == "jessie" %}
 plugin = python34
 {% elif debian_version == "stretch" %}
@@ -46,9 +43,9 @@ plugin = {{uwsgi_python_plugin|mandatory}}{# or add new debian versions here #}
 {% else %}
 plugin = {{uwsgi_python_plugin|mandatory}}{# or add new python versions here #}
 {% endif %}
-virtualenv = {{uwsgi_venv|default(uwsgi_path)}}
-wsgi-file = {{uwsgi_path}}/{{uwsgi_program}}
-callable = {{uwsgi_callable}}
-pyargv = {{uwsgi_program}} {{uwsgi_command}}
+virtualenv = {{app_venv|default(app_path)}}
+wsgi-file = {{app_path}}/{{app_program}}
+callable = {{uapp_callable}}
+pyargv = {{app_program}} {{app_command}}
 manage-script-name =
-mount={{uwsgi_mountpoint}}={{uwsgi_path}}/{{uwsgi_program}}
+mount={{app_mountpoint}}={{app_path}}/{{app_program}}

uwsgi-python/tmpfiles.conf

+d /run/uwsgi 0755 root root - -
+d /run/uwsgi/app 0755 root root - -

uwsgi-python/vars/shorturl.yml

+app_name: shorturl
+app_user: shortlinks
+app_group: shortlinks
+app_home: /var/www/shorturl
+app_path: /var/www/shorturl
+app_python_version: 3
+app_venv: /var/www/shorturl/venv
+app_program: shorturl.py
+app_callable: app
+app_command: ""
+app_mountpoint: /
+
+app_db_type: postgres
+
+app_deploy_key: "{{ inventory_dir }}/files/deploy-keys/shorturl"
+app_git_url: "git@git.fsmpi.rwth-aachen.de:infra/shorturl.git"
+app_git_version: HEAD
+
+app_config_file: config.py
+app_secret_config: true
+
+shorturl_default_redirect: "https://www.example.com"
+shorturl_admin_group: admin
+shorturl_user_group: users
+
+shorturl_ad_host: 'ad.example.com'
+shorturl_ad_domain: 'EXAMPLE'
+shorturl_ad_user_dn: "cn=users,dc=example,dc=com"
+shorturl_ad_group_dn: "cn=users,dc=example,dc=com"
+shorturl_ad_cacert: "/etc/ssl/certs/example_cacert.pem"
+
+shorturl_target_regex: '^https://([a-zA-Z0-9-]+\.)*example\.com(/(.*))?$'
+shorturl_branding_name: 'Example'
+shorturl_branding_domain: 'short.example'
+shorturl_branding_domain_regex: '^(?!(https?://)?(www\.)?(short\.example)/?)(.*)'
+shorturl_branding_contact: 'contact@example.com'
+
+shorturl_mail_subject: 'confirmation request ShortURL service'
+shorturl_mail_domain: 'example.com'
+shorturl_mail_admin: 'contact@example.com'
+shorturl_mail_host: 'mail.example.com'
+

uwsgi-python/vars/template.yml

+name: 
+user
+group
+home
+path
+python_version
+venv
+program = server.py
+callable = app
+command 
+mountpoint
+
+db_type
+
+
+
+uwsgi_options 
+uwsgi_harakiri
+uwsgi_mules
+uwsgi_enable_threads
+
+app_deploy_key
+app_git_url
+app_git_version
+
+config_file
+secret_config = T/F
+

[Robin Sonnabend]

Da ist ein u zu viel.